Commit 590c2629 authored by Carl Schreck's avatar Carl Schreck

Automated Nightly Commit - Tue May 7 00:01:12 EDT 2019

parent 91522a1e
#!/bin/bash --login
#if [ -z "$1" ]; then
# QUEUE=`pick_queue`
#else
# QUEUE=$1
#fi
QUEUE=allnodes
echo $QUEUE
PY_SCRIPT=extract_interim
PY_DIR=`pwd`
LOG_DIR=$PY_DIR/log
mkdir -p $LOG_DIR
for YEAR in {1979..2018}; do
JOB_NAME=$PY_SCRIPT"_"$YEAR
echo $JOB_NAME `date`
LOG_FILE=$LOG_DIR/$JOB_NAME.log
ERR_FILE=$LOG_DIR/$JOB_NAME.err
rm $LOG_FILE $ERR_FILE
bsub \
-J $JOB_NAME \
-o $LOG_FILE \
-e $ERR_FILE \
-q $QUEUE \
-n 1 -R "span[hosts=1]" -W 48:00 \
-sp 75 \
python -u $PY_DIR/$PY_SCRIPT.py $YEAR
done
......@@ -18,13 +18,34 @@ else:
print('Year = ', year)
cjs.tstamp('Here we go!')
basedir = '/home/carl/data/era-interim/incoming/'
os.makedirs(basedir + year, exist_ok=True)
tar_files = glob.glob(basedir + 'tar/*' + year + '*.tar')
print(tar_files)
#tar = tarfile.open(basedir +
cjs.tstamp('Untar')
os.makedirs(basedir+year, exist_ok=True)
tar_files = glob.glob(basedir + 'tar/*' + year + '*.tar')
for tar_file in tar_files:
tar = tarfile.open(tar_file)
tar.extractall(basedir+year)
cjs.tstamp('Open')
grb_files = glob.glob(basedir + year + '/*' + year + '*schreck368330')
grb_files.sort()
ds = xr.open_mfdataset(grb_files, engine='cfgrib', concat_dim='time')
cjs.tstamp('Read/Write')
ds.time.encoding['units'] = 'days since 1800-01-01'
ds.time.encoding['dtype'] = 'float64'
#ds.encoding['_FillValue'] = -9999.
ds.to_netcdf(basedir + 'u850.' + year + '.nc' )
cjs.tstamp('Closing')
ds.close()
cjs.tstamp('Cleanup')
files = glob.glob(basedir + year + '/*')
for file in files:
os.remove(file)
os.rmdir(basedir + year)
cjs.tstamp('Thank you, come again.')
""" Read a latitude strip from annual files and write it into a single file."""
__author__ = "Carl Schreck"
__email__ = "cjschrec@ncsu.edu"
__copyright__ = "Copyright 2019, North Carolina State University"
__license__ = "BSD-3.0"
import xarray as xr
import sys
sys.path.append('/home/carl/lib/python')
import cjs
import tarfile, os, glob
cjs.tstamp('Here we go!')
var_name = 'u850'
basedir = '/home/carl/data/era-interim/'
cjs.tstamp('Open')
in_files = glob.glob(basedir + '/' + var_name + '.201*.nc')
in_files.sort()
ds = xr.open_mfdataset(in_files, concat_dim='time')
cjs.tstamp('Read/Write')
ds.encoding['_FillValue'] = -9999.
ds.to_netcdf(basedir + 'total/' + var_name + '.nc' )
cjs.tstamp('Closing')
ds.close()
cjs.tstamp('Thank you, come again.')
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment