Automated Nightly Commit - Tue May 7 00:01:12 EDT 2019

era-interim/bsub_py.sh

+#!/bin/bash --login
+#if [ -z "$1" ]; then
+#  QUEUE=`pick_queue`
+#else
+#  QUEUE=$1
+#fi
+QUEUE=allnodes
+echo $QUEUE
+
+PY_SCRIPT=extract_interim
+PY_DIR=`pwd`
+LOG_DIR=$PY_DIR/log
+mkdir -p $LOG_DIR
+
+for YEAR in {1979..2018}; do
+  JOB_NAME=$PY_SCRIPT"_"$YEAR
+  echo $JOB_NAME `date`
+  LOG_FILE=$LOG_DIR/$JOB_NAME.log
+  ERR_FILE=$LOG_DIR/$JOB_NAME.err
+
+
+  rm $LOG_FILE $ERR_FILE
+  bsub \
+    -J $JOB_NAME \
+    -o $LOG_FILE \
+    -e $ERR_FILE \
+    -q $QUEUE \
+    -n 1 -R "span[hosts=1]" -W 48:00 \
+    -sp 75 \
+    python -u $PY_DIR/$PY_SCRIPT.py $YEAR
+
+
+done
+

era-interim/extract_interim.py

@@ -18,13 +18,34 @@ else:
 print('Year = ', year)
 
 cjs.tstamp('Here we go!')
-
 basedir = '/home/carl/data/era-interim/incoming/'
-os.makedirs(basedir + year, exist_ok=True)
-tar_files = glob.glob(basedir + 'tar/*' + year + '*.tar')
-print(tar_files)
-#tar = tarfile.open(basedir + 
 
+cjs.tstamp('Untar')
+os.makedirs(basedir+year, exist_ok=True)
+tar_files = glob.glob(basedir + 'tar/*' + year + '*.tar')
+for tar_file in tar_files:
+    tar = tarfile.open(tar_file)
+    tar.extractall(basedir+year)
+
+cjs.tstamp('Open')
+grb_files = glob.glob(basedir + year + '/*' + year + '*schreck368330')
+grb_files.sort()
+ds = xr.open_mfdataset(grb_files, engine='cfgrib', concat_dim='time')
+
+cjs.tstamp('Read/Write')
+ds.time.encoding['units'] = 'days since 1800-01-01'
+ds.time.encoding['dtype'] = 'float64'
+#ds.encoding['_FillValue'] = -9999.
+ds.to_netcdf(basedir + 'u850.' + year + '.nc' )
+
+cjs.tstamp('Closing')
+ds.close()
+
+cjs.tstamp('Cleanup')
+files = glob.glob(basedir + year + '/*')
+for file in files:
+    os.remove(file)
+os.rmdir(basedir + year)
 
 cjs.tstamp('Thank you, come again.')
 

era-interim/subset_interim.py

+""" Read a latitude strip from annual files and write it into a single file."""
+
+__author__ = "Carl Schreck"
+__email__ = "cjschrec@ncsu.edu"
+__copyright__ = "Copyright 2019, North Carolina State University"
+__license__ = "BSD-3.0"
+
+import xarray as xr
+import sys
+sys.path.append('/home/carl/lib/python')
+import cjs
+import tarfile, os, glob
+
+cjs.tstamp('Here we go!')
+var_name = 'u850'
+basedir = '/home/carl/data/era-interim/'
+
+cjs.tstamp('Open')
+in_files = glob.glob(basedir + '/' + var_name + '.201*.nc')
+in_files.sort()
+ds = xr.open_mfdataset(in_files, concat_dim='time')
+
+cjs.tstamp('Read/Write')
+ds.encoding['_FillValue'] = -9999.
+ds.to_netcdf(basedir + 'total/' + var_name + '.nc' )
+
+cjs.tstamp('Closing')
+ds.close()
+
+cjs.tstamp('Thank you, come again.')
+