Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Carl Schreck
Carl NCL tools
Commits
590c2629
Commit
590c2629
authored
May 07, 2019
by
Carl Schreck
Browse files
Automated Nightly Commit - Tue May 7 00:01:12 EDT 2019
parent
91522a1e
Changes
3
Hide whitespace changes
Inline
Side-by-side
era-interim/bsub_py.sh
0 → 100755
View file @
590c2629
#!/bin/bash --login
#if [ -z "$1" ]; then
# QUEUE=`pick_queue`
#else
# QUEUE=$1
#fi
QUEUE
=
allnodes
echo
$QUEUE
PY_SCRIPT
=
extract_interim
PY_DIR
=
`
pwd
`
LOG_DIR
=
$PY_DIR
/log
mkdir
-p
$LOG_DIR
for
YEAR
in
{
1979..2018
}
;
do
JOB_NAME
=
$PY_SCRIPT
"_"
$YEAR
echo
$JOB_NAME
`
date
`
LOG_FILE
=
$LOG_DIR
/
$JOB_NAME
.log
ERR_FILE
=
$LOG_DIR
/
$JOB_NAME
.err
rm
$LOG_FILE
$ERR_FILE
bsub
\
-J
$JOB_NAME
\
-o
$LOG_FILE
\
-e
$ERR_FILE
\
-q
$QUEUE
\
-n
1
-R
"span[hosts=1]"
-W
48:00
\
-sp
75
\
python
-u
$PY_DIR
/
$PY_SCRIPT
.py
$YEAR
done
era-interim/extract_interim.py
View file @
590c2629
...
...
@@ -18,13 +18,34 @@ else:
print
(
'Year = '
,
year
)
cjs
.
tstamp
(
'Here we go!'
)
basedir
=
'/home/carl/data/era-interim/incoming/'
os
.
makedirs
(
basedir
+
year
,
exist_ok
=
True
)
tar_files
=
glob
.
glob
(
basedir
+
'tar/*'
+
year
+
'*.tar'
)
print
(
tar_files
)
#tar = tarfile.open(basedir +
cjs
.
tstamp
(
'Untar'
)
os
.
makedirs
(
basedir
+
year
,
exist_ok
=
True
)
tar_files
=
glob
.
glob
(
basedir
+
'tar/*'
+
year
+
'*.tar'
)
for
tar_file
in
tar_files
:
tar
=
tarfile
.
open
(
tar_file
)
tar
.
extractall
(
basedir
+
year
)
cjs
.
tstamp
(
'Open'
)
grb_files
=
glob
.
glob
(
basedir
+
year
+
'/*'
+
year
+
'*schreck368330'
)
grb_files
.
sort
()
ds
=
xr
.
open_mfdataset
(
grb_files
,
engine
=
'cfgrib'
,
concat_dim
=
'time'
)
cjs
.
tstamp
(
'Read/Write'
)
ds
.
time
.
encoding
[
'units'
]
=
'days since 1800-01-01'
ds
.
time
.
encoding
[
'dtype'
]
=
'float64'
#ds.encoding['_FillValue'] = -9999.
ds
.
to_netcdf
(
basedir
+
'u850.'
+
year
+
'.nc'
)
cjs
.
tstamp
(
'Closing'
)
ds
.
close
()
cjs
.
tstamp
(
'Cleanup'
)
files
=
glob
.
glob
(
basedir
+
year
+
'/*'
)
for
file
in
files
:
os
.
remove
(
file
)
os
.
rmdir
(
basedir
+
year
)
cjs
.
tstamp
(
'Thank you, come again.'
)
era-interim/subset_interim.py
0 → 100644
View file @
590c2629
""" Read a latitude strip from annual files and write it into a single file."""
__author__
=
"Carl Schreck"
__email__
=
"cjschrec@ncsu.edu"
__copyright__
=
"Copyright 2019, North Carolina State University"
__license__
=
"BSD-3.0"
import
xarray
as
xr
import
sys
sys
.
path
.
append
(
'/home/carl/lib/python'
)
import
cjs
import
tarfile
,
os
,
glob
cjs
.
tstamp
(
'Here we go!'
)
var_name
=
'u850'
basedir
=
'/home/carl/data/era-interim/'
cjs
.
tstamp
(
'Open'
)
in_files
=
glob
.
glob
(
basedir
+
'/'
+
var_name
+
'.201*.nc'
)
in_files
.
sort
()
ds
=
xr
.
open_mfdataset
(
in_files
,
concat_dim
=
'time'
)
cjs
.
tstamp
(
'Read/Write'
)
ds
.
encoding
[
'_FillValue'
]
=
-
9999.
ds
.
to_netcdf
(
basedir
+
'total/'
+
var_name
+
'.nc'
)
cjs
.
tstamp
(
'Closing'
)
ds
.
close
()
cjs
.
tstamp
(
'Thank you, come again.'
)
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment