Commit 76405d47 authored by Carl Schreck's avatar Carl Schreck

working version on shiva

parent 05f24172
""" Kick off the cluster jobs to make monitoring maps for custom periods in
month."""
__author__ = "Carl Schreck"
__email__ = "carl_schreck@ncsu.edu"
__copyright__ = "Copyright 2020, North Carolina State University"
__license__ = "BSD-3.0"
import numpy as np
import os
import argparse
import subprocess
from datetime import datetime
import glob
import calendar
import cjs # '~carl/lib/python'
# These are some parameters that could be useful to have up top
parser = argparse.ArgumentParser()
parser.add_argument('--year', default=(datetime.today().year))
parser.add_argument('--month', default=(datetime.today().month - 1))
parser.add_argument('--dpi', default=200)
args = parser.parse_args()
year = int(args.year)
month = int(args.month)
dpi = int(args.dpi)
if month < 1:
month += 12
year -= 1
first_days = np.array([1, 7, 12, 19, 28])
days_in_month = calendar.monthrange(year, month)[1]
clobber = False
var_names = ['z500', 'prcp', 'tavg']
last_days = first_days[1:] - 1
last_days = np.append(last_days, days_in_month)
first_days = np.append(first_days, 1)
last_days = np.append(last_days, days_in_month)
cjs.tstamp('Here we go!')
fig_dir = f'figures/final/'
os.makedirs(fig_dir, exist_ok=True)
old_files = glob.glob(f'{fig_dir}/*')
for file in old_files:
os.remove(file)
for var_name in var_names:
cjs.tstamp(f'Downloading {var_name}')
if var_name == 'z500':
file_name = f'hgt.{year}.nc'
in_path = f'{os.environ["DATA_DIR"]}/ncep1/esrl/daily/{file_name}'
ftp_path = (
f'ftp://ftp.cdc.noaa.gov/Datasets/ncep.reanalysis.dailyavgs/'
f'pressure/{file_name}')
download_err = cjs.download_file(ftp_path, in_path, clobber=False)
else:
file_type = 'scaled'
file_name = f'{var_name}-{year}{month:02}-grd-{file_type}.nc'
in_path = f'{os.environ["DATA_DIR"]}/nclimgrid/beta/{var_name}/' \
f'{file_name}'
ftp_path = (
f'ftp://ftp.ncdc.noaa.gov/pub/data/daily-grids/beta/by-month/'
f'{year}/{month:02}/{file_name}')
web_path = (
f'https://www1.ncdc.noaa.gov/pub/data/daily-grids/beta/by-month/'
f'{year}/{month:02}/{file_name}')
download_err = cjs.download_file(ftp_path, in_path, clobber=False)
if download_err:
file_type = 'prelim'
file_name = f'{var_name}-{year}{month:02}-grd-{file_type}.nc'
in_path = f'{os.environ["DATA_DIR"]}/nclimgrid/beta/{var_name}/' \
f'{file_name}'
web_path = (
f'https://www1.ncdc.noaa.gov/pub/data/daily-grids/beta/by-month/'
f'{year}/{month:02}/{file_name}')
ftp_path = (
f'ftp://ftp.ncdc.noaa.gov/pub/data/daily-grids/beta/by-month/'
f'{year}/{month:02}/{file_name}')
download_err = cjs.download_file(ftp_path, in_path, clobber=True)
for first_day, last_day in zip(first_days, last_days):
py_dir = os.getcwd()
py_script = f'{var_name}_map.py'
log_dir = f'{py_dir}/log'
os.makedirs(log_dir, exist_ok=True)
job_name = f'{var_name}_{first_day}_{last_day}'
log_file = f'{log_dir}/{job_name}.log'
err_file = f'{log_dir}/{job_name}.err'
try:
os.remove(log_file)
except OSError:
pass
try:
os.remove(err_file)
except OSError:
pass
py_option = f'--year {year} --month {month} ' \
f'--first_day {first_day} --last_day {last_day} ' \
f'--dpi {dpi} --fig_dir {fig_dir}'
command = f'bsub -J {job_name} -o {log_file} -e {err_file} ' \
f'-q allnodes -sp 75 /home/carl/template/run_py.sh ' \
f'{py_dir} {py_script} "{py_option}"'
subprocess.Popen(command, shell=True)
cjs.tstamp('Thank you, come again.')
""" Kick off the cluster jobs to make monitoring maps for the month."""
""" Kick off the cluster jobs to make monitoring maps for each day of the
month."""
__author__ = "Carl Schreck"
__email__ = "carl_schreck@ncsu.edu"
__copyright__ = "Copyright 2020, North Carolina State University"
__license__ = "BSD-3.0"
import numpy as np
import os
import argparse
import subprocess
from datetime import datetime
import glob
import calendar
import cjs # '~carl/lib/python'
# These are some parameters that could be useful to have up top
parser = argparse.ArgumentParser()
parser.add_argument('--year', default=(datetime.today().year))
parser.add_argument('--month', default=(datetime.today().month - 1))
parser.add_argument('--dpi', default=200)
parser.add_argument('--dpi', default=100)
args = parser.parse_args()
year = int(args.year)
month = int(args.month)
dpi = int(args.dpi)
......@@ -22,15 +28,49 @@ if month < 1:
month += 12
year -= 1
start_day = 1
days_in_month = calendar.monthrange(year, month)[1]
clobber = False
days_in_month = 6 #######################
var_names = ['z500', 'prcp', 'temp']
var_names = ['z500', 'prcp', 'tavg']
delta_days = [1, 5]
cjs.tstamp('Here we go!')
for var_name in var_names:
cjs.tstamp(f'Downloading {var_name}')
if var_name == 'z500':
file_name = f'hgt.{year}.nc'
in_path = f'{os.environ["DATA_DIR"]}/ncep1/esrl/daily/{file_name}'
ftp_path = (
f'ftp://ftp.cdc.noaa.gov/Datasets/ncep.reanalysis.dailyavgs/'
f'pressure/{file_name}')
download_err = cjs.download_file(ftp_path, in_path, clobber=False)
else:
file_type = 'scaled'
file_name = f'{var_name}-{year}{month:02}-grd-{file_type}.nc'
in_path = f'{os.environ["DATA_DIR"]}/nclimgrid/beta/{var_name}/' \
f'{file_name}'
ftp_path = (
f'ftp://ftp.ncdc.noaa.gov/pub/data/daily-grids/beta/by-month/'
f'{year}/{month:02}/{file_name}')
web_path = (
f'https://www1.ncdc.noaa.gov/pub/data/daily-grids/beta/by-month/'
f'{year}/{month:02}/{file_name}')
download_err = cjs.download_file(ftp_path, in_path, clobber=False)
if download_err:
file_type = 'prelim'
file_name = f'{var_name}-{year}{month:02}-grd-{file_type}.nc'
in_path = f'{os.environ["DATA_DIR"]}/nclimgrid/beta/{var_name}/' \
f'{file_name}'
web_path = (
f'https://www1.ncdc.noaa.gov/pub/data/daily-grids/beta/by-month/'
f'{year}/{month:02}/{file_name}')
ftp_path = (
f'ftp://ftp.ncdc.noaa.gov/pub/data/daily-grids/beta/by-month/'
f'{year}/{month:02}/{file_name}')
download_err = cjs.download_file(ftp_path, in_path, clobber=True)
for delta_day in delta_days:
fig_dir = f'figures/{var_name}-{delta_day}'
os.makedirs(fig_dir, exist_ok=True)
......@@ -38,8 +78,8 @@ for var_name in var_names:
for file in old_files:
os.remove(file)
first_days = np.arange(start_day, days_in_month - delta_days + 2, 1)
last_days = np.arange(start_day + delta_days - 1, days_in_month + 1, 1)
first_days = np.arange(start_day, days_in_month - delta_day + 2, 1)
last_days = np.arange(start_day + delta_day - 1, days_in_month + 1, 1)
for first_day, last_day in zip(first_days, last_days):
py_dir = os.getcwd()
......@@ -49,17 +89,25 @@ for var_name in var_names:
job_name = f'{var_name}_{first_day}_{last_day}'
log_file = f'{log_dir}/{job_name}.log'
err_file = f'{log_dir}/{job_name}.err'
os.remove(log_file)
os.remove(err_file)
try:
os.remove(log_file)
except OSError:
pass
try:
os.remove(err_file)
except OSError:
pass
py_option = f'--year {year} --month {month} ' \
f'--first_day {first_day} --last_day {last_day} ' \
f'--dpi {dpi} --fig_dir {fig_dir} --no_terrain'
f'--dpi {dpi} --fig_dir {fig_dir}'
if var_name != 'z500':
py_option = f'{py_option} --no_terrain'
command = f'bsub -J {job_name} -o {log_file} -e {err_file} ' \
f'-q allnodes -sp 75 /home/carl/template/run_py.sh' \
f'-q allnodes -sp 75 /home/carl/template/run_py.sh ' \
f'{py_dir} {py_script} "{py_option}"'
subprocess.run(command, shell=True)
subprocess.Popen(command, shell=True)
......
Here we go! | 2020-07-09 14:04:48
Downloading/opening data | 2020-07-09 14:04:48
Downloading: IBTrACS.ALL.v04r00.nc | 2020-07-09 14:04:48
North Atlantic: June - June : 1851 | 2020-07-09 14:04:51
Calculating ACE | 2020-07-09 14:04:51
Count 35 | 2020-07-09 14:04:54
Count 64 | 2020-07-09 14:04:54
Count 96 | 2020-07-09 14:04:55
North Atlantic: Tropical Storms
2020: 1
Rank: 8
Tied with: 14
Average: 0.6
Record: 3
Years: [1886, 1909, 1936, 1968]
North Atlantic: Hurricanes
2020: 0
Rank: 6
Tied with: 34
Average: 0.13333333333333333
Record: 3
Years: [1886]
North Atlantic: Major Hurricanes
2020: 0
Rank: 1
Tied with: 39
Average: 0.0
Record: 1
Years: [1957, 1966]
North Atlantic: ACE
2020: 3.809
Rank: 5
Tied with: 0
Average: 1.203892615740883
Record: 20.577100000041284
Years: [1886]
Individual North Atlantic Storms: | 2020-07-09 14:04:56
id: 2020150N09269
name: Amanda:Cristobal
category: TS
max wind: 50.0 kt / 57.49999999999999 mph
min pressure: 992.0
Jun 2 - Jun 8
id: 2020173N34287
name: Dolly
category: TS
max wind: 40.0 kt / 46.0 mph
min pressure: 1002.0
Jun 23 - Jun 24
Eastern Pacific: June - June : 1949 | 2020-07-09 14:04:57
Calculating ACE | 2020-07-09 14:04:57
Count 35 | 2020-07-09 14:04:57
Count 64 | 2020-07-09 14:04:58
Count 96 | 2020-07-09 14:04:59
Eastern Pacific: Tropical Storms
2020: 1
Rank: 26
Tied with: 10
Average: 1.9
Record: 5
Years: [1985, 2018]
Eastern Pacific: Hurricanes
2020: 0
Rank: 27
Tied with: 13
Average: 0.8666666666666667
Record: 3
Years: [1984]
Eastern Pacific: Major Hurricanes
2020: 0
Rank: 14
Tied with: 26
Average: 0.36666666666666664
Record: 2
Years: [1978, 1984, 2010, 2018]
Eastern Pacific: ACE
2020: 0.18375000000000002
Rank: 37
Tied with: 0
Average: 10.921143055558606
Record: 40.224652083409396
Years: [2015]
Individual Eastern Pacific Storms: | 2020-07-09 14:04:59
id: 2020174N11231
name: Boris
category: TS
max wind: 35.0 kt / 40.25 mph
min pressure: 1005.0
Jun 25 - Jun 26
Global Category 5s | 2020-07-09 14:05:00
Thank you, come again. | 2020-07-09 14:05:00
Here we go! | 2020-07-09 14:04:33
Downloading/opening data | 2020-07-09 14:04:33
Downloading: IBTrACS.ALL.v04r00.nc | 2020-07-09 14:04:33
North Atlantic: January - June : 1851 | 2020-07-09 14:04:38
Calculating ACE | 2020-07-09 14:04:38
Count 35 | 2020-07-09 14:04:46
Count 64 | 2020-07-09 14:04:49
Count 96 | 2020-07-09 14:04:52
North Atlantic: Tropical Storms
2020: 3
Rank: 3
Tied with: 1
Average: 0.7666666666666667
Record: 4
Years: [2012, 2016]
North Atlantic: Hurricanes
2020: 0
Rank: 7
Tied with: 33
Average: 0.13333333333333333
Record: 3
Years: [1886]
North Atlantic: Major Hurricanes
2020: 0
Rank: 1
Tied with: 39
Average: 0.0
Record: 1
Years: [1957, 1966]
North Atlantic: ACE
2020: 5.99545
Rank: 4
Tied with: 0
Average: 1.5708192824075493
Record: 21.4852
Years: [1951]
Individual North Atlantic Storms: | 2020-07-09 14:04:55
id: 2020137N24278
name: Arthur
category: TS
max wind: 52.0 kt / 59.8 mph
min pressure: 990.0
May 17 - May 19
id: 2020147N27281
name: Bertha
category: TS
max wind: 45.0 kt / 51.74999999999999 mph
min pressure: 1007.0
May 27 - May 27
id: 2020150N09269
name: Amanda:Cristobal
category: TS
max wind: 50.0 kt / 57.49999999999999 mph
min pressure: 992.0
Jun 2 - Jun 8
id: 2020173N34287
name: Dolly
category: TS
max wind: 40.0 kt / 46.0 mph
min pressure: 1002.0
Jun 23 - Jun 24
Eastern Pacific: January - June : 1949 | 2020-07-09 14:04:55
Calculating ACE | 2020-07-09 14:04:55
Count 35 | 2020-07-09 14:04:58
Count 64 | 2020-07-09 14:04:59
Count 96 | 2020-07-09 14:05:01
Eastern Pacific: Tropical Storms
2020: 2
Rank: 22
Tied with: 11
Average: 2.6666666666666665
Record: 5
Years: [1984, 1985, 1990, 1992, 2014, 2018]
Eastern Pacific: Hurricanes
2020: 0
Rank: 31
Tied with: 9
Average: 1.1666666666666667
Record: 4
Years: [1984]
Eastern Pacific: Major Hurricanes
2020: 0
Rank: 17
Tied with: 23
Average: 0.4666666666666667
Record: 2
Years: [1978, 1984, 1992, 2010, 2014, 2015, 2018]
Eastern Pacific: ACE
2020: 0.36750000000000005
Rank: 40
Tied with: 0
Average: 14.076890138891605
Record: 49.5312020834094
Years: [2015]
Individual Eastern Pacific Storms: | 2020-07-09 14:05:02
id: 2020150N09269
name: Amanda:Cristobal
category: TS
max wind: 35.0 kt / 40.25 mph
min pressure: 1003.0
May 31 - May 31
id: 2020174N11231
name: Boris
category: TS
max wind: 35.0 kt / 40.25 mph
min pressure: 1005.0
Jun 25 - Jun 26
Global Category 5s | 2020-07-09 14:05:03
name: Harold
id: 2020094S10160
max wind: 165.6 mph
min pressure: 912.0
basin: SP
Apr 2 - Apr 10
name: Amphan
id: 2020138N10086
max wind: 165.6 mph
min pressure: 901.0
basin: NI
May 16 - May 21
Thank you, come again. | 2020-07-09 14:05:03
......@@ -26,7 +26,7 @@ end_year = int(args.year)
end_month = int(args.month)
end_month = np.where(end_month < 1, end_month+12, end_month)
do_ytd = True
do_ytd = False
if do_ytd:
months = range(1, end_month + 1)
else:
......
......@@ -25,6 +25,7 @@ parser.add_argument('--dpi', default=200)
parser.add_argument('--fig_dir', default='figures/')
parser.add_argument('--no_terrain', dest='terrain', action='store_false')
parser.set_defaults(terrain=True)
args = parser.parse_args()
year = int(args.year)
month = int(args.month)
if month < 1:
......
......@@ -25,6 +25,7 @@ parser.add_argument('--dpi', default=200)
parser.add_argument('--fig_dir', default='figures/')
parser.add_argument('--no_terrain', dest='terrain', action='store_false')
parser.set_defaults(terrain=True)
args = parser.parse_args()
year = int(args.year)
month = int(args.month)
if month < 1:
......
......@@ -74,7 +74,7 @@ plt.legend(labels=[f'{x.upper()}: '
plt.tight_layout()
plt.savefig(f'figures/teleconnections-{year}-{month:02}.png', dpi=200)
plt.show()
# plt.show()
plt.close()
cjs.tstamp('Thank you, come again.')
......
......@@ -22,9 +22,10 @@ parser = argparse.ArgumentParser()
parser.add_argument('--year', default=(datetime.today().year))
parser.add_argument('--month', default=(datetime.today().month - 1))
parser.add_argument('--first_day', default=25)
parser.add_argument('--last_day', default=31)
parser.add_argument('--last_day', default=30)
parser.add_argument('--dpi', default=200)
parser.add_argument('--fig_dir', default='figures/')
args = parser.parse_args()
year = int(args.year)
month = int(args.month)
if month < 1:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment