Skip to content

GitLab

Commit 848d62da authored by abuddenberg's avatar abuddenberg
Browse files

Work product snapshot

parent 4c3cb68f
Hide whitespace changes
Inline Side-by-side
bin/problems
View file @ 848d62da
......@@ -20,6 +20,8 @@ whitelisted_dupes = [
'8c679de0-4a6c-4fc9-bbd1-59251ffe3d7f', #2.31 #34.21
'a1bd40f2-424e-4af8-8d3d-f44ba7ba36b6', #33.17 #34.13
'834f7624-40de-4a09-9ced-faaca30d4d6d', #2.27 #18.7
]
def main():
......
bin/sync_figures
View file @ 848d62da
......@@ -7,109 +7,109 @@ from collections import OrderedDict
webform = WebformClient('http://resources.assessment.globalchange.gov', webform_token)
# gcis = GcisClient('http://data.gcis-dev-front.joss.ucar.edu', *gcis_dev_auth)
gcis = GcisClient('http://data-stage.globalchange.gov', *gcis_stage_auth)
gcis = GcisClient('http://data.gcis-dev-front.joss.ucar.edu', *gcis_dev_auth)
# gcis = GcisClient('http://data-stage.globalchange.gov', *gcis_stage_auth)
sync_metadata_tree = {
#Reports
'nca3': OrderedDict([
#Chapter 2
('our-changing-climate', [
# (webform_url, gcis_id)
('/metadata/figures/3074', 'ten-indicators-of-a-warming-world'), # 2.1
('/metadata/figures/3293', 'observed-increase-in-frostfree-season-length'), # 2.10
('/metadata/figures/3294', 'projected-changes-in-frostfree-season-length'), # 2.11
('/metadata/figures/2677', 'observed-us-precipitation-change'), # 2.12
('/metadata/figures/3298', 'observed-us-trend-in-heavy-precipitation'), # 2.16
('/metadata/figures/2506', 'observed-change-in-very-heavy-precipitation'), # 2.17
('/metadata/figures/2997', 'observed-change-in-very-heavy-precipitation-2'), # 2.18
('/metadata/figures/3170', 'global-temperature-and-carbon-dioxide'), # 2.2
('/metadata/figures/2939', 'projected-changes-in-soil-moisture-for-the-western-us'), # 2.22
('/metadata/figures/3305', 'variation-of-storm-frequency-and-intensity-during-the-cold-season-november--march'), # 2.24
('/metadata/figures/3067', 'ice-cover-in-the-great-lakes'), # 2.27
('/metadata/figures/3073', 'projected-arctic-sea-ice-decline'), # 2.29
('/metadata/figures/2940', 'as-oceans-absorb-co-they-become-more-acidic'), # 2.30
('/metadata/figures/2523', 'shells-dissolve-in-acidified-ocean-water'), # 2.31
('/metadata/figures/3175', 'observed-us-temperature-change'), # 2.7
]),
#Chapter 3
('water-resources', [
('/metadata/figures/2676', 'projected-changes-in-snow-runoff-and-soil-moisture'), # 3.1
('/metadata/figures/3303', 'trends-in-flood-magnitude'), # 3.5
]),
#Chapter 4
('energy-supply-and-use', [
('/metadata/figures/3082', 'paths-of-hurricanes-katrina-and-rita-relative-to-oil-and-gas-production-facilities'), # 4.1
('/metadata/figures/3292', 'cooling-degree-days'), # 4.3
]),
#Chapter 5
('transportation', [
('/metadata/figures/3088', 'possible-future-flood-depths-in-mobile-al-with-rising-sea-level'), # 5.1
('/metadata/figures/2905', 'gulf-coast-transportation-hubs-at-risk'), # 5.3
('/metadata/figures/3568', 'hurricane-sandy-causes-flooding-in-new-york-city-subway-stations'), # 5.4
('/metadata/figures/2952', 'role-of-adaptive-strategies-and-tactics-in-reducing-impacts-and-consequences'), # 5.5
('/metadata/figures/3169', 'tropical-storm-impact-on-vermont-road'), # 5.6
]),
#Chapter 6
('agriculture', [
('/metadata/figures/2691', 'projected-changes-in-key-climate-variables-affecting-agricultural-productivity'), # 6.5
('/metadata/figures/2872', 'drainage-tiles'), # 6.8
]),
#Chapter 7
('forests', [
('/metadata/figures/2887', 'forest-ecosystem-disturbances'), # 7.1
('/metadata/figures/2977', 'effectiveness-of-forest-management-in-reducing-wildfire-risk'), # 7.2
('/metadata/figures/2978', 'forest-vulnerability-to-changing-climate'), # 7.3
('/metadata/figures/2985', 'forests-can-be-a-source--or-a-sink--for-carbon'), # 7.4
('/metadata/figures/3130', 'public-and-private-forestlands'), # 7.8
]),
#Chapter 8
('ecosystems', [
('/metadata/figures/2456', 'adaptation-planning-and-implementation-framework'), # 8.3
('/metadata/figures/3574', 'biological-responses-to-climate-change'), # 8.4
]),
#Chapter 9
('human-health', [
('/metadata/figures/3162', 'projected-climate-change-worsens-asthma'), # 9.1
('/metadata/figures/2896', 'heavy-downpours-disease'), # 9.7
('/metadata/figures/2897', 'harmful-bloom-of-algae'), # 9.8
]),
#Chapter 10
('water-energy-land-use', [
('/metadata/figures/2601', 'energy-water-land-and-climate-interactions'), # 10.1
('/metadata/figures/2917', 'the-columbia-river-basin-land-use-and-land-cover'), # 10.10
('/metadata/figures/2410', 'coasttocoast-100degree-days-in-2011'), # 10.2
('/metadata/figures/3158', 'projected-landuse-intensity-in-2030'), # 10.6
('/metadata/figures/2986', 'hydraulic-fracturing-and-water-use'), # 10.7
('/metadata/figures/2916', 'renewable-energy-and-land-use'), # 10.8
('/metadata/figures/2918', 'water-stress-in-the-us'), # 10.9
]),
#Chapter 11
('urban-systems-infrastructure-vulnerability', [
('/metadata/figures/3569', 'urban-support-systems-are-interconnected'), # 11.2
('/metadata/figures/3090', 'new-york-city-and-sea-level-rise'), # 11.3
]),
#Chapter 12
('tribal-indigenous-native-lands-resources', [
('/metadata/figures/3131', 'indigenous-populations-extend-beyond-reservation-lands'), # 12.1
('/metadata/figures/2594', 'many-tribes-many-climate-change-initiatives'), # 12.2
('/metadata/figures/2909', 'sand-dune-expansion'), # 12.3
('/metadata/figures/2911', 'arctic-marine-food-web'), # 12.5
]),
#Chapter 13
('land-use-land-cover-change', [
('/metadata/figures/2902', 'building-loss-by-fires-at-california-wildlandurban-interfaces'), # 13.4
]),
#Chapter 14
('rural', [
('/metadata/figures/2662', 'rural-counties'), # 14.1
('/metadata/figures/2661', 'economic-dependence-varies-by-region'), # 14.2
('/metadata/figures/3306', 'growing-season-lengthens'), # 14.3
('/metadata/figures/2904', 'many-rural-areas-are-losing-population'), # 14.5
]),
# ('our-changing-climate', [
# # (webform_url, gcis_id)
# ('/metadata/figures/3074', 'ten-indicators-of-a-warming-world'), # 2.1
# ('/metadata/figures/3293', 'observed-increase-in-frostfree-season-length'), # 2.10
# ('/metadata/figures/3294', 'projected-changes-in-frostfree-season-length'), # 2.11
# ('/metadata/figures/2677', 'observed-us-precipitation-change'), # 2.12
# ('/metadata/figures/3298', 'observed-us-trend-in-heavy-precipitation'), # 2.16
# ('/metadata/figures/2506', 'observed-change-in-very-heavy-precipitation'), # 2.17
# ('/metadata/figures/2997', 'observed-change-in-very-heavy-precipitation-2'), # 2.18
# ('/metadata/figures/3170', 'global-temperature-and-carbon-dioxide'), # 2.2
# ('/metadata/figures/2939', 'projected-changes-in-soil-moisture-for-the-western-us'), # 2.22
# ('/metadata/figures/3305', 'variation-of-storm-frequency-and-intensity-during-the-cold-season-november--march'), # 2.24
# ('/metadata/figures/3067', 'ice-cover-in-the-great-lakes'), # 2.27
# ('/metadata/figures/3073', 'projected-arctic-sea-ice-decline'), # 2.29
# ('/metadata/figures/2940', 'as-oceans-absorb-co-they-become-more-acidic'), # 2.30
# ('/metadata/figures/2523', 'shells-dissolve-in-acidified-ocean-water'), # 2.31
# ('/metadata/figures/3175', 'observed-us-temperature-change'), # 2.7
#
# ]),
# #Chapter 3
# ('water-resources', [
# ('/metadata/figures/2676', 'projected-changes-in-snow-runoff-and-soil-moisture'), # 3.1
# ('/metadata/figures/3303', 'trends-in-flood-magnitude'), # 3.5
# ]),
# #Chapter 4
# ('energy-supply-and-use', [
# ('/metadata/figures/3082', 'paths-of-hurricanes-katrina-and-rita-relative-to-oil-and-gas-production-facilities'), # 4.1
# ('/metadata/figures/3292', 'cooling-degree-days'), # 4.3
# ]),
# #Chapter 5
# ('transportation', [
# ('/metadata/figures/3088', 'possible-future-flood-depths-in-mobile-al-with-rising-sea-level'), # 5.1
# ('/metadata/figures/2905', 'gulf-coast-transportation-hubs-at-risk'), # 5.3
# ('/metadata/figures/3568', 'hurricane-sandy-causes-flooding-in-new-york-city-subway-stations'), # 5.4
# ('/metadata/figures/2952', 'role-of-adaptive-strategies-and-tactics-in-reducing-impacts-and-consequences'), # 5.5
# ('/metadata/figures/3169', 'tropical-storm-impact-on-vermont-road'), # 5.6
# ]),
# #Chapter 6
# ('agriculture', [
# ('/metadata/figures/2691', 'projected-changes-in-key-climate-variables-affecting-agricultural-productivity'), # 6.5
# ('/metadata/figures/2872', 'drainage-tiles'), # 6.8
# ]),
# #Chapter 7
# ('forests', [
# ('/metadata/figures/2887', 'forest-ecosystem-disturbances'), # 7.1
# ('/metadata/figures/2977', 'effectiveness-of-forest-management-in-reducing-wildfire-risk'), # 7.2
# ('/metadata/figures/2978', 'forest-vulnerability-to-changing-climate'), # 7.3
# ('/metadata/figures/2985', 'forests-can-be-a-source--or-a-sink--for-carbon'), # 7.4
# ('/metadata/figures/3130', 'public-and-private-forestlands'), # 7.8
#
# ]),
# #Chapter 8
# ('ecosystems', [
# ('/metadata/figures/2456', 'adaptation-planning-and-implementation-framework'), # 8.3
# ('/metadata/figures/3574', 'biological-responses-to-climate-change'), # 8.4
# ]),
# #Chapter 9
# ('human-health', [
# ('/metadata/figures/3162', 'projected-climate-change-worsens-asthma'), # 9.1
# ('/metadata/figures/2896', 'heavy-downpours-disease'), # 9.7
# ('/metadata/figures/2897', 'harmful-bloom-of-algae'), # 9.8
# ]),
# #Chapter 10
# ('water-energy-land-use', [
# ('/metadata/figures/2601', 'energy-water-land-and-climate-interactions'), # 10.1
# ('/metadata/figures/2917', 'the-columbia-river-basin-land-use-and-land-cover'), # 10.10
# ('/metadata/figures/2410', 'coasttocoast-100degree-days-in-2011'), # 10.2
# ('/metadata/figures/3158', 'projected-landuse-intensity-in-2030'), # 10.6
# ('/metadata/figures/2986', 'hydraulic-fracturing-and-water-use'), # 10.7
# ('/metadata/figures/2916', 'renewable-energy-and-land-use'), # 10.8
# ('/metadata/figures/2918', 'water-stress-in-the-us'), # 10.9
#
# ]),
# #Chapter 11
# ('urban-systems-infrastructure-vulnerability', [
# ('/metadata/figures/3569', 'urban-support-systems-are-interconnected'), # 11.2
# ('/metadata/figures/3090', 'new-york-city-and-sea-level-rise'), # 11.3
# ]),
# #Chapter 12
# ('tribal-indigenous-native-lands-resources', [
# ('/metadata/figures/3131', 'indigenous-populations-extend-beyond-reservation-lands'), # 12.1
# ('/metadata/figures/2594', 'many-tribes-many-climate-change-initiatives'), # 12.2
# ('/metadata/figures/2909', 'sand-dune-expansion'), # 12.3
# ('/metadata/figures/2911', 'arctic-marine-food-web'), # 12.5
# ]),
# #Chapter 13
# ('land-use-land-cover-change', [
# ('/metadata/figures/2902', 'building-loss-by-fires-at-california-wildlandurban-interfaces'), # 13.4
# ]),
# #Chapter 14
# ('rural', [
# ('/metadata/figures/2662', 'rural-counties'), # 14.1
# ('/metadata/figures/2661', 'economic-dependence-varies-by-region'), # 14.2
# ('/metadata/figures/3306', 'growing-season-lengthens'), # 14.3
# ('/metadata/figures/2904', 'many-rural-areas-are-losing-population'), # 14.5
# ]),
# Chapter 15
('biogeochemical-cycles', [
('/metadata/figures/2874', 'major-north-american-carbon-dioxide-sources-and-sinks'), # 15.1
......@@ -126,6 +126,7 @@ sync_metadata_tree = {
]),
#Chapter 17
('southeast', [
('/metadata/figures/2562', 'southeast-temperature-observed-and-projected'), # 17.3
('/metadata/figures/2998', 'projected-change-in-number-of-days-over-95-f'), # 17.4
('/metadata/figures/2999', 'projected-change-in-number-of-nights-below-32f'), # 17.5
('/metadata/figures/2857', 'local-planning'), # 17.9
......@@ -134,6 +135,8 @@ sync_metadata_tree = {
('midwest', [
('/metadata/figures/2550', 'temperatures-are-rising-in-the-midwest'), # 18.1
('/metadata/figures/2992', 'projected-midcentury-temperature-changes-in-the-midwest'), # 18.2
('/metadata/figures/2841', 'crop-yields-decline-under-higher-temperatures'), #18.3
('/metadata/figures/2993', 'reducing-emissions-improving-health'), # 18.5
('/metadata/figures/2994', 'when-it-rains-it-pours'), # 18.6
('/metadata/figures/3159', 'midwest-ice-cover-in-the-great-lakes') #18.7
]),
......@@ -151,6 +154,7 @@ sync_metadata_tree = {
]),
#Chapter 21
('northwest', [
('/metadata/figures/2851', 'adapting-the-nisqually-river-delta-to-sea-level-rise'), #21.5
('/metadata/figures/2558', 'forest-mortality'), # 21.6
]),
#Chapter 22
......@@ -234,7 +238,17 @@ def main():
print gcis.test_login()
# sync_dataset_metadata(gcis, webform.get_aggregated_datasets())
# sync(replace=True)
# sync(replace=False)
# print gcis.associate_figure_with_report('ten-indicators-of-a-warming-world', 'nca3', 'noaa-stateofclim-2009')
# from gcis_clients.domain import Dataset
# for ds in [d['identifier'] for d in gcis.get_dataset_list().json() if 'the-world-climate-research-programme-s--wcrp-s--coupled-model-intercomparison-project-phase-3--cmip3--multi-model-dataset' in d['identifier']]:
# d = Dataset({})
# d.identifier = ds
# print gcis.delete_dataset(d)
print gcis.lookup_publication('report', 'NOAA 2009 State of the Climate Report')
def sync(replace=False):
......
gcis_clients/domain.py
View file @ 848d62da
......@@ -233,40 +233,37 @@ class Dataset(GcisObject):
#This desperately needs to get added to the webform
self._identifiers = {
'Global Historical Climatology Network - Daily': 'ghcn-daily',
'Global Historical Climatology Network - Monthly': 'ghcn-monthly',
'NCDC Merged Land and Ocean Surface Temperature': 'mlost',
'U.S. Climate Divisional Dataset Version 2': 'cddv2',
'Climate Division Database Version 2': 'cddv2',
'Eighth degree-CONUS Daily Downscaled Climate Projections by Katharine Hayhoe': 'cmip3-downscaled', #Problem
'Eighth degree-CONUS Daily Downscaled Climate Projections': 'cmip3-downscaled', #Problem
'Earth Policy Institute Atmospheric Carbon Dioxide Concentration, 1000-2012': 'epi-co2',
'Daily 1/8-degree gridded meteorological data [1 Jan 1949 - 31 Dec 2010]': 'maurer',
'NCEP/NCAR Reanalysis': 'ncep-ncar',
'NCDC Global Surface Temperature Anomalies': 'ncdc-gst-anomalies',
'GRACE Static Field Geopotential Coefficients JPL Release 5.0 GSM': 'grace',
'UW/NCDC Satellite Derived Hurricane Intensity Dataset': 'hurricane-intensity',
'Bias-Corrected and Spatially Downscaled Surface Water Projections Hydrologic Data': 'water-projections',
'International Best Track Archive for Climate Stewardship (IBTrACS)': 'ibtracs',
'the World Climate Research Programme\'s (WCRP\'s) Coupled Model Intercomparison Project phase 3 (CMIP3) multi-model dataset': 'cmip3',
'World Climate Research Programme\'s (WCRP\'s) Coupled Model Intercomparison Project phase 3 (CMIP3) multi-model dataset': 'cmip3',
'World Climate Research Program\'s (WCRP\'s) Coupled Model Intercomparison Project phase 3 (CMIP3) multi-model dataset': 'cmip3',
'North American Regional Climate Change Assessment Program dataset': 'narccap',
'Gridded Population of the World Version 3 (GPWv3): Population Count Grid': 'gpwv3',
'ETCCDI Extremes Indices Archive': 'etccdi',
'Historical Climatology Network Monthly (USHCN) Version 2.5': 'ushcn',
'Annual Maximum Ice Coverage (AMIC)': 'amic',
'Global Historical Climatology Network-Daily (GHCN-D) Monthly Summaries: North American subset': 'ghcnd-monthly-summaries',
'Global Sea Level From TOPEX & Jason Altimetry': 'topex-jason-altimetry',
'World Climate Research Program\'s (WCRP\'s) Coupled Model Intercomparison Project phase 5 (CMIP5) multi-model ensemble': 'cmip5',
'Global Historical Climatology Network - Daily': 'nca3-ghcn-daily-r201305',
'Global Historical Climatology Network - Monthly': 'nca3-ghcn-monthly-r201305',
'NCDC Merged Land and Ocean Surface Temperature': 'nca3-mlost',
'U.S. Climate Divisional Dataset Version 2': 'nca3-cddv2-r1',
'Climate Division Database Version 2': 'nca3-cddv2-r1',
'Eighth degree-CONUS Daily Downscaled Climate Projections by Katharine Hayhoe': 'nca3-cmip3-downscaled-r201304',
'Eighth degree-CONUS Daily Downscaled Climate Projections': 'nca3-cmip3-downscaled-r201304',
'Earth Policy Institute Atmospheric Carbon Dioxide Concentration, 1000-2012': 'nca3-epi-co2-r201307',
'Daily 1/8-degree gridded meteorological data [1 Jan 1949 - 31 Dec 2010]': 'nca3-maurer-r201304',
'NCEP/NCAR Reanalysis': 'nca3-ncep-ncar-r1',
'NCDC Global Surface Temperature Anomalies': 'nca3-ncdc-gst-anomalies-r201307',
'GRACE Static Field Geopotential Coefficients JPL Release 5.0 GSM': 'nca3-grace-r201307',
'UW/NCDC Satellite Derived Hurricane Intensity Dataset': 'nca3-hurricane-intensity-r1',
'Bias-Corrected and Spatially Downscaled Surface Water Projections Hydrologic Data': 'nca3-water-projections-r201208',
'International Best Track Archive for Climate Stewardship (IBTrACS)': 'nca3-ibtracs-r201311',
'the World Climate Research Programme\'s (WCRP\'s) Coupled Model Intercomparison Project phase 3 (CMIP3) multi-model dataset': 'nca3-cmip3-r201205',
'World Climate Research Programme\'s (WCRP\'s) Coupled Model Intercomparison Project phase 3 (CMIP3) multi-model dataset': 'nca3-cmip3-r201205',
'World Climate Research Program\'s (WCRP\'s) Coupled Model Intercomparison Project phase 3 (CMIP3) multi-model dataset': 'nca3-cmip3-r201205',
'North American Regional Climate Change Assessment Program dataset': 'nca3-narccap-r201205',
'Gridded Population of the World Version 3 (GPWv3): Population Count Grid': 'nca3-gpwv3-r201211',
'ETCCDI Extremes Indices Archive': 'nca3-etccdi-r201305',
'Historical Climatology Network Monthly (USHCN) Version 2.5': 'nca3-ushcn',
'Annual Maximum Ice Coverage (AMIC)': 'nca3-amic-r201308',
'Global Historical Climatology Network-Daily (GHCN-D) Monthly Summaries: North American subset': 'nca3-ghcnd-monthly-summaries-r201401',
'Global Sea Level From TOPEX & Jason Altimetry': 'nca3-topex-jason-altimetry-r1',
'World Climate Research Program\'s (WCRP\'s) Coupled Model Intercomparison Project phase 5 (CMIP5) multi-model ensemble': 'nca3-cmip5-r1',
#Surely we can do better
'Proxy Data': 'proxy-data',
'Tide Gauge Data': 'tide-gauge-data',
'Projected Sea Level Rise': 'projected-sea-level-rise',
'Proxy Data': 'nca3-proxy-data-r1',
'Tide Gauge Data': 'nca3-tide-gauge-data-r1',
'Projected Sea Level Rise': 'nca3-projected-sea-level-rise-r1',
}
#Private attributes for handling date parsing
......
gcis_clients/gcis_client.py
View file @ 848d62da
......@@ -290,18 +290,26 @@ class GcisClient(object):
url = '{b}/dataset/{ds}'.format(b=self.base_url, ds=dataset_id)
return self.s.head(url, verify=False)
@http_resp
def create_dataset(self, dataset):
url = '{b}/dataset/'.format(b=self.base_url)
return self.s.post(url, data=dataset.as_json(), verify=False)
@http_resp
def update_dataset(self, dataset, old_id=None):
url = '{b}/dataset/{ds}'.format(b=self.base_url, ds=old_id or dataset.identifier)
return self.s.post(url, data=dataset.as_json(), verify=False)
@http_resp
def delete_dataset(self, dataset):
url = '{b}/dataset/{ds}'.format(b=self.base_url, ds=dataset.identifier)
return self.s.delete(url, verify=False)
@http_resp
def get_dataset_list(self):
url = '{b}/dataset/'.format(b=self.base_url)
return self.s.get(url, params={'all': 1}, verify=False)
def associate_dataset_with_image(self, dataset_id, image_id, activity_id=None):
url = '{b}/image/prov/{img}'.format(b=self.base_url, img=image_id)
......@@ -525,3 +533,22 @@ class GcisClient(object):
return self.s.post(url, data=json.dumps(data), verify=False)
@http_resp
def associate_figure_with_report(self, figure_id, report_id, other_report_id):
url = '{b}/report/{rpt}/figure/prov/{fig}'.format(b=self.base_url, rpt=report_id, fig=figure_id)
data = {
'parent_uri': '/report/' + other_report_id,
'parent_rel': 'prov:wasDerivedFrom'
}
return self.s.post(url, data=json.dumps(data), verify=False)
def lookup_publication(self, pub_type, name):
url = '{b}/autocomplete'.format(b=self.base_url)
resp = self.s.get(url, params={'q': name, 'items': 15, 'type': pub_type}, verify=False)
if resp.status_code == 200:
return [re.match(r'\[.+\] \{(.+)\} (.*)', r).groups() for r in resp.json()]
else:
raise Exception(resp.text)
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment