Commit 962c225d authored by abuddenberg's avatar abuddenberg
Browse files

Added support for T2 fields (datasets, activities, contributors) to figures...

Added support for T2 fields (datasets, activities, contributors) to figures AND images. Added support for copyright limits
parent 0e6a3749
...@@ -167,7 +167,6 @@ class GcisClient(object): ...@@ -167,7 +167,6 @@ class GcisClient(object):
@http_resp @http_resp
def upload_figure_file(self, report_id, chapter_id, figure_id, local_path): def upload_figure_file(self, report_id, chapter_id, figure_id, local_path):
# report/usgcrp-climate-and-health-assessment-draft/chapter/climate-change-and-human-health/figure/files/emissions_levels_determine_temperature_rises/
url = '{b}/report/{rpt}/chapter/{chp}/figure/files/{id}/{fn}'.format(b=self.base_url, rpt=report_id, chp=chapter_id, id=figure_id, fn=basename(local_path)) url = '{b}/report/{rpt}/chapter/{chp}/figure/files/{id}/{fn}'.format(b=self.base_url, rpt=report_id, chp=chapter_id, id=figure_id, fn=basename(local_path))
# For future multi-part encoding support # For future multi-part encoding support
# return self.s.put(url, headers=headers, files={'file': (filename, open(filepath, 'rb'))}) # return self.s.put(url, headers=headers, files={'file': (filename, open(filepath, 'rb'))})
......
...@@ -5,7 +5,7 @@ import requests ...@@ -5,7 +5,7 @@ import requests
import re import re
from os.path import join, basename from os.path import join, basename
from gcis_clients.domain import Figure, Image, Dataset, Parent, Contributor, Person, Organization from gcis_clients.domain import Figure, Image, Dataset, Parent, Contributor, Person, Organization, Activity
import survey_transforms as trans import survey_transforms as trans
...@@ -59,12 +59,15 @@ def populate_image(img_json): ...@@ -59,12 +59,15 @@ def populate_image(img_json):
def populate_dataset(ds_json): def populate_dataset(ds_json):
ds = Dataset({})
try: try:
ds.name = ds_json['dataset_name'] ds = Dataset({
ds.url = ds_json['dataset_url'] 'name': ds_json['dataset_name'],
'url': ds_json['dataset_url']
}, known_ids=trans.DATASET_IDS)
except Exception, e: except Exception, e:
print 'Dataset exception: ', e print 'Dataset exception: ', e
ds = Dataset({})
image_select = ds_json['imageSelect'] if 'imageSelect' in ds_json else [] image_select = ds_json['imageSelect'] if 'imageSelect' in ds_json else []
associated_images = [idx for idx, value in enumerate(image_select) if value == 'on'] associated_images = [idx for idx, value in enumerate(image_select) if value == 'on']
...@@ -72,6 +75,27 @@ def populate_dataset(ds_json): ...@@ -72,6 +75,27 @@ def populate_dataset(ds_json):
return ds, associated_images return ds, associated_images
def populate_activity(mthd_json):
act = Activity({})
try:
act.methodology = mthd_json['dataset_methods_used']
act.data_usage = mthd_json['dataset_how_visualized']
files = [mthd_json['dataset_output_file']]
files.extend([f for f in mthd_json['dataset_files'] if f])
act.output_artifacts = ', '.join(files)
act.duration = mthd_json['dataset_creation_time']
act.computing_environment = mthd_json['dataset_os_used']
act.software = ', '.join([s for s in mthd_json['dataset_software_used'] if s])
act.visualization_software = ', '.join([vs for vs in mthd_json['dataset_visualization_software'] if vs])
except Exception, e:
print 'Activity exception: ', e
return act, mthd_json['image_name'], mthd_json['dataset']
def populate_parent(pub_json): def populate_parent(pub_json):
p = Parent({}) p = Parent({})
try: try:
...@@ -124,6 +148,7 @@ class SurveyClient: ...@@ -124,6 +148,7 @@ class SurveyClient:
full_url = '{b}{url}?token={t}'.format(b=self.base_url, url=fig_url, t=self.token) full_url = '{b}{url}?token={t}'.format(b=self.base_url, url=fig_url, t=self.token)
survey_json = requests.get(full_url).json() survey_json = requests.get(full_url).json()
tier1_json = survey_json[0]['t1'] if len(survey_json) > 0 and survey_json[0]['t1'] is not None else [] tier1_json = survey_json[0]['t1'] if len(survey_json) > 0 and survey_json[0]['t1'] is not None else []
tier2_json = survey_json[0]['t2'] if len(survey_json) > 0 and survey_json[0]['t2'] is not None else []
f = None f = None
...@@ -134,26 +159,58 @@ class SurveyClient: ...@@ -134,26 +159,58 @@ class SurveyClient:
f.remote_path = survey_json[0]['filepath'] f.remote_path = survey_json[0]['filepath']
f.local_path = join(self.local_download_dir, basename(f.remote_path)) if f.remote_path else None f.local_path = join(self.local_download_dir, basename(f.remote_path)) if f.remote_path else None
if 'copyright' in survey_json[0]:
f.usage_limits = trans.COPYRIGHT_TRANSLATIONS[survey_json[0]['copyright']]
if 'origination' in figure_json and figure_json['origination'] not in ('Original',) and 'publication' in figure_json: if 'origination' in figure_json and figure_json['origination'] not in ('Original',) and 'publication' in figure_json:
f.parents.append(populate_parent(figure_json['publication'])) f.parents.append(populate_parent(figure_json['publication']))
# elif 'origination' in figure_json and figure_json['origination'] == 'Original':
# f.add_contributor(populate_contributors(figure_json))
if 'images' in tier1_json: if 'images' in tier1_json:
images = [populate_image(img) for img in tier1_json['images']] for img_json in tier1_json['images']:
f.images.extend(images) image_obj = populate_image(img_json)
elif 'figure' in tier1_json:
default_image = populate_image(tier1_json['figure']) #Populate image contributor info, if available
f.images.append(default_image) if 'origination' in img_json and img_json['origination'] not in ('Original',) and 'publication' in img_json:
image_obj.parents.append(populate_parent(img_json['publication']))
elif 'origination' in img_json and img_json['origination'] == 'Original':
image_obj.add_contributor(populate_contributors(img_json['original_agency']))
f.images.append(image_obj)
# # Recent decision: No default images
# elif 'figure' in tier1_json:
# default_image = populate_image(tier1_json['figure'])
# f.images.append(default_image)
if 'datasets' in tier1_json: if 'datasets' in tier1_json:
datasets = [populate_dataset(ds) for ds in tier1_json['datasets']] datasets = [populate_dataset(ds) for ds in tier1_json['datasets']]
#Associate datasets with images # Create activities
activities = [populate_activity(m) for m in tier2_json['methods']] if tier2_json and 'methods' in tier2_json else []
for ds, img_idxs in datasets: for ds, img_idxs in datasets:
for idx in img_idxs: # Associate datasets with images if we have images
try: if 'images' in tier1_json:
f.images[idx].datasets.append(ds) for idx in img_idxs:
except Exception, e: # Associate activities with datasets
print 'Association exception: ', e for act, img_name, ds_name in activities:
if img_name == f.images[idx].title and ds_name == ds.name:
ds.activity = act
try:
f.images[idx].datasets.append(ds)
except Exception, e:
print 'Dataset / Image association exception: ', e
# Else associate the datasets with the figure
else:
f.add_parent(Parent.from_obj(ds))
# Associate activities with datasets
for act, img_name, ds_name in activities:
if img_name == f.title and ds_name == ds.name:
ds.activity = act
if 'poc' in tier1_json: if 'poc' in tier1_json:
f.add_contributor(populate_contributors(tier1_json['poc'])) f.add_contributor(populate_contributors(tier1_json['poc']))
......
__author__ = 'abuddenberg' __author__ = 'abuddenberg'
DATASET_IDS = {
'U.S. Climate Divisional Dataset Version 2': 'nca3-cddv2-r1',
'World Climate Research Program\'s (WCRP\'s) Coupled Model Intercomparison Project phase 5 (CMIP5) multi-model ensemble': 'nca3-cmip5-r1',
'World Climate Research Program\'s (WCRP\'s) Coupled Model Intercomparison Project phase 5 (CMIP5) multi-model ensemble': 'nca3-cmip5-r1'
}
COPYRIGHT_TRANSLATIONS = {
None: None,
'requested': 'Copyright protected. Obtain permission from the original figure source.',
'denied': 'Copyright protected. Obtain permission from the original figure source.',
'obtained': 'Copyright protected. Obtain permission from the original figure source.',
'original_work_nr': 'Free to use with credit to the original figure source.'
}
PARENT_TRANSLATIONS = { PARENT_TRANSLATIONS = {
'publicationType': 'publication_type_identifier', 'publicationType': 'publication_type_identifier',
'report_name': 'label', 'report_name': 'label',
......
...@@ -14,81 +14,81 @@ surveys = SurveyClient('https://healthresources.globalchange.gov', survey_token) ...@@ -14,81 +14,81 @@ surveys = SurveyClient('https://healthresources.globalchange.gov', survey_token)
sync_metadata_tree = { sync_metadata_tree = {
'usgcrp-climate-and-health-assessment-draft': OrderedDict([ 'usgcrp-climate-and-health-assessment-draft': OrderedDict([
('executive-summary', []), # ('executive-summary', []),
('climate-change-and-human-health', [ # ('climate-change-and-human-health', [
('/metadata/figures/3698', 'major_u_s__climate_trends'), #1.1 #climate-change-and-human-health # ('/metadata/figures/3698', 'major_u_s__climate_trends'), #1.1 #climate-change-and-human-health
('/metadata/figures/3632', 'percent_changes_in_the_annual_number_of_extreme_precipitation_events_'), #1.2 #climate-change-and-human-health # ('/metadata/figures/3632', 'percent_changes_in_the_annual_number_of_extreme_precipitation_events_'), #1.2 #climate-change-and-human-health
('/metadata/figures/3635', 'projected_changes_in_temperature_and_precipitation_by_mid_century'), #1.3 #climate-change-and-human-health # ('/metadata/figures/3635', 'projected_changes_in_temperature_and_precipitation_by_mid_century'), #1.3 #climate-change-and-human-health
('/metadata/figures/3633', 'projected_changes_in_the_hottest_coldest_and_wettest_driest_day_of_the_year'), #1.4 #climate-change-and-human-health # ('/metadata/figures/3633', 'projected_changes_in_the_hottest_coldest_and_wettest_driest_day_of_the_year'), #1.4 #climate-change-and-human-health
('/metadata/figures/3757', 'climate_change_and_health'), #1.5 #climate-change-and-human-health # ('/metadata/figures/3757', 'climate_change_and_health'), #1.5 #climate-change-and-human-health
]), # ]),
('temperature-related-death-and-illness', [ # ('temperature-related-death-and-illness', [
('/metadata/figures/3811', 'climate_change_and_health__extreme_heat'), #2.1 #temperature-related-death-and-illness # ('/metadata/figures/3811', 'climate_change_and_health__extreme_heat'), #2.1 #temperature-related-death-and-illness
('/metadata/figures/3585', 'heat_related_deaths_during_the_1995_chicago_heat_wave'), #2.2 #temperature-related-death-and-illness # ('/metadata/figures/3585', 'heat_related_deaths_during_the_1995_chicago_heat_wave'), #2.2 #temperature-related-death-and-illness
('/metadata/figures/3643', 'projected_net_changes_in_extreme_temperature_related_deaths'), #2.3 #temperature-related-death-and-illness # ('/metadata/figures/3643', 'projected_net_changes_in_extreme_temperature_related_deaths'), #2.3 #temperature-related-death-and-illness
('/metadata/figures/3653', 'projected_changes_in_deaths_in_u_s__cities_by_season'), #2.4 #temperature-related-death-and-illness # ('/metadata/figures/3653', 'projected_changes_in_deaths_in_u_s__cities_by_season'), #2.4 #temperature-related-death-and-illness
]), # ]),
('air-quality-impacts', [ # ('air-quality-impacts', [
('/metadata/figures/3812', 'climate_change_and_health__outdoor_air_quality'), #3.1 #air-quality-impacts # ('/metadata/figures/3812', 'climate_change_and_health__outdoor_air_quality'), #3.1 #air-quality-impacts
('/metadata/figures/3647', 'projected_change_in_average_daily_maximum_temperature__seasonal_average_maximum_daily_8_hr_ozone__and_ozone_related_premature_deaths_in_2013'), #3.2 #air-quality-impacts # ('/metadata/figures/3647', 'projected_change_in_average_daily_maximum_temperature__seasonal_average_maximum_daily_8_hr_ozone__and_ozone_related_premature_deaths_in_2013'), #3.2 #air-quality-impacts
('/metadata/figures/3649', 'projected_change_in_ozone_related_premature_deaths'), #3.3 #air-quality-impacts # ('/metadata/figures/3649', 'projected_change_in_ozone_related_premature_deaths'), #3.3 #air-quality-impacts
('/metadata/figures/3650', 'ragweed_pollen_season_lengthens'), #3.4 #air-quality-impacts # ('/metadata/figures/3650', 'ragweed_pollen_season_lengthens'), #3.4 #air-quality-impacts
]), # ]),
('vectorborne-diseases', [ # ('vectorborne-diseases', [
('/metadata/figures/3807', 'climate_change_and_health__lyme_disease'), #4.1 #vectorborne-diseases # ('/metadata/figures/3807', 'climate_change_and_health__lyme_disease'), #4.1 #vectorborne-diseases
('/metadata/figures/3659', 'changes_in_lyme_disease_case_report_distribution'), #4.2 #vectorborne-diseases # ('/metadata/figures/3659', 'changes_in_lyme_disease_case_report_distribution'), #4.2 #vectorborne-diseases
('/metadata/figures/3658', 'life_cycle_of_blacklegged_ticks__ixodes_scapularis'), #4.3 #vectorborne-diseases # ('/metadata/figures/3658', 'life_cycle_of_blacklegged_ticks__ixodes_scapularis'), #4.3 #vectorborne-diseases
('/metadata/figures/3747', 'lyme_disease_onset_week_modeling_scenarios'), #4.4 #vectorborne-diseases # ('/metadata/figures/3747', 'lyme_disease_onset_week_modeling_scenarios'), #4.4 #vectorborne-diseases
('/metadata/figures/3674', 'incidence_of_west_nile_neuroinvasive_disease_in_the_united_states'), #4.5 #vectorborne-diseases # ('/metadata/figures/3674', 'incidence_of_west_nile_neuroinvasive_disease_in_the_united_states'), #4.5 #vectorborne-diseases
('/metadata/figures/3675', 'climate_impacts_on_west_nile_virus_transmission'), #4.6 #vectorborne-diseases # ('/metadata/figures/3675', 'climate_impacts_on_west_nile_virus_transmission'), #4.6 #vectorborne-diseases
]), # ]),
('water-related-illnesses', [ # ('water-related-illnesses', [
('/metadata/figures/3824', 'climate_change_and_health___vibrio'), #5.1 #water-related-illnesses # ('/metadata/figures/3824', 'climate_change_and_health___vibrio'), #5.1 #water-related-illnesses
('/metadata/figures/3700', 'links_between_climate_change__water_quantity_and_quality__and_human_exposure_to_water_related_illness'), #5.2 #water-related-illnesses # ('/metadata/figures/3700', 'links_between_climate_change__water_quantity_and_quality__and_human_exposure_to_water_related_illness'), #5.2 #water-related-illnesses
('/metadata/figures/3671', 'locations_of_livestock_and_projections_of_heavy_precipitation'), #5.3 #water-related-illnesses # ('/metadata/figures/3671', 'locations_of_livestock_and_projections_of_heavy_precipitation'), #5.3 #water-related-illnesses
# ('/metadata/figures/3673', 'potential_routes_of_manure_borne_microbial_contaminants_to_ground_and_surface_water_supplies_'), #5.3 #water-related-illnesses UNUSED? # # ('/metadata/figures/3673', 'potential_routes_of_manure_borne_microbial_contaminants_to_ground_and_surface_water_supplies_'), #5.3 #water-related-illnesses UNUSED?
('/metadata/figures/3709', 'projections_of_vibrio_occurrence_and_abundance_in_chesapeake_bay'), #5.4 #water-related-illnesses # ('/metadata/figures/3709', 'projections_of_vibrio_occurrence_and_abundance_in_chesapeake_bay'), #5.4 #water-related-illnesses
('/metadata/figures/3704', 'changes_in_suitable_coastal_vibrio_habitat_in_alaska'), #5.5 #water-related-illnesses # ('/metadata/figures/3704', 'changes_in_suitable_coastal_vibrio_habitat_in_alaska'), #5.5 #water-related-illnesses
('/metadata/figures/3734', 'projected_changes_in_caribbean_gambierdiscus_species'), #5.6 #water-related-illnesses # ('/metadata/figures/3734', 'projected_changes_in_caribbean_gambierdiscus_species'), #5.6 #water-related-illnesses
('/metadata/figures/3712', 'projections_of_growth_of_alexandrium_fundyense_in_puget_sound'), #5.7 #water-related-illnesses # ('/metadata/figures/3712', 'projections_of_growth_of_alexandrium_fundyense_in_puget_sound'), #5.7 #water-related-illnesses
]), # ]),
('food-safety--nutrition--and-distribution', [ # ('food-safety--nutrition--and-distribution', [
('/metadata/figures/3579', 'farm_to_table'), #6.1 #food-safety--nutrition--and-distribution # ('/metadata/figures/3579', 'farm_to_table'), #6.1 #food-safety--nutrition--and-distribution
# ('/metadata/figures/3600', 'mycotoxin_in_corn'), #6.1 #food-safety--nutrition--and-distribution BOX 1? # # ('/metadata/figures/3600', 'mycotoxin_in_corn'), #6.1 #food-safety--nutrition--and-distribution BOX 1?
('/metadata/figures/3809', 'climate_change_and_health__salmonella'), #6.2 #food-safety--nutrition--and-distribution # ('/metadata/figures/3809', 'climate_change_and_health__salmonella'), #6.2 #food-safety--nutrition--and-distribution
('/metadata/figures/3748', 'seasonality_of_human_illnesses_associated_with_foodborne_pathogens'), #6.3 #food-safety--nutrition--and-distribution # ('/metadata/figures/3748', 'seasonality_of_human_illnesses_associated_with_foodborne_pathogens'), #6.3 #food-safety--nutrition--and-distribution
('/metadata/figures/3688', 'effects_of_carbon_dioxide_on_protein_and_minerals'), #6.4 #food-safety--nutrition--and-distribution # ('/metadata/figures/3688', 'effects_of_carbon_dioxide_on_protein_and_minerals'), #6.4 #food-safety--nutrition--and-distribution
('/metadata/figures/3597', 'mississippi_river_level_at_st__louis__missouri'), #6.5 #food-safety--nutrition--and-distribution # ('/metadata/figures/3597', 'mississippi_river_level_at_st__louis__missouri'), #6.5 #food-safety--nutrition--and-distribution
]), # ]),
('extreme-weather', [ # ('extreme-weather', [
('/metadata/figures/3810', 'estimated_deaths_and_billion_dollar_losses_from_extreme_weather_events_in_the_u_s__2004_2013'), #7.1 #extreme-weather # ('/metadata/figures/3810', 'estimated_deaths_and_billion_dollar_losses_from_extreme_weather_events_in_the_u_s__2004_2013'), #7.1 #extreme-weather
# ('/metadata/figures/3772', 'trends_in_flood_magnitude'), #7.2 #extreme-weather NOT USED # # ('/metadata/figures/3772', 'trends_in_flood_magnitude'), #7.2 #extreme-weather NOT USED
('/metadata/figures/3808', 'climate_change_and_health__flooding'), #7.2 #extreme-weather # ('/metadata/figures/3808', 'climate_change_and_health__flooding'), #7.2 #extreme-weather
('/metadata/figures/3760', 'hurricane_induced_flood_effects_in_eastern_and_central_united_states'), #7.3 #extreme-weather # ('/metadata/figures/3760', 'hurricane_induced_flood_effects_in_eastern_and_central_united_states'), #7.3 #extreme-weather
]), # ]),
('mental-health-and-well-being', [ # ('mental-health-and-well-being', [
('/metadata/figures/3789', 'climate_change_and_mental_health'), #8.1 #mental-health-and-well-being # ('/metadata/figures/3789', 'climate_change_and_mental_health'), #8.1 #mental-health-and-well-being
('/metadata/figures/3722', 'the_impact_of_climate_change_on_physical__mental__and_community_health'), #8.2 #mental-health-and-well-being # ('/metadata/figures/3722', 'the_impact_of_climate_change_on_physical__mental__and_community_health'), #8.2 #mental-health-and-well-being
]), # ]),
('populations-of-concern', [ ('populations-of-concern', [
('/metadata/figures/3696', 'determinants_of_vulnerability'), #9.1 #populations-of-concern # ('/metadata/figures/3696', 'determinants_of_vulnerability'), #9.1 #populations-of-concern
('/metadata/figures/3694', 'social_determinants_of_health'), #9.2 #populations-of-concern ('/metadata/figures/3694', 'social_determinants_of_health'), #9.2 #populations-of-concern
('/metadata/figures/3758', 'children_at_different_lifestages_experience_unique_vulnerabilities_to_climate_change'), #9.3 #populations-of-concern # ('/metadata/figures/3758', 'children_at_different_lifestages_experience_unique_vulnerabilities_to_climate_change'), #9.3 #populations-of-concern
('/metadata/figures/3714', 'mapping_social_vulnerability'), #9.4 #populations-of-concern # ('/metadata/figures/3714', 'mapping_social_vulnerability'), #9.4 #populations-of-concern
('/metadata/figures/3717', 'mapping_communities_vulnerable_to_heat_in_georgia'), #9.5 #populations-of-concern # ('/metadata/figures/3717', 'mapping_communities_vulnerable_to_heat_in_georgia'), #9.5 #populations-of-concern
]), ]),
('appendix-1--technical-support-document', [ # ('appendix-1--technical-support-document', [
('/metadata/figures/3623', 'emissions_levels_determine_temperature_rises'), #1.1 #climate-change-and-human-health # ('/metadata/figures/3623', 'emissions_levels_determine_temperature_rises'), #1.1 #climate-change-and-human-health
('/metadata/figures/3759', 'the_shared_socioeconomic_pathways'), #1.2 #climate-change-and-human-health # ('/metadata/figures/3759', 'the_shared_socioeconomic_pathways'), #1.2 #climate-change-and-human-health
('/metadata/figures/3726', 'example_spatial_resolution_of_climate_models'), #1.3 #climate-change-and-human-health # ('/metadata/figures/3726', 'example_spatial_resolution_of_climate_models'), #1.3 #climate-change-and-human-health
('/metadata/figures/3638', 'sensitivity_analysis_of_differences_in_modeling_approaches'), #1.4 #climate-change-and-human-health # ('/metadata/figures/3638', 'sensitivity_analysis_of_differences_in_modeling_approaches'), #1.4 #climate-change-and-human-health
]) # ])
]) ])
} }
def main(): def main():
regenerate_image_id_map() # regenerate_image_id_map()
image_id_map = pickle.load(open('image_id_cache.pk1', 'r')) image_id_map = pickle.load(open('image_id_cache.pk1', 'r'))
for report_id in sync_metadata_tree: for report_id in sync_metadata_tree:
...@@ -97,16 +97,36 @@ def main(): ...@@ -97,16 +97,36 @@ def main():
print survey_url print survey_url
s = surveys.get_survey(survey_url, do_download=False) s = surveys.get_survey(survey_url, do_download=False)
print s.contributors
print s.parents
print s.usage_limits
# for i in s.images:
# i.identifier = image_id_map[i.identifier]
#
# print i.contributors
# print i.parents
#
# for ds in i.datasets:
# if ds.activity and ds.activity.identifier is None:
# ds.activity.identifier = generate_activity_id(i, ds)
# print ds
# realize_parents(gcis, s.parents) # realize_parents(gcis, s.parents)
# realize_contributors(gcis, s.contributors) # realize_contributors(gcis, s.contributors)
for i in s.images:
i.identifier = image_id_map[i.identifier]
i.datasets = []
print gcis.create_image(i, report_id=report_id, figure_id=figure_id)
# for i in s.images:
# i.identifier = image_id_map[i.identifier]
# i.datasets = []
# print gcis.create_image(i, report_id=report_id, figure_id=figure_id)
def generate_activity_id(image, dataset):
return '-'.join([image.identifier.split('-')[0], dataset.identifier, '-process'])
def regenerate_image_id_map(existing=None): def regenerate_image_id_map(existing=None):
from uuid import uuid4 from uuid import uuid4
image_id_map = existing if existing else {} image_id_map = existing if existing else {}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment