Skip to content

GitLab

Commit 962c225d authored by abuddenberg's avatar abuddenberg
Browse files

Added support for T2 fields (datasets, activities, contributors) to figures... 

Added support for T2 fields (datasets, activities, contributors) to figures AND images. Added support for copyright limits
parent 0e6a3749
Hide whitespace changes
Inline Side-by-side
Showing with 177 additions and 87 deletions
gcis_clients/gcis_client.py
View file @ 962c225d
......@@ -167,7 +167,6 @@ class GcisClient(object):
@http_resp
def upload_figure_file(self, report_id, chapter_id, figure_id, local_path):
# report/usgcrp-climate-and-health-assessment-draft/chapter/climate-change-and-human-health/figure/files/emissions_levels_determine_temperature_rises/
url = '{b}/report/{rpt}/chapter/{chp}/figure/files/{id}/{fn}'.format(b=self.base_url, rpt=report_id, chp=chapter_id, id=figure_id, fn=basename(local_path))
# For future multi-part encoding support
# return self.s.put(url, headers=headers, files={'file': (filename, open(filepath, 'rb'))})
......
gcis_clients/survey_client.py
View file @ 962c225d
......@@ -5,7 +5,7 @@ import requests
import re
from os.path import join, basename
from gcis_clients.domain import Figure, Image, Dataset, Parent, Contributor, Person, Organization
from gcis_clients.domain import Figure, Image, Dataset, Parent, Contributor, Person, Organization, Activity
import survey_transforms as trans
......@@ -59,12 +59,15 @@ def populate_image(img_json):
def populate_dataset(ds_json):
ds = Dataset({})
try:
ds.name = ds_json['dataset_name']
ds.url = ds_json['dataset_url']
ds = Dataset({
'name': ds_json['dataset_name'],
'url': ds_json['dataset_url']
}, known_ids=trans.DATASET_IDS)
except Exception, e:
print 'Dataset exception: ', e
ds = Dataset({})
image_select = ds_json['imageSelect'] if 'imageSelect' in ds_json else []
associated_images = [idx for idx, value in enumerate(image_select) if value == 'on']
......@@ -72,6 +75,27 @@ def populate_dataset(ds_json):
return ds, associated_images
def populate_activity(mthd_json):
act = Activity({})
try:
act.methodology = mthd_json['dataset_methods_used']
act.data_usage = mthd_json['dataset_how_visualized']
files = [mthd_json['dataset_output_file']]
files.extend([f for f in mthd_json['dataset_files'] if f])
act.output_artifacts = ', '.join(files)
act.duration = mthd_json['dataset_creation_time']
act.computing_environment = mthd_json['dataset_os_used']
act.software = ', '.join([s for s in mthd_json['dataset_software_used'] if s])
act.visualization_software = ', '.join([vs for vs in mthd_json['dataset_visualization_software'] if vs])
except Exception, e:
print 'Activity exception: ', e
return act, mthd_json['image_name'], mthd_json['dataset']
def populate_parent(pub_json):
p = Parent({})
try:
......@@ -124,6 +148,7 @@ class SurveyClient:
full_url = '{b}{url}?token={t}'.format(b=self.base_url, url=fig_url, t=self.token)
survey_json = requests.get(full_url).json()
tier1_json = survey_json[0]['t1'] if len(survey_json) > 0 and survey_json[0]['t1'] is not None else []
tier2_json = survey_json[0]['t2'] if len(survey_json) > 0 and survey_json[0]['t2'] is not None else []
f = None
......@@ -134,26 +159,58 @@ class SurveyClient:
f.remote_path = survey_json[0]['filepath']
f.local_path = join(self.local_download_dir, basename(f.remote_path)) if f.remote_path else None
if 'copyright' in survey_json[0]:
f.usage_limits = trans.COPYRIGHT_TRANSLATIONS[survey_json[0]['copyright']]
if 'origination' in figure_json and figure_json['origination'] not in ('Original',) and 'publication' in figure_json:
f.parents.append(populate_parent(figure_json['publication']))
# elif 'origination' in figure_json and figure_json['origination'] == 'Original':
# f.add_contributor(populate_contributors(figure_json))
if 'images' in tier1_json:
images = [populate_image(img) for img in tier1_json['images']]
f.images.extend(images)
elif 'figure' in tier1_json:
default_image = populate_image(tier1_json['figure'])
f.images.append(default_image)
for img_json in tier1_json['images']:
image_obj = populate_image(img_json)
#Populate image contributor info, if available
if 'origination' in img_json and img_json['origination'] not in ('Original',) and 'publication' in img_json:
image_obj.parents.append(populate_parent(img_json['publication']))
elif 'origination' in img_json and img_json['origination'] == 'Original':
image_obj.add_contributor(populate_contributors(img_json['original_agency']))
f.images.append(image_obj)
# # Recent decision: No default images
# elif 'figure' in tier1_json:
# default_image = populate_image(tier1_json['figure'])
# f.images.append(default_image)
if 'datasets' in tier1_json:
datasets = [populate_dataset(ds) for ds in tier1_json['datasets']]
#Associate datasets with images
# Create activities
activities = [populate_activity(m) for m in tier2_json['methods']] if tier2_json and 'methods' in tier2_json else []
for ds, img_idxs in datasets:
for idx in img_idxs:
try:
f.images[idx].datasets.append(ds)
except Exception, e:
print 'Association exception: ', e
# Associate datasets with images if we have images
if 'images' in tier1_json:
for idx in img_idxs:
# Associate activities with datasets
for act, img_name, ds_name in activities:
if img_name == f.images[idx].title and ds_name == ds.name:
ds.activity = act
try:
f.images[idx].datasets.append(ds)
except Exception, e:
print 'Dataset / Image association exception: ', e
# Else associate the datasets with the figure
else:
f.add_parent(Parent.from_obj(ds))
# Associate activities with datasets
for act, img_name, ds_name in activities:
if img_name == f.title and ds_name == ds.name:
ds.activity = act
if 'poc' in tier1_json:
f.add_contributor(populate_contributors(tier1_json['poc']))
......
gcis_clients/survey_transforms.py
View file @ 962c225d
__author__ = 'abuddenberg'
DATASET_IDS = {
'U.S. Climate Divisional Dataset Version 2': 'nca3-cddv2-r1',
'World Climate Research Program\'s (WCRP\'s) Coupled Model Intercomparison Project phase 5 (CMIP5) multi-model ensemble': 'nca3-cmip5-r1',
'World Climate Research Program\'s (WCRP\'s) Coupled Model Intercomparison Project phase 5 (CMIP5) multi-model ensemble': 'nca3-cmip5-r1'
}
COPYRIGHT_TRANSLATIONS = {
None: None,
'requested': 'Copyright protected. Obtain permission from the original figure source.',
'denied': 'Copyright protected. Obtain permission from the original figure source.',
'obtained': 'Copyright protected. Obtain permission from the original figure source.',
'original_work_nr': 'Free to use with credit to the original figure source.'
}
PARENT_TRANSLATIONS = {
'publicationType': 'publication_type_identifier',
'report_name': 'label',
......
sync_surveys.py
View file @ 962c225d
......@@ -14,81 +14,81 @@ surveys = SurveyClient('https://healthresources.globalchange.gov', survey_token)
sync_metadata_tree = {
'usgcrp-climate-and-health-assessment-draft': OrderedDict([
('executive-summary', []),
('climate-change-and-human-health', [
('/metadata/figures/3698', 'major_u_s__climate_trends'), #1.1 #climate-change-and-human-health
('/metadata/figures/3632', 'percent_changes_in_the_annual_number_of_extreme_precipitation_events_'), #1.2 #climate-change-and-human-health
('/metadata/figures/3635', 'projected_changes_in_temperature_and_precipitation_by_mid_century'), #1.3 #climate-change-and-human-health
('/metadata/figures/3633', 'projected_changes_in_the_hottest_coldest_and_wettest_driest_day_of_the_year'), #1.4 #climate-change-and-human-health
('/metadata/figures/3757', 'climate_change_and_health'), #1.5 #climate-change-and-human-health
]),
('temperature-related-death-and-illness', [
('/metadata/figures/3811', 'climate_change_and_health__extreme_heat'), #2.1 #temperature-related-death-and-illness
('/metadata/figures/3585', 'heat_related_deaths_during_the_1995_chicago_heat_wave'), #2.2 #temperature-related-death-and-illness
('/metadata/figures/3643', 'projected_net_changes_in_extreme_temperature_related_deaths'), #2.3 #temperature-related-death-and-illness
('/metadata/figures/3653', 'projected_changes_in_deaths_in_u_s__cities_by_season'), #2.4 #temperature-related-death-and-illness
]),
('air-quality-impacts', [
('/metadata/figures/3812', 'climate_change_and_health__outdoor_air_quality'), #3.1 #air-quality-impacts
('/metadata/figures/3647', 'projected_change_in_average_daily_maximum_temperature__seasonal_average_maximum_daily_8_hr_ozone__and_ozone_related_premature_deaths_in_2013'), #3.2 #air-quality-impacts
('/metadata/figures/3649', 'projected_change_in_ozone_related_premature_deaths'), #3.3 #air-quality-impacts
('/metadata/figures/3650', 'ragweed_pollen_season_lengthens'), #3.4 #air-quality-impacts
]),
('vectorborne-diseases', [
('/metadata/figures/3807', 'climate_change_and_health__lyme_disease'), #4.1 #vectorborne-diseases
('/metadata/figures/3659', 'changes_in_lyme_disease_case_report_distribution'), #4.2 #vectorborne-diseases
('/metadata/figures/3658', 'life_cycle_of_blacklegged_ticks__ixodes_scapularis'), #4.3 #vectorborne-diseases
('/metadata/figures/3747', 'lyme_disease_onset_week_modeling_scenarios'), #4.4 #vectorborne-diseases
('/metadata/figures/3674', 'incidence_of_west_nile_neuroinvasive_disease_in_the_united_states'), #4.5 #vectorborne-diseases
('/metadata/figures/3675', 'climate_impacts_on_west_nile_virus_transmission'), #4.6 #vectorborne-diseases
]),
('water-related-illnesses', [
('/metadata/figures/3824', 'climate_change_and_health___vibrio'), #5.1 #water-related-illnesses
('/metadata/figures/3700', 'links_between_climate_change__water_quantity_and_quality__and_human_exposure_to_water_related_illness'), #5.2 #water-related-illnesses
('/metadata/figures/3671', 'locations_of_livestock_and_projections_of_heavy_precipitation'), #5.3 #water-related-illnesses
# ('/metadata/figures/3673', 'potential_routes_of_manure_borne_microbial_contaminants_to_ground_and_surface_water_supplies_'), #5.3 #water-related-illnesses UNUSED?
('/metadata/figures/3709', 'projections_of_vibrio_occurrence_and_abundance_in_chesapeake_bay'), #5.4 #water-related-illnesses
('/metadata/figures/3704', 'changes_in_suitable_coastal_vibrio_habitat_in_alaska'), #5.5 #water-related-illnesses
('/metadata/figures/3734', 'projected_changes_in_caribbean_gambierdiscus_species'), #5.6 #water-related-illnesses
('/metadata/figures/3712', 'projections_of_growth_of_alexandrium_fundyense_in_puget_sound'), #5.7 #water-related-illnesses
]),
('food-safety--nutrition--and-distribution', [
('/metadata/figures/3579', 'farm_to_table'), #6.1 #food-safety--nutrition--and-distribution
# ('/metadata/figures/3600', 'mycotoxin_in_corn'), #6.1 #food-safety--nutrition--and-distribution BOX 1?
('/metadata/figures/3809', 'climate_change_and_health__salmonella'), #6.2 #food-safety--nutrition--and-distribution
('/metadata/figures/3748', 'seasonality_of_human_illnesses_associated_with_foodborne_pathogens'), #6.3 #food-safety--nutrition--and-distribution
('/metadata/figures/3688', 'effects_of_carbon_dioxide_on_protein_and_minerals'), #6.4 #food-safety--nutrition--and-distribution
('/metadata/figures/3597', 'mississippi_river_level_at_st__louis__missouri'), #6.5 #food-safety--nutrition--and-distribution
]),
('extreme-weather', [
('/metadata/figures/3810', 'estimated_deaths_and_billion_dollar_losses_from_extreme_weather_events_in_the_u_s__2004_2013'), #7.1 #extreme-weather
# ('/metadata/figures/3772', 'trends_in_flood_magnitude'), #7.2 #extreme-weather NOT USED
('/metadata/figures/3808', 'climate_change_and_health__flooding'), #7.2 #extreme-weather
('/metadata/figures/3760', 'hurricane_induced_flood_effects_in_eastern_and_central_united_states'), #7.3 #extreme-weather
]),
('mental-health-and-well-being', [
('/metadata/figures/3789', 'climate_change_and_mental_health'), #8.1 #mental-health-and-well-being
('/metadata/figures/3722', 'the_impact_of_climate_change_on_physical__mental__and_community_health'), #8.2 #mental-health-and-well-being
]),
# ('executive-summary', []),
# ('climate-change-and-human-health', [
# ('/metadata/figures/3698', 'major_u_s__climate_trends'), #1.1 #climate-change-and-human-health
# ('/metadata/figures/3632', 'percent_changes_in_the_annual_number_of_extreme_precipitation_events_'), #1.2 #climate-change-and-human-health
# ('/metadata/figures/3635', 'projected_changes_in_temperature_and_precipitation_by_mid_century'), #1.3 #climate-change-and-human-health
# ('/metadata/figures/3633', 'projected_changes_in_the_hottest_coldest_and_wettest_driest_day_of_the_year'), #1.4 #climate-change-and-human-health
# ('/metadata/figures/3757', 'climate_change_and_health'), #1.5 #climate-change-and-human-health
# ]),
# ('temperature-related-death-and-illness', [
# ('/metadata/figures/3811', 'climate_change_and_health__extreme_heat'), #2.1 #temperature-related-death-and-illness
# ('/metadata/figures/3585', 'heat_related_deaths_during_the_1995_chicago_heat_wave'), #2.2 #temperature-related-death-and-illness
# ('/metadata/figures/3643', 'projected_net_changes_in_extreme_temperature_related_deaths'), #2.3 #temperature-related-death-and-illness
# ('/metadata/figures/3653', 'projected_changes_in_deaths_in_u_s__cities_by_season'), #2.4 #temperature-related-death-and-illness
# ]),
# ('air-quality-impacts', [
# ('/metadata/figures/3812', 'climate_change_and_health__outdoor_air_quality'), #3.1 #air-quality-impacts
# ('/metadata/figures/3647', 'projected_change_in_average_daily_maximum_temperature__seasonal_average_maximum_daily_8_hr_ozone__and_ozone_related_premature_deaths_in_2013'), #3.2 #air-quality-impacts
# ('/metadata/figures/3649', 'projected_change_in_ozone_related_premature_deaths'), #3.3 #air-quality-impacts
# ('/metadata/figures/3650', 'ragweed_pollen_season_lengthens'), #3.4 #air-quality-impacts
# ]),
# ('vectorborne-diseases', [
# ('/metadata/figures/3807', 'climate_change_and_health__lyme_disease'), #4.1 #vectorborne-diseases
# ('/metadata/figures/3659', 'changes_in_lyme_disease_case_report_distribution'), #4.2 #vectorborne-diseases
# ('/metadata/figures/3658', 'life_cycle_of_blacklegged_ticks__ixodes_scapularis'), #4.3 #vectorborne-diseases
# ('/metadata/figures/3747', 'lyme_disease_onset_week_modeling_scenarios'), #4.4 #vectorborne-diseases
# ('/metadata/figures/3674', 'incidence_of_west_nile_neuroinvasive_disease_in_the_united_states'), #4.5 #vectorborne-diseases
# ('/metadata/figures/3675', 'climate_impacts_on_west_nile_virus_transmission'), #4.6 #vectorborne-diseases
# ]),
# ('water-related-illnesses', [
# ('/metadata/figures/3824', 'climate_change_and_health___vibrio'), #5.1 #water-related-illnesses
# ('/metadata/figures/3700', 'links_between_climate_change__water_quantity_and_quality__and_human_exposure_to_water_related_illness'), #5.2 #water-related-illnesses
# ('/metadata/figures/3671', 'locations_of_livestock_and_projections_of_heavy_precipitation'), #5.3 #water-related-illnesses
# # ('/metadata/figures/3673', 'potential_routes_of_manure_borne_microbial_contaminants_to_ground_and_surface_water_supplies_'), #5.3 #water-related-illnesses UNUSED?
# ('/metadata/figures/3709', 'projections_of_vibrio_occurrence_and_abundance_in_chesapeake_bay'), #5.4 #water-related-illnesses
# ('/metadata/figures/3704', 'changes_in_suitable_coastal_vibrio_habitat_in_alaska'), #5.5 #water-related-illnesses
# ('/metadata/figures/3734', 'projected_changes_in_caribbean_gambierdiscus_species'), #5.6 #water-related-illnesses
# ('/metadata/figures/3712', 'projections_of_growth_of_alexandrium_fundyense_in_puget_sound'), #5.7 #water-related-illnesses
# ]),
# ('food-safety--nutrition--and-distribution', [
# ('/metadata/figures/3579', 'farm_to_table'), #6.1 #food-safety--nutrition--and-distribution
# # ('/metadata/figures/3600', 'mycotoxin_in_corn'), #6.1 #food-safety--nutrition--and-distribution BOX 1?
# ('/metadata/figures/3809', 'climate_change_and_health__salmonella'), #6.2 #food-safety--nutrition--and-distribution
# ('/metadata/figures/3748', 'seasonality_of_human_illnesses_associated_with_foodborne_pathogens'), #6.3 #food-safety--nutrition--and-distribution
# ('/metadata/figures/3688', 'effects_of_carbon_dioxide_on_protein_and_minerals'), #6.4 #food-safety--nutrition--and-distribution
# ('/metadata/figures/3597', 'mississippi_river_level_at_st__louis__missouri'), #6.5 #food-safety--nutrition--and-distribution
# ]),
# ('extreme-weather', [
# ('/metadata/figures/3810', 'estimated_deaths_and_billion_dollar_losses_from_extreme_weather_events_in_the_u_s__2004_2013'), #7.1 #extreme-weather
# # ('/metadata/figures/3772', 'trends_in_flood_magnitude'), #7.2 #extreme-weather NOT USED
# ('/metadata/figures/3808', 'climate_change_and_health__flooding'), #7.2 #extreme-weather
# ('/metadata/figures/3760', 'hurricane_induced_flood_effects_in_eastern_and_central_united_states'), #7.3 #extreme-weather
# ]),
# ('mental-health-and-well-being', [
# ('/metadata/figures/3789', 'climate_change_and_mental_health'), #8.1 #mental-health-and-well-being
# ('/metadata/figures/3722', 'the_impact_of_climate_change_on_physical__mental__and_community_health'), #8.2 #mental-health-and-well-being
# ]),
('populations-of-concern', [
('/metadata/figures/3696', 'determinants_of_vulnerability'), #9.1 #populations-of-concern
# ('/metadata/figures/3696', 'determinants_of_vulnerability'), #9.1 #populations-of-concern
('/metadata/figures/3694', 'social_determinants_of_health'), #9.2 #populations-of-concern
('/metadata/figures/3758', 'children_at_different_lifestages_experience_unique_vulnerabilities_to_climate_change'), #9.3 #populations-of-concern
('/metadata/figures/3714', 'mapping_social_vulnerability'), #9.4 #populations-of-concern
('/metadata/figures/3717', 'mapping_communities_vulnerable_to_heat_in_georgia'), #9.5 #populations-of-concern
# ('/metadata/figures/3758', 'children_at_different_lifestages_experience_unique_vulnerabilities_to_climate_change'), #9.3 #populations-of-concern
# ('/metadata/figures/3714', 'mapping_social_vulnerability'), #9.4 #populations-of-concern
# ('/metadata/figures/3717', 'mapping_communities_vulnerable_to_heat_in_georgia'), #9.5 #populations-of-concern
]),
('appendix-1--technical-support-document', [
('/metadata/figures/3623', 'emissions_levels_determine_temperature_rises'), #1.1 #climate-change-and-human-health
('/metadata/figures/3759', 'the_shared_socioeconomic_pathways'), #1.2 #climate-change-and-human-health
('/metadata/figures/3726', 'example_spatial_resolution_of_climate_models'), #1.3 #climate-change-and-human-health
('/metadata/figures/3638', 'sensitivity_analysis_of_differences_in_modeling_approaches'), #1.4 #climate-change-and-human-health
])
# ('appendix-1--technical-support-document', [
# ('/metadata/figures/3623', 'emissions_levels_determine_temperature_rises'), #1.1 #climate-change-and-human-health
# ('/metadata/figures/3759', 'the_shared_socioeconomic_pathways'), #1.2 #climate-change-and-human-health
# ('/metadata/figures/3726', 'example_spatial_resolution_of_climate_models'), #1.3 #climate-change-and-human-health
# ('/metadata/figures/3638', 'sensitivity_analysis_of_differences_in_modeling_approaches'), #1.4 #climate-change-and-human-health
# ])
])
}
def main():
regenerate_image_id_map()
# regenerate_image_id_map()
image_id_map = pickle.load(open('image_id_cache.pk1', 'r'))
for report_id in sync_metadata_tree:
......@@ -97,16 +97,36 @@ def main():
print survey_url
s = surveys.get_survey(survey_url, do_download=False)
print s.contributors
print s.parents
print s.usage_limits
# for i in s.images:
# i.identifier = image_id_map[i.identifier]
#
# print i.contributors
# print i.parents
#
# for ds in i.datasets:
# if ds.activity and ds.activity.identifier is None:
# ds.activity.identifier = generate_activity_id(i, ds)
# print ds
# realize_parents(gcis, s.parents)
# realize_contributors(gcis, s.contributors)
for i in s.images:
i.identifier = image_id_map[i.identifier]
i.datasets = []
print gcis.create_image(i, report_id=report_id, figure_id=figure_id)
# for i in s.images:
# i.identifier = image_id_map[i.identifier]
# i.datasets = []
# print gcis.create_image(i, report_id=report_id, figure_id=figure_id)
def generate_activity_id(image, dataset):
return '-'.join([image.identifier.split('-')[0], dataset.identifier, '-process'])
def regenerate_image_id_map(existing=None):
from uuid import uuid4
image_id_map = existing if existing else {}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment