Commit 09d19feb authored by abuddenberg's avatar abuddenberg
Browse files

Added support for Activities

parent 865e1605
......@@ -286,4 +286,26 @@ class Dataset(Gcisbase):
if match:
self._publication_year = match.group()
else:
self._publication_year = None
\ No newline at end of file
self._publication_year = None
class Activity(Gcisbase):
def __init__(self, data):
self.gcis_fields = ['start_time', 'uri', 'methodology', 'data_usage', 'href', 'metholodogies', 'end_time',
'output_artifacts', 'duration', 'identifier', 'publication_maps', 'computing_environment']
self.translations = {
'how_much_time_was_invested_in_creating_the_image': 'duration',
'35_what_are_all_of_the_files_names_and_extensions_associated_with_this_image': 'output_artifacts',
'what_operating_systems_and_platforms_were_used': 'computing_environment',
'what_analytical_statistical_methods_were_employed_to_the_data': 'methodology',
'describe_how_the_data_was_used_in_the_image_figure_creation': 'data_usage'
}
super(Activity, self).__init__(data, fields=self.gcis_fields, trans=self.translations)
def as_json(self, indent=0):
return super(Activity, self).as_json(omit_fields=['metholodogies', 'publication_maps'])
......@@ -5,7 +5,7 @@ import urllib
import json
import requests
from os.path import exists, basename
from domain import Figure, Image, Dataset
from domain import Figure, Image, Dataset, Activity
def check_image(fn):
......@@ -31,6 +31,16 @@ def exists(fn):
return wrapped
def http_resp(fn):
def wrapped(*args, **kwargs):
resp = fn(*args, **kwargs)
if resp.status_code == 200:
return resp
else:
raise Exception(resp.text)
return wrapped
class GcisClient(object):
def __init__(self, url, username, password):
self.headers = {
......@@ -94,21 +104,25 @@ class GcisClient(object):
@check_image
def create_image(self, image, report_id=None, figure_id=None):
url = '{b}/image/'.format(b=self.base_url, img=image.identifier)
responses = [requests.post(url, image.as_json(), headers=self.headers)]
resp = requests.post(url, image.as_json(), headers=self.headers)
if image.local_path is not None:
responses.append(self.upload_image_file(image.identifier, image.local_path))
self.upload_image_file(image.identifier, image.local_path)
if figure_id and report_id:
responses.append(self.associate_image_with_figure(image.identifier, report_id, figure_id))
self.associate_image_with_figure(image.identifier, report_id, figure_id)
for dataset in image.datasets:
self.associate_dataset_with_image(dataset.identifier, image.identifier)
self.create_activity(dataset.activity)
self.associate_dataset_with_image(dataset.identifier, image.identifier,
activity_id=dataset.activity.identifier)
return responses
return resp
@check_image
def update_image(self, image):
update_url = '{b}/image/{img}'.format(b=self.base_url, img=image.identifier)
for dataset in image.datasets:
self.associate_dataset_with_image(dataset.identifier, image.identifier)
self.update_activity(dataset.activity)
self.associate_dataset_with_image(dataset.identifier, image.identifier,
activity_id=dataset.activity.identifier)
return requests.post(update_url, image.as_json(), headers=self.headers)
......@@ -240,20 +254,78 @@ class GcisClient(object):
url = '{b}/dataset/{ds}'.format(b=self.base_url, ds=dataset.identifier)
return requests.delete(url, headers=self.headers)
def associate_dataset_with_image(self, dataset_id, image_id):
def associate_dataset_with_image(self, dataset_id, image_id, activity_id=None):
url = '{b}/image/prov/{img}'.format(b=self.base_url, img=image_id)
data = {
'parent_uri': '/dataset/' + dataset_id,
'parent_rel': 'prov:wasDerivedFrom'
}
if activity_id:
data['activity'] = activity_id
self.delete_dataset_image_assoc(dataset_id, image_id)
resp = requests.post(url, data=json.dumps(data), headers=self.headers)
if resp.status_code == 200:
return resp
#TODO: Change to 409 in next release
elif resp.status_code == 400:
print resp.text
print 'Duplicate dataset association {ds} for image: {img}'.format(ds=dataset_id, img=image_id)
return resp
else:
raise Exception('Dataset association failed:\n{url}\n{resp}'.format(url=url, resp=resp.text))
def delete_dataset_image_assoc(self, dataset_id, image_id):
url = '{b}/image/prov/{img}'.format(b=self.base_url, img=image_id)
data = {
'delete': {
'parent_uri': '/dataset/' + dataset_id,
'parent_rel': 'prov:wasDerivedFrom'
}
}
print data
print json.dumps(data)
resp = requests.post(url, data=json.dumps(data), headers=self.headers)
if resp.status_code == 200:
return resp
else:
print resp.status_code
raise Exception('Dataset dissociation failed:\n{url}\n{resp}'.format(url=url, resp=resp.text))
# @exists
def activity_exists(self, activity_id):
url = '{b}/activity/{act}'.format(b=self.base_url, act=activity_id)
resp = requests.head(url, headers=self.headers)
if resp.status_code == 200:
return True
else:
return False
def get_activity(self, activity_id):
url = '{b}/activity/{act}'.format(b=self.base_url, act=activity_id)
resp = requests.get(url, headers=self.headers, verify=False)
try:
return Activity(resp.json())
except ValueError:
raise Exception(resp.text())
@http_resp
def create_activity(self, activity):
url = '{b}/activity/'.format(b=self.base_url)
return requests.post(url, data=activity.as_json(), headers=self.headers)
@http_resp
def update_activity(self, activity):
url = '{b}/activity/{act}'.format(b=self.base_url, act=activity.identifier)
return requests.post(url, data=activity.as_json(), headers=self.headers)
@http_resp
def delete_activity(self, activity):
url = '{b}/activity/{act}'.format(b=self.base_url, act=activity.identifier)
return requests.delete(url, headers=self.headers)
......@@ -4,6 +4,7 @@ from webform_client import WebformClient
from gcis_client import GcisClient
from os.path import exists
import json
import pickle
# webform_dev = ('http://dev.nemac.org/asides10/metadata/figures/all?token=A2PNYxRuG9')
......@@ -19,72 +20,84 @@ sync_metadata_tree = {
#Chapter 2
'our-changing-climate': [
#(webform_url, gcis_id)
('/metadata/figures/2506', 'observed-change-in-very-heavy-precipitation'),
('/metadata/figures/2997', 'observed-change-in-very-heavy-precipitation-2'),
# ('/metadata/figures/2506', 'observed-change-in-very-heavy-precipitation'),
# ('/metadata/figures/2997', 'observed-change-in-very-heavy-precipitation-2'),
('/metadata/figures/2677', 'observed-us-precipitation-change'),
# ('/metadata/figures/3175', 'observed-us-temperature-change'),
# ('/metadata/figures/3074', 'ten-indicators-of-a-warming-world'),
# ('/metadata/figures/3170', 'global-temperature-and-carbon-dioxide'),
# ('/metadata/figures/3293', 'observed-increase-in-frostfree-season-length'),
('/metadata/figures/3294', 'projected-changes-in-frostfree-season-length'), #Good
# # ('/metadata/figures/3175', 'observed-us-temperature-change'),
# # ('/metadata/figures/3074', 'ten-indicators-of-a-warming-world'),
# # ('/metadata/figures/3170', 'global-temperature-and-carbon-dioxide'),
# # ('/metadata/figures/3293', 'observed-increase-in-frostfree-season-length'),
# ('/metadata/figures/3294', 'projected-changes-in-frostfree-season-length'), #Good
# ('/metadata/figures/3305', 'variation-of-storm-frequency') #incomplete
],
#Chapter 4
'energy-supply-and-use': [
# ('/metadata/figures/3292', 'cooling-degree-days')
],
#Chapter 6
'agriculture': [
# ('/metadata/figures/2872', 'drainage'),
# ('/metadata/figures/2691', 'variables-affecting-ag') #Needs images redone
('/metadata/figures/2691', 'projected-changes-in-key-climate-variables-affecting-agricultural-productivity')
],
#Chapter 9
# 'human-health': [
# ('/metadata/figures/2896', 'heavy-downpours-disease') #Needs images redone
# #Chapter 4
# 'energy-supply-and-use': [
# # ('/metadata/figures/3292', 'cooling-degree-days')
# ],
#Chapter 10
'water-energy-land-use': [
('/metadata/figures/2410', 'coasttocoast-100degree-days-in-2011')
],
#Chapter 14
'rural': [
('/metadata/figures/3306', 'growing-season-lengthens') #Needs images redone
],
#Chapter 16
'northeast': [
('/metadata/figures/2995', 'projected-increases-in-the-number-of-days-over-90f')
],
#Chapter 17
'southeast': [
('/metadata/figures/2998', 'projected-change-in-number-of-days-over-95-f'),
('/metadata/figures/2999', 'projected-change-in-number-of-nights-below-32f')
],
#Chapter 18
'midwest': [
('/metadata/figures/2992', 'projected-midcentury-temperature-changes-in-the-midwest'),
('/metadata/figures/2994', 'when-it-rains-it-pours')
],
#Chapter 19
# 'great-plains': [
# ('/metadata/figures/2697', 'temperature-and-precipitation-distribution-in-the-great-plains') #Needs images redone
#
# #Chapter 6
# 'agriculture': [
# # ('/metadata/figures/2872', 'drainage'),
# # ('/metadata/figures/2691', 'variables-affecting-ag') #Needs images redone
# ('/metadata/figures/2691', 'projected-changes-in-key-climate-variables-affecting-agricultural-productivity')
# ],
#Chapter 25
'coastal-zone': [
('/metadata/figures/2543', 'coastal-ecosystem-services')
],
# Climate Science Appendix
'appendix-climate-science': [
('/metadata/figures/3147', 'ice-loss-from-greenland-and-antarctica')
]
# #Chapter 9
# # 'human-health': [
# # ('/metadata/figures/2896', 'heavy-downpours-disease') #Needs images redone
# # ],
# #Chapter 10
# 'water-energy-land-use': [
# ('/metadata/figures/2410', 'coasttocoast-100degree-days-in-2011')
# ],
# #Chapter 14
# 'rural': [
# ('/metadata/figures/3306', 'growing-season-lengthens') #Needs images redone
# ],
# #Chapter 16
# 'northeast': [
# ('/metadata/figures/2995', 'projected-increases-in-the-number-of-days-over-90f')
# ],
# #Chapter 17
# 'southeast': [
# ('/metadata/figures/2998', 'projected-change-in-number-of-days-over-95-f'),
# ('/metadata/figures/2999', 'projected-change-in-number-of-nights-below-32f')
# ],
# #Chapter 18
# 'midwest': [
# ('/metadata/figures/2992', 'projected-midcentury-temperature-changes-in-the-midwest'),
# ('/metadata/figures/2994', 'when-it-rains-it-pours')
# ],
# #Chapter 19
# # 'great-plains': [
# # ('/metadata/figures/2697', 'temperature-and-precipitation-distribution-in-the-great-plains') #Needs images redone
# #
# # ],
# #Chapter 25
# 'coastal-zone': [
# ('/metadata/figures/2543', 'coastal-ecosystem-services')
# ],
# # Climate Science Appendix
# 'appendix-climate-science': [
# ('/metadata/figures/3147', 'ice-loss-from-greenland-and-antarctica')
# ]
}
}
def main():
sync(replace=True)
# sync_dataset_metadata(aggregate_webform_datasets())
sync(replace=False)
# with open('../datasets.pk1', 'wb') as outfile:
# pickle.dump(aggregate_webform_datasets(), outfile)
#
# datasets = pickle.load(open('../datasets.pk1', 'r'))
# # sync_dataset_metadata(datasets)
# for ds in datasets:
# print ds.identifier, ': '
# print ds.activity.as_json(indent=4)
# print ds.activity.uri
def sync(replace=False):
......@@ -146,6 +159,8 @@ def sync_dataset_metadata(datasets):
print 'Creating: {ds}'.format(ds=ds)
gcis.create_dataset(ds)
create_or_update_activity(ds.activity)
def aggregate_webform_datasets():
dataset_map = {}
......@@ -162,9 +177,19 @@ def aggregate_webform_datasets():
dataset_map[dataset.identifier] = dataset
else:
dataset_map[dataset.identifier].merge(dataset)
dataset_map[dataset.identifier].activity.merge(dataset.activity)
return dataset_map.values()
def create_or_update_activity(activity):
if gcis.activity_exists(activity.identifier):
print 'Updating: {act}'.format(act=activity)
gcis.update_activity(activity)
else:
print 'Creating: {act}'.format(act=activity)
gcis.create_activity(activity)
if __name__ == '__main__':
main()
\ No newline at end of file
......@@ -6,7 +6,7 @@ import re
from os.path import join
from dateutil.parser import parse
from domain import Figure, Image, Dataset
from domain import Figure, Image, Dataset, Activity
def sanitized(pattern):
......@@ -72,6 +72,15 @@ class WebformClient:
dataset.spatial_extent = ' '.join(['{k}: {v};'.format(k=key, v=dataset_json[key]) for key in
['maximum_latitude', 'minimum_latitude', 'maximum_longitude',
'minimum_longitude']])
#Filter overlapping Dataset keys out
activity_json = {k: dataset_json[k] for k in dataset_json if
k not in ['href', 'uri', 'identifier', 'start_time', 'end_time']}
#Add synthetic identifier
activity_json['identifier'] = dataset.identifier + '-process'
dataset.activity = Activity(activity_json)
#TODO: Extract DOIs from citation
image_obj.datasets.append(dataset)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment