Commit ef618aa6 authored by abuddenberg's avatar abuddenberg
Browse files

Analyzing network problem

parent 84690aae
......@@ -5,8 +5,8 @@
.settings
*.pyc
*.pk1
src/blah.py
src/Blah.py
blah.py
Blah.py
dist/images/*.png
dist/*.tar.gz
gcis-py-client.iml
......
v0.68, Thu mar 27 11:55:43 EDT 2014 -- Renamed datasets identifiers to lowercase now that GCIS enforces that
v0.67, Mon Mar 24 13:24:01 EDT 2014 -- Initial release.
......@@ -4,7 +4,69 @@ gcis-py-client
gcis-py-client provides a python-based abstraction of the Global Change
Information System Structured Data Server [API](http://data.globalchange.gov).
Installation
------------
### Requirements
These should be available through the package manager of your choice
(Macports, apt, yum, etc.):
* Python 2.7.x (though it might work with previous versions)
* Setuptools 2.0.x
* pip is highly recommended
To install from git with pip:
pip install git+ssh://gcis-dev-back/usr/local/projects/repos/gcis-py-client.git/
To install from tarball:
tar -xzvf GcisPyClient-0.67.tar.gz
cd GcisPyClient-0.67
python setup.py test (to run the test suite)
python setup.py install (to install)
Usage
-----
More to follow. This is my first distributable python package.
from gcis_clients import GcisClient
base_url = 'http://data.gcis-dev-front.joss.ucar.edu'
gcis = GcisClient(base_url, 'andrew.buddenberg@noaa.gov', 'your api key here')
Make sure our credentials work:
status_code, resp_text = gcis.test_login()
print status_code, resp_text
assert 'auth_required' not in resp_text
Let's pull a list of all figures in Chapter 2 of the NCA3draft:
for partial_figure in gcis.get_figure_listing('nca3draft', chapter_id='our-changing-climate'):
full_figure = gcis.get_figure('nca3draft', partial_figure.identifier, chapter_id='our-changing-climate')
print full_figure
Let's work with the infamous temperature figure:
fig2_7 = gcis.get_figure('nca3draft', 'observed-us-temperature-change')
Warning: Images and Chapters are specifically excluded from JSON output. This is what gets sent to GCIS. So...
print fig2_7.as_json(indent=4)
...you might want to view the original input:
print fig2_7.original
Let's see about the images:
for image in fig2_7.images:
print image
How about the whole Image?:
print gcis.get_image('69da6d93-4426-4061-a2a1-7b3d01f2dc1c').as_json(indent=4)
......@@ -6,7 +6,12 @@ base_url = 'http://data.gcis-dev-front.joss.ucar.edu'
gcis = GcisClient(base_url, 'andrew.buddenberg@noaa.gov', 'fcee8e3f11f36313e463ece51aab15242f71f3d552d565be')
# #Let's pull a list of all figures in Chapter 2 of the NCA3draft
#Make sure our credentials work
status_code, resp_text = gcis.test_login()
print status_code, resp_text
assert 'auth_required' not in resp_text
#Let's pull a list of all figures in Chapter 2 of the NCA3draft
for partial_figure in gcis.get_figure_listing('nca3draft', chapter_id='our-changing-climate'):
#The listing doesn't provide all available fields for the figure (Images, for instance).
#There aren't very many figures, so let's go ahead and grab a complete version of each
......
......@@ -4,13 +4,14 @@ __author__ = 'abuddenberg'
import pickle
from gcis_clients import GcisClient
from gcis_clients import WebformClient
from sync_figures import upload_images_to_gcis
from gcis_clients.sync_utils import move_images_to_gcis
webform = WebformClient('http://resources.assessment.globalchange.gov', 'mgTD63FAjG')
gcis_url = 'http://data.gcis-dev-front.joss.ucar.edu'
gcis = GcisClient(gcis_url, 'andrew.buddenberg@noaa.gov', 'd9fcfd947c1785ab1cd329a9920e05e5c5d3d7f35315f164')
# gcis = GcisClient('http://data-stage.globalchange.gov', 'andrew.buddenberg@noaa.gov', 'a6efcc7cf39c55e9329a8b027e0817e3354bada65310d192')
# gcis = GcisClient('http://data-stage.globalchange.gov', 'andrew.buddenberg@noaa.gov', 'b4f1458c3cf28248c982428c46e170019327bd4c533c23dd')
def main():
hitlist_file = '../hitlist.pk1'
......@@ -42,7 +43,7 @@ def solve_problems(path, report_id):
assocs = img_assoc_broken - creates
if len(creates) > 0:
upload_images_to_gcis(webform_id, gcis_id, report_id, subset_images=creates)
move_images_to_gcis(webform_id, gcis_id, report_id, subset_images=creates)
for image_id in assocs:
print 'Associating image: {i} with figure: {f}'.format(i=image_id, f=gcis_id)
......
......@@ -3,99 +3,135 @@ __author__ = 'abuddenberg'
from gcis_clients import WebformClient
from gcis_clients import GcisClient
from os.path import exists
from gcis_clients.sync_utils import move_images_to_gcis, sync_dataset_metadata
from collections import OrderedDict
import json
import pickle
# webform_dev = ('http://dev.nemac.org/asides10/metadata/figures/all?token=A2PNYxRuG9')
webform = WebformClient('http://resources.assessment.globalchange.gov', 'mgTD63FAjG')
gcis_url = 'http://data.gcis-dev-front.joss.ucar.edu'
# gcis = GcisClient(gcis_url, 'andrew.buddenberg@noaa.gov', 'd9fcfd947c1785ab1cd329a9920e05e5c5d3d7f35315f164')
gcis = GcisClient('http://data-stage.globalchange.gov', 'andrew.buddenberg@noaa.gov', 'a6efcc7cf39c55e9329a8b027e0817e3354bada65310d192')
gcis = GcisClient('http://data-stage.globalchange.gov', 'andrew.buddenberg@noaa.gov', 'b4f1458c3cf28248c982428c46e170019327bd4c533c23dd')
sync_metadata_tree = {
#Reports
'nca3': {
'nca3': OrderedDict([
#Chapter 2
'our-changing-climate': [
#(webform_url, gcis_id)
('our-changing-climate', [
# (webform_url, gcis_id)
('/metadata/figures/2506', 'observed-change-in-very-heavy-precipitation'),
('/metadata/figures/2997', 'observed-change-in-very-heavy-precipitation-2'),
('/metadata/figures/2677', 'observed-us-precipitation-change'),
# # ('/metadata/figures/3175', 'observed-us-temperature-change'),
# # ('/metadata/figures/3074', 'ten-indicators-of-a-warming-world'),
# # ('/metadata/figures/3170', 'global-temperature-and-carbon-dioxide'),
# # ('/metadata/figures/3293', 'observed-increase-in-frostfree-season-length'),
# ('/metadata/figures/3294', 'projected-changes-in-frostfree-season-length'), #Good
# ('/metadata/figures/3305', 'variation-of-storm-frequency') #incomplete
],
# #Chapter 4
# 'energy-supply-and-use': [
# # ('/metadata/figures/3292', 'cooling-degree-days')
# ],
# #Chapter 6
# 'agriculture': [
# # ('/metadata/figures/2872', 'drainage'),
# # ('/metadata/figures/2691', 'variables-affecting-ag') #Needs images redone
# ('/metadata/figures/2691', 'projected-changes-in-key-climate-variables-affecting-agricultural-productivity')
# ],
# #Chapter 9
# # 'human-health': [
# # ('/metadata/figures/2896', 'heavy-downpours-disease') #Needs images redone
# # ],
# #Chapter 10
# 'water-energy-land-use': [
# ('/metadata/figures/2410', 'coasttocoast-100degree-days-in-2011')
# ],
# #Chapter 14
# 'rural': [
# ('/metadata/figures/3306', 'growing-season-lengthens') #Needs images redone
# ],
# #Chapter 16
# 'northeast': [
# ('/metadata/figures/2995', 'projected-increases-in-the-number-of-days-over-90f')
# ],
# #Chapter 17
# 'southeast': [
# ('/metadata/figures/2998', 'projected-change-in-number-of-days-over-95-f'),
# ('/metadata/figures/2999', 'projected-change-in-number-of-nights-below-32f')
# ],
# #Chapter 18
# 'midwest': [
# ('/metadata/figures/2992', 'projected-midcentury-temperature-changes-in-the-midwest'),
# ('/metadata/figures/2994', 'when-it-rains-it-pours')
# ],
# #Chapter 19
# # 'great-plains': [
# # ('/metadata/figures/2697', 'temperature-and-precipitation-distribution-in-the-great-plains') #Needs images redone
# #
# # ],
# #Chapter 25
# 'coastal-zone': [
# ('/metadata/figures/2543', 'coastal-ecosystem-services')
# ],
# # Climate Science Appendix
# 'appendix-climate-science': [
# ('/metadata/figures/3147', 'ice-loss-from-greenland-and-antarctica')
# ]
}
('/metadata/figures/3175', 'observed-us-temperature-change'),
('/metadata/figures/3074', 'ten-indicators-of-a-warming-world'),
('/metadata/figures/3170', 'global-temperature-and-carbon-dioxide'),
('/metadata/figures/3293', 'observed-increase-in-frostfree-season-length'),
('/metadata/figures/3294', 'projected-changes-in-frostfree-season-length'), #Good
('/metadata/figures/3305', 'variation-of-storm-frequency-and-intensity-during-the-cold-season-november--march'),
('/metadata/figures/2939', 'projected-changes-in-soil-moisture-for-the-western-us'),
('/metadata/figures/3298', 'observed-us-trend-in-heavy-precipitation'),
('/metadata/figures/2523', 'shells-dissolve-in-acidified-ocean-water')
]),
#Chapter 4
('energy-supply-and-use', [
('/metadata/figures/3292', 'cooling-degree-days')
]),
#Chapter 5
('transportation', [
('/metadata/figures/3568', 'hurricane-sandy-causes-flooding-in-new-york-city-subway-stations'),
('/metadata/figures/3169', 'tropical-storm-impact-on-vermont-road'),
('/metadata/figures/2952', 'role-of-adaptive-strategies-and-tactics-in-reducing-impacts-and-consequences')
]),
#Chapter 6
('agriculture', [
('/metadata/figures/2872', 'drainage-tiles'),
('/metadata/figures/2691', 'projected-changes-in-key-climate-variables-affecting-agricultural-productivity')
]),
#Chapter 7
('forests', [
('/metadata/figures/2977', 'effectiveness-of-forest-management-in-reducing-wildfire-risk'),
('/metadata/figures/2978', 'forest-vulnerability-to-changing-climate'), #Create
('/metadata/figures/2985', 'forests-can-be-a-source--or-a-sink--for-carbon')
]),
#Chapter 8
('ecosystems', [
('/metadata/figures/2456', 'adaptation-planning-and-implementation-framework'),
('/metadata/figures/3574', 'biological-responses-to-climate-change')
]),
#Chapter 9
('human-health', [
('/metadata/figures/2896', 'heavy-downpours-disease') #Needs images redone
]),
#Chapter 10
('water-energy-land-use', [
('/metadata/figures/2410', 'coasttocoast-100degree-days-in-2011'),
('/metadata/figures/2986', 'hydraulic-fracturing-and-water-use'),
('/metadata/figures/2916', 'renewable-energy-and-land-use')
]),
#Chapter 11
('urban-systems-infrastructure-vulnerability', [
('/metadata/figures/3569', 'urban-support-systems-are-interconnected')
]),
#Chapter 12
('tribal-indigenous-native-lands-resources', [
('/metadata/figures/2911', 'arctic-marine-food-web')
]),
#Chapter 14
('rural', [
('/metadata/figures/3306', 'growing-season-lengthens') #Needs images redone
]),
# Chapter 15
('biogeochemical-cycles', [
('/metadata/figures/2575', 'many-factors-combine-to-affect-biogeochemical-cycles')
]),
#Chapter 16
('northeast', [
('/metadata/figures/2995', 'projected-increases-in-the-number-of-days-over-90f'),
('/metadata/figures/2844', 'coney-island-after-hurricane-irene'),
('/metadata/figures/2846', 'storm-surge-barrier') #Wrong figurenum in GCIS
]),
#Chapter 17
('southeast', [
('/metadata/figures/2998', 'projected-change-in-number-of-days-over-95-f'),
('/metadata/figures/2999', 'projected-change-in-number-of-nights-below-32f'),
('/metadata/figures/2857', 'local-planning')
]),
#Chapter 18
('midwest', [
('/metadata/figures/2992', 'projected-midcentury-temperature-changes-in-the-midwest'),
('/metadata/figures/2994', 'when-it-rains-it-pours'),
('/metadata/figures/2550', 'temperatures-are-rising-in-the-midwest')
]),
#Chapter 19
('great-plains', [
('/metadata/figures/2697', 'temperature-and-precipitation-distribution-in-the-great-plains'), #Needs images redone
('/metadata/figures/2989', 'projected-change-in-number-of-hot-days'),
('/metadata/figures/2991', 'projected-change-in-number-of-heavy-precipitation-days'),
('/metadata/figures/3002', 'projected-change-in-number-of-consecutive-dry-days'),
('/metadata/figures/2990', 'projected-change-in-number-of-warm-nights')
]),
#Chapter 25
('coastal-zone', [
('/metadata/figures/2543', 'coastal-ecosystem-services')
]),
# Climate Science Appendix
('appendix-climate-science', [
('/metadata/figures/3147', 'ice-loss-from-greenland-and-antarctica')
])
])
}
def main():
# print gcis.test_login()
# sync_dataset_metadata(aggregate_webform_datasets())
sync(replace=False)
# with open('../datasets.pk1', 'wb') as outfile:
# pickle.dump(aggregate_webform_datasets(), outfile)
# datasets = pickle.load(open('../datasets.pk1', 'r'))
# sync_dataset_metadata(datasets)
sync(replace=True)
# f = webform.get_webform('/metadata/figures/2575', download_images=True)
# print gcis.create_figure('nca3', 'biogeochemical-cycles', f)
def sync(replace=False):
......@@ -118,60 +154,12 @@ def sync(replace=False):
gcis.delete_image(image)
print 'Attempting to upload: {id}'.format(id=gcis_id)
upload_images_to_gcis(webform_url, gcis_id, report_id)
move_images_to_gcis(webform, gcis, webform_url, gcis_id, report_id)
#...then send it.
gcis.update_figure(report_id, chapter_id, figure_obj)
print 'Success!'
#This function is for adding images to existing figures
def upload_images_to_gcis(webform_url, gcis_id, report_id, subset_images=None):
figure = webform.get_webform(webform_url, download_images=True)
#Now identifiers don't need to be matched
figure.identifier = gcis_id
#If a subset of identifiers has been provided, only process those
if subset_images:
images_to_process = [image for image in figure.images if image.identifier in subset_images]
else:
images_to_process = figure.images
for image in images_to_process:
if not exists(image.local_path):
raise Exception('Local file missing ' + image.local_path)
if not gcis.image_exists(image.identifier):
print 'Creating image: {img}'.format(img=image.identifier)
gcis.create_image(image, report_id=report_id, figure_id=figure.identifier)
def sync_dataset_metadata(datasets):
for ds in datasets:
gcis.create_or_update_dataset(ds)
gcis.create_or_update_activity(ds.activity)
def aggregate_webform_datasets():
dataset_map = {}
for item in webform.get_list():
webform_url = item['url']
f = webform.get_webform(webform_url)
#aggregate datasets
for image in f.images:
for dataset in image.datasets:
if dataset.identifier not in dataset_map:
dataset_map[dataset.identifier] = dataset
else:
dataset_map[dataset.identifier].merge(dataset)
dataset_map[dataset.identifier].activity.merge(dataset.activity)
return dataset_map.values()
if __name__ == '__main__':
main()
\ No newline at end of file
......@@ -4,6 +4,7 @@ import json
from os.path import exists, basename
import requests
from requests.exceptions import ConnectionError
from domain import Figure, Image, Dataset, Activity
......@@ -93,7 +94,7 @@ class GcisClient(object):
b=self.base_url, rpt=report_id, chp=chapter_id, fig=figure.identifier
)
resp = requests.post(update_url, figure.as_json(), headers=self.headers, verify=False)
resp = requests.post(update_url, data=figure.as_json(), headers=self.headers, verify=False)
if skip_images is False:
for image in figure.images:
......@@ -108,18 +109,20 @@ class GcisClient(object):
@check_image
def create_image(self, image, report_id=None, figure_id=None):
url = '{b}/image/'.format(b=self.base_url, img=image.identifier)
resp = requests.post(url, image.as_json(), headers=self.headers, verify=False)
if image.local_path is not None:
self.upload_image_file(image.identifier, image.local_path)
if figure_id and report_id:
self.associate_image_with_figure(image.identifier, report_id, figure_id)
for dataset in image.datasets:
self.create_or_update_dataset(dataset)
self.create_or_update_activity(dataset.activity)
self.associate_dataset_with_image(dataset.identifier, image.identifier,
activity_id=dataset.activity.identifier)
url = '{b}/image/'.format(b=self.base_url)
resp = requests.post(url, data=image.as_json(), headers=self.headers, verify=False)
try:
if image.local_path is not None:
self.upload_image_file(image.identifier, image.local_path)
if figure_id and report_id:
self.associate_image_with_figure(image.identifier, report_id, figure_id)
for dataset in image.datasets:
self.create_or_update_dataset(dataset)
self.create_or_update_activity(dataset.activity)
self.associate_dataset_with_image(dataset.identifier, image.identifier,
activity_id=dataset.activity.identifier)
except ConnectionError, e:
print e
return resp
@check_image
......@@ -130,7 +133,7 @@ class GcisClient(object):
self.associate_dataset_with_image(dataset.identifier, image.identifier,
activity_id=dataset.activity.identifier)
return requests.post(update_url, image.as_json(), headers=self.headers, verify=False)
return requests.post(update_url, data=image.as_json(), headers=self.headers, verify=False)
@check_image
def delete_image(self, image):
......@@ -139,7 +142,7 @@ class GcisClient(object):
def associate_image_with_figure(self, image_id, report_id, figure_id):
url = '{b}/report/{rpt}/figure/rel/{fig}'.format(b=self.base_url, rpt=report_id, fig=figure_id)
return requests.post(url, json.dumps({'add_image_identifier': image_id}), headers=self.headers, verify=False)
return requests.post(url, data=json.dumps({'add_image_identifier': image_id}), headers=self.headers, verify=False)
def upload_image_file(self, image_id, local_path):
url = '{b}/image/files/{id}/{fn}'.format(b=self.base_url, id=image_id, fn=basename(local_path))
......
__author__ = 'abuddenberg'
from os.path import exists
#This function is for adding images to existing figures
def move_images_to_gcis(webform_client, gcis_client, webform_url, gcis_id, report_id, subset_images=None):
figure = webform_client.get_webform(webform_url, download_images=True)
#Now identifiers don't need to be matched
figure.identifier = gcis_id
#If a subset of identifiers has been provided, only process those
if subset_images:
images_to_process = [image for image in figure.images if image.identifier in subset_images]
else:
images_to_process = figure.images
for image in images_to_process:
if not exists(image.local_path):
raise Exception('Local file missing ' + image.local_path)
if not gcis_client.image_exists(image.identifier):
print 'Creating image: {img}'.format(img=image.identifier)
gcis_client.create_image(image, report_id=report_id, figure_id=figure.identifier)
def sync_dataset_metadata(gcis_client, datasets):
for ds in datasets:
gcis_client.create_or_update_dataset(ds)
gcis_client.create_or_update_activity(ds.activity)
\ No newline at end of file
......@@ -9,7 +9,7 @@ import json
import pytest
from test_data import test_figure_json, test_image_json, webform_json_precip, test_dataset_json
from gcis_client.domain import Gcisbase, Figure, Image, Dataset, Chapter
from gcis_clients.domain import Gcisbase, Figure, Image, Dataset, Chapter
def test_gcis_client_version():
......
......@@ -130,4 +130,21 @@ class WebformClient:
for image in figure.images:
self.download_image(image)
def get_aggregated_datasets(self):
dataset_map = {}
for item in self.get_list():
webform_url = item['url']
f = self.get_webform(webform_url)
#aggregate datasets
for image in f.images:
for dataset in image.datasets:
if dataset.identifier not in dataset_map:
dataset_map[dataset.identifier] = dataset
else:
dataset_map[dataset.identifier].merge(dataset)
dataset_map[dataset.identifier].activity.merge(dataset.activity)
return dataset_map.values()
\ No newline at end of file
......@@ -19,11 +19,11 @@ class PyTest(TestCommand):
setup(
name='GcisPyClient',
version='0.67',
version='0.68',
author='Andrew Buddenberg',
author_email='andrew.buddenberg@noaa.gov',
packages=find_packages(),
scripts=['bin/example', 'bin/problems', 'bin/sync_figures'],
scripts=['bin/example', 'bin/problems', 'bin/sync_figures.py'],
url='http://data.globalchange.gov',
description='CLient for GCIS webservices',
long_description=open('README.txt').read(),
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment