Commit 763b5445 authored by abuddenberg's avatar abuddenberg
Browse files

Got basic image upload workflow going. Needs refinement

parent a2166c04
......@@ -30,6 +30,12 @@ class Gcisbase(object):
finally:
setattr(self, k, data[k])
def merge(self, other):
for k in self.__dict__:
if self.__dict__[k] in (None, '') and hasattr(other, k):
self.__dict__[k] = getattr(other, k)
return self
class Figure(Gcisbase):
_gcis_fields = [
......@@ -109,7 +115,13 @@ class Image(Gcisbase):
#Hack
self.identifier = self.identifier.replace('/image/', '')
self.filepath = filepath
webform_filename = data.pop('what_is_the_file_name_extension_of_the_image', None)
if filepath is not None:
self.filepath = filepath
elif webform_filename is not None:
self.filepath = '/system/files/' + webform_filename.lower()
else:
self.filepath = None
def as_json(self, indent=0):
out_fields = self._gcis_fields
......
......@@ -28,6 +28,14 @@ for image in fig2_6.images:
#How about the whole Image?
print gcis.get_image('69da6d93-4426-4061-a2a1-7b3d01f2dc1c').as_json(indent=4)
#Let's assign some GCMD keywords (The first 4 from the list)
keyword_ids = [k['identifier'] for k in gcis.get_keyword_listing()[0:4]]
for keyword_id in keyword_ids:
resp = gcis.associate_keyword_with_figure(keyword_id, 'nca3draft', 'observed-us-temperature-change')
print resp.status_code, resp.text
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager" inherit-compiler-output="true">
<exclude-output />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$" isTestSource="false" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="PyDocumentationSettings">
<option name="myDocStringFormat" value="Plain" />
</component>
<component name="TestRunnerService">
<option name="projectConfiguration" value="Unittests" />
<option name="PROJECT_TEST_RUNNER" value="Unittests" />
</component>
</module>
......@@ -3,6 +3,7 @@
from base64 import b64encode
import urllib
import json
from wsgiref import headers
import requests
from domain import Figure, Image
......@@ -76,8 +77,8 @@ class GcisClient(object):
def create_image(self, image, report_id=None, figure_id=None):
url = '{b}/image/'.format(b=self.base_url, img=image.identifier)
responses = [requests.post(url, image.as_json(), headers=self.headers)]
if image.filename is not None:
responses.append(self.upload_image_file(image.identifier, image.filename))
if image.filepath is not None:
responses.append(self.upload_image_file(image.identifier, image.filepath))
if figure_id and report_id:
responses.append(self.associate_image_with_figure(image.identifier, report_id, figure_id))
......@@ -112,7 +113,10 @@ class GcisClient(object):
)
resp = requests.get(url, headers=self.headers)
return [Figure(figure) for figure in resp.json()]
try:
return [Figure(figure) for figure in resp.json()]
except ValueError:
raise Exception('Add a better exception string here')
def get_figure(self, report_id, figure_id, chapter_id=None):
chapter_filter = '/chapter/' + chapter_id if chapter_id else ''
......@@ -135,4 +139,22 @@ class GcisClient(object):
def test_login(self):
url = '{b}/login.json'.format(b=self.base_url)
resp = requests.get(url, headers=self.headers)
return resp.status_code, resp.text
\ No newline at end of file
return resp.status_code, resp.text
def get_keyword_listing(self):
url = '{b}/gcmd_keyword?{p}'.format(b=self.base_url, p=urllib.urlencode({'all': '1'}))
# print url
resp = requests.get(url, headers=self.headers)
# print resp.headers
return resp.json()
def get_keyword(self, key_id):
url = '{b}/gcmd_keyword/{k}'.format(b=self.base_url, k=key_id)
return requests.get(url, headers=self.headers).json()
def associate_keyword_with_figure(self, keyword_id, report_id, figure_id):
url = '{b}/report/{rpt}/figure/keywords/{fig}'.format(b=self.base_url, rpt=report_id, fig=figure_id)
print url
# print json.dumps()
return requests.post(url, data=json.dumps({'identifier': keyword_id}), headers=self.headers)
\ No newline at end of file
......@@ -2,38 +2,57 @@ __author__ = 'abuddenberg'
from webform_client import WebformClient
from gcis_client import GcisClient
import json
from subprocess import call
# webform_dev = ('http://dev.nemac.org/asides10/metadata/figures/all?token=A2PNYxRuG9')
webform = WebformClient('http://resources.assessment.globalchange.gov', 'mgTD63FAjG')
gcis_url = 'http://data.gcis-dev-front.joss.ucar.edu'
gcis = GcisClient(gcis_url, 'andrew.buddenberg@noaa.gov', 'fcee8e3f11f36313e463ece51aab15242f71f3d552d565be')
gcis = GcisClient(gcis_url, 'andrew.buddenberg@noaa.gov', '4cd31dc7173eb47b26f616fb07db607f25ab861552e81195')
# stage = GcisClient('http://data-stage.globalchange.gov', 'andrew.buddenberg@noaa.gov', 'ef427a895acf26d4f0b1f053ba7d922791b76f7852e7efee')
#Hack
file_map = {
'69da6d93-4426-4061-a2a1-7b3d01f2dc1c': '../AK.jpg',
'230cb2f8-92e0-4897-ab5f-4d6339673832': '../US.jpg',
'1f5a3cdd-fc45-403e-bf11-d1772005b430': '../GPN.jpg',
'b180cfd9-b064-4644-a9a1-d2c3660c1be7': '../MW.jpg',
'fa83c34b-7b67-4b74-bcba-5bf60ba7730f': '../NE.jpg',
'ca983a87-53a7-4c42-b0e9-18d26fad40ba': '../SE.jpg',
'68537d68-b14c-4811-908a-5dc0ab73879b': '../GPS.jpg',
'26a28c2a-75f2-47f7-a40f-becfc468d3d6': '../SW.jpg',
'f69194e8-397d-4f9c-836c-335d259ee09c': '../HI.jpg',
'db4d291d-17c5-4e10-b760-6c8799a8d709': '../NW.jpg',
'8e74f576-a5af-46c0-b33a-f30072118b86': '../usgcrp_draft-038-012.jpg',
}
figure = webform.get_webform('/metadata/figures/3175')
# print gcis.update_figure('nca3draft', figure)
print webform.get_list()
# for image in figure.images[2:]:
# print gcis.update_figure('nca3draft', 'our-changing-climate', figure, skip_images=True)
# for item in webform.get_list():
# webform_url = item['url']
# print webform_url
# print webform.get_webform(webform_url)
heavy_precip = webform.get_webform('/metadata/figures/2506').merge(
gcis.get_figure('nca3draft', 'observed-change-in-very-heavy-precipitation', chapter_id='our-changing-climate')
)
# for i in heavy_precip.images[1:]:
# gcis.create_image(i, report_id='nca3draft', figure_id='observed-change-in-very-heavy-precipitation')
gcis.update_figure('nca3draft', 'our-changing-climate', heavy_precip)
# heavy_precip = webform.get_webform('/metadata/figures/2506')
#
# for i in heavy_precip.images:
# # print i.as_json()
# # print i.filepath, webform.image_exists(i.filepath)
# image = webform.download_image(i.filepath)
# call(['/opt/local/bin/convert', image, image.replace('.eps', '.png')])
# print json.dumps(webform.get_webform('/metadata/figures/3294').original, indent=4)
# for image in figure.images[1:]:
# print image.identifier
# image.filename = file_map[image.identifier]
#
# print gcis.create_image(image, report_id='nca3draft', figure_id='temperature-change')
# print gcis.associate_image_with_figure(image.identifier, 'nca3draft', figure.identifier)
# print gcis.create_image(image, report_id='nca3draft', figure_id='temperature-change')
......@@ -3,6 +3,7 @@
import urllib
import requests
import re
import os.path
from domain import Figure, Image
......@@ -20,9 +21,10 @@ def sanitized(pattern):
class WebformClient:
def __init__(self, url, token):
def __init__(self, url, token, local_image_repo='../dist/images/'):
self.base_url = url
self.token = token
self.images_dir = local_image_repo
def get_list(self):
url = '{b}/metadata/list?token={t}'.format(b=self.base_url, t=self.token)
......@@ -40,19 +42,38 @@ class WebformClient:
#TODO: refactor the service so this isn't necessary
id = figure_json.keys()[0]
f = Figure(figure_json[id]['figure'][0])
f.images = [Image(image) for image in figure_json[id]['images']]
# f.images = [Image(image) for image in figure_json[id]['images']]
for i in figure_json[id]['images']:
image = Image(i)
if image.filepath not in (None, ''):
#TODO: this sucks in every way; make it better
png_image = image.filepath.split('/')[-1].replace('.eps', '.png')
image.filepath = os.path.join(self.images_dir, png_image) if self.local_image_exists(png_image) else image.filepath
f.images.append(image)
return f
def local_image_exists(self, filename):
return os.path.exists(os.path.join(self.images_dir, filename))
def remote_image_exists(self, path):
url = '{b}{path}?token={t}'.format(b=self.base_url, path=path, t=self.token)
resp = requests.head(url)
print resp.status_code, resp.text
return True if resp.status_code == 200 else False
def download_image(self, path):
url = '{b}{path}?token={t}'.format(b=self.base_url, path=path, t=self.token)
resp = requests.get(url, stream=True)
if resp.status_code == 200:
filename = path.split('/')[-1]
with open('../dist/images/' + filename, 'wb') as image_out:
filepath = os.path.join(self.images_dir, path.split('/')[-1])
with open(filepath, 'wb') as image_out:
for bytes in resp.iter_content(chunk_size=4096):
image_out.write(bytes)
print 'Downloaded: ' + filename
return filepath
else:
return resp
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment