Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Andrew Buddenberg
gcis-py-client
Commits
a7388f8e
Commit
a7388f8e
authored
Apr 11, 2014
by
abuddenberg
Browse files
Snapshot
parent
31ff700a
Changes
3
Hide whitespace changes
Inline
Side-by-side
bin/sync_figures
View file @
a7388f8e
...
...
@@ -3,15 +3,15 @@ __author__ = 'abuddenberg'
from
gcis_clients
import
WebformClient
from
gcis_clients
import
GcisClient
from
gcis_clients.sync_utils
import
move_images_to_gcis
,
sync_dataset_metadata
from
gcis_clients.sync_utils
import
move_images_to_gcis
,
sync_dataset_metadata
,
populate_contributors
from
collections
import
OrderedDict
import
json
import
pickle
webform
=
WebformClient
(
'http://resources.assessment.globalchange.gov'
,
'mgTD63FAjG'
)
gcis_url
=
'http://data.gcis-dev-front.joss.ucar.edu'
#gcis = GcisClient(gcis_url, 'andrew.buddenberg@noaa.gov', 'ad90c05b37d4128ae514bc6caa7a41911d2f1de353443a54')
#
gcis_url = 'http://data.gcis-dev-front.joss.ucar.edu'
#
gcis = GcisClient(gcis_url, 'andrew.buddenberg@noaa.gov', 'ad90c05b37d4128ae514bc6caa7a41911d2f1de353443a54')
gcis
=
GcisClient
(
'http://data-stage.globalchange.gov'
,
'andrew.buddenberg@noaa.gov'
,
'b4f1458c3cf28248c982428c46e170019327bd4c533c23dd'
)
sync_metadata_tree
=
{
...
...
@@ -53,7 +53,7 @@ sync_metadata_tree = {
#Chapter 7
(
'forests'
,
[
(
'/metadata/figures/2977'
,
'effectiveness-of-forest-management-in-reducing-wildfire-risk'
),
(
'/metadata/figures/2978'
,
'forest-vulnerability-to-changing-climate'
),
(
'/metadata/figures/2978'
,
'forest-vulnerability-to-changing-climate'
),
(
'/metadata/figures/2985'
,
'forests-can-be-a-source--or-a-sink--for-carbon'
)
]),
...
...
@@ -108,7 +108,7 @@ sync_metadata_tree = {
]),
#Chapter 19
(
'great-plains'
,
[
(
'/metadata/figures/2697'
,
'temperature-and-precipitation-distribution-in-the-great-plains'
),
(
'/metadata/figures/2697'
,
'temperature-and-precipitation-distribution-in-the-great-plains'
),
(
'/metadata/figures/2989'
,
'projected-change-in-number-of-hot-days'
),
(
'/metadata/figures/2990'
,
'projected-change-in-number-of-warm-nights'
),
(
'/metadata/figures/2991'
,
'projected-change-in-number-of-heavy-precipitation-days'
),
...
...
@@ -120,7 +120,7 @@ sync_metadata_tree = {
]),
#Chapter 33: Climate Science Appendix
(
'appendix-climate-science'
,
[
(
'/metadata/figures/3301'
,
'human-influences-apparent-in-many-aspects-of-the-changing-climate'
),
(
'/metadata/figures/3301'
,
'human-influences-apparent-in-many-aspects-of-the-changing-climate'
),
(
'/metadata/figures/2798'
,
'detection-and-attribution-as-forensics'
),
(
'/metadata/figures/2799'
,
'human-influences-apparent-in-many-climate-variables'
),
(
'/metadata/figures/2802'
,
'modeling-the-climate-system'
),
...
...
@@ -132,9 +132,9 @@ sync_metadata_tree = {
#Chapter 34: FAQ/CAQ
(
'appendix-climate-science-faqs'
,
[
(
'/metadata/figures/2663'
,
'us-annual-temperature'
),
# ('/metadata/figures/3300', 'human-influences-apparent-in-many-aspects-of-the-changing-climate'),#Manually associate existing figure
# ('/metadata/figures/3075', 'ten-indicators-of-a-warming-world'), #Manually associate existing figure
(
'/metadata/figures/2520'
,
'ocean-acidification-and-the-food-web'
),
# ('/metadata/figures/3300', 'human-influences-apparent-in-many-aspects-of-the-changing-climate'),#Manually associate existing figure
# ('/metadata/figures/3075', 'ten-indicators-of-a-warming-world'), #Manually associate existing figure
(
'/metadata/figures/2520'
,
'ocean-acidification-and-the-food-web'
),
(
'/metadata/figures/3134'
,
'temperature-trends-1900-2012'
),
])
])
...
...
@@ -144,7 +144,17 @@ sync_metadata_tree = {
def
main
():
# print gcis.test_login()
# sync_dataset_metadata(aggregate_webform_datasets())
sync
(
replace
=
True
)
sync
(
replace
=
False
)
# s = 'Vose, R.S., S. Applequist, M.A. Bourassa, S.C. Pryor, R.J. Barthelmie, B. Blanton, P.D. Bromirski, H.E. Brooks, A.T. D,'
#
# for author in s.split(','):
# frags = author.strip().rsplit()
# if len(frags) > 0 and len(frags[-1]) > 1:
# # print frags
# print '{t}: {r}'.format(t=frags[-1], r=gcis.lookup_person(frags[-1]))
def
sync
(
replace
=
False
):
...
...
@@ -153,26 +163,32 @@ def sync(replace=False):
for
figure_ids
in
sync_metadata_tree
[
report_id
][
chapter_id
]:
webform_url
,
gcis_id
=
figure_ids
print
'Attempting to sync: {id}'
.
format
(
id
=
gcis_id
)
#Merge data from both systems into one object...
figure_obj
=
webform
.
get_webform
(
webform_url
).
merge
(
gcis
.
get_figure
(
report_id
,
gcis_id
,
chapter_id
=
chapter_id
)
)
if
replace
:
for
image
in
figure_obj
.
images
:
#TODO: There are better ways to do this. Build File support.
print
'Deleting {img}'
.
format
(
img
=
image
.
identifier
)
gcis
.
delete_image
(
image
)
print
'Attempting to upload: {id}'
.
format
(
id
=
gcis_id
)
move_images_to_gcis
(
webform
,
gcis
,
webform_url
,
gcis_id
,
report_id
)
#...then send it.
gcis
.
update_figure
(
report_id
,
chapter_id
,
figure_obj
)
print
'Success!'
print
'Attempting to sync: {id}'
.
format
(
id
=
gcis_id
),
' '
,
webform_url
f
=
webform
.
get_webform
(
webform_url
)
populate_contributors
(
gcis
,
f
.
contributors
)
print
f
.
contributors
# #Merge data from both systems into one object...
# gcis_fig = gcis.get_figure(report_id, gcis_id, chapter_id=chapter_id)
# figure_obj = webform.get_webform(webform_url).merge(gcis_fig)
#
# #Hack: override webform's title in favor of gcis
# figure_obj.title = gcis_fig.title if gcis_fig.title not in (None, '') else figure_obj.title
#
# if replace:
# for image in figure_obj.images:
# #TODO: There are better ways to do this. Build File support.
# print 'Deleting {img}'.format(img=image.identifier)
# gcis.delete_image(image)
#
# print 'Attempting to upload: {id}'.format(id=gcis_id)
# move_images_to_gcis(webform, gcis, webform_url, gcis_id, report_id)
#
# #...then send it.
# gcis.update_figure(report_id, chapter_id, figure_obj)
#
# print 'Success!'
if
__name__
==
'__main__'
:
main
()
gcis_clients/sync_utils.py
View file @
a7388f8e
...
...
@@ -26,4 +26,19 @@ def move_images_to_gcis(webform_client, gcis_client, webform_url, gcis_id, repor
def
sync_dataset_metadata
(
gcis_client
,
datasets
):
for
ds
in
datasets
:
gcis_client
.
create_or_update_dataset
(
ds
)
gcis_client
.
create_or_update_activity
(
ds
.
activity
)
\ No newline at end of file
gcis_client
.
create_or_update_activity
(
ds
.
activity
)
def
populate_contributors
(
gcis_client
,
contributors
):
for
cont
in
contributors
:
person
=
cont
.
person
org
=
cont
.
organization
matches
=
gcis_client
.
lookup_person
(
person
.
last_name
)
if
len
(
matches
)
==
1
:
person
.
id
=
matches
[
0
][
0
]
elif
len
(
matches
)
==
0
:
print
'No ID found for '
+
person
.
last_name
else
:
print
'Ambiguous results for '
+
person
.
last_name
print
'
\t
'
,
matches
\ No newline at end of file
gcis_clients/webform_client.py
View file @
a7388f8e
...
...
@@ -7,7 +7,7 @@ from os.path import join
import
requests
from
dateutil.parser
import
parse
from
domain
import
Figure
,
Image
,
Dataset
,
Activity
from
domain
import
Figure
,
Image
,
Dataset
,
Activity
,
Contributor
,
Person
,
Organization
def
sanitized
(
pattern
):
...
...
@@ -21,6 +21,19 @@ def sanitized(pattern):
return
dec
def
parse_creators
(
field
):
s
=
field
.
split
(
','
)
name
,
rest
=
s
[
0
],
s
[
1
:]
name_split
=
name
.
split
()
first_name
,
last_name
=
name_split
[
0
],
name_split
[
-
1
]
org_name
=
rest
[
0
]
if
len
(
rest
)
>
0
else
None
return
Contributor
(
Person
({
'first_name'
:
first_name
,
'last_name'
:
last_name
}),
Organization
({
'name'
:
org_name
})
)
class
WebformClient
:
...
...
@@ -50,13 +63,23 @@ class WebformClient:
#TODO: refactor the service so this isn't necessary
webform_nid
=
webform_json
.
keys
()[
0
]
f
=
Figure
(
webform_json
[
webform_nid
][
'figure'
][
0
])
figure_json
=
webform_json
[
webform_nid
][
'figure'
][
0
]
f
=
Figure
(
figure_json
)
#Add contributor info
if
'list_the_creator_of_the_figure'
in
figure_json
:
f
.
add_contributor
(
parse_creators
(
figure_json
[
'list_the_creator_of_the_figure'
]))
if
'images'
in
webform_json
[
webform_nid
]:
for
img_idx
,
image
in
enumerate
(
webform_json
[
webform_nid
][
'images'
]):
image_obj
=
Image
(
image
,
local_path
=
self
.
get_local_image_path
(
image
),
remote_path
=
self
.
get_remote_image_path
(
image
))
#Add contributor info
if
'list_the_creator_of_the_image'
in
figure_json
:
f
.
add_contributor
(
parse_creators
(
image
[
'list_the_creator_of_the_image'
]))
#TODO: this just keeps getting worse
if
'datasources'
in
webform_json
[
webform_nid
][
'images'
][
img_idx
]:
for
dataset_json
in
webform_json
[
webform_nid
][
'images'
][
img_idx
][
'datasources'
]:
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment