Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Andrew Buddenberg
gcis-py-client
Commits
56bebe06
Commit
56bebe06
authored
Mar 07, 2014
by
abuddenberg
Browse files
Refactoring sync; building separate "Solve Problems" module
parent
73d5a8cf
Changes
2
Hide whitespace changes
Inline
Side-by-side
src/problems.py
View file @
56bebe06
__author__
=
'abuddenberg'
import
pickle
from
gcis_client
import
GcisClient
from
webform_client
import
WebformClient
webform
=
WebformClient
(
'http://resources.assessment.globalchange.gov'
,
'mgTD63FAjG'
)
gcis_url
=
'http://data.gcis-dev-front.joss.ucar.edu'
gcis
=
GcisClient
(
gcis_url
,
'andrew.buddenberg@noaa.gov'
,
'd9fcfd947c1785ab1cd329a9920e05e5c5d3d7f35315f164'
)
def
main
():
hitlist_file
=
'../hitlist.pk1'
# create_problem_list('nca3', hitlist_file)
print_problem_list
(
hitlist_file
)
solve_problems
(
hitlist_file
)
def
solve_problems
(
path
):
problems
=
load_problem_list
(
path
)
for
webform
in
problems
:
if
problems
[
webform
][
'figure_id_not_found'
]:
print
'Unable to resolve figure identifier: '
+
problems
[
webform
][
'figure_id_not_found'
]
continue
def
create_problem_list
(
report_id
,
path
):
pickle
.
dump
(
sort_webform_list
(
report_id
),
open
(
path
,
'wb'
))
def
load_problem_list
(
path
):
all_forms
,
ready
,
problems
=
pickle
.
load
(
open
(
path
,
'r'
))
return
problems
def
load_ready_list
(
path
):
all_forms
,
ready
,
problems
=
pickle
.
load
(
open
(
path
,
'r'
))
return
ready
def
print_problem_list
(
path
):
problems
=
load_problem_list
(
path
)
for
webform
in
problems
:
print
webform
for
problem_type
in
problems
[
webform
]:
print
'
\t
'
,
problem_type
,
problems
[
webform
][
problem_type
]
def
sort_webform_list
(
report_id
):
all_forms
=
[]
ready
=
[]
problems
=
{}
for
item
in
webform
.
get_list
():
webform_url
=
item
[
'url'
]
f
=
webform
.
get_webform
(
webform_url
)
all_forms
.
append
(
f
)
#Check the ready for publication flag
if
'ready_for_publication'
in
f
.
original
and
f
.
original
[
'ready_for_publication'
]
==
'yes'
:
#Check if the figure exists in GCIS
if
not
gcis
.
figure_exists
(
report_id
,
f
.
identifier
):
problems
.
setdefault
(
webform_url
,
{}).
setdefault
(
'figure_id_not_found'
,
[]).
append
((
f
.
identifier
,
f
.
figure_num
,
f
.
title
))
#Check if each image exists in GCIS
for
image
in
f
.
images
:
if
not
gcis
.
image_exists
(
image
.
identifier
):
problems
.
setdefault
(
webform_url
,
{}).
setdefault
(
'image_id_not_found'
,
[]).
append
(
image
.
identifier
)
#Check if each image's dataset exists in GCIS
for
dataset
in
image
.
datasets
:
if
not
gcis
.
dataset_exists
(
dataset
.
identifier
):
problems
.
setdefault
(
webform_url
,
{}).
setdefault
(
'dataset_id_not_found'
,
[]).
append
(
dataset
.
identifier
)
#Check if the filename fields are filled out and correct for what's been uploaded
if
image
.
remote_path
in
(
None
,
''
)
or
not
webform
.
remote_image_exists
(
image
.
remote_path
):
problems
.
setdefault
(
webform_url
,
{}).
setdefault
(
'missing_image_files'
,
[]).
append
(
image
.
identifier
)
#Check for broken image associations
if
not
gcis
.
has_all_associated_images
(
report_id
,
f
.
identifier
,
f
.
images
):
problems
.
setdefault
(
webform_url
,
{}).
setdefault
(
'broken_image_assoc'
,
[]).
append
(
image
.
identifier
)
if
webform_url
not
in
problems
:
ready
.
append
((
webform_url
,
f
.
identifier
))
return
all_forms
,
ready
,
problems
if
__name__
==
'__main__'
:
main
()
\ No newline at end of file
src/sync_figures.py
View file @
56bebe06
...
...
@@ -11,52 +11,54 @@ import pickle
webform
=
WebformClient
(
'http://resources.assessment.globalchange.gov'
,
'mgTD63FAjG'
)
gcis_url
=
'http://data.gcis-dev-front.joss.ucar.edu'
gcis
=
GcisClient
(
gcis_url
,
'andrew.buddenberg@noaa.gov'
,
'4cd31dc7173eb47b26f616fb07db607f25ab861552e81195'
)
# gcis = GcisClient('http://data-stage.globalchange.gov', 'andrew.buddenberg@noaa.gov', 'ef427a895acf26d4f0b1f053ba7d922791b76f7852e7efee')
global_report_name
=
'nca3draft'
gcis
=
GcisClient
(
gcis_url
,
'andrew.buddenberg@noaa.gov'
,
'd9fcfd947c1785ab1cd329a9920e05e5c5d3d7f35315f164'
)
# gcis = GcisClient('http://data-stage.globalchange.gov', 'andrew.buddenberg@noaa.gov', 'a6efcc7cf39c55e9329a8b027e0817e3354bada65310d192')
sync_metadata_tree
=
{
#Reports
global_report_name
:
{
'nca3'
:
{
#Chapter 2
'our-changing-climate'
:
[
#(webform_url, gcis_id)
#
('/metadata/figures/2506', 'observed-change-in-very-heavy-precipitation'),
#
('/metadata/figures/2997', 'observed-change-in-very-heavy-precipitation-2'),
#
('/metadata/figures/2677', 'observed-us-precipitation-change'),
#
('/metadata/figures/3175', 'observed-us-temperature-change'),
#
('/metadata/figures/3074', 'ten-indicators-of-a-warming-world'),
#
('/metadata/figures/3170', 'global-temperature-and-carbon-dioxide'),
#
('/metadata/figures/3293', 'observed-increase-in-frostfree-season-length'),
#
('/metadata/figures/3294', 'projected-changes-in-frostfree-season-length'),
(
'/metadata/figures/2506'
,
'observed-change-in-very-heavy-precipitation'
),
(
'/metadata/figures/2997'
,
'observed-change-in-very-heavy-precipitation-2'
),
(
'/metadata/figures/2677'
,
'observed-us-precipitation-change'
),
(
'/metadata/figures/3175'
,
'observed-us-temperature-change'
),
(
'/metadata/figures/3074'
,
'ten-indicators-of-a-warming-world'
),
(
'/metadata/figures/3170'
,
'global-temperature-and-carbon-dioxide'
),
(
'/metadata/figures/3293'
,
'observed-increase-in-frostfree-season-length'
),
(
'/metadata/figures/3294'
,
'projected-changes-in-frostfree-season-length'
),
# ('/metadata/figures/3305', 'variation-of-storm-frequency-and-intensity-during-the-cold-season-november--march') #incomplete
],
#Chapter 4
'energy-supply-and-use'
:
[
#
('/metadata/figures/3292', 'cooling-degree-days')
(
'/metadata/figures/3292'
,
'cooling-degree-days'
)
],
#Chapter 6
'agriculture'
:
[
#
('/metadata/figures/2872', 'drainage')
#
('/metadata/figures/2691', 'variables-affecting-ag') #Needs images redone
(
'/metadata/figures/2872'
,
'drainage'
)
,
(
'/metadata/figures/2691'
,
'variables-affecting-ag'
)
#Needs images redone
],
#Chapter 9
''
:
[
#
('/metadata/figures/2896', 'heavy-downpours-disease') #Needs images redone
'
human-health
'
:
[
(
'/metadata/figures/2896'
,
'heavy-downpours-disease'
)
#Needs images redone
],
#Chapter 14
'rural'
:
[
# ('/metadata/figures/3306', 'length-growing-season') #Needs images redone
(
'/metadata/figures/3306'
,
'length-growing-season'
)
#Needs images redone
],
#Chapter 18
''
:
[
(
'/metadata/figures/2992'
,
'projected-midcentury-temperature-changes-in-the-midwest'
)
],
#Chapter 19
'great-plains'
:
[
#
('/metadata/figures/2697', 'mean-annual-temp-and-precip') #Needs images redone
(
'/metadata/figures/2697'
,
'mean-annual-temp-and-precip'
)
#Needs images redone
],
#Chapter 25
'coastal-zone'
:
[
#
('/metadata/figures/2543', 'coastal-ecosystem-services')
(
'/metadata/figures/2543'
,
'coastal-ecosystem-services'
)
],
#Climate Science Appendix
'appendix-climate-science'
:
[
...
...
@@ -66,56 +68,9 @@ sync_metadata_tree = {
}
}
#These are artifacts from our collection efforts; largely duplicates
# webform_skip_list = []
def
main
():
pickle
.
dump
(
sort_webform_list
(),
open
(
'../hitlist.pk1'
,
'wb'
))
all_forms
,
ready
,
problems
=
pickle
.
load
(
open
(
'../hitlist.pk1'
,
'r'
))
print
ready
# for ds in aggregate_datasets():
# gcis.update_dataset(ds)
# sync(uploads=False)
# f = webform.get_webform('/metadata/figures/3147').merge(gcis.get_figure('nca3draft', 'ice-loss-from-greenland-and-antarctica', chapter_id='appendix-climate-science'))
def
sort_webform_list
():
all_forms
=
[]
ready
=
[]
problems
=
{}
for
item
in
webform
.
get_list
():
webform_url
=
item
[
'url'
]
f
=
webform
.
get_webform
(
webform_url
)
all_forms
.
append
(
f
)
#Check the ready for publication flag
if
'ready_for_publication'
in
f
.
original
and
f
.
original
[
'ready_for_publication'
]
==
'yes'
:
#Check if the figure exists in GCIS
if
not
gcis
.
figure_exists
(
global_report_name
,
f
.
identifier
):
problems
.
setdefault
(
webform_url
,
{}).
setdefault
(
'figure_id'
,
[]).
append
(
f
.
identifier
)
#Check if each image exists in GCIS
for
image
in
f
.
images
:
if
not
gcis
.
image_exists
(
image
.
identifier
):
problems
.
setdefault
(
webform_url
,
{}).
setdefault
(
'image_id'
,
[]).
append
(
image
.
identifier
)
#Check if each image's dataset exists in GCIS
for
dataset
in
image
.
datasets
:
if
not
gcis
.
dataset_exists
(
dataset
.
identifier
):
problems
.
setdefault
(
webform_url
,
{}).
setdefault
(
'dataset_id'
,
[]).
append
(
dataset
.
identifier
)
#Check if the filename fields are filled out and correct for what's been uploaded
if
image
.
remote_path
in
(
None
,
''
)
or
not
webform
.
remote_image_exists
(
image
.
remote_path
):
problems
.
setdefault
(
webform_url
,
{}).
setdefault
(
'missing_image_files'
,
[]).
append
(
image
.
identifier
)
if
webform_url
not
in
problems
:
ready
.
append
((
webform_url
,
f
.
identifier
))
return
all_forms
,
ready
,
problems
sync
(
uploads
=
False
)
def
sync
(
uploads
=
True
):
...
...
@@ -128,11 +83,11 @@ def sync(uploads=True):
print
'Attempting to upload: '
+
gcis_id
upload_images_to_gcis
(
webform_url
,
gcis_id
,
report_id
)
print
'Attempting to sync: '
+
gcis_id
sync_metadata
(
report_id
,
chapter_id
,
webform_url
,
gcis_id
)
sync_
figure_
metadata
(
report_id
,
chapter_id
,
webform_url
,
gcis_id
)
print
'Success!'
def
sync_metadata
(
report_id
,
chapter_id
,
webform_url
,
gcis_id
):
def
sync_
figure_
metadata
(
report_id
,
chapter_id
,
webform_url
,
gcis_id
):
#Merge data from both systems into one object...
figure_obj
=
webform
.
get_webform
(
webform_url
).
merge
(
gcis
.
get_figure
(
report_id
,
gcis_id
,
chapter_id
=
chapter_id
)
...
...
@@ -143,25 +98,30 @@ def sync_metadata(report_id, chapter_id, webform_url, gcis_id):
#This function is for adding images to existing figures
def
upload_images_to_gcis
(
webform_url
,
gcis_id
,
report_id
):
figure
=
webform
.
get_webform
(
webform_url
)
figure
=
webform
.
get_webform
(
webform_url
,
download_images
=
True
)
#Now identifiers don't need to be matched
figure
.
identifier
=
gcis_id
webform
.
download_all_images
(
figure
)
#Make sure we have all the images required for a COMPLETE update
for
image
in
figure
.
images
:
if
not
exists
(
image
.
local_path
):
raise
Exception
(
'Local file missing '
+
image
.
local_path
)
for
image
in
figure
.
images
:
for
resp
in
gcis
.
create_image
(
image
,
report_id
=
report_id
,
figure_id
=
figure
.
identifier
):
print
resp
.
status_code
,
resp
.
text
if
not
gcis
.
image_exists
(
image
.
identifier
):
gcis
.
create_image
(
image
,
report_id
=
report_id
,
figure_id
=
figure
.
identifier
)
# for dataset in image.datasets:
# gcis.associate_dataset_with_image(dataset.identifier, report_id, image.identifier)
def
sync_dataset_metadata
(
datasets
):
for
ds
in
datasets
:
if
gcis
.
dataset_exists
(
ds
.
identifier
):
print
'Updating: {ds}'
.
format
(
ds
=
ds
)
gcis
.
update_dataset
(
ds
)
else
:
print
'Creating: {ds}'
.
format
(
ds
=
ds
)
gcis
.
create_dataset
(
ds
)
def
aggregate_datasets
():
def
aggregate_
webform_
datasets
():
dataset_map
=
{}
for
item
in
webform
.
get_list
():
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment