Commit 24870b09 authored by abuddenberg's avatar abuddenberg
Browse files

Built automated problem solver out. Used to expand the list of figures that we can sync over

parent cd1f2d30
......@@ -3,33 +3,54 @@ __author__ = 'abuddenberg'
import pickle
from gcis_client import GcisClient
from webform_client import WebformClient
from sync_figures import upload_images_to_gcis
webform = WebformClient('http://resources.assessment.globalchange.gov', 'mgTD63FAjG')
gcis_url = 'http://data.gcis-dev-front.joss.ucar.edu'
gcis = GcisClient(gcis_url, 'andrew.buddenberg@noaa.gov', 'd9fcfd947c1785ab1cd329a9920e05e5c5d3d7f35315f164')
# gcis = GcisClient('http://data-stage.globalchange.gov', 'andrew.buddenberg@noaa.gov', 'a6efcc7cf39c55e9329a8b027e0817e3354bada65310d192')
def main():
hitlist_file = '../hitlist.pk1'
# create_problem_list('nca3', hitlist_file)
create_problem_list('nca3', hitlist_file)
print_problem_list(hitlist_file)
solve_problems(hitlist_file)
# solve_problems(hitlist_file, 'nca3')
#WTF:
#requests.exceptions.ConnectionError: HTTPConnectionPool(host='data.gcis-dev-front.joss.ucar.edu', port=80): Max retries exceeded with url: /report/nca3/figure/rel/observed-change-in-very-heavy-precipitation (Caused by <class 'httplib.BadStatusLine'>: '')
def solve_problems(path):
def solve_problems(path, report_id):
problems = load_problem_list(path)
for webform in problems:
if problems[webform]['figure_id_not_found']:
print 'Unable to resolve figure identifier: ' + problems[webform]['figure_id_not_found']
webform_id, fig_num, gcis_id = webform
#Without a valid GCIS figure_id, nothing else can happen
if 'figure_id_not_found' in problems[webform]:
print 'Unable to resolve figure identifier: {fig}'.format(fig=problems[webform]['figure_id_not_found'])
continue
img_id_missing = set(problems[webform]['image_id_not_found']) if 'image_id_not_found' in problems[webform] else set()
img_file_missing = set(problems[webform]['missing_image_files']) if 'missing_image_files' in problems[webform] else set()
creates = img_id_missing - img_file_missing
img_assoc_broken = set(problems[webform]['broken_image_assoc']) if 'broken_image_assoc' in problems[webform] else set()
assocs = img_assoc_broken - creates
if len(creates) > 0:
upload_images_to_gcis(webform_id, gcis_id, report_id, subset_images=creates)
for image_id in assocs:
print 'Associating image: {i} with figure: {f}'.format(i=image_id, f=gcis_id)
print gcis.associate_image_with_figure(image_id, report_id, gcis_id)
def create_problem_list(report_id, path):
pickle.dump(sort_webform_list(report_id), open(path, 'wb'))
with open(path, 'wb') as problem_file:
pickle.dump(sort_webform_list(report_id), problem_file)
def load_problem_list(path):
......@@ -58,37 +79,45 @@ def sort_webform_list(report_id):
for item in webform.get_list():
webform_url = item['url']
f = webform.get_webform(webform_url)
all_forms.append(f)
key = (webform_url, f.figure_num, f.identifier)
all_forms.append(key)
#Check the ready for publication flag
if 'ready_for_publication' in f.original and f.original['ready_for_publication'] == 'yes':
#Check if the figure exists in GCIS
if not gcis.figure_exists(report_id, f.identifier):
problems.setdefault(webform_url, {}).setdefault('figure_id_not_found', []).append((f.identifier, f.figure_num, f.title))
problems.setdefault(key, {}).setdefault('figure_id_not_found', []).append(
(f.identifier, f.figure_num, f.title))
#Check if each image exists in GCIS
for image in f.images:
if not gcis.image_exists(image.identifier):
problems.setdefault(webform_url, {}).setdefault('image_id_not_found', []).append(image.identifier)
problems.setdefault(key, {}).setdefault('image_id_not_found', []).append(
image.identifier)
#Check if each image's dataset exists in GCIS
for dataset in image.datasets:
if not gcis.dataset_exists(dataset.identifier):
problems.setdefault(webform_url, {}).setdefault('dataset_id_not_found', []).append(dataset.identifier)
problems.setdefault(key, {}).setdefault('dataset_id_not_found',
[]).append(dataset.identifier)
#Check if the filename fields are filled out and correct for what's been uploaded
if image.remote_path in (None, '') or not webform.remote_image_exists(image.remote_path):
problems.setdefault(webform_url, {}).setdefault('missing_image_files', []).append(image.identifier)
problems.setdefault(key, {}).setdefault('missing_image_files', []).append(
image.identifier)
#Check for broken image associations
if not gcis.has_all_associated_images(report_id, f.identifier, f.images):
problems.setdefault(webform_url, {}).setdefault('broken_image_assoc', []).append(image.identifier)
has_all_images, image_deltas = gcis.has_all_associated_images(report_id, f.identifier,
[i.identifier for i in f.images])
if not has_all_images and len(image_deltas) > 0:
problems.setdefault(key, {}).setdefault('broken_image_assoc', []).extend(
image_deltas)
if webform_url not in problems:
ready.append((webform_url, f.identifier))
if key not in problems:
ready.append(key)
return all_forms, ready, problems
if __name__ == '__main__':
main()
\ No newline at end of file
......@@ -4,7 +4,6 @@ from webform_client import WebformClient
from gcis_client import GcisClient
from os.path import exists
import json
import pickle
# webform_dev = ('http://dev.nemac.org/asides10/metadata/figures/all?token=A2PNYxRuG9')
......@@ -23,46 +22,61 @@ sync_metadata_tree = {
('/metadata/figures/2506', 'observed-change-in-very-heavy-precipitation'),
('/metadata/figures/2997', 'observed-change-in-very-heavy-precipitation-2'),
('/metadata/figures/2677', 'observed-us-precipitation-change'),
('/metadata/figures/3175', 'observed-us-temperature-change'),
('/metadata/figures/3074', 'ten-indicators-of-a-warming-world'),
('/metadata/figures/3170', 'global-temperature-and-carbon-dioxide'),
('/metadata/figures/3293', 'observed-increase-in-frostfree-season-length'),
('/metadata/figures/3294', 'projected-changes-in-frostfree-season-length'),
# ('/metadata/figures/3305', 'variation-of-storm-frequency-and-intensity-during-the-cold-season-november--march') #incomplete
# ('/metadata/figures/3175', 'observed-us-temperature-change'),
# ('/metadata/figures/3074', 'ten-indicators-of-a-warming-world'),
# ('/metadata/figures/3170', 'global-temperature-and-carbon-dioxide'),
# ('/metadata/figures/3293', 'observed-increase-in-frostfree-season-length'),
('/metadata/figures/3294', 'projected-changes-in-frostfree-season-length'), #Good
# ('/metadata/figures/3305', 'variation-of-storm-frequency') #incomplete
],
#Chapter 4
'energy-supply-and-use': [
('/metadata/figures/3292', 'cooling-degree-days')
# ('/metadata/figures/3292', 'cooling-degree-days')
],
#Chapter 6
'agriculture': [
('/metadata/figures/2872', 'drainage'),
('/metadata/figures/2691', 'variables-affecting-ag') #Needs images redone
# ('/metadata/figures/2872', 'drainage'),
# ('/metadata/figures/2691', 'variables-affecting-ag') #Needs images redone
('/metadata/figures/2691', 'projected-changes-in-key-climate-variables-affecting-agricultural-productivity')
],
#Chapter 9
'human-health': [
('/metadata/figures/2896', 'heavy-downpours-disease') #Needs images redone
# 'human-health': [
# ('/metadata/figures/2896', 'heavy-downpours-disease') #Needs images redone
# ],
#Chapter 10
'water-energy-land-use': [
('/metadata/figures/2410', 'coasttocoast-100degree-days-in-2011')
],
#Chapter 14
'rural': [
('/metadata/figures/3306', 'length-growing-season') #Needs images redone
('/metadata/figures/3306', 'growing-season-lengthens') #Needs images redone
],
#Chapter 16
'northeast': [
('/metadata/figures/2995', 'projected-increases-in-the-number-of-days-over-90f')
],
#Chapter 17
'southeast': [
('/metadata/figures/2998', 'projected-change-in-number-of-days-over-95-f'),
('/metadata/figures/2999', 'projected-change-in-number-of-nights-below-32f')
],
#Chapter 18
'': [
('/metadata/figures/2992', 'projected-midcentury-temperature-changes-in-the-midwest')
'midwest': [
('/metadata/figures/2992', 'projected-midcentury-temperature-changes-in-the-midwest'),
('/metadata/figures/2994', 'when-it-rains-it-pours')
],
#Chapter 19
'great-plains': [
('/metadata/figures/2697', 'mean-annual-temp-and-precip') #Needs images redone
],
# 'great-plains': [
# ('/metadata/figures/2697', 'temperature-and-precipitation-distribution-in-the-great-plains') #Needs images redone
#
# ],
#Chapter 25
'coastal-zone': [
('/metadata/figures/2543', 'coastal-ecosystem-services')
],
#Climate Science Appendix
# Climate Science Appendix
'appendix-climate-science': [
# ('/metadata/figures/3147', 'ice-loss-from-greenland-and-antarctica')
('/metadata/figures/3147', 'ice-loss-from-greenland-and-antarctica')
]
}
......@@ -70,44 +84,56 @@ sync_metadata_tree = {
def main():
sync(uploads=False)
sync(replace=True)
def sync(uploads=True):
def sync(replace=False):
for report_id in sync_metadata_tree:
for chapter_id in sync_metadata_tree[report_id]:
for figure_ids in sync_metadata_tree[report_id][chapter_id]:
webform_url, gcis_id = figure_ids
if uploads:
print 'Attempting to upload: ' + gcis_id
print 'Attempting to sync: {id}'.format(id=gcis_id)
#Merge data from both systems into one object...
figure_obj = webform.get_webform(webform_url).merge(
gcis.get_figure(report_id, gcis_id, chapter_id=chapter_id)
)
if replace:
for image in figure_obj.images:
#TODO: There are better ways to do this. Build File support.
print 'Deleting {img}'.format(img=image.identifier)
gcis.delete_image(image)
print 'Attempting to upload: {id}'.format(id=gcis_id)
upload_images_to_gcis(webform_url, gcis_id, report_id)
print 'Attempting to sync: ' + gcis_id
sync_figure_metadata(report_id, chapter_id, webform_url, gcis_id)
print 'Success!'
def sync_figure_metadata(report_id, chapter_id, webform_url, gcis_id):
#Merge data from both systems into one object...
figure_obj = webform.get_webform(webform_url).merge(
gcis.get_figure(report_id, gcis_id, chapter_id=chapter_id)
)
#...then send it.
gcis.update_figure(report_id, chapter_id, figure_obj)
#...then send it.
gcis.update_figure(report_id, chapter_id, figure_obj)
print 'Success!'
#This function is for adding images to existing figures
def upload_images_to_gcis(webform_url, gcis_id, report_id):
def upload_images_to_gcis(webform_url, gcis_id, report_id, subset_images=None):
figure = webform.get_webform(webform_url, download_images=True)
#Now identifiers don't need to be matched
figure.identifier = gcis_id
#Make sure we have all the images required for a COMPLETE update
for image in figure.images:
#If a subset of identifiers has been provided, only process those
if subset_images:
images_to_process = [image for image in figure.images if image.identifier in subset_images]
else:
images_to_process = figure.images
for image in images_to_process:
if not exists(image.local_path):
raise Exception('Local file missing ' + image.local_path)
for image in figure.images:
if not gcis.image_exists(image.identifier):
print 'Creating image: {img}'.format(img=image.identifier)
gcis.create_image(image, report_id=report_id, figure_id=figure.identifier)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment