Skip to content

GitLab

Commit 24870b09 authored by abuddenberg's avatar abuddenberg
Browse files

Built automated problem solver out. Used to expand the list of figures that we can sync over

parent cd1f2d30
Hide whitespace changes
Inline Side-by-side
src/problems.py
View file @ 24870b09
......@@ -3,33 +3,54 @@ __author__ = 'abuddenberg'
import pickle
from gcis_client import GcisClient
from webform_client import WebformClient
from sync_figures import upload_images_to_gcis
webform = WebformClient('http://resources.assessment.globalchange.gov', 'mgTD63FAjG')
gcis_url = 'http://data.gcis-dev-front.joss.ucar.edu'
gcis = GcisClient(gcis_url, 'andrew.buddenberg@noaa.gov', 'd9fcfd947c1785ab1cd329a9920e05e5c5d3d7f35315f164')
# gcis = GcisClient('http://data-stage.globalchange.gov', 'andrew.buddenberg@noaa.gov', 'a6efcc7cf39c55e9329a8b027e0817e3354bada65310d192')
def main():
hitlist_file = '../hitlist.pk1'
# create_problem_list('nca3', hitlist_file)
create_problem_list('nca3', hitlist_file)
print_problem_list(hitlist_file)
solve_problems(hitlist_file)
# solve_problems(hitlist_file, 'nca3')
#WTF:
#requests.exceptions.ConnectionError: HTTPConnectionPool(host='data.gcis-dev-front.joss.ucar.edu', port=80): Max retries exceeded with url: /report/nca3/figure/rel/observed-change-in-very-heavy-precipitation (Caused by <class 'httplib.BadStatusLine'>: '')
def solve_problems(path):
def solve_problems(path, report_id):
problems = load_problem_list(path)
for webform in problems:
if problems[webform]['figure_id_not_found']:
print 'Unable to resolve figure identifier: ' + problems[webform]['figure_id_not_found']
webform_id, fig_num, gcis_id = webform
#Without a valid GCIS figure_id, nothing else can happen
if 'figure_id_not_found' in problems[webform]:
print 'Unable to resolve figure identifier: {fig}'.format(fig=problems[webform]['figure_id_not_found'])
continue
img_id_missing = set(problems[webform]['image_id_not_found']) if 'image_id_not_found' in problems[webform] else set()
img_file_missing = set(problems[webform]['missing_image_files']) if 'missing_image_files' in problems[webform] else set()
creates = img_id_missing - img_file_missing
img_assoc_broken = set(problems[webform]['broken_image_assoc']) if 'broken_image_assoc' in problems[webform] else set()
assocs = img_assoc_broken - creates
if len(creates) > 0:
upload_images_to_gcis(webform_id, gcis_id, report_id, subset_images=creates)
for image_id in assocs:
print 'Associating image: {i} with figure: {f}'.format(i=image_id, f=gcis_id)
print gcis.associate_image_with_figure(image_id, report_id, gcis_id)
def create_problem_list(report_id, path):
pickle.dump(sort_webform_list(report_id), open(path, 'wb'))
with open(path, 'wb') as problem_file:
pickle.dump(sort_webform_list(report_id), problem_file)
def load_problem_list(path):
......@@ -58,37 +79,45 @@ def sort_webform_list(report_id):
for item in webform.get_list():
webform_url = item['url']
f = webform.get_webform(webform_url)
all_forms.append(f)
key = (webform_url, f.figure_num, f.identifier)
all_forms.append(key)
#Check the ready for publication flag
if 'ready_for_publication' in f.original and f.original['ready_for_publication'] == 'yes':
#Check if the figure exists in GCIS
if not gcis.figure_exists(report_id, f.identifier):
problems.setdefault(webform_url, {}).setdefault('figure_id_not_found', []).append((f.identifier, f.figure_num, f.title))
problems.setdefault(key, {}).setdefault('figure_id_not_found', []).append(
(f.identifier, f.figure_num, f.title))
#Check if each image exists in GCIS
for image in f.images:
if not gcis.image_exists(image.identifier):
problems.setdefault(webform_url, {}).setdefault('image_id_not_found', []).append(image.identifier)
problems.setdefault(key, {}).setdefault('image_id_not_found', []).append(
image.identifier)
#Check if each image's dataset exists in GCIS
for dataset in image.datasets:
if not gcis.dataset_exists(dataset.identifier):
problems.setdefault(webform_url, {}).setdefault('dataset_id_not_found', []).append(dataset.identifier)
problems.setdefault(key, {}).setdefault('dataset_id_not_found',
[]).append(dataset.identifier)
#Check if the filename fields are filled out and correct for what's been uploaded
if image.remote_path in (None, '') or not webform.remote_image_exists(image.remote_path):
problems.setdefault(webform_url, {}).setdefault('missing_image_files', []).append(image.identifier)
problems.setdefault(key, {}).setdefault('missing_image_files', []).append(
image.identifier)
#Check for broken image associations
if not gcis.has_all_associated_images(report_id, f.identifier, f.images):
problems.setdefault(webform_url, {}).setdefault('broken_image_assoc', []).append(image.identifier)
has_all_images, image_deltas = gcis.has_all_associated_images(report_id, f.identifier,
[i.identifier for i in f.images])
if not has_all_images and len(image_deltas) > 0:
problems.setdefault(key, {}).setdefault('broken_image_assoc', []).extend(
image_deltas)
if webform_url not in problems:
ready.append((webform_url, f.identifier))
if key not in problems:
ready.append(key)
return all_forms, ready, problems
if __name__ == '__main__':
main()
\ No newline at end of file
src/sync_figures.py
View file @ 24870b09
......@@ -4,7 +4,6 @@ from webform_client import WebformClient
from gcis_client import GcisClient
from os.path import exists
import json
import pickle
# webform_dev = ('http://dev.nemac.org/asides10/metadata/figures/all?token=A2PNYxRuG9')
......@@ -23,46 +22,61 @@ sync_metadata_tree = {
('/metadata/figures/2506', 'observed-change-in-very-heavy-precipitation'),
('/metadata/figures/2997', 'observed-change-in-very-heavy-precipitation-2'),
('/metadata/figures/2677', 'observed-us-precipitation-change'),
('/metadata/figures/3175', 'observed-us-temperature-change'),
('/metadata/figures/3074', 'ten-indicators-of-a-warming-world'),
('/metadata/figures/3170', 'global-temperature-and-carbon-dioxide'),
('/metadata/figures/3293', 'observed-increase-in-frostfree-season-length'),
('/metadata/figures/3294', 'projected-changes-in-frostfree-season-length'),
# ('/metadata/figures/3305', 'variation-of-storm-frequency-and-intensity-during-the-cold-season-november--march') #incomplete
# ('/metadata/figures/3175', 'observed-us-temperature-change'),
# ('/metadata/figures/3074', 'ten-indicators-of-a-warming-world'),
# ('/metadata/figures/3170', 'global-temperature-and-carbon-dioxide'),
# ('/metadata/figures/3293', 'observed-increase-in-frostfree-season-length'),
('/metadata/figures/3294', 'projected-changes-in-frostfree-season-length'), #Good
# ('/metadata/figures/3305', 'variation-of-storm-frequency') #incomplete
],
#Chapter 4
'energy-supply-and-use': [
('/metadata/figures/3292', 'cooling-degree-days')
# ('/metadata/figures/3292', 'cooling-degree-days')
],
#Chapter 6
'agriculture': [
('/metadata/figures/2872', 'drainage'),
('/metadata/figures/2691', 'variables-affecting-ag') #Needs images redone
# ('/metadata/figures/2872', 'drainage'),
# ('/metadata/figures/2691', 'variables-affecting-ag') #Needs images redone
('/metadata/figures/2691', 'projected-changes-in-key-climate-variables-affecting-agricultural-productivity')
],
#Chapter 9
'human-health': [
('/metadata/figures/2896', 'heavy-downpours-disease') #Needs images redone
# 'human-health': [
# ('/metadata/figures/2896', 'heavy-downpours-disease') #Needs images redone
# ],
#Chapter 10
'water-energy-land-use': [
('/metadata/figures/2410', 'coasttocoast-100degree-days-in-2011')
],
#Chapter 14
'rural': [
('/metadata/figures/3306', 'length-growing-season') #Needs images redone
('/metadata/figures/3306', 'growing-season-lengthens') #Needs images redone
],
#Chapter 16
'northeast': [
('/metadata/figures/2995', 'projected-increases-in-the-number-of-days-over-90f')
],
#Chapter 17
'southeast': [
('/metadata/figures/2998', 'projected-change-in-number-of-days-over-95-f'),
('/metadata/figures/2999', 'projected-change-in-number-of-nights-below-32f')
],
#Chapter 18
'': [
('/metadata/figures/2992', 'projected-midcentury-temperature-changes-in-the-midwest')
'midwest': [
('/metadata/figures/2992', 'projected-midcentury-temperature-changes-in-the-midwest'),
('/metadata/figures/2994', 'when-it-rains-it-pours')
],
#Chapter 19
'great-plains': [
('/metadata/figures/2697', 'mean-annual-temp-and-precip') #Needs images redone
],
# 'great-plains': [
# ('/metadata/figures/2697', 'temperature-and-precipitation-distribution-in-the-great-plains') #Needs images redone
#
# ],
#Chapter 25
'coastal-zone': [
('/metadata/figures/2543', 'coastal-ecosystem-services')
],
#Climate Science Appendix
# Climate Science Appendix
'appendix-climate-science': [
# ('/metadata/figures/3147', 'ice-loss-from-greenland-and-antarctica')
('/metadata/figures/3147', 'ice-loss-from-greenland-and-antarctica')
]
}
......@@ -70,44 +84,56 @@ sync_metadata_tree = {
def main():
sync(uploads=False)
sync(replace=True)
def sync(uploads=True):
def sync(replace=False):
for report_id in sync_metadata_tree:
for chapter_id in sync_metadata_tree[report_id]:
for figure_ids in sync_metadata_tree[report_id][chapter_id]:
webform_url, gcis_id = figure_ids
if uploads:
print 'Attempting to upload: ' + gcis_id
print 'Attempting to sync: {id}'.format(id=gcis_id)
#Merge data from both systems into one object...
figure_obj = webform.get_webform(webform_url).merge(
gcis.get_figure(report_id, gcis_id, chapter_id=chapter_id)
)
if replace:
for image in figure_obj.images:
#TODO: There are better ways to do this. Build File support.
print 'Deleting {img}'.format(img=image.identifier)
gcis.delete_image(image)
print 'Attempting to upload: {id}'.format(id=gcis_id)
upload_images_to_gcis(webform_url, gcis_id, report_id)
print 'Attempting to sync: ' + gcis_id
sync_figure_metadata(report_id, chapter_id, webform_url, gcis_id)
print 'Success!'
def sync_figure_metadata(report_id, chapter_id, webform_url, gcis_id):
#Merge data from both systems into one object...
figure_obj = webform.get_webform(webform_url).merge(
gcis.get_figure(report_id, gcis_id, chapter_id=chapter_id)
)
#...then send it.
gcis.update_figure(report_id, chapter_id, figure_obj)
#...then send it.
gcis.update_figure(report_id, chapter_id, figure_obj)
print 'Success!'
#This function is for adding images to existing figures
def upload_images_to_gcis(webform_url, gcis_id, report_id):
def upload_images_to_gcis(webform_url, gcis_id, report_id, subset_images=None):
figure = webform.get_webform(webform_url, download_images=True)
#Now identifiers don't need to be matched
figure.identifier = gcis_id
#Make sure we have all the images required for a COMPLETE update
for image in figure.images:
#If a subset of identifiers has been provided, only process those
if subset_images:
images_to_process = [image for image in figure.images if image.identifier in subset_images]
else:
images_to_process = figure.images
for image in images_to_process:
if not exists(image.local_path):
raise Exception('Local file missing ' + image.local_path)
for image in figure.images:
if not gcis.image_exists(image.identifier):
print 'Creating image: {img}'.format(img=image.identifier)
gcis.create_image(image, report_id=report_id, figure_id=figure.identifier)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment