problems 4.98 KB
Newer Older
1
#!/usr/bin/env python
2
__author__ = 'abuddenberg'
3
4
5
6

import pickle
from gcis_client import GcisClient
from webform_client import WebformClient
7
from sync_figures import upload_images_to_gcis
8
9
10
11
12

webform = WebformClient('http://resources.assessment.globalchange.gov', 'mgTD63FAjG')

gcis_url = 'http://data.gcis-dev-front.joss.ucar.edu'
gcis = GcisClient(gcis_url, 'andrew.buddenberg@noaa.gov', 'd9fcfd947c1785ab1cd329a9920e05e5c5d3d7f35315f164')
13
# gcis = GcisClient('http://data-stage.globalchange.gov', 'andrew.buddenberg@noaa.gov', 'a6efcc7cf39c55e9329a8b027e0817e3354bada65310d192')
14
15
16
17

def main():
    hitlist_file = '../hitlist.pk1'

18
    create_problem_list('nca3', hitlist_file)
19
    print_problem_list(hitlist_file)
20
21
22
23
    # solve_problems(hitlist_file, 'nca3')

    #WTF:
    #requests.exceptions.ConnectionError: HTTPConnectionPool(host='data.gcis-dev-front.joss.ucar.edu', port=80): Max retries exceeded with url: /report/nca3/figure/rel/observed-change-in-very-heavy-precipitation (Caused by <class 'httplib.BadStatusLine'>: '')
24
25


26
def solve_problems(path, report_id):
27
28
29
    problems = load_problem_list(path)

    for webform in problems:
30
31
32
33
34
        webform_id, fig_num, gcis_id = webform

        #Without a valid GCIS figure_id, nothing else can happen
        if 'figure_id_not_found' in problems[webform]:
            print 'Unable to resolve figure identifier: {fig}'.format(fig=problems[webform]['figure_id_not_found'])
35
            continue
36
37
38
39
40
41
42
43
44
45
46
47
48
49

        img_id_missing = set(problems[webform]['image_id_not_found']) if 'image_id_not_found' in problems[webform] else set()
        img_file_missing = set(problems[webform]['missing_image_files']) if 'missing_image_files' in problems[webform] else set()
        creates = img_id_missing - img_file_missing

        img_assoc_broken = set(problems[webform]['broken_image_assoc']) if 'broken_image_assoc' in problems[webform] else set()
        assocs = img_assoc_broken - creates

        if len(creates) > 0:
            upload_images_to_gcis(webform_id, gcis_id, report_id, subset_images=creates)

        for image_id in assocs:
            print 'Associating image: {i} with figure: {f}'.format(i=image_id, f=gcis_id)
            print gcis.associate_image_with_figure(image_id, report_id, gcis_id)
50
51
52


def create_problem_list(report_id, path):
53
54
    with open(path, 'wb') as problem_file:
        pickle.dump(sort_webform_list(report_id), problem_file)
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82


def load_problem_list(path):
    all_forms, ready, problems = pickle.load(open(path, 'r'))
    return problems


def load_ready_list(path):
    all_forms, ready, problems = pickle.load(open(path, 'r'))
    return ready


def print_problem_list(path):
    problems = load_problem_list(path)
    for webform in problems:
        print webform
        for problem_type in problems[webform]:
            print '\t', problem_type, problems[webform][problem_type]


def sort_webform_list(report_id):
    all_forms = []
    ready = []
    problems = {}

    for item in webform.get_list():
        webform_url = item['url']
        f = webform.get_webform(webform_url)
83
84
85
        key = (webform_url, f.figure_num, f.identifier)

        all_forms.append(key)
86
87
88
89
90

        #Check the ready for publication flag
        if 'ready_for_publication' in f.original and f.original['ready_for_publication'] == 'yes':
            #Check if the figure exists in GCIS
            if not gcis.figure_exists(report_id, f.identifier):
91
92
                problems.setdefault(key, {}).setdefault('figure_id_not_found', []).append(
                    (f.identifier, f.figure_num, f.title))
93
94
95
            #Check if each image exists in GCIS
            for image in f.images:
                if not gcis.image_exists(image.identifier):
96
97
                    problems.setdefault(key, {}).setdefault('image_id_not_found', []).append(
                        image.identifier)
98
99
100
101

                #Check if each image's dataset exists in GCIS
                for dataset in image.datasets:
                    if not gcis.dataset_exists(dataset.identifier):
102
103
                        problems.setdefault(key, {}).setdefault('dataset_id_not_found',
                            []).append(dataset.identifier)
104
105
106

                #Check if the filename fields are filled out and correct for what's been uploaded
                if image.remote_path in (None, '') or not webform.remote_image_exists(image.remote_path):
107
108
                    problems.setdefault(key, {}).setdefault('missing_image_files', []).append(
                        image.identifier)
109
110

            #Check for broken image associations
111
112
113
114
115
            has_all_images, image_deltas = gcis.has_all_associated_images(report_id, f.identifier,
                                                                               [i.identifier for i in f.images])
            if not has_all_images and len(image_deltas) > 0:
                problems.setdefault(key, {}).setdefault('broken_image_assoc', []).extend(
                    image_deltas)
116

117
118
            if key not in problems:
                ready.append(key)
119
120
121
122
123
124

    return all_forms, ready, problems


if __name__ == '__main__':
    main()