sync_figures.py 6.26 KB
Newer Older
1
2
3
4
__author__ = 'abuddenberg'

from webform_client import WebformClient
from gcis_client import GcisClient
5
from os.path import exists
6
import json
7
8
9
10
11
12

# webform_dev = ('http://dev.nemac.org/asides10/metadata/figures/all?token=A2PNYxRuG9')

webform = WebformClient('http://resources.assessment.globalchange.gov', 'mgTD63FAjG')

gcis_url = 'http://data.gcis-dev-front.joss.ucar.edu'
13
14
gcis = GcisClient(gcis_url, 'andrew.buddenberg@noaa.gov', 'd9fcfd947c1785ab1cd329a9920e05e5c5d3d7f35315f164')
# gcis = GcisClient('http://data-stage.globalchange.gov', 'andrew.buddenberg@noaa.gov', 'a6efcc7cf39c55e9329a8b027e0817e3354bada65310d192')
15

16
17
sync_metadata_tree = {
    #Reports
18
    'nca3': {
19
        #Chapter 2
20
21
        'our-changing-climate': [
            #(webform_url, gcis_id)
22
23
24
            ('/metadata/figures/2506', 'observed-change-in-very-heavy-precipitation'),
            ('/metadata/figures/2997', 'observed-change-in-very-heavy-precipitation-2'),
            ('/metadata/figures/2677', 'observed-us-precipitation-change'),
25
26
27
28
29
30
            # ('/metadata/figures/3175', 'observed-us-temperature-change'),
            # ('/metadata/figures/3074', 'ten-indicators-of-a-warming-world'),
            # ('/metadata/figures/3170', 'global-temperature-and-carbon-dioxide'),
            # ('/metadata/figures/3293', 'observed-increase-in-frostfree-season-length'),
            ('/metadata/figures/3294', 'projected-changes-in-frostfree-season-length'), #Good
            # ('/metadata/figures/3305', 'variation-of-storm-frequency') #incomplete
31
        ],
32
33
        #Chapter 4
        'energy-supply-and-use': [
34
            # ('/metadata/figures/3292', 'cooling-degree-days')
35
        ],
36
37
        #Chapter 6
        'agriculture': [
38
39
40
        #     ('/metadata/figures/2872', 'drainage'),
        #     ('/metadata/figures/2691', 'variables-affecting-ag') #Needs images redone
            ('/metadata/figures/2691', 'projected-changes-in-key-climate-variables-affecting-agricultural-productivity')
41
42
        ],
        #Chapter 9
43
44
45
46
47
48
        # 'human-health': [
        #     ('/metadata/figures/2896', 'heavy-downpours-disease') #Needs images redone
        # ],
        #Chapter 10
        'water-energy-land-use': [
            ('/metadata/figures/2410', 'coasttocoast-100degree-days-in-2011')
49
50
        ],
        #Chapter 14
51
        'rural': [
52
53
54
55
56
57
58
59
60
61
            ('/metadata/figures/3306', 'growing-season-lengthens') #Needs images redone
        ],
        #Chapter 16
        'northeast': [
            ('/metadata/figures/2995', 'projected-increases-in-the-number-of-days-over-90f')
        ],
        #Chapter 17
        'southeast': [
            ('/metadata/figures/2998', 'projected-change-in-number-of-days-over-95-f'),
            ('/metadata/figures/2999', 'projected-change-in-number-of-nights-below-32f')
62
63
        ],
        #Chapter 18
64
65
66
        'midwest': [
            ('/metadata/figures/2992', 'projected-midcentury-temperature-changes-in-the-midwest'),
            ('/metadata/figures/2994', 'when-it-rains-it-pours')
67
68
        ],
        #Chapter 19
69
70
71
72
        # 'great-plains': [
        #     ('/metadata/figures/2697', 'temperature-and-precipitation-distribution-in-the-great-plains') #Needs images redone
        #
        # ],
73
74
        #Chapter 25
        'coastal-zone': [
75
            ('/metadata/figures/2543', 'coastal-ecosystem-services')
abuddenberg's avatar
abuddenberg committed
76
        ],
77
        # Climate Science Appendix
abuddenberg's avatar
abuddenberg committed
78
        'appendix-climate-science': [
79
            ('/metadata/figures/3147', 'ice-loss-from-greenland-and-antarctica')
80
        ]
abuddenberg's avatar
abuddenberg committed
81

82
83
84
    }
}

85

86
def main():
87
    sync(replace=True)
88
89


90
def sync(replace=False):
91
92
93
    for report_id in sync_metadata_tree:
        for chapter_id in sync_metadata_tree[report_id]:
            for figure_ids in sync_metadata_tree[report_id][chapter_id]:
94
                webform_url, gcis_id = figure_ids
95

96
97
98
99
100
101
102
103
104
105
106
107
108
109
                print 'Attempting to sync: {id}'.format(id=gcis_id)

                #Merge data from both systems into one object...
                figure_obj = webform.get_webform(webform_url).merge(
                    gcis.get_figure(report_id, gcis_id, chapter_id=chapter_id)
                )

                if replace:
                    for image in figure_obj.images:
                        #TODO: There are better ways to do this. Build File support.
                        print 'Deleting {img}'.format(img=image.identifier)
                        gcis.delete_image(image)

                    print 'Attempting to upload: {id}'.format(id=gcis_id)
110
                    upload_images_to_gcis(webform_url, gcis_id, report_id)
111
112


113
114
115
116
                #...then send it.
                gcis.update_figure(report_id, chapter_id, figure_obj)

                print 'Success!'
117

118
119

#This function is for adding images to existing figures
120
def upload_images_to_gcis(webform_url, gcis_id, report_id, subset_images=None):
121
    figure = webform.get_webform(webform_url, download_images=True)
122
123
    #Now identifiers don't need to be matched
    figure.identifier = gcis_id
124

125
126
127
128
129
130
131
    #If a subset of identifiers has been provided, only process those
    if subset_images:
        images_to_process = [image for image in figure.images if image.identifier in subset_images]
    else:
        images_to_process = figure.images

    for image in images_to_process:
132
133
        if not exists(image.local_path):
            raise Exception('Local file missing ' + image.local_path)
134

135
        if not gcis.image_exists(image.identifier):
136
            print 'Creating image: {img}'.format(img=image.identifier)
137
138
            gcis.create_image(image, report_id=report_id, figure_id=figure.identifier)

139

140
141
142
143
144
145
146
147
def sync_dataset_metadata(datasets):
    for ds in datasets:
        if gcis.dataset_exists(ds.identifier):
            print 'Updating: {ds}'.format(ds=ds)
            gcis.update_dataset(ds)
        else:
            print 'Creating: {ds}'.format(ds=ds)
            gcis.create_dataset(ds)
148
149


150
def aggregate_webform_datasets():
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
    dataset_map = {}

    for item in webform.get_list():
        webform_url = item['url']

        f = webform.get_webform(webform_url)

        #aggregate datasets
        for image in f.images:
            for dataset in image.datasets:
                if dataset.identifier not in dataset_map:
                    dataset_map[dataset.identifier] = dataset
                else:
                    dataset_map[dataset.identifier].merge(dataset)

    return dataset_map.values()
abuddenberg's avatar
abuddenberg committed
167

168

169
170
if __name__ == '__main__':
    main()