sync_figures 19.4 KB
Newer Older
1
#!/usr/bin/env python
2
3
__author__ = 'abuddenberg'

4
from gcis_clients import WebformClient, GcisClient, gcis_dev_auth, gcis_stage_auth, webform_token, AssociationException
5
from gcis_clients.sync_utils import move_images_to_gcis, sync_dataset_metadata, realize_contributors, realize_parents
abuddenberg's avatar
abuddenberg committed
6
from collections import OrderedDict
7

8
webform = WebformClient('http://resources.assessment.globalchange.gov', webform_token)
9

10
11
# gcis = GcisClient('http://data.gcis-dev-front.joss.ucar.edu', *gcis_dev_auth)
gcis = GcisClient('http://data-stage.globalchange.gov', *gcis_stage_auth)
12

13
14
sync_metadata_tree = {
    #Reports
abuddenberg's avatar
abuddenberg committed
15
    'nca3': OrderedDict([
16
        #Chapter 2
17
18
19
20
21
22
        ('our-changing-climate', [
            # (webform_url, gcis_id)
            ('/metadata/figures/3074', 'ten-indicators-of-a-warming-world'),  # 2.1
            ('/metadata/figures/3293', 'observed-increase-in-frostfree-season-length'),  # 2.10
            ('/metadata/figures/3294', 'projected-changes-in-frostfree-season-length'),  # 2.11
            ('/metadata/figures/2677', 'observed-us-precipitation-change'),  # 2.12
23
24
25
            ('/metadata/figures/3296', 'ann-max-precip-cons-dry-day'),  # 2.13
            ('/metadata/figures/3297', 'projected-precipitation-change-by-season'),  # 2.14
            ('/metadata/figures/2955', 'proj-precip-change'),  # 2.15
26
27
28
29
30
            ('/metadata/figures/3298', 'observed-us-trend-in-heavy-precipitation'),  # 2.16
            ('/metadata/figures/2506', 'observed-change-in-very-heavy-precipitation'),  # 2.17
            ('/metadata/figures/2997', 'observed-change-in-very-heavy-precipitation-2'),  # 2.18
            ('/metadata/figures/3170', 'global-temperature-and-carbon-dioxide'),  # 2.2
            ('/metadata/figures/2939', 'projected-changes-in-soil-moisture-for-the-western-us'),  # 2.22
31
            ('/metadata/figures/3304', 'observed-trends-in-hurricane-power-dissipation'),  # 2.23
32
33
34
35
36
            ('/metadata/figures/3305', 'variation-of-storm-frequency-and-intensity-during-the-cold-season-november--march'),  # 2.24
            ('/metadata/figures/3067', 'ice-cover-in-the-great-lakes'),  # 2.27
            ('/metadata/figures/3073', 'projected-arctic-sea-ice-decline'),  # 2.29
            ('/metadata/figures/2940', 'as-oceans-absorb-co-they-become-more-acidic'),  # 2.30
            ('/metadata/figures/2523', 'shells-dissolve-in-acidified-ocean-water'),  # 2.31
37
38
39
            ('/metadata/figures/3078', 'emissions-levels-determine-temperature-rises'),  # 2.4
            ('/metadata/figures/3172', 'projected-change-in-average-annual-temperature'),  #2.5
            ('/metadata/figures/3176', 'projected-change-in-average-annual-precipitation'),  # 2.6
40
            ('/metadata/figures/3175', 'observed-us-temperature-change'),  # 2.7
41
            ('/metadata/figures/3001', 'projected-temperature-change'),  # 2.8
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
            ('/metadata/figures/3000', 'projected-temperature-change-by-20712099-cmip5-models'),  #2.9

        ]),
        #Chapter 3
        ('water-resources', [
            ('/metadata/figures/2676', 'projected-changes-in-snow-runoff-and-soil-moisture'),  # 3.1
            ('/metadata/figures/3303', 'trends-in-flood-magnitude'),  # 3.5
        ]),
        #Chapter 4
        ('energy-supply-and-use', [
            ('/metadata/figures/3082', 'paths-of-hurricanes-katrina-and-rita-relative-to-oil-and-gas-production-facilities'),  # 4.1
            ('/metadata/figures/3292', 'cooling-degree-days'),  # 4.3
        ]),
        #Chapter 5
        ('transportation', [
57
58
            ('/metadata/figures/3088', 'possible-future-flood-depths-in-mobile-al-with-rising-sea-level'),  # 5.1
            ('/metadata/figures/2905', 'gulf-coast-transportation-hubs-at-risk'),  # 5.3
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
            ('/metadata/figures/3568', 'hurricane-sandy-causes-flooding-in-new-york-city-subway-stations'),  # 5.4
            ('/metadata/figures/2952', 'role-of-adaptive-strategies-and-tactics-in-reducing-impacts-and-consequences'),  # 5.5
            ('/metadata/figures/3169', 'tropical-storm-impact-on-vermont-road'),  # 5.6
        ]),
        #Chapter 6
        ('agriculture', [
            ('/metadata/figures/3582', 'crop-yield-response-to-warming-in-californias-central-valley'),  #6.4
            ('/metadata/figures/2691', 'projected-changes-in-key-climate-variables-affecting-agricultural-productivity'),  # 6.5
            ('/metadata/figures/2872', 'drainage-tiles'),  # 6.8
        ]),
        #Chapter 7
        ('forests', [
            ('/metadata/figures/2887', 'forest-ecosystem-disturbances'),  # 7.1
            ('/metadata/figures/2977', 'effectiveness-of-forest-management-in-reducing-wildfire-risk'),  # 7.2
            ('/metadata/figures/2978', 'forest-vulnerability-to-changing-climate'),  # 7.3
            ('/metadata/figures/2985', 'forests-can-be-a-source--or-a-sink--for-carbon'),  # 7.4
            ('/metadata/figures/3130', 'public-and-private-forestlands'),  # 7.8

        ]),
        #Chapter 8
        ('ecosystems', [
80
            ('/metadata/figures/2576', 'the-aftermath-of-hurricanes'),  # 8.2
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
            ('/metadata/figures/2456', 'adaptation-planning-and-implementation-framework'),  # 8.3
            ('/metadata/figures/3574', 'biological-responses-to-climate-change'),  # 8.4
        ]),
        #Chapter 9
        ('human-health', [
            ('/metadata/figures/3162', 'projected-climate-change-worsens-asthma'),  # 9.1
            ('/metadata/figures/2896', 'heavy-downpours-disease'),  # 9.7
            ('/metadata/figures/2897', 'harmful-bloom-of-algae'),  # 9.8
        ]),
        #Chapter 10
        ('water-energy-land-use', [
            ('/metadata/figures/2601', 'energy-water-land-and-climate-interactions'),  # 10.1
            ('/metadata/figures/2917', 'the-columbia-river-basin-land-use-and-land-cover'),  # 10.10
            ('/metadata/figures/2410', 'coasttocoast-100degree-days-in-2011'),  # 10.2
            ('/metadata/figures/3158', 'projected-landuse-intensity-in-2030'),  # 10.6
            ('/metadata/figures/2986', 'hydraulic-fracturing-and-water-use'),  # 10.7
            ('/metadata/figures/2916', 'renewable-energy-and-land-use'),  # 10.8
            ('/metadata/figures/2918', 'water-stress-in-the-us'),  # 10.9

        ]),
        #Chapter 11
        ('urban-systems-infrastructure-vulnerability', [
            ('/metadata/figures/3569', 'urban-support-systems-are-interconnected'),  # 11.2
            ('/metadata/figures/3090', 'new-york-city-and-sea-level-rise'),  # 11.3
        ]),
        #Chapter 12
        ('tribal-indigenous-native-lands-resources', [
            ('/metadata/figures/3131', 'indigenous-populations-extend-beyond-reservation-lands'),  # 12.1
            ('/metadata/figures/2594', 'many-tribes-many-climate-change-initiatives'),  # 12.2
            ('/metadata/figures/2909', 'sand-dune-expansion'),  # 12.3
111
            ('/metadata/figures/3572', 'sea-ice-cover-reaches-record-low'),  # 12.4
112
113
114
115
116
117
118
119
120
121
122
123
124
            ('/metadata/figures/2911', 'arctic-marine-food-web'),  # 12.5
        ]),
        #Chapter 13
        ('land-use-land-cover-change', [
            ('/metadata/figures/2902', 'building-loss-by-fires-at-california-wildlandurban-interfaces'),  # 13.4
        ]),
        #Chapter 14
        ('rural', [
            ('/metadata/figures/2662', 'rural-counties'),  # 14.1
            ('/metadata/figures/2661', 'economic-dependence-varies-by-region'),  # 14.2
            ('/metadata/figures/3306', 'growing-season-lengthens'),  # 14.3
            ('/metadata/figures/2904', 'many-rural-areas-are-losing-population'),  # 14.5
        ]),
abuddenberg's avatar
abuddenberg committed
125
126
        # Chapter 15
        ('biogeochemical-cycles', [
127
            ('/metadata/figures/2874', 'major-north-american-carbon-dioxide-sources-and-sinks'),  # 15.1
128
            ('/metadata/figures/2574', 'nitrogen-emissions'),  # 15.3
129
            ('/metadata/figures/2575', 'many-factors-combine-to-affect-biogeochemical-cycles'),  # 15.4
abuddenberg's avatar
abuddenberg committed
130
131
132
        ]),
        #Chapter 16
        ('northeast', [
133
134
135
136
137
138
            ('/metadata/figures/2995', 'projected-increases-in-the-number-of-days-over-90f'),  # 16.2
            ('/metadata/figures/3512', 'flooding-and-hurricane-irene'),  # 16.3
            ('/metadata/figures/3104', 'coastal-flooding-along-new-jerseys-shore'),  # 16.4
            ('/metadata/figures/2844', 'coney-island-after-hurricane-irene'),  # 16.6
            ('/metadata/figures/2846', 'storm-surge-barrier'),  # 16.8

abuddenberg's avatar
abuddenberg committed
139
140
141
        ]),
        #Chapter 17
        ('southeast', [
abuddenberg's avatar
abuddenberg committed
142
            ('/metadata/figures/2562', 'southeast-temperature-observed-and-projected'),  # 17.3
143
144
145
            ('/metadata/figures/2998', 'projected-change-in-number-of-days-over-95-f'),  # 17.4
            ('/metadata/figures/2999', 'projected-change-in-number-of-nights-below-32f'),  # 17.5
            ('/metadata/figures/2857', 'local-planning'),  # 17.9
abuddenberg's avatar
abuddenberg committed
146
147
148
        ]),
        #Chapter 18
        ('midwest', [
149
150
            ('/metadata/figures/2550', 'temperatures-are-rising-in-the-midwest'),  # 18.1
            ('/metadata/figures/2992', 'projected-midcentury-temperature-changes-in-the-midwest'),  # 18.2
abuddenberg's avatar
abuddenberg committed
151
152
            ('/metadata/figures/2841', 'crop-yields-decline-under-higher-temperatures'),  #18.3
            ('/metadata/figures/2993', 'reducing-emissions-improving-health'),  # 18.5
153
154
            ('/metadata/figures/2994', 'when-it-rains-it-pours'),  # 18.6
            ('/metadata/figures/3159', 'midwest-ice-cover-in-the-great-lakes')  #18.7
abuddenberg's avatar
abuddenberg committed
155
156
157
        ]),
        #Chapter 19
        ('great-plains', [
158
159
160
161
162
            ('/metadata/figures/2697', 'temperature-and-precipitation-distribution-in-the-great-plains'),  # 19.1
            ('/metadata/figures/2989', 'projected-change-in-number-of-hot-days'),  # 19.2
            ('/metadata/figures/2990', 'projected-change-in-number-of-warm-nights'),  # 19.3
            ('/metadata/figures/2991', 'projected-change-in-number-of-heavy-precipitation-days'),  # 19.4
            ('/metadata/figures/3002', 'projected-change-in-number-of-consecutive-dry-days'),  # 19.5
163
            ('/metadata/figures/3280', 'historical-and-current-range-of-sage-grouse-habitat')  #19.7
164
165
166
        ]),
        #Chapter 20
        ('southwest', [
167
            ('/metadata/figures/2568', 'coastal-risks-posed-by-sea-level-rise-and-high-tides'),  #20.5
168
169
170
171
            ('/metadata/figures/3484', 'urban-heat-and-public-health'),  # 20.6
        ]),
        #Chapter 21
        ('northwest', [
abuddenberg's avatar
abuddenberg committed
172
            ('/metadata/figures/2851', 'adapting-the-nisqually-river-delta-to-sea-level-rise'),  #21.5
173
174
175
176
177
178
179
180
181
            ('/metadata/figures/2558', 'forest-mortality'),  # 21.6
        ]),
        #Chapter 22
        ('alaska', [
            ('/metadata/figures/2817', 'newtok-alaska'),  # 22.4
        ]),
        #Chapter 23
        ('hawaii', [
            ('/metadata/figures/2834', 'us-pacific-islands-region'),  # 23.1
182
            ('/metadata/figures/3283', 'high-and-low-pacific-islands-face-different-threats'),  #23.2
183
184
185
186
187
188
            ('/metadata/figures/3284', 'increased-acidification-decreases-suitable-coral-habitat'),  # 23.3
            ('/metadata/figures/2837', 'observed-changes-in-annual-rainfall-in-the-western-north-pacific'),  # 23.4
            ('/metadata/figures/2838', 'native-plants-at-risk'),  # 23.5
            ('/metadata/figures/3285', 'saltwater-intrusion-destroys-crops'),  # 23.6
            ('/metadata/figures/2840', 'residents-of-lowlying-islands-at-risk'),  # 23.7
            ('/metadata/figures/3286', 'higher-sea-level-rise-in-western-pacific'),  # 23.8
abuddenberg's avatar
abuddenberg committed
189
        ]),
190
191
192
193
194
195
        #Chapter 24
        ('oceans-marine-resources', [
            ('/metadata/figures/3106', 'observed-ocean-warming'),  # 24.1
            ('/metadata/figures/2560', 'ocean-acidification-reduces-size-of-clams'),  #24.3
            ('/metadata/figures/2853', 'warming-seas-are-a-doubleblow-to-corals')  # 24.4
        ]),
abuddenberg's avatar
abuddenberg committed
196
197
        #Chapter 25
        ('coastal-zone', [
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
            ('/metadata/figures/2540', 'adapting-coastal-infrastructure-to-sea-level-rise-and-land-loss'),  # 25.5
            ('/metadata/figures/3430', 'coasttoinland-economic-connections'),  # 25.7
            ('/metadata/figures/2543', 'coastal-ecosystem-services'),  # 25.8
        ]),
        #Chapter 26
        ('decision-support', [
            ('/metadata/figures/3571', 'decisionmaking-elements-and-outcomes'),  # 26.1
            ('/metadata/figures/2949', 'boundary-processes-linking-decisionmakers-and-scientifictechnical-experts'),  # 26.2
            ('/metadata/figures/2950', 'decisionmaking-framework'),  # 26.3
            ('/metadata/figures/2611', 'linking-risk-assessment-and-risk-perception-with-risk-management-of-climate-change'),  # 26.4
            ('/metadata/figures/2962', 'land-use-planning-tool-for-the-upper-santa-cruz-watershed'),  # 26.5
        ]),
        #Chapter 27
        ('mitigation', [
            ('/metadata/figures/3080', 'drivers-of-us-fossil-emissions'),  # 27.2
        ]),
        #Chapter 28
        ('adaptation', [
            ('/metadata/figures/2862', 'status-of-state-climate-adaptation-plans'),  # 28.1
            ('/metadata/figures/2863', 'risk-disk'),  # 28.2
            ('/metadata/figures/2864', 'adaptation-process'),  # 28.3
            ('/metadata/figures/2675', 'us-drought-monitor'),  # 28.5
            ('/metadata/figures/2865', 'northwoods-climate-change-response-framework'),  # 28.6
abuddenberg's avatar
abuddenberg committed
221
        ]),
222
        #Chapter 33: Climate Science Appendix
223
        ('appendix-climate-science-supplement', [
224
            ('/metadata/figures/3112', 'indicators-of-warming-from-multiple-data-sets'),  # 33.10
225
            ('/metadata/figures/3097', 'human-influence-on-the-greenhouse-effect'),  #33.1
226
227
228
229
230
231
232
233
234
235
236
            ('/metadata/figures/3138', 'warming-trend-and-effects-of-el-nino-la-nina'),  # 33.14
            ('/metadata/figures/2798', 'detection-and-attribution-as-forensics'),  # 33.16
            ('/metadata/figures/3301', 'human-influences-apparent-in-many-aspects-of-the-changing-climate'),  # 33.17
            ('/metadata/figures/2813', 'earths-energy-balance'),  # 33.2
            ('/metadata/figures/2802', 'modeling-the-climate-system'),  # 33.24
            ('/metadata/figures/2803', 'increasing-model-resolution'),  # 33.25
            ('/metadata/figures/3575', 'increasing-climate-model-components'),  # 33.26
            ('/metadata/figures/2533', 'us-seasonal-temperatures'),  # 33.28
            ('/metadata/figures/2806', 'ice-cover-on-lake-mendota'),  # 33.29
            ('/metadata/figures/2534', 'app-extreme-precipitation'),  # 33.32
            ('/metadata/figures/2809', 'extreme-drought-in-the-us-and-mexico-past-and-future'),  # 33.35
237
            ('/metadata/figures/3148', 'arctic-sea-ice-decline'),  # 33.37
238
239
240
241
242
243
            ('/metadata/figures/2393', 'permafrost-temperatures-rising'),  # 33.38
            ('/metadata/figures/3144', 'melting-of-arctic-landbased-ice'),  # 33.39
            ('/metadata/figures/3145', 'melting-glaciers-lead-to-sea-level-rise'),  # 33.40
            ('/metadata/figures/3147', 'ice-loss-from-greenland-and-antarctica'),  # 33.42
            ('/metadata/figures/3110', 'relative-strengths-of-warming-and-cooling-influences'),  # 33.6
            ('/metadata/figures/2796', 'development-of-observing-capabilities'),  # 33.7
244
245
        ]),
        #Chapter 34: FAQ/CAQ
246
        ('appendix-faqs', [
247
            ('/metadata/figures/2663', 'us-annual-temperature'),  # 34.1
248
            ('/metadata/figures/3096', 'caq-human-influence-on-the-greenhouse-effect'),  # 34.11
249
250
251
252
253
254
255
256
            ('/metadata/figures/3300', 'caq-human-influences-apparent-in-many-aspects-of-the-changing-climate'),  # 34.13
            ('/metadata/figures/3091', 'published-climate-change-research-papers'),  # 34.18
            ('/metadata/figures/3075', 'caq-ten-indicators-of-a-warming-world'),  # 34.2
            ('/metadata/figures/2520', 'ocean-acidification-and-the-food-web'),  # 34.21
            ('/metadata/figures/2790', 'potential-tipping-points'),  # 34.24
            ('/metadata/figures/2635', 'potential-effects-of-climate-change'),  # 34.25
            ('/metadata/figures/2791', 'two-us-emissions-reduction-pathways'),  # 34.27
            ('/metadata/figures/3134', 'temperature-trends-1900-2012'),  # 34.8
abuddenberg's avatar
abuddenberg committed
257
258
        ])
    ])
259
260
}

261
262
263
264
265
266
267
268
269
270
images_with_multiple_figures = {
    'e13cdf2e-0699-4c52-a642-56c8fc3317c4': ['human-influence-on-the-greenhouse-effect', 'caq-human-influence-on-the-greenhouse-effect'], #33.1 #34.11
    '03937439-cc5a-45c7-a0fd-78b863c49979': ['human-influence-on-the-greenhouse-effect', 'caq-human-influence-on-the-greenhouse-effect'], #33.1 #34.11
    '0bcb7984-50c0-4f94-9946-c3fa635125a4': ['ten-indicators-of-a-warming-world', 'caq-ten-indicators-of-a-warming-world'], #2.1 #34.2
    'eb9bd7dc-3e3a-4001-9a64-f2a0763b3f64': ['shells-dissolve-in-acidified-ocean-water', 'ocean-acidification-and-the-food-web'], #2.31 #34.21
    '8c679de0-4a6c-4fc9-bbd1-59251ffe3d7f': ['shells-dissolve-in-acidified-ocean-water', 'ocean-acidification-and-the-food-web'], #2.31 #34.21
    'a1bd40f2-424e-4af8-8d3d-f44ba7ba36b6': ['human-influences-apparent-in-many-aspects-of-the-changing-climate', 'caq-human-influences-apparent-in-many-aspects-of-the-changing-climate'], #33.17 #34.13
    '834f7624-40de-4a09-9ced-faaca30d4d6d': ['ice-cover-in-the-great-lakes', 'midwest-ice-cover-in-the-great-lakes'], #2.27 #18.7
}

271

272
def main():
273
    print gcis.test_login()
274
    # sync_dataset_metadata(gcis, webform.get_aggregated_datasets())
275
276
    # sync(replace=True)
    cleanup_after_replace()
abuddenberg's avatar
abuddenberg committed
277

278

279
def sync(replace=False):
280
281
    count = 0

282
283
284
    for report_id in sync_metadata_tree:
        for chapter_id in sync_metadata_tree[report_id]:
            for figure_ids in sync_metadata_tree[report_id][chapter_id]:
285
                webform_url, gcis_id = figure_ids
286

287
288
289
290
291
                print 'Attempting to sync: {id}'.format(id=gcis_id), webform_url

                #Merge data from both systems into one object...
                gcis_fig = gcis.get_figure(report_id, gcis_id, chapter_id=chapter_id)
                figure_obj = webform.get_webform(webform_url).merge(gcis_fig)
292
                realize_parents(gcis, figure_obj.parents)
293
294
295
296
297
298
299
300
301

                #Lookup and populate contributor information for figures and images
                realize_contributors(gcis, figure_obj.contributors)
                for i in figure_obj.images:
                    realize_contributors(gcis, i.contributors)

                #Hack: override webform's title in favor of gcis
                figure_obj.title = gcis_fig.title if gcis_fig.title not in (None, '') else figure_obj.title

302
303
304
305

                if len(gcis_fig.contributors) > 1:
                    print '*** Multiple contributors: ', gcis_id

306
                if replace:
307
308
309
310
311
312
313
314
315
                    #Hack: removing existing contributors to address orphans
                    for c in gcis_fig.contributors:
                        gcis.delete_contributor_figure_assoc(c, report_id, chapter_id, gcis_id)

                    # #Remove existing parents
                    for p in gcis_fig.parents:
                        gcis.delete_figure_pub_assoc(report_id, gcis_id, p)

                    for image in gcis_fig.images:
316
317
318
319
320
321
322
323
324
325
326
                        #TODO: There are better ways to do this. Build File support.
                        print 'Deleting {img}'.format(img=image.identifier)
                        gcis.delete_image(image)

                    print 'Attempting to upload: {id}'.format(id=gcis_id)
                    move_images_to_gcis(webform, gcis, webform_url, gcis_id, report_id)

                #...then send it.
                gcis.update_figure(report_id, chapter_id, figure_obj)

                print 'Success!'
327
328
                count += 1
    print count
329

330
331
332
333
334
335
336

#Hack
def cleanup_after_replace():
    for image_id in images_with_multiple_figures:
        for figure_id in images_with_multiple_figures[image_id]:
            print gcis.associate_image_with_figure(image_id, 'nca3', figure_id)

337
if __name__ == '__main__':
338
    main()