sync_figures.py 6.62 KB
Newer Older
1
#!/usr/bin/env python
2
3
__author__ = 'abuddenberg'

4
5
from gcis_clients import WebformClient
from gcis_clients import GcisClient
6
from os.path import exists
7
import json
abuddenberg's avatar
abuddenberg committed
8
import pickle
9
10
11
12
13
14

# webform_dev = ('http://dev.nemac.org/asides10/metadata/figures/all?token=A2PNYxRuG9')

webform = WebformClient('http://resources.assessment.globalchange.gov', 'mgTD63FAjG')

gcis_url = 'http://data.gcis-dev-front.joss.ucar.edu'
15
16
# gcis = GcisClient(gcis_url, 'andrew.buddenberg@noaa.gov', 'd9fcfd947c1785ab1cd329a9920e05e5c5d3d7f35315f164')
gcis = GcisClient('http://data-stage.globalchange.gov', 'andrew.buddenberg@noaa.gov', 'a6efcc7cf39c55e9329a8b027e0817e3354bada65310d192')
17

18
19
sync_metadata_tree = {
    #Reports
20
    'nca3': {
21
        #Chapter 2
22
23
        'our-changing-climate': [
            #(webform_url, gcis_id)
24
25
            ('/metadata/figures/2506', 'observed-change-in-very-heavy-precipitation'),
            ('/metadata/figures/2997', 'observed-change-in-very-heavy-precipitation-2'),
26
            ('/metadata/figures/2677', 'observed-us-precipitation-change'),
abuddenberg's avatar
abuddenberg committed
27
28
29
30
31
            # # ('/metadata/figures/3175', 'observed-us-temperature-change'),
            # # ('/metadata/figures/3074', 'ten-indicators-of-a-warming-world'),
            # # ('/metadata/figures/3170', 'global-temperature-and-carbon-dioxide'),
            # # ('/metadata/figures/3293', 'observed-increase-in-frostfree-season-length'),
            # ('/metadata/figures/3294', 'projected-changes-in-frostfree-season-length'), #Good
32
            # ('/metadata/figures/3305', 'variation-of-storm-frequency') #incomplete
33
        ],
abuddenberg's avatar
abuddenberg committed
34
35
36
        # #Chapter 4
        # 'energy-supply-and-use': [
        #     # ('/metadata/figures/3292', 'cooling-degree-days')
37
        # ],
abuddenberg's avatar
abuddenberg committed
38
39
40
41
42
        # #Chapter 6
        # 'agriculture': [
        # #     ('/metadata/figures/2872', 'drainage'),
        # #     ('/metadata/figures/2691', 'variables-affecting-ag') #Needs images redone
        #     ('/metadata/figures/2691', 'projected-changes-in-key-climate-variables-affecting-agricultural-productivity')
43
        # ],
abuddenberg's avatar
abuddenberg committed
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
        # #Chapter 9
        # # 'human-health': [
        # #     ('/metadata/figures/2896', 'heavy-downpours-disease') #Needs images redone
        # # ],
        # #Chapter 10
        # 'water-energy-land-use': [
        #     ('/metadata/figures/2410', 'coasttocoast-100degree-days-in-2011')
        # ],
        # #Chapter 14
        # 'rural': [
        #     ('/metadata/figures/3306', 'growing-season-lengthens') #Needs images redone
        # ],
        # #Chapter 16
        # 'northeast': [
        #     ('/metadata/figures/2995', 'projected-increases-in-the-number-of-days-over-90f')
        # ],
        # #Chapter 17
        # 'southeast': [
        #     ('/metadata/figures/2998', 'projected-change-in-number-of-days-over-95-f'),
        #     ('/metadata/figures/2999', 'projected-change-in-number-of-nights-below-32f')
        # ],
        # #Chapter 18
        # 'midwest': [
        #     ('/metadata/figures/2992', 'projected-midcentury-temperature-changes-in-the-midwest'),
        #     ('/metadata/figures/2994', 'when-it-rains-it-pours')
        # ],
        # #Chapter 19
        # # 'great-plains': [
        # #     ('/metadata/figures/2697', 'temperature-and-precipitation-distribution-in-the-great-plains') #Needs images redone
        # #
        # # ],
        # #Chapter 25
        # 'coastal-zone': [
        #     ('/metadata/figures/2543', 'coastal-ecosystem-services')
        # ],
        # # Climate Science Appendix
        # 'appendix-climate-science': [
        #     ('/metadata/figures/3147', 'ice-loss-from-greenland-and-antarctica')
        # ]
abuddenberg's avatar
abuddenberg committed
83

84
85
86
    }
}

87

88
def main():
abuddenberg's avatar
abuddenberg committed
89
90
91
92
93
    # sync_dataset_metadata(aggregate_webform_datasets())
    sync(replace=False)

    # with open('../datasets.pk1', 'wb') as outfile:
    #     pickle.dump(aggregate_webform_datasets(), outfile)
94

abuddenberg's avatar
abuddenberg committed
95
    # datasets = pickle.load(open('../datasets.pk1', 'r'))
96
97
    # sync_dataset_metadata(datasets)

abuddenberg's avatar
abuddenberg committed
98

99
100


101
def sync(replace=False):
102
103
104
    for report_id in sync_metadata_tree:
        for chapter_id in sync_metadata_tree[report_id]:
            for figure_ids in sync_metadata_tree[report_id][chapter_id]:
105
                webform_url, gcis_id = figure_ids
106

107
108
109
110
111
112
113
114
115
116
117
118
119
120
                print 'Attempting to sync: {id}'.format(id=gcis_id)

                #Merge data from both systems into one object...
                figure_obj = webform.get_webform(webform_url).merge(
                    gcis.get_figure(report_id, gcis_id, chapter_id=chapter_id)
                )

                if replace:
                    for image in figure_obj.images:
                        #TODO: There are better ways to do this. Build File support.
                        print 'Deleting {img}'.format(img=image.identifier)
                        gcis.delete_image(image)

                    print 'Attempting to upload: {id}'.format(id=gcis_id)
121
                    upload_images_to_gcis(webform_url, gcis_id, report_id)
122
123


124
125
126
127
                #...then send it.
                gcis.update_figure(report_id, chapter_id, figure_obj)

                print 'Success!'
128

129
130

#This function is for adding images to existing figures
131
def upload_images_to_gcis(webform_url, gcis_id, report_id, subset_images=None):
132
    figure = webform.get_webform(webform_url, download_images=True)
133
134
    #Now identifiers don't need to be matched
    figure.identifier = gcis_id
135

136
137
138
139
140
141
142
    #If a subset of identifiers has been provided, only process those
    if subset_images:
        images_to_process = [image for image in figure.images if image.identifier in subset_images]
    else:
        images_to_process = figure.images

    for image in images_to_process:
143
144
        if not exists(image.local_path):
            raise Exception('Local file missing ' + image.local_path)
145

146
        if not gcis.image_exists(image.identifier):
147
            print 'Creating image: {img}'.format(img=image.identifier)
148
149
            gcis.create_image(image, report_id=report_id, figure_id=figure.identifier)

150

151
152
def sync_dataset_metadata(datasets):
    for ds in datasets:
153
154
        gcis.create_or_update_dataset(ds)
        gcis.create_or_update_activity(ds.activity)
abuddenberg's avatar
abuddenberg committed
155

156

157
def aggregate_webform_datasets():
158
159
160
161
162
163
164
165
166
167
168
169
170
171
    dataset_map = {}

    for item in webform.get_list():
        webform_url = item['url']

        f = webform.get_webform(webform_url)

        #aggregate datasets
        for image in f.images:
            for dataset in image.datasets:
                if dataset.identifier not in dataset_map:
                    dataset_map[dataset.identifier] = dataset
                else:
                    dataset_map[dataset.identifier].merge(dataset)
abuddenberg's avatar
abuddenberg committed
172
                    dataset_map[dataset.identifier].activity.merge(dataset.activity)
173
174

    return dataset_map.values()
abuddenberg's avatar
abuddenberg committed
175

176
177
if __name__ == '__main__':
    main()