sync_figures.py 5.11 KB
Newer Older
1
2
3
4
__author__ = 'abuddenberg'

from webform_client import WebformClient
from gcis_client import GcisClient
5
from os.path import exists
6
import json
7
import pickle
8
9
10
11
12
13

# webform_dev = ('http://dev.nemac.org/asides10/metadata/figures/all?token=A2PNYxRuG9')

webform = WebformClient('http://resources.assessment.globalchange.gov', 'mgTD63FAjG')

gcis_url = 'http://data.gcis-dev-front.joss.ucar.edu'
14
15
gcis = GcisClient(gcis_url, 'andrew.buddenberg@noaa.gov', 'd9fcfd947c1785ab1cd329a9920e05e5c5d3d7f35315f164')
# gcis = GcisClient('http://data-stage.globalchange.gov', 'andrew.buddenberg@noaa.gov', 'a6efcc7cf39c55e9329a8b027e0817e3354bada65310d192')
16

17
18
sync_metadata_tree = {
    #Reports
19
    'nca3': {
20
        #Chapter 2
21
22
        'our-changing-climate': [
            #(webform_url, gcis_id)
23
24
25
26
27
28
29
30
            ('/metadata/figures/2506', 'observed-change-in-very-heavy-precipitation'),
            ('/metadata/figures/2997', 'observed-change-in-very-heavy-precipitation-2'),
            ('/metadata/figures/2677', 'observed-us-precipitation-change'),
            ('/metadata/figures/3175', 'observed-us-temperature-change'),
            ('/metadata/figures/3074', 'ten-indicators-of-a-warming-world'),
            ('/metadata/figures/3170', 'global-temperature-and-carbon-dioxide'),
            ('/metadata/figures/3293', 'observed-increase-in-frostfree-season-length'),
            ('/metadata/figures/3294', 'projected-changes-in-frostfree-season-length'),
31
            # ('/metadata/figures/3305', 'variation-of-storm-frequency-and-intensity-during-the-cold-season-november--march') #incomplete
32
        ],
33
34
        #Chapter 4
        'energy-supply-and-use': [
35
            ('/metadata/figures/3292', 'cooling-degree-days')
36
        ],
37
38
        #Chapter 6
        'agriculture': [
39
40
            ('/metadata/figures/2872', 'drainage'),
            ('/metadata/figures/2691', 'variables-affecting-ag') #Needs images redone
41
42
        ],
        #Chapter 9
43
44
        'human-health': [
            ('/metadata/figures/2896', 'heavy-downpours-disease') #Needs images redone
45
46
        ],
        #Chapter 14
47
        'rural': [
48
49
50
51
52
            ('/metadata/figures/3306', 'length-growing-season') #Needs images redone
        ],
        #Chapter 18
        '': [
            ('/metadata/figures/2992', 'projected-midcentury-temperature-changes-in-the-midwest')
53
54
55
        ],
        #Chapter 19
        'great-plains': [
56
            ('/metadata/figures/2697', 'mean-annual-temp-and-precip') #Needs images redone
57
58
59
60

        ],
        #Chapter 25
        'coastal-zone': [
61
            ('/metadata/figures/2543', 'coastal-ecosystem-services')
abuddenberg's avatar
abuddenberg committed
62
63
64
        ],
        #Climate Science Appendix
        'appendix-climate-science': [
65
            # ('/metadata/figures/3147', 'ice-loss-from-greenland-and-antarctica')
66
        ]
abuddenberg's avatar
abuddenberg committed
67

68
69
70
    }
}

71

72
def main():
73
    sync(uploads=False)
74
75


76
def sync(uploads=True):
77
78
79
    for report_id in sync_metadata_tree:
        for chapter_id in sync_metadata_tree[report_id]:
            for figure_ids in sync_metadata_tree[report_id][chapter_id]:
80
                webform_url, gcis_id = figure_ids
81

82
83
                if uploads:
                    print 'Attempting to upload: ' + gcis_id
84
                    upload_images_to_gcis(webform_url, gcis_id, report_id)
85
                print 'Attempting to sync: ' + gcis_id
86
                sync_figure_metadata(report_id, chapter_id, webform_url, gcis_id)
87
                print 'Success!'
88
89


90
def sync_figure_metadata(report_id, chapter_id, webform_url, gcis_id):
91
92
93
94
95
96
    #Merge data from both systems into one object...
    figure_obj = webform.get_webform(webform_url).merge(
        gcis.get_figure(report_id, gcis_id, chapter_id=chapter_id)
    )
    #...then send it.
    gcis.update_figure(report_id, chapter_id, figure_obj)
97

98
99
100

#This function is for adding images to existing figures
def upload_images_to_gcis(webform_url, gcis_id, report_id):
101
    figure = webform.get_webform(webform_url, download_images=True)
102
103
    #Now identifiers don't need to be matched
    figure.identifier = gcis_id
104

105
106
107
108
109
    #Make sure we have all the images required for a COMPLETE update
    for image in figure.images:
        if not exists(image.local_path):
            raise Exception('Local file missing ' + image.local_path)
    for image in figure.images:
110
111
112
        if not gcis.image_exists(image.identifier):
            gcis.create_image(image, report_id=report_id, figure_id=figure.identifier)

113

114
115
116
117
118
119
120
121
def sync_dataset_metadata(datasets):
    for ds in datasets:
        if gcis.dataset_exists(ds.identifier):
            print 'Updating: {ds}'.format(ds=ds)
            gcis.update_dataset(ds)
        else:
            print 'Creating: {ds}'.format(ds=ds)
            gcis.create_dataset(ds)
122
123


124
def aggregate_webform_datasets():
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
    dataset_map = {}

    for item in webform.get_list():
        webform_url = item['url']

        f = webform.get_webform(webform_url)

        #aggregate datasets
        for image in f.images:
            for dataset in image.datasets:
                if dataset.identifier not in dataset_map:
                    dataset_map[dataset.identifier] = dataset
                else:
                    dataset_map[dataset.identifier].merge(dataset)

    return dataset_map.values()
abuddenberg's avatar
abuddenberg committed
141

142

143
144
if __name__ == '__main__':
    main()