sync_states.py 4.89 KB
Newer Older
1
2
3
4
5
6
7
8
__author__ = 'abuddenberg'

import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning

requests.packages.urllib3.disable_warnings(InsecureRequestWarning)

from gcis_clients import GcisClient, SurveyClient, survey_token, gcis_dev_auth, gcis_stage_auth
abuddenberg's avatar
abuddenberg committed
9
from domain import Report, Chapter
10
11
12
13
14
15
from sync_utils import realize_parents, realize_contributors
from states import sync_metadata_tree

import pickle
import sys
import re
16
17
import traceback

18

19
20
# gcis = GcisClient('https://data-stage.globalchange.gov', *gcis_stage_auth)
gcis = GcisClient('https://data-review.globalchange.gov', *gcis_stage_auth)
21
22
23
24
25
surveys = SurveyClient('https://state-resources.cicsnc.org', survey_token)


def main():
    print(gcis.test_login())
26

27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
    cssr = Report({
        'identifier': 'climate-science-special-report',
        'report_type_identifier': 'report',
        'title': 'Climate Science Special Report',
        # 'url': 'https://statesummaries.cicsnc.org/',
        'publication_year': '2017',
        'contact_email': ''
    })

    print(gcis.create_report(cssr));
    # for report_id in sync_metadata_tree:
    #     for chapter_id in sync_metadata_tree[report_id]:
    #         for survey_url, figure_id, figure_num in sync_metadata_tree[report_id][chapter_id]:
    #             figure, datasets = surveys.get_survey(survey_url, do_download=False)
    #
    #             resp = gcis.post_figure_original(report_id, figure_id, figure.original, chapter_id=chapter_id)
    #             print(resp.status_code, resp.text)
    #
    #             #Fix misspelling
    #             figure.identifier = figure_id
    #             figure.title = figure.title.replace('precipitaton', 'precipitation')
    #             figure.ordinal = figure_num
    #
    #             print(survey_url)
    #             print(figure, datasets)
    #
    #             realize_parents(gcis, figure.parents)
    #             realize_contributors(gcis, figure.contributors)
    #
    #             print('Contributors: ', figure.contributors)
    #             print('Parents: ', figure.parents)
    #             # gcis_fig = gcis.get_figure(report_id, figure_id, chapter_id=chapter_id)
    #
    #             for ds in [p for p in figure.parents if p.publication_type_identifier == 'dataset']:
    #                 # Assign synthetic activity identifier to for datasets associated with figure
    #                 if ds.activity and ds.activity.identifier is None:
    #                     ds.activity.identifier = generate_activity_id(figure, ds.publication)
    #                 print('Dataset: ', ds.activity)
    #
    #             print('Creating figure... ', gcis.create_figure(report_id, chapter_id, figure, skip_images=True, skip_upload=False))
    #             # print('Updating figure... ', gcis.update_figure(report_id, chapter_id, figure, skip_images=True))
68
69
70
71
72
73
74
75

    
def generate_activity_id(image, dataset):
    try:
        return '-'.join([image.identifier.split('-')[0], dataset.identifier, 'process'])
    except Exception, e:
        sys.stderr.write('WARNING: Activity identifier generation failed\n')
        traceback.print_exc()
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110


def gen_survey_list():
    # with open('survey_list.pk', 'wb') as out:
    #     pickle.dump(gen_survey_list(), out)
    # surveys = pickle.load(open('survey_list.pk'))
    #
    # for st in sync_metadata_tree['noaa-led-state-summaries-2016']:
    #     print(st)
    #     for f in sorted(surveys[st], key=lambda x: x[1]):
    #         print("('{0}', '{1}', '{2}'),".format(f[0], f[2], f[1]))
    #     print('')

    realized_list = {}

    survey_list = surveys.get_list()
    for i, survey in enumerate(survey_list):
        url = survey['url']
        match = re.match('group/([a-z-]+)', survey['node_title'])
        chapter = match.group(1) if match else ''

        print('Processing: {b}{url} ({i}/{total})'.format(b=surveys.base_url, url=url, i=i + 1, total=len(survey_list)))

        s, ds = surveys.get_survey(url)
        if s:
            print(s.identifier)
            print(chapter, s.ordinal, s.title)

            realized_list.setdefault(chapter, []).append((url, s.ordinal, s.identifier, s.title))
        print('')
    return realized_list


def create_nlss_report():
    nlss = Report({
111
        'identifier': 'noaa-led-state-summaries-2017',
112
        'report_type_identifier': 'report',
113
        'title': 'NOAA-led State Summaries 2017',
114
        'url': 'https://statesummaries.cicsnc.org/',
115
        'publication_year': '2017',
116
117
118
        'contact_email': ''
    })

119
    chapters = [(id, i + 1, ' '.join([w.capitalize() for w in id.split('-')])) for i, id in enumerate(sync_metadata_tree['noaa-led-state-summaries-2017'])]
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134

    print(gcis.create_report(nlss))

    for id, num, title in chapters:
        ch = Chapter({
            'identifier': id,
            'number': num,
            'title': title,
            'report_identifier': nlss.identifier
        })

        print(gcis.create_chapter(nlss.identifier, ch))


main()