gcis_client.py 20.9 KB
Newer Older
1
import json
2
3
import os
from os.path import basename, expanduser
abuddenberg's avatar
abuddenberg committed
4
import re
5
import requests
6
7
import yaml
import getpass
8

abuddenberg's avatar
abuddenberg committed
9
from domain import Figure, Image, Dataset, Activity, Person, Organization
10
11
12
13
14
15


def check_image(fn):
    def wrapped(*args, **kwargs):
        # if len(args) < 1 or not isinstance(args[0], Image):
        #     raise Exception('Invalid Image')
16
        if args[1].identifier in (None, ''):
17
            raise Exception('Invalid identifier', args[0].identifier)
18
        return fn(*args, **kwargs)
19
20
21
22

    return wrapped


23
24
25
26
27
28
29
30
31
32
33
34
def exists(fn):
    def wrapped(*args, **kwargs):
        resp = fn(*args, **kwargs)
        if resp.status_code == 200:
            return True
        elif resp.status_code == 404:
            return False
        else:
            raise Exception(resp.text)
    return wrapped


abuddenberg's avatar
abuddenberg committed
35
36
37
38
39
40
41
42
43
44
def http_resp(fn):
    def wrapped(*args, **kwargs):
        resp = fn(*args, **kwargs)
        if resp.status_code == 200:
            return resp
        else:
            raise Exception(resp.text)
    return wrapped


45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
def get_credentials(url):
    #First check our magic enviroment variables (GCIS_USER and GCIS_KEY)
    from gcis_clients import gcis_auth
    env_user, env_key = gcis_auth

    if env_user is not None and env_key is not None:
        return env_user, env_key

    #Next, see if we can find Gcis.conf somewhere
    conf_possibilities = [expanduser(f) for f in ['~/etc/Gcis.conf', '~/.gcis-py-client/Gcis.conf'] if
                          os.path.exists(expanduser(f))]

    for gcis_config in conf_possibilities:
        print 'Using {gc} for credentials...'.format(gc=gcis_config)
        all_creds = yaml.load(open(gcis_config, 'r'))
        instance_creds = [c for c in all_creds if c['url'] == url][0]

        return instance_creds['userinfo'].split(':')[0], instance_creds['key']

    #Else prompt for credentials
    #Wow, I managed to use it
    else:
        username = raw_input('Username: ')
        api_key = getpass.getpass('API key: ')

        return username, api_key


73
74
75
76
77
78
79
80
class AssociationException(Exception):
    def __init__(self, value):
        self.value = value

    def __str__(self):
        return repr(self.value)


81
class GcisClient(object):
82
    def __init__(self, url, username, api_key):
83
        self.base_url = url
84
85
86
87
88

        #If credentials were not provided, obtain them
        if username is None or api_key is None:
            username, api_key = get_credentials(url)

89
        self.s = requests.Session()
90
        self.s.auth = (username, api_key)
91
        self.s.headers.update({'Accept': 'application/json'})
92

93
    @http_resp
94
95
    def create_figure(self, report_id, chapter_id, figure, skip_images=False):
        if figure.identifier in (None, ''):
96
97
98
99
100
            raise Exception('Invalid figure identifier', figure.identifier)

        #Is GCIS not inferring this from the url parameter?
        if figure.chapter_identifier in (None, ''):
            figure.chapter_identifier = chapter_id
101

102
103
        url = '{b}/report/{rpt}/chapter/{chp}/figure/'.format(
            b=self.base_url, rpt=report_id, chp=chapter_id
104
105
        )

106
        resp = self.s.post(url, data=figure.as_json(), verify=False)
107
108

        if skip_images is False:
109
            for image in figure.images:
110
111
                self.create_image(image),
                self.associate_image_with_figure(image.identifier, report_id, figure.identifier)
112

113
114
115
        for p in figure.parents:
            self.associate_figure_with_parent(report_id, figure.identifier, p)

116
        return resp
117

118
    @http_resp
119
    def update_figure(self, report_id, chapter_id, figure, skip_images=False, old_id=None):
120
121
        if figure.identifier in (None, ''):
            raise Exception('Invalid identifier', figure.identifier)
122
123
124
125
126

        #Is GCIS not inferring this from the url parameter?
        if figure.chapter_identifier in (None, ''):
            figure.chapter_identifier = chapter_id

127
128
        url = '{b}/report/{rpt}/chapter/{chp}/figure/{fig}'.format(
            b=self.base_url, rpt=report_id, chp=chapter_id, fig=old_id or figure.identifier
129
130
        )

131
        resp = self.s.post(url, data=figure.as_json(), verify=False)
132

133
        if skip_images is False:
134
            for image in figure.images:
135
136
                self.update_image(image)

137
138
139
        for c in figure.contributors:
            self.associate_contributor_with_figure(c, report_id, chapter_id, figure.identifier)

140
141
142
        for p in figure.parents:
            self.associate_figure_with_parent(report_id, figure.identifier, p)

143
        return resp
144

145
    @http_resp
146
147
    def delete_figure(self, report_id, figure_id):
        url = '{b}/report/{rpt}/figure/{fig}'.format(b=self.base_url, rpt=report_id, fig=figure_id)
148
        return self.s.delete(url, verify=False)
149

150
    @check_image
151
    def create_image(self, image, report_id=None, figure_id=None):
abuddenberg's avatar
abuddenberg committed
152
        url = '{b}/image/'.format(b=self.base_url)
153
        resp = self.s.post(url, data=image.as_json(), verify=False)
154
155
156
157
158
159
        
        if image.local_path is not None:
            self.upload_image_file(image.identifier, image.local_path)
        if figure_id and report_id:
            self.associate_image_with_figure(image.identifier, report_id, figure_id)
        for dataset in image.datasets:
abuddenberg's avatar
abuddenberg committed
160
161
            if not self.dataset_exists(dataset.identifier):
                self.create_dataset(dataset)
162
163
164
            # if not self.activity_exists(dataset.activity.identifier):
            #     self.create_activity(dataset.activity))
            self.create_or_update_activity(dataset.activity)
165
166
            self.associate_dataset_with_image(dataset.identifier, image.identifier,
                                              activity_id=dataset.activity.identifier)
abuddenberg's avatar
abuddenberg committed
167
        return resp
168

169
    @check_image
170
171
    def update_image(self, image, old_id=None):
        url = '{b}/image/{img}'.format(b=self.base_url, img=old_id or image.identifier)
172
        for dataset in image.datasets:
173
174
            # self.update_activity(dataset.activity)
            self.create_or_update_activity(dataset.activity)
abuddenberg's avatar
abuddenberg committed
175
176
            self.associate_dataset_with_image(dataset.identifier, image.identifier,
                                              activity_id=dataset.activity.identifier)
177
178
        for c in image.contributors:
            self.associate_contributor_with_image(c, image.identifier)
179

180
        return self.s.post(url, data=image.as_json(), verify=False)
181

182
    @check_image
183
    @http_resp
184
185
    def delete_image(self, image):
        delete_url = '{b}/image/{img}'.format(b=self.base_url, img=image.identifier)
186
        return self.s.delete(delete_url, verify=False)
187

188
    @http_resp
189
190
    def associate_image_with_figure(self, image_id, report_id, figure_id):
        url = '{b}/report/{rpt}/figure/rel/{fig}'.format(b=self.base_url, rpt=report_id, fig=figure_id)
191
        return self.s.post(url, data=json.dumps({'add_image_identifier': image_id}), verify=False)
192

193
    @http_resp
194
    def upload_image_file(self, image_id, local_path):
195
        url = '{b}/image/files/{id}/{fn}'.format(b=self.base_url, id=image_id, fn=basename(local_path))
196
        # For future multi-part encoding support
197
        # return self.s.put(url, headers=headers, files={'file': (filename, open(filepath, 'rb'))})
198
        if not os.path.exists(local_path):
199
200
            raise Exception('File not found: ' + local_path)

201
        return self.s.put(url, data=open(local_path, 'rb'), verify=False)
202

203
204
205
    #Full listing
    def get_figure_listing(self, report_id, chapter_id=None):
        chapter_filter = '/chapter/' + chapter_id if chapter_id else ''
206

207
208
        url = '{b}/report/{rpt}{chap}/figure'.format(b=self.base_url, rpt=report_id, chap=chapter_filter)
        resp = self.s.get(url, params={'all': '1'}, verify=False)
209

210
211
212
        try:
            return [Figure(figure) for figure in resp.json()]
        except ValueError:
213
            raise Exception(resp.text)
214

215
216
    def get_figure(self, report_id, figure_id, chapter_id=None):
        chapter_filter = '/chapter/' + chapter_id if chapter_id else ''
217

218
219
        url = '{b}/report/{rpt}{chap}/figure/{fig}'.format(
            b=self.base_url, rpt=report_id, chap=chapter_filter, fig=figure_id
220
        )
221
        resp = self.s.get(url, params={'all': '1'}, verify=False)
222

223
224
225
226
        try:
            return Figure(resp.json())
        except ValueError:
            raise Exception(resp.text)
227

228
229
230
231
    @exists
    def figure_exists(self, report_id, figure_id, chapter_id=None):
        chapter_filter = '/chapter/' + chapter_id if chapter_id else ''

232
233
        url = '{b}/report/{rpt}{chap}/figure/{fig}'.format(
            b=self.base_url, rpt=report_id, chap=chapter_filter, fig=figure_id
234
        )
235
        return self.s.head(url, verify=False)
236

237
238
    def get_image(self, image_id):
        url = '{b}/image/{img}'.format(b=self.base_url, img=image_id)
239
        resp = self.s.get(url, verify=False)
240

241
242
243
244
245
246
247
248
        try:
            return Image(resp.json())
        except ValueError:
            raise Exception(resp.text)

    @exists
    def image_exists(self, image_id):
        url = '{b}/image/{img}'.format(b=self.base_url, img=image_id)
249
        return self.s.head(url, verify=False)
250

251
252
    def has_all_associated_images(self, report_id, figure_id, target_image_ids):
        try:
253
254
255
256
            figure_image_ids = [i.identifier for i in self.get_figure(report_id, figure_id).images]
        except Exception, e:
            print e.message
            return False, set()
257

258
259
260
261
262
263
        target_set = set(target_image_ids)
        gcis_set = set(figure_image_ids)
        deltas = target_set - gcis_set

        if target_set.issubset(gcis_set):
            return True, deltas
264
        else:
265
            return False, deltas
266

267
268
    def test_login(self):
        url = '{b}/login.json'.format(b=self.base_url)
269
        resp = self.s.get(url, verify=False)
270
271
272
        return resp.status_code, resp.text

    def get_keyword_listing(self):
273
274
        url = '{b}/gcmd_keyword'.format(b=self.base_url)
        resp = self.s.get(url, params={'all': '1'}, verify=False)
275

276
277
278
279
        return resp.json()

    def get_keyword(self, key_id):
        url = '{b}/gcmd_keyword/{k}'.format(b=self.base_url, k=key_id)
280
        return self.s.get(url, verify=False).json()
281
282
283

    def associate_keyword_with_figure(self, keyword_id, report_id, figure_id):
        url = '{b}/report/{rpt}/figure/keywords/{fig}'.format(b=self.base_url, rpt=report_id, fig=figure_id)
284
        return self.s.post(url, data=json.dumps({'identifier': keyword_id}), verify=False)
285
286
287

    def get_dataset(self, dataset_id):
        url = '{b}/dataset/{ds}'.format(b=self.base_url, ds=dataset_id)
288
        resp = self.s.get(url, verify=False)
289
290
291
        try:
            return Dataset(resp.json())
        except ValueError:
abuddenberg's avatar
abuddenberg committed
292
            raise Exception(resp.text)
293

294
295
296
    @exists
    def dataset_exists(self, dataset_id):
        url = '{b}/dataset/{ds}'.format(b=self.base_url, ds=dataset_id)
297
        return self.s.head(url, verify=False)
298

abuddenberg's avatar
abuddenberg committed
299
    @http_resp
abuddenberg's avatar
abuddenberg committed
300
    def create_dataset(self, dataset):
301
        url = '{b}/dataset/'.format(b=self.base_url)
302
        return self.s.post(url, data=dataset.as_json(), verify=False)
303

abuddenberg's avatar
abuddenberg committed
304
    @http_resp
305
306
    def update_dataset(self, dataset, old_id=None):
        url = '{b}/dataset/{ds}'.format(b=self.base_url, ds=old_id or dataset.identifier)
307
        return self.s.post(url, data=dataset.as_json(), verify=False)
308

abuddenberg's avatar
abuddenberg committed
309
    @http_resp
310
311
    def delete_dataset(self, dataset):
        url = '{b}/dataset/{ds}'.format(b=self.base_url, ds=dataset.identifier)
312
        return self.s.delete(url, verify=False)
313

abuddenberg's avatar
abuddenberg committed
314
315
316
317
318
    @http_resp
    def get_dataset_list(self):
        url = '{b}/dataset/'.format(b=self.base_url)
        return self.s.get(url, params={'all': 1}, verify=False)

abuddenberg's avatar
abuddenberg committed
319
    def associate_dataset_with_image(self, dataset_id, image_id, activity_id=None):
320
        url = '{b}/image/prov/{img}'.format(b=self.base_url, img=image_id)
abuddenberg's avatar
abuddenberg committed
321

322
323
324
325
        data = {
            'parent_uri': '/dataset/' + dataset_id,
            'parent_rel': 'prov:wasDerivedFrom'
        }
abuddenberg's avatar
abuddenberg committed
326
327
328
        if activity_id:
            data['activity'] = activity_id

329
330
331
332
333
        try:
            self.delete_dataset_image_assoc(dataset_id, image_id)
        except AssociationException as e:
            print e.value

334
        resp = self.s.post(url, data=json.dumps(data), verify=False)
335
336
337
338

        if resp.status_code == 200:
            return resp
        else:
339
            raise Exception('Dataset association failed:\n{url}\n{resp}'.format(url=url, resp=resp.text))
340

abuddenberg's avatar
abuddenberg committed
341
342
343
344
345
346
347
348
349
    def delete_dataset_image_assoc(self, dataset_id, image_id):
        url = '{b}/image/prov/{img}'.format(b=self.base_url, img=image_id)

        data = {
            'delete': {
                'parent_uri': '/dataset/' + dataset_id,
                'parent_rel': 'prov:wasDerivedFrom'
            }
        }
350
        resp = self.s.post(url, data=json.dumps(data), verify=False)
abuddenberg's avatar
abuddenberg committed
351
352
353
354

        if resp.status_code == 200:
            return resp
        else:
355
356
357
358
359
360
            raise AssociationException(
                'Dataset dissociation failed:\n{url}\n{resp}\n{d}'.format(url=url, resp=resp.text, d=data))

    def create_or_update_dataset(self, dataset):
        if self.dataset_exists(dataset.identifier):
            print 'Updating dataset: ' + dataset.identifier
361
            self.update_dataset(dataset)
362
363
        else:
            print 'Creating dataset: ' + dataset.identifier
364
            self.create_dataset(dataset)
365

abuddenberg's avatar
abuddenberg committed
366
367
368
    # @exists
    def activity_exists(self, activity_id):
        url = '{b}/activity/{act}'.format(b=self.base_url, act=activity_id)
369
        resp = self.s.head(url, verify=False)
abuddenberg's avatar
abuddenberg committed
370
371
372
373
374
375
376
        if resp.status_code == 200:
            return True
        else:
            return False

    def get_activity(self, activity_id):
        url = '{b}/activity/{act}'.format(b=self.base_url, act=activity_id)
377
        resp = self.s.get(url, verify=False)
abuddenberg's avatar
abuddenberg committed
378
379
380
        try:
            return Activity(resp.json())
        except ValueError:
abuddenberg's avatar
abuddenberg committed
381
            raise Exception(resp.text)
abuddenberg's avatar
abuddenberg committed
382
383
384
385

    @http_resp
    def create_activity(self, activity):
        url = '{b}/activity/'.format(b=self.base_url)
386
        return self.s.post(url, data=activity.as_json(), verify=False)
abuddenberg's avatar
abuddenberg committed
387
388

    @http_resp
389
390
    def update_activity(self, activity, old_id=None):
        url = '{b}/activity/{act}'.format(b=self.base_url, act=old_id or activity.identifier)
391
        return self.s.post(url, data=activity.as_json(), verify=False)
abuddenberg's avatar
abuddenberg committed
392
393
394
395

    @http_resp
    def delete_activity(self, activity):
        url = '{b}/activity/{act}'.format(b=self.base_url, act=activity.identifier)
396
        return self.s.delete(url, verify=False)
abuddenberg's avatar
abuddenberg committed
397

398
399
400
401
402
    def create_or_update_activity(self, activity):
        if self.activity_exists(activity.identifier):
            self.update_activity(activity)
        else:
            self.create_activity(activity)
abuddenberg's avatar
abuddenberg committed
403

404
405
406
407
408
    def get_activity_list(self):
        url = '{b}/activity'.format(b=self.base_url)

        return self.s.get(url, params={'all': 1}, verify=False).json()

abuddenberg's avatar
abuddenberg committed
409
410
411
    @exists
    def person_exists(self, person_id):
        url = '{b}/person/{pid}'.format(b=self.base_url, pid=person_id)
412
        return self.s.head(url, verfiy=False)
abuddenberg's avatar
abuddenberg committed
413
414
415

    def get_person(self, person_id):
        url = '{b}/person/{pid}'.format(b=self.base_url, pid=person_id)
416
        resp = self.s.get(url, verify=False)
abuddenberg's avatar
abuddenberg committed
417
418
419
420
421
422
423
        try:
            return Person(resp.json())
        except ValueError:
            raise Exception(resp.text)

    def lookup_person(self, name):
        url = '{b}/autocomplete'.format(b=self.base_url)
424
        resp = self.s.get(url, params={'q': name, 'items': 15, 'type': 'person'}, verify=False)
abuddenberg's avatar
abuddenberg committed
425
426
427
428
429
430
431
432
433

        if resp.status_code == 200:
            return [re.match(r'\[person\] \{(\d+)\} (.*)', r).groups() for r in resp.json()]
        else:
            raise Exception(resp.text)

    @http_resp
    def create_person(self, person):
        url = '{b}/person/'.format(b=self.base_url)
434
        return self.s.post(url, data=person.as_json(), verify=False)
abuddenberg's avatar
abuddenberg committed
435
436

    @http_resp
437
438
    def update_person(self, person, old_id=None):
        url = '{b}/person/{pid}'.format(b=self.base_url, pid=old_id or person.identifier)
439
        return self.s.post(url, data=person.as_json(), verify=False)
abuddenberg's avatar
abuddenberg committed
440
441
442
443

    @http_resp
    def delete_person(self, person):
        url = '{b}/person/{pid}'.format(b=self.base_url, pid=person.identifier)
444
        return self.s.delete(url, verify=False)
abuddenberg's avatar
abuddenberg committed
445
446
447
448

    @exists
    def organization_exists(self, org_id):
        url = '{b}/organization/{org_id)'.format(b=self.base_url, org_id=org_id)
449
        return self.s.head(url, verify=False)
abuddenberg's avatar
abuddenberg committed
450
451

    def get_organization(self, org_id):
452
453
        url = '{b}/organization/{org_id}'.format(b=self.base_url, org_id=org_id)
        resp = self.s.get(url, verify=False)
abuddenberg's avatar
abuddenberg committed
454
455
456
457
458
459
460

        try:
            return Organization(resp.json())
        except ValueError:
            raise Exception(resp.text)

    def lookup_organization(self, name):
461
        url = '{b}/autocomplete'.format(b=self.base_url)
462
        resp = self.s.get(url, params={'q': name, 'items': 15, 'type': 'organization'}, verify=False)
463
464
465
466
467
        
        if resp.status_code == 200:
            return [re.match(r'\[organization\] \{(.*)\} (.*)', r).groups() for r in resp.json()]
        else:
            raise Exception(resp.text)
abuddenberg's avatar
abuddenberg committed
468
469
470
471

    @http_resp
    def create_organization(self, org):
        url = '{b}/organization/'.format(b=self.base_url)
472
        return self.s.post(url, data=org.as_json(), verify=False)
abuddenberg's avatar
abuddenberg committed
473
474

    @http_resp
475
476
    def update_organization(self, org, old_id=None):
        url = '{b}/organization/{org_id}'.format(b=self.base_url, org_id=old_id or org.identifier)
477
        return self.s.post(url, data=org.as_json(), verify=False)
abuddenberg's avatar
abuddenberg committed
478
479
480
481

    @http_resp
    def delete_organization(self, org):
        url = '{b}/organization/{org_id}'.format(b=self.base_url, org_id=org.identifier)
482
        return self.s.delete(url, verify=False)
483
484
485
486
487

    @http_resp
    def associate_contributor_with_figure(self, contrib, report_id, chapter_id, figure_id):
        url = '{b}/report/{rpt}/chapter/{chp}/figure/contributors/{fig}'.format(b=self.base_url, rpt=report_id, chp=chapter_id, fig=figure_id)

488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
        data = {
            'role': contrib.role.type_id,
        }

        if contrib.person is not None and contrib.person.id is not None:
            data['person_id'] = contrib.person.id
        if contrib.organization is not None and contrib.organization.identifier:
            data['organization_identifier'] = contrib.organization.identifier

        resp = self.s.post(url, data=json.dumps(data), verify=False)
        return resp

    @http_resp
    def delete_contributor_figure_assoc(self, contrib, report_id, chapter_id, figure_id):
        url = '{b}/report/{rpt}/chapter/{chp}/figure/contributors/{fig}'.format(b=self.base_url, rpt=report_id, chp=chapter_id, fig=figure_id)

        data = {
505
            'delete_contributor': contrib.id
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
        }

        return self.s.post(url, data=json.dumps(data), verify=False)

    @http_resp
    def associate_contributor_with_image(self, contrib, image_id):
        url = '{b}/image/contributors/{img}'.format(b=self.base_url, img=image_id)

        data = {
            'role': contrib.role.type_id,
        }
        if contrib.person is not None and contrib.person.id is not None:
            data['person_id'] = contrib.person.id
        if contrib.organization is not None and contrib.organization.identifier:
            data['organization_identifier'] = contrib.organization.identifier

        return self.s.post(url, data=json.dumps(data), verify=False)

    @http_resp
    def delete_contributor_image_assoc(self, contrib, image_id):
        url = '{b}/image/contributors/{img}'.format(b=self.base_url, img=image_id)

        data = {
            'delete': {
                'role': contrib.role.type_id,
                'organization_identifier': contrib.organization.identifier,
                'person_id': contrib.person.identifier
            }
        }
535

536
        return self.s.post(url, data=json.dumps(data), verify=False)
537

abuddenberg's avatar
abuddenberg committed
538
    @http_resp
539
    def associate_figure_with_parent(self, report_id, figure_id, parent):
abuddenberg's avatar
abuddenberg committed
540
541
542
        url = '{b}/report/{rpt}/figure/prov/{fig}'.format(b=self.base_url, rpt=report_id, fig=figure_id)

        data = {
543
544
            'parent_uri': parent.url,
            'parent_rel': parent.relationship
abuddenberg's avatar
abuddenberg committed
545
546
        }

547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
        try:
            self.delete_figure_pub_assoc(report_id, figure_id, parent)
        except AssociationException as e:
            print e.value

        resp = self.s.post(url, data=json.dumps(data), verify=False)
        return resp

    def delete_figure_pub_assoc(self, report_id, figure_id, parent):
        url = '{b}/report/{rpt}/figure/prov/{fig}'.format(b=self.base_url, rpt=report_id, fig=figure_id)

        data = {
            'delete': {
                'parent_uri': parent.url,
                'parent_rel': parent.relationship
            }
        }
        resp = self.s.post(url, data=json.dumps(data), verify=False)

        if resp.status_code == 200:
            return resp
        else:
            raise AssociationException(
                'Parent dissociation failed:\n{url}\n{resp}\n{d}'.format(url=url, resp=resp.text, d=data))
abuddenberg's avatar
abuddenberg committed
571
572
573
574
575
576
577

    def lookup_publication(self, pub_type, name):
        url = '{b}/autocomplete'.format(b=self.base_url)
        resp = self.s.get(url, params={'q': name, 'items': 15, 'type': pub_type}, verify=False)

        if resp.status_code == 200:
            return [re.match(r'\[.+\] \{(.+)\} (.*)', r).groups() for r in resp.json()]
578
            # return resp.json()
abuddenberg's avatar
abuddenberg committed
579
        else:
580
            raise Exception(resp.text)