Commit ef1d5c16 authored by abuddenberg's avatar abuddenberg
Browse files

Removed hardcoded auth tokens and api keys from everything. Added support for...

Removed hardcoded auth tokens and api keys from everything. Added support for env variables and config files for persisting credentials
parent b6a94b04
...@@ -23,12 +23,35 @@ To install from git with pip: ...@@ -23,12 +23,35 @@ To install from git with pip:
To install from tarball: To install from tarball:
tar -xzvf GcisPyClient-0.67.tar.gz tar -xzvf GcisPyClient-x.y.tar.gz
cd GcisPyClient-0.67 cd GcisPyClient-x.y
python setup.py test (to run the test suite) python setup.py test (to run the test suite)
python setup.py install (to install) python setup.py install (to install)
Setup
-----
### User Credentials
GcisClient will use the first credentials it finds while searching the following places:
1. Strings passed directly to the GcisClient constructor.
2. Environment variables "GCIS_USER" and "GCIS_KEY" (both are required)
3. Config file at $HOME/etc/Gcis.conf (A sample of this file is included below)
4. Config file at $HOME/.gcis-py-client/Gcis.conf
5. Interactively, with a username and key prompt
Sample Gcis.conf file:
- url : http://data-stage.globalchange.gov
userinfo : me@example.com:298015f752D99E789056EF826A7D97afc38a8bbd6e3e23b3
key : M2FiLtG2n2qTyJHIztvHm5zweTYjNkM5ZWEtYjNkMS00LTgS00LTg2N2QtYZDFhzQyNGUxCg==
- url : http://data.globalchange.gov
userinfo : username:pass
key : key
Usage Usage
----- -----
...@@ -43,16 +66,16 @@ Make sure our credentials work: ...@@ -43,16 +66,16 @@ Make sure our credentials work:
print status_code, resp_text print status_code, resp_text
assert 'auth_required' not in resp_text assert 'auth_required' not in resp_text
Let's pull a list of all figures in Chapter 2 of the NCA3draft: Let's pull a list of all figures in Chapter 2 of the nca3:
for partial_figure in gcis.get_figure_listing('nca3draft', chapter_id='our-changing-climate'): for partial_figure in gcis.get_figure_listing('nca3', chapter_id='our-changing-climate'):
full_figure = gcis.get_figure('nca3draft', partial_figure.identifier, chapter_id='our-changing-climate') full_figure = gcis.get_figure('nca3', partial_figure.identifier, chapter_id='our-changing-climate')
print full_figure print full_figure
Let's work with the infamous temperature figure: Let's work with the infamous temperature figure:
fig2_7 = gcis.get_figure('nca3draft', 'observed-us-temperature-change') fig2_7 = gcis.get_figure('nca3', 'observed-us-temperature-change')
Warning: Images and Chapters are specifically excluded from JSON output. This is what gets sent to GCIS. So... Warning: Images and Chapters are specifically excluded from JSON output. This is what gets sent to GCIS. So...
......
#!/usr/bin/env python #!/usr/bin/env python
from gcis_clients import GcisClient from gcis_clients import GcisClient, gcis_dev_auth
base_url = 'http://data.gcis-dev-front.joss.ucar.edu' base_url = 'http://data.gcis-dev-front.joss.ucar.edu'
gcis = GcisClient(base_url, 'andrew.buddenberg@noaa.gov', 'fcee8e3f11f36313e463ece51aab15242f71f3d552d565be') #These are provided by environment variables GCIS_DEV_USER and GCIS_DEV_KEY
username, api_key = gcis_dev_auth
gcis = GcisClient(base_url, username, api_key)
#Make sure our credentials work #Make sure our credentials work
status_code, resp_text = gcis.test_login() status_code, resp_text = gcis.test_login()
...@@ -12,15 +15,15 @@ print status_code, resp_text ...@@ -12,15 +15,15 @@ print status_code, resp_text
assert 'auth_required' not in resp_text assert 'auth_required' not in resp_text
#Let's pull a list of all figures in Chapter 2 of the NCA3draft #Let's pull a list of all figures in Chapter 2 of the NCA3draft
for partial_figure in gcis.get_figure_listing('nca3draft', chapter_id='our-changing-climate'): for partial_figure in gcis.get_figure_listing('nca3', chapter_id='our-changing-climate'):
#The listing doesn't provide all available fields for the figure (Images, for instance). #The listing doesn't provide all available fields for the figure (Images, for instance).
#There aren't very many figures, so let's go ahead and grab a complete version of each #There aren't very many figures, so let's go ahead and grab a complete version of each
full_figure = gcis.get_figure('nca3draft', partial_figure.identifier, chapter_id='our-changing-climate') full_figure = gcis.get_figure('nca3', partial_figure.identifier, chapter_id='our-changing-climate')
print full_figure print full_figure
#Let's work with the infamous temperature figure #Let's work with the infamous temperature figure
fig2_7 = gcis.get_figure('nca3draft', 'observed-us-temperature-change') fig2_7 = gcis.get_figure('nca3', 'observed-us-temperature-change')
#Warning: Images and Chapters are specifically excluded from JSON output. This is what gets sent to GCIS. So... #Warning: Images and Chapters are specifically excluded from JSON output. This is what gets sent to GCIS. So...
print fig2_7.as_json(indent=4) print fig2_7.as_json(indent=4)
......
...@@ -2,23 +2,21 @@ ...@@ -2,23 +2,21 @@
__author__ = 'abuddenberg' __author__ = 'abuddenberg'
import pickle import pickle
from gcis_clients import GcisClient from gcis_clients import GcisClient, WebformClient, gcis_dev_auth, gcis_stage_auth, webform_token
from gcis_clients import WebformClient
from gcis_clients.sync_utils import move_images_to_gcis from gcis_clients.sync_utils import move_images_to_gcis
webform_client = WebformClient('http://resources.assessment.globalchange.gov', 'mgTD63FAjG') webform_client = WebformClient('http://resources.assessment.globalchange.gov', webform_token)
gcis_url = 'http://data.gcis-dev-front.joss.ucar.edu' gcis = GcisClient('http://data.gcis-dev-front.joss.ucar.edu', *gcis_dev_auth)
#gcis = GcisClient(gcis_url, 'andrew.buddenberg@noaa.gov', 'ad90c05b37d4128ae514bc6caa7a41911d2f1de353443a54') gcis = GcisClient('http://data-stage.globalchange.gov', *gcis_stage_auth)
gcis = GcisClient('http://data-stage.globalchange.gov', 'andrew.buddenberg@noaa.gov', 'b4f1458c3cf28248c982428c46e170019327bd4c533c23dd')
def main(): def main():
hitlist_file = 'hitlist.pk1' hitlist_file = 'hitlist.pk1'
# create_problem_list('nca3', hitlist_file) create_problem_list('nca3', hitlist_file)
# print_problem_list(hitlist_file) print_problem_list(hitlist_file)
solve_problems(hitlist_file, 'nca3') # solve_problems(hitlist_file, 'nca3')
# print_ready_list(hitlist_file) # print_ready_list(hitlist_file)
...@@ -99,7 +97,7 @@ def sort_webform_list(report_id): ...@@ -99,7 +97,7 @@ def sort_webform_list(report_id):
#Check if organizations have been proper identified #Check if organizations have been proper identified
for cont in f.contributors: for cont in f.contributors:
if cont.identifier is None: if cont.organization.identifier is None:
problems.setdefault(key, {}).setdefault('org_id_not_found', []).append(cont) problems.setdefault(key, {}).setdefault('org_id_not_found', []).append(cont)
for image in f.images: for image in f.images:
...@@ -124,7 +122,7 @@ def sort_webform_list(report_id): ...@@ -124,7 +122,7 @@ def sort_webform_list(report_id):
#Check if organizations have been proper identified #Check if organizations have been proper identified
for cont in image.contributors: for cont in image.contributors:
if cont.identifier is None: if cont.organization.identifier is None:
problems.setdefault(key, {}).setdefault('org_id_not_found', []).append(cont) problems.setdefault(key, {}).setdefault('org_id_not_found', []).append(cont)
#Check for broken image associations #Check for broken image associations
......
#!/usr/bin/env python #!/usr/bin/env python
__author__ = 'abuddenberg' __author__ = 'abuddenberg'
from gcis_clients import WebformClient from gcis_clients import WebformClient, GcisClient, gcis_dev_auth, gcis_stage_auth, webform_token
from gcis_clients import GcisClient
from gcis_clients.sync_utils import move_images_to_gcis, sync_dataset_metadata, realize_contributors from gcis_clients.sync_utils import move_images_to_gcis, sync_dataset_metadata, realize_contributors
from collections import OrderedDict from collections import OrderedDict
import json
import pickle
webform = WebformClient('http://resources.assessment.globalchange.gov', 'mgTD63FAjG') webform = WebformClient('http://resources.assessment.globalchange.gov', webform_token)
# gcis = GcisClient('http://data.gcis-dev-front.joss.ucar.edu', 'andrew.buddenberg@noaa.gov', 'ad90c05b37d4128ae514bc6caa7a41911d2f1de353443a54') # gcis = GcisClient('http://data.gcis-dev-front.joss.ucar.edu', *gcis_dev_auth)
gcis = GcisClient('http://data-stage.globalchange.gov', 'andrew.buddenberg@noaa.gov', 'b4f1458c3cf28248c982428c46e170019327bd4c533c23dd') gcis = GcisClient('http://data-stage.globalchange.gov', *gcis_stage_auth)
sync_metadata_tree = { sync_metadata_tree = {
#Reports #Reports
...@@ -141,8 +138,8 @@ sync_metadata_tree = { ...@@ -141,8 +138,8 @@ sync_metadata_tree = {
def main(): def main():
# print gcis.test_login() print gcis.test_login()
# sync_dataset_metadata(gcis, webform.get_aggregated_datasets(), skip=['Proxy Data', 'Projected Sea Level Rise', 'Tide Gauge Data']) sync_dataset_metadata(gcis, webform.get_aggregated_datasets(), skip=['Proxy Data', 'Projected Sea Level Rise', 'Tide Gauge Data'])
sync(replace=False) sync(replace=False)
......
from gcis_client import GcisClient from gcis_client import GcisClient
from webform_client import WebformClient from webform_client import WebformClient
from nca3_client import Nca3Client
from os.path import expanduser, exists from os.path import expanduser, exists
from os import makedirs from os import makedirs, getenv
def default_image_dir(): def default_image_dir():
...@@ -11,3 +12,11 @@ def default_image_dir(): ...@@ -11,3 +12,11 @@ def default_image_dir():
makedirs(image_dir) makedirs(image_dir)
return image_dir return image_dir
#Magic environment variables
gcis_prod_auth = (getenv('GCIS_PROD_USER'), getenv('GCIS_PROD_KEY'))
gcis_stage_auth = (getenv('GCIS_STAGE_USER'), getenv('GCIS_STAGE_KEY'))
gcis_dev_auth = (getenv('GCIS_DEV_USER'), getenv('GCIS_DEV_KEY'))
gcis_auth = (getenv('GCIS_USER'), getenv('GCIS_KEY'))
webform_token = getenv('WEBFORM_TOKEN')
\ No newline at end of file
import urllib
import json import json
from os.path import exists, basename import os
from os.path import basename, expanduser
import re import re
import requests import requests
import yaml
import getpass
from domain import Figure, Image, Dataset, Activity, Person, Organization from domain import Figure, Image, Dataset, Activity, Person, Organization
...@@ -40,6 +42,34 @@ def http_resp(fn): ...@@ -40,6 +42,34 @@ def http_resp(fn):
return wrapped return wrapped
def get_credentials(url):
#First check our magic enviroment variables (GCIS_USER and GCIS_KEY)
from gcis_clients import gcis_auth
env_user, env_key = gcis_auth
if env_user is not None and env_key is not None:
return env_user, env_key
#Next, see if we can find Gcis.conf somewhere
conf_possibilities = [expanduser(f) for f in ['~/etc/Gcis.conf', '~/.gcis-py-client/Gcis.conf'] if
os.path.exists(expanduser(f))]
for gcis_config in conf_possibilities:
print 'Using {gc} for credentials...'.format(gc=gcis_config)
all_creds = yaml.load(open(gcis_config, 'r'))
instance_creds = [c for c in all_creds if c['url'] == url][0]
return instance_creds['userinfo'].split(':')[0], instance_creds['key']
#Else prompt for credentials
#Wow, I managed to use it
else:
username = raw_input('Username: ')
api_key = getpass.getpass('API key: ')
return username, api_key
class AssociationException(Exception): class AssociationException(Exception):
def __init__(self, value): def __init__(self, value):
self.value = value self.value = value
...@@ -49,10 +79,15 @@ class AssociationException(Exception): ...@@ -49,10 +79,15 @@ class AssociationException(Exception):
class GcisClient(object): class GcisClient(object):
def __init__(self, url, username, password): def __init__(self, url, username, api_key):
self.base_url = url self.base_url = url
#If credentials were not provided, obtain them
if username is None or api_key is None:
username, api_key = get_credentials(url)
self.s = requests.Session() self.s = requests.Session()
self.s.auth = (username, password) self.s.auth = (username, api_key)
self.s.headers.update({'Accept': 'application/json'}) self.s.headers.update({'Accept': 'application/json'})
@http_resp @http_resp
...@@ -152,7 +187,7 @@ class GcisClient(object): ...@@ -152,7 +187,7 @@ class GcisClient(object):
url = '{b}/image/files/{id}/{fn}'.format(b=self.base_url, id=image_id, fn=basename(local_path)) url = '{b}/image/files/{id}/{fn}'.format(b=self.base_url, id=image_id, fn=basename(local_path))
# For future multi-part encoding support # For future multi-part encoding support
# return self.s.put(url, headers=headers, files={'file': (filename, open(filepath, 'rb'))}) # return self.s.put(url, headers=headers, files={'file': (filename, open(filepath, 'rb'))})
if not exists(local_path): if not os.path.exists(local_path):
raise Exception('File not found: ' + local_path) raise Exception('File not found: ' + local_path)
return self.s.put(url, data=open(local_path, 'rb'), verify=False) return self.s.put(url, data=open(local_path, 'rb'), verify=False)
...@@ -161,15 +196,13 @@ class GcisClient(object): ...@@ -161,15 +196,13 @@ class GcisClient(object):
def get_figure_listing(self, report_id, chapter_id=None): def get_figure_listing(self, report_id, chapter_id=None):
chapter_filter = '/chapter/' + chapter_id if chapter_id else '' chapter_filter = '/chapter/' + chapter_id if chapter_id else ''
url = '{b}/report/{rpt}{chap}/figure?{p}'.format( url = '{b}/report/{rpt}{chap}/figure'.format(b=self.base_url, rpt=report_id, chap=chapter_filter)
b=self.base_url, rpt=report_id, chap=chapter_filter, p=urllib.urlencode({'all': '1'}) resp = self.s.get(url, params={'all': '1'}, verify=False)
)
resp = self.s.get(url, verify=False)
try: try:
return [Figure(figure) for figure in resp.json()] return [Figure(figure) for figure in resp.json()]
except ValueError: except ValueError:
raise Exception('Add a better exception string here') raise Exception(resp.text)
def get_figure(self, report_id, figure_id, chapter_id=None): def get_figure(self, report_id, figure_id, chapter_id=None):
chapter_filter = '/chapter/' + chapter_id if chapter_id else '' chapter_filter = '/chapter/' + chapter_id if chapter_id else ''
...@@ -188,8 +221,8 @@ class GcisClient(object): ...@@ -188,8 +221,8 @@ class GcisClient(object):
def figure_exists(self, report_id, figure_id, chapter_id=None): def figure_exists(self, report_id, figure_id, chapter_id=None):
chapter_filter = '/chapter/' + chapter_id if chapter_id else '' chapter_filter = '/chapter/' + chapter_id if chapter_id else ''
url = '{b}/report/{rpt}{chap}/figure/{fig}?{p}'.format( url = '{b}/report/{rpt}{chap}/figure/{fig}'.format(
b=self.base_url, rpt=report_id, chap=chapter_filter, fig=figure_id, p=urllib.urlencode({'all': '1'}) b=self.base_url, rpt=report_id, chap=chapter_filter, fig=figure_id
) )
return self.s.head(url, verify=False) return self.s.head(url, verify=False)
...@@ -229,8 +262,8 @@ class GcisClient(object): ...@@ -229,8 +262,8 @@ class GcisClient(object):
return resp.status_code, resp.text return resp.status_code, resp.text
def get_keyword_listing(self): def get_keyword_listing(self):
url = '{b}/gcmd_keyword?{p}'.format(b=self.base_url, p=urllib.urlencode({'all': '1'})) url = '{b}/gcmd_keyword'.format(b=self.base_url)
resp = self.s.get(url, verify=False) resp = self.s.get(url, params={'all': '1'}, verify=False)
return resp.json() return resp.json()
......
__author__ = 'abuddenberg'
import requests
def http_resp(fn):
def wrapped(*args, **kwargs):
resp = fn(*args, **kwargs)
if resp.status_code == 200:
return resp
else:
raise Exception('Status: {code} \n{txt}'.format(code=resp.status_code, txt=resp.text))
return wrapped
class Nca3Client(object):
def __init__(self, url, username, password, http_basic_user=None, http_basic_pass=None):
self.base_url = url
self.s = requests.Session()
self.s.auth = (http_basic_user, http_basic_pass)
self.drupal_user = username
self.drupal_pass = password
def do_login(self):
url = '{b}/user'.format(b=self.base_url)
resp = self.s.post(
url,
data={
'name': self.drupal_user,
'pass': self.drupal_pass,
'form_id': 'user_login',
'op': 'Log in'
},
allow_redirects=False
)
return resp
@http_resp
def get_all_captions(self):
self.do_login()
url = '{b}/gcis/figure-table-captions'.format(b=self.base_url)
resp = self.s.get(url, verify=False)
return resp
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
import urllib import urllib
import re import re
from os.path import join from os.path import join
import getpass
import requests import requests
from dateutil.parser import parse from dateutil.parser import parse
...@@ -16,7 +16,7 @@ def sanitized(pattern): ...@@ -16,7 +16,7 @@ def sanitized(pattern):
if re.match(pattern, urllib.quote(args[1])): if re.match(pattern, urllib.quote(args[1])):
return fn(*args, **kwargs) return fn(*args, **kwargs)
else: else:
print 'Shitlisted: ', args[1] print 'Rejected: ', args[1]
return wrapped return wrapped
return dec return dec
...@@ -36,10 +36,26 @@ def parse_creators(field): ...@@ -36,10 +36,26 @@ def parse_creators(field):
return contributor return contributor
def get_credentials():
#First check our magic enviroment variable (WEBFORM_TOKEN)
from gcis_clients import webform_token
if webform_token is not None:
return webform_token
else:
return getpass.getpass('Webform token: ')
class WebformClient: class WebformClient:
def __init__(self, url, token, local_image_dir=None, remote_dir='/system/files/'): def __init__(self, url, token, local_image_dir=None, remote_dir='/system/files/'):
self.base_url = url self.base_url = url
#If token was not provided, obtain it
if token is None:
token = get_credentials()
self.token = token self.token = token
if local_image_dir: if local_image_dir:
......
...@@ -19,7 +19,7 @@ class PyTest(TestCommand): ...@@ -19,7 +19,7 @@ class PyTest(TestCommand):
setup( setup(
name='GcisPyClient', name='GcisPyClient',
version='0.68', version='1.0',
author='Andrew Buddenberg', author='Andrew Buddenberg',
author_email='andrew.buddenberg@noaa.gov', author_email='andrew.buddenberg@noaa.gov',
packages=find_packages(), packages=find_packages(),
...@@ -29,7 +29,8 @@ setup( ...@@ -29,7 +29,8 @@ setup(
long_description=open('README.txt').read(), long_description=open('README.txt').read(),
install_requires=[ install_requires=[
"requests >= 2.1.0", "requests >= 2.1.0",
"python-dateutil >= 2.2" "python-dateutil >= 2.2",
"PyYAML >= 3.11"
], ],
tests_require=["pytest >= 2.5.2"], tests_require=["pytest >= 2.5.2"],
cmdclass={'test': PyTest}, cmdclass={'test': PyTest},
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment