Commit ef1d5c16 authored by abuddenberg's avatar abuddenberg
Browse files

Removed hardcoded auth tokens and api keys from everything. Added support for...

Removed hardcoded auth tokens and api keys from everything. Added support for env variables and config files for persisting credentials
parent b6a94b04
......@@ -23,12 +23,35 @@ To install from git with pip:
To install from tarball:
tar -xzvf GcisPyClient-0.67.tar.gz
cd GcisPyClient-0.67
tar -xzvf GcisPyClient-x.y.tar.gz
cd GcisPyClient-x.y
python setup.py test (to run the test suite)
python setup.py install (to install)
Setup
-----
### User Credentials
GcisClient will use the first credentials it finds while searching the following places:
1. Strings passed directly to the GcisClient constructor.
2. Environment variables "GCIS_USER" and "GCIS_KEY" (both are required)
3. Config file at $HOME/etc/Gcis.conf (A sample of this file is included below)
4. Config file at $HOME/.gcis-py-client/Gcis.conf
5. Interactively, with a username and key prompt
Sample Gcis.conf file:
- url : http://data-stage.globalchange.gov
userinfo : me@example.com:298015f752D99E789056EF826A7D97afc38a8bbd6e3e23b3
key : M2FiLtG2n2qTyJHIztvHm5zweTYjNkM5ZWEtYjNkMS00LTgS00LTg2N2QtYZDFhzQyNGUxCg==
- url : http://data.globalchange.gov
userinfo : username:pass
key : key
Usage
-----
......@@ -43,16 +66,16 @@ Make sure our credentials work:
print status_code, resp_text
assert 'auth_required' not in resp_text
Let's pull a list of all figures in Chapter 2 of the NCA3draft:
Let's pull a list of all figures in Chapter 2 of the nca3:
for partial_figure in gcis.get_figure_listing('nca3draft', chapter_id='our-changing-climate'):
full_figure = gcis.get_figure('nca3draft', partial_figure.identifier, chapter_id='our-changing-climate')
for partial_figure in gcis.get_figure_listing('nca3', chapter_id='our-changing-climate'):
full_figure = gcis.get_figure('nca3', partial_figure.identifier, chapter_id='our-changing-climate')
print full_figure
Let's work with the infamous temperature figure:
fig2_7 = gcis.get_figure('nca3draft', 'observed-us-temperature-change')
fig2_7 = gcis.get_figure('nca3', 'observed-us-temperature-change')
Warning: Images and Chapters are specifically excluded from JSON output. This is what gets sent to GCIS. So...
......
#!/usr/bin/env python
from gcis_clients import GcisClient
from gcis_clients import GcisClient, gcis_dev_auth
base_url = 'http://data.gcis-dev-front.joss.ucar.edu'
gcis = GcisClient(base_url, 'andrew.buddenberg@noaa.gov', 'fcee8e3f11f36313e463ece51aab15242f71f3d552d565be')
#These are provided by environment variables GCIS_DEV_USER and GCIS_DEV_KEY
username, api_key = gcis_dev_auth
gcis = GcisClient(base_url, username, api_key)
#Make sure our credentials work
status_code, resp_text = gcis.test_login()
......@@ -12,15 +15,15 @@ print status_code, resp_text
assert 'auth_required' not in resp_text
#Let's pull a list of all figures in Chapter 2 of the NCA3draft
for partial_figure in gcis.get_figure_listing('nca3draft', chapter_id='our-changing-climate'):
for partial_figure in gcis.get_figure_listing('nca3', chapter_id='our-changing-climate'):
#The listing doesn't provide all available fields for the figure (Images, for instance).
#There aren't very many figures, so let's go ahead and grab a complete version of each
full_figure = gcis.get_figure('nca3draft', partial_figure.identifier, chapter_id='our-changing-climate')
full_figure = gcis.get_figure('nca3', partial_figure.identifier, chapter_id='our-changing-climate')
print full_figure
#Let's work with the infamous temperature figure
fig2_7 = gcis.get_figure('nca3draft', 'observed-us-temperature-change')
fig2_7 = gcis.get_figure('nca3', 'observed-us-temperature-change')
#Warning: Images and Chapters are specifically excluded from JSON output. This is what gets sent to GCIS. So...
print fig2_7.as_json(indent=4)
......
......@@ -2,23 +2,21 @@
__author__ = 'abuddenberg'
import pickle
from gcis_clients import GcisClient
from gcis_clients import WebformClient
from gcis_clients import GcisClient, WebformClient, gcis_dev_auth, gcis_stage_auth, webform_token
from gcis_clients.sync_utils import move_images_to_gcis
webform_client = WebformClient('http://resources.assessment.globalchange.gov', 'mgTD63FAjG')
webform_client = WebformClient('http://resources.assessment.globalchange.gov', webform_token)
gcis_url = 'http://data.gcis-dev-front.joss.ucar.edu'
#gcis = GcisClient(gcis_url, 'andrew.buddenberg@noaa.gov', 'ad90c05b37d4128ae514bc6caa7a41911d2f1de353443a54')
gcis = GcisClient('http://data-stage.globalchange.gov', 'andrew.buddenberg@noaa.gov', 'b4f1458c3cf28248c982428c46e170019327bd4c533c23dd')
gcis = GcisClient('http://data.gcis-dev-front.joss.ucar.edu', *gcis_dev_auth)
gcis = GcisClient('http://data-stage.globalchange.gov', *gcis_stage_auth)
def main():
hitlist_file = 'hitlist.pk1'
# create_problem_list('nca3', hitlist_file)
# print_problem_list(hitlist_file)
solve_problems(hitlist_file, 'nca3')
create_problem_list('nca3', hitlist_file)
print_problem_list(hitlist_file)
# solve_problems(hitlist_file, 'nca3')
# print_ready_list(hitlist_file)
......@@ -99,7 +97,7 @@ def sort_webform_list(report_id):
#Check if organizations have been proper identified
for cont in f.contributors:
if cont.identifier is None:
if cont.organization.identifier is None:
problems.setdefault(key, {}).setdefault('org_id_not_found', []).append(cont)
for image in f.images:
......@@ -124,7 +122,7 @@ def sort_webform_list(report_id):
#Check if organizations have been proper identified
for cont in image.contributors:
if cont.identifier is None:
if cont.organization.identifier is None:
problems.setdefault(key, {}).setdefault('org_id_not_found', []).append(cont)
#Check for broken image associations
......
#!/usr/bin/env python
__author__ = 'abuddenberg'
from gcis_clients import WebformClient
from gcis_clients import GcisClient
from gcis_clients import WebformClient, GcisClient, gcis_dev_auth, gcis_stage_auth, webform_token
from gcis_clients.sync_utils import move_images_to_gcis, sync_dataset_metadata, realize_contributors
from collections import OrderedDict
import json
import pickle
webform = WebformClient('http://resources.assessment.globalchange.gov', 'mgTD63FAjG')
webform = WebformClient('http://resources.assessment.globalchange.gov', webform_token)
# gcis = GcisClient('http://data.gcis-dev-front.joss.ucar.edu', 'andrew.buddenberg@noaa.gov', 'ad90c05b37d4128ae514bc6caa7a41911d2f1de353443a54')
gcis = GcisClient('http://data-stage.globalchange.gov', 'andrew.buddenberg@noaa.gov', 'b4f1458c3cf28248c982428c46e170019327bd4c533c23dd')
# gcis = GcisClient('http://data.gcis-dev-front.joss.ucar.edu', *gcis_dev_auth)
gcis = GcisClient('http://data-stage.globalchange.gov', *gcis_stage_auth)
sync_metadata_tree = {
#Reports
......@@ -141,8 +138,8 @@ sync_metadata_tree = {
def main():
# print gcis.test_login()
# sync_dataset_metadata(gcis, webform.get_aggregated_datasets(), skip=['Proxy Data', 'Projected Sea Level Rise', 'Tide Gauge Data'])
print gcis.test_login()
sync_dataset_metadata(gcis, webform.get_aggregated_datasets(), skip=['Proxy Data', 'Projected Sea Level Rise', 'Tide Gauge Data'])
sync(replace=False)
......
from gcis_client import GcisClient
from webform_client import WebformClient
from nca3_client import Nca3Client
from os.path import expanduser, exists
from os import makedirs
from os import makedirs, getenv
def default_image_dir():
......@@ -11,3 +12,11 @@ def default_image_dir():
makedirs(image_dir)
return image_dir
#Magic environment variables
gcis_prod_auth = (getenv('GCIS_PROD_USER'), getenv('GCIS_PROD_KEY'))
gcis_stage_auth = (getenv('GCIS_STAGE_USER'), getenv('GCIS_STAGE_KEY'))
gcis_dev_auth = (getenv('GCIS_DEV_USER'), getenv('GCIS_DEV_KEY'))
gcis_auth = (getenv('GCIS_USER'), getenv('GCIS_KEY'))
webform_token = getenv('WEBFORM_TOKEN')
\ No newline at end of file
import urllib
import json
from os.path import exists, basename
import os
from os.path import basename, expanduser
import re
import requests
import yaml
import getpass
from domain import Figure, Image, Dataset, Activity, Person, Organization
......@@ -40,6 +42,34 @@ def http_resp(fn):
return wrapped
def get_credentials(url):
#First check our magic enviroment variables (GCIS_USER and GCIS_KEY)
from gcis_clients import gcis_auth
env_user, env_key = gcis_auth
if env_user is not None and env_key is not None:
return env_user, env_key
#Next, see if we can find Gcis.conf somewhere
conf_possibilities = [expanduser(f) for f in ['~/etc/Gcis.conf', '~/.gcis-py-client/Gcis.conf'] if
os.path.exists(expanduser(f))]
for gcis_config in conf_possibilities:
print 'Using {gc} for credentials...'.format(gc=gcis_config)
all_creds = yaml.load(open(gcis_config, 'r'))
instance_creds = [c for c in all_creds if c['url'] == url][0]
return instance_creds['userinfo'].split(':')[0], instance_creds['key']
#Else prompt for credentials
#Wow, I managed to use it
else:
username = raw_input('Username: ')
api_key = getpass.getpass('API key: ')
return username, api_key
class AssociationException(Exception):
def __init__(self, value):
self.value = value
......@@ -49,10 +79,15 @@ class AssociationException(Exception):
class GcisClient(object):
def __init__(self, url, username, password):
def __init__(self, url, username, api_key):
self.base_url = url
#If credentials were not provided, obtain them
if username is None or api_key is None:
username, api_key = get_credentials(url)
self.s = requests.Session()
self.s.auth = (username, password)
self.s.auth = (username, api_key)
self.s.headers.update({'Accept': 'application/json'})
@http_resp
......@@ -152,7 +187,7 @@ class GcisClient(object):
url = '{b}/image/files/{id}/{fn}'.format(b=self.base_url, id=image_id, fn=basename(local_path))
# For future multi-part encoding support
# return self.s.put(url, headers=headers, files={'file': (filename, open(filepath, 'rb'))})
if not exists(local_path):
if not os.path.exists(local_path):
raise Exception('File not found: ' + local_path)
return self.s.put(url, data=open(local_path, 'rb'), verify=False)
......@@ -161,15 +196,13 @@ class GcisClient(object):
def get_figure_listing(self, report_id, chapter_id=None):
chapter_filter = '/chapter/' + chapter_id if chapter_id else ''
url = '{b}/report/{rpt}{chap}/figure?{p}'.format(
b=self.base_url, rpt=report_id, chap=chapter_filter, p=urllib.urlencode({'all': '1'})
)
resp = self.s.get(url, verify=False)
url = '{b}/report/{rpt}{chap}/figure'.format(b=self.base_url, rpt=report_id, chap=chapter_filter)
resp = self.s.get(url, params={'all': '1'}, verify=False)
try:
return [Figure(figure) for figure in resp.json()]
except ValueError:
raise Exception('Add a better exception string here')
raise Exception(resp.text)
def get_figure(self, report_id, figure_id, chapter_id=None):
chapter_filter = '/chapter/' + chapter_id if chapter_id else ''
......@@ -188,8 +221,8 @@ class GcisClient(object):
def figure_exists(self, report_id, figure_id, chapter_id=None):
chapter_filter = '/chapter/' + chapter_id if chapter_id else ''
url = '{b}/report/{rpt}{chap}/figure/{fig}?{p}'.format(
b=self.base_url, rpt=report_id, chap=chapter_filter, fig=figure_id, p=urllib.urlencode({'all': '1'})
url = '{b}/report/{rpt}{chap}/figure/{fig}'.format(
b=self.base_url, rpt=report_id, chap=chapter_filter, fig=figure_id
)
return self.s.head(url, verify=False)
......@@ -229,8 +262,8 @@ class GcisClient(object):
return resp.status_code, resp.text
def get_keyword_listing(self):
url = '{b}/gcmd_keyword?{p}'.format(b=self.base_url, p=urllib.urlencode({'all': '1'}))
resp = self.s.get(url, verify=False)
url = '{b}/gcmd_keyword'.format(b=self.base_url)
resp = self.s.get(url, params={'all': '1'}, verify=False)
return resp.json()
......
__author__ = 'abuddenberg'
import requests
def http_resp(fn):
def wrapped(*args, **kwargs):
resp = fn(*args, **kwargs)
if resp.status_code == 200:
return resp
else:
raise Exception('Status: {code} \n{txt}'.format(code=resp.status_code, txt=resp.text))
return wrapped
class Nca3Client(object):
def __init__(self, url, username, password, http_basic_user=None, http_basic_pass=None):
self.base_url = url
self.s = requests.Session()
self.s.auth = (http_basic_user, http_basic_pass)
self.drupal_user = username
self.drupal_pass = password
def do_login(self):
url = '{b}/user'.format(b=self.base_url)
resp = self.s.post(
url,
data={
'name': self.drupal_user,
'pass': self.drupal_pass,
'form_id': 'user_login',
'op': 'Log in'
},
allow_redirects=False
)
return resp
@http_resp
def get_all_captions(self):
self.do_login()
url = '{b}/gcis/figure-table-captions'.format(b=self.base_url)
resp = self.s.get(url, verify=False)
return resp
......@@ -3,7 +3,7 @@
import urllib
import re
from os.path import join
import getpass
import requests
from dateutil.parser import parse
......@@ -16,7 +16,7 @@ def sanitized(pattern):
if re.match(pattern, urllib.quote(args[1])):
return fn(*args, **kwargs)
else:
print 'Shitlisted: ', args[1]
print 'Rejected: ', args[1]
return wrapped
return dec
......@@ -36,10 +36,26 @@ def parse_creators(field):
return contributor
def get_credentials():
#First check our magic enviroment variable (WEBFORM_TOKEN)
from gcis_clients import webform_token
if webform_token is not None:
return webform_token
else:
return getpass.getpass('Webform token: ')
class WebformClient:
def __init__(self, url, token, local_image_dir=None, remote_dir='/system/files/'):
self.base_url = url
#If token was not provided, obtain it
if token is None:
token = get_credentials()
self.token = token
if local_image_dir:
......
......@@ -19,7 +19,7 @@ class PyTest(TestCommand):
setup(
name='GcisPyClient',
version='0.68',
version='1.0',
author='Andrew Buddenberg',
author_email='andrew.buddenberg@noaa.gov',
packages=find_packages(),
......@@ -29,7 +29,8 @@ setup(
long_description=open('README.txt').read(),
install_requires=[
"requests >= 2.1.0",
"python-dateutil >= 2.2"
"python-dateutil >= 2.2",
"PyYAML >= 3.11"
],
tests_require=["pytest >= 2.5.2"],
cmdclass={'test': PyTest},
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment