Removed hardcoded auth tokens and api keys from everything. Added support for...

Removed hardcoded auth tokens and api keys from everything. Added support for env variables and config files for persisting credentials

README.txt

@@ -23,12 +23,35 @@ To install from git with pip:
 
 To install from tarball:
 
-    tar -xzvf GcisPyClient-0.67.tar.gz
-    cd GcisPyClient-0.67
+    tar -xzvf GcisPyClient-x.y.tar.gz
+    cd GcisPyClient-x.y
 
     python setup.py test (to run the test suite)
     python setup.py install (to install)
 
+Setup
+-----
+
+### User Credentials
+
+GcisClient will use the first credentials it finds while searching the following places:
+
+1. Strings passed directly to the GcisClient constructor.
+2. Environment variables "GCIS_USER" and "GCIS_KEY" (both are required)
+3. Config file at $HOME/etc/Gcis.conf (A sample of this file is included below)
+4. Config file at $HOME/.gcis-py-client/Gcis.conf
+5. Interactively, with a username and key prompt
+
+Sample Gcis.conf file:
+
+    - url      : http://data-stage.globalchange.gov
+      userinfo : me@example.com:298015f752D99E789056EF826A7D97afc38a8bbd6e3e23b3
+      key      : M2FiLtG2n2qTyJHIztvHm5zweTYjNkM5ZWEtYjNkMS00LTgS00LTg2N2QtYZDFhzQyNGUxCg==
+
+    - url      : http://data.globalchange.gov
+      userinfo : username:pass
+      key      : key
+
 Usage
 -----
 
@@ -43,16 +66,16 @@ Make sure our credentials work:
     print status_code, resp_text
     assert 'auth_required' not in resp_text
 
-Let's pull a list of all figures in Chapter 2 of the NCA3draft:
+Let's pull a list of all figures in Chapter 2 of the nca3:
 
-    for partial_figure in gcis.get_figure_listing('nca3draft', chapter_id='our-changing-climate'):
-        full_figure = gcis.get_figure('nca3draft', partial_figure.identifier, chapter_id='our-changing-climate')
+    for partial_figure in gcis.get_figure_listing('nca3', chapter_id='our-changing-climate'):
+        full_figure = gcis.get_figure('nca3', partial_figure.identifier, chapter_id='our-changing-climate')
         print full_figure
 
 
 Let's work with the infamous temperature figure:
 
-    fig2_7 = gcis.get_figure('nca3draft', 'observed-us-temperature-change')
+    fig2_7 = gcis.get_figure('nca3', 'observed-us-temperature-change')
 
 Warning: Images and Chapters are specifically excluded from JSON output.  This is what gets sent to GCIS. So...
 

bin/example

 #!/usr/bin/env python
 
-from gcis_clients import GcisClient
+from gcis_clients import GcisClient, gcis_dev_auth
 
 base_url = 'http://data.gcis-dev-front.joss.ucar.edu'
 
-gcis = GcisClient(base_url, 'andrew.buddenberg@noaa.gov', 'fcee8e3f11f36313e463ece51aab15242f71f3d552d565be')
+#These are provided by environment variables GCIS_DEV_USER and GCIS_DEV_KEY
+username, api_key = gcis_dev_auth
+
+gcis = GcisClient(base_url, username, api_key)
 
 #Make sure our credentials work
 status_code, resp_text = gcis.test_login()
@@ -12,15 +15,15 @@ print status_code, resp_text
 assert 'auth_required' not in resp_text
 
 #Let's pull a list of all figures in Chapter 2 of the NCA3draft
-for partial_figure in gcis.get_figure_listing('nca3draft', chapter_id='our-changing-climate'):
+for partial_figure in gcis.get_figure_listing('nca3', chapter_id='our-changing-climate'):
     #The listing doesn't provide all available fields for the figure (Images, for instance).
     #There aren't very many figures, so let's go ahead and grab a complete version of each
-    full_figure = gcis.get_figure('nca3draft', partial_figure.identifier, chapter_id='our-changing-climate')
+    full_figure = gcis.get_figure('nca3', partial_figure.identifier, chapter_id='our-changing-climate')
     print full_figure
 
 
 #Let's work with the infamous temperature figure
-fig2_7 = gcis.get_figure('nca3draft', 'observed-us-temperature-change')
+fig2_7 = gcis.get_figure('nca3', 'observed-us-temperature-change')
 
 #Warning: Images and Chapters are specifically excluded from JSON output.  This is what gets sent to GCIS. So...
 print fig2_7.as_json(indent=4)

bin/problems

@@ -2,23 +2,21 @@
 __author__ = 'abuddenberg'
 
 import pickle
-from gcis_clients import GcisClient
-from gcis_clients import WebformClient
+from gcis_clients import GcisClient, WebformClient, gcis_dev_auth, gcis_stage_auth, webform_token
 from gcis_clients.sync_utils import move_images_to_gcis
 
-webform_client = WebformClient('http://resources.assessment.globalchange.gov', 'mgTD63FAjG')
+webform_client = WebformClient('http://resources.assessment.globalchange.gov', webform_token)
 
-gcis_url = 'http://data.gcis-dev-front.joss.ucar.edu'
-#gcis = GcisClient(gcis_url, 'andrew.buddenberg@noaa.gov', 'ad90c05b37d4128ae514bc6caa7a41911d2f1de353443a54')
-gcis = GcisClient('http://data-stage.globalchange.gov', 'andrew.buddenberg@noaa.gov', 'b4f1458c3cf28248c982428c46e170019327bd4c533c23dd')
+gcis = GcisClient('http://data.gcis-dev-front.joss.ucar.edu', *gcis_dev_auth)
+gcis = GcisClient('http://data-stage.globalchange.gov', *gcis_stage_auth)
 
 
 def main():
     hitlist_file = 'hitlist.pk1'
 
-#    create_problem_list('nca3', hitlist_file)
-#    print_problem_list(hitlist_file)
-    solve_problems(hitlist_file, 'nca3')
+    create_problem_list('nca3', hitlist_file)
+    print_problem_list(hitlist_file)
+#     solve_problems(hitlist_file, 'nca3')
 #    print_ready_list(hitlist_file)
 
 
@@ -99,7 +97,7 @@ def sort_webform_list(report_id):
 
             #Check if organizations have been proper identified
             for cont in f.contributors:
-                if cont.identifier is None:
+                if cont.organization.identifier is None:
                     problems.setdefault(key, {}).setdefault('org_id_not_found', []).append(cont)
 
             for image in f.images:
@@ -124,7 +122,7 @@ def sort_webform_list(report_id):
 
                 #Check if organizations have been proper identified
                 for cont in image.contributors:
-                    if cont.identifier is None:
+                    if cont.organization.identifier is None:
                         problems.setdefault(key, {}).setdefault('org_id_not_found', []).append(cont)
 
             #Check for broken image associations

bin/sync_figures

 #!/usr/bin/env python
 __author__ = 'abuddenberg'
 
-from gcis_clients import WebformClient
-from gcis_clients import GcisClient
+from gcis_clients import WebformClient, GcisClient, gcis_dev_auth, gcis_stage_auth, webform_token
 from gcis_clients.sync_utils import move_images_to_gcis, sync_dataset_metadata, realize_contributors
 from collections import OrderedDict
-import json
-import pickle
 
-webform = WebformClient('http://resources.assessment.globalchange.gov', 'mgTD63FAjG')
+webform = WebformClient('http://resources.assessment.globalchange.gov', webform_token)
 
-# gcis = GcisClient('http://data.gcis-dev-front.joss.ucar.edu', 'andrew.buddenberg@noaa.gov', 'ad90c05b37d4128ae514bc6caa7a41911d2f1de353443a54')
-gcis = GcisClient('http://data-stage.globalchange.gov', 'andrew.buddenberg@noaa.gov', 'b4f1458c3cf28248c982428c46e170019327bd4c533c23dd')
+# gcis = GcisClient('http://data.gcis-dev-front.joss.ucar.edu', *gcis_dev_auth)
+gcis = GcisClient('http://data-stage.globalchange.gov', *gcis_stage_auth)
 
 sync_metadata_tree = {
     #Reports
@@ -141,8 +138,8 @@ sync_metadata_tree = {
 
 
 def main():
-    # print gcis.test_login()
-    # sync_dataset_metadata(gcis, webform.get_aggregated_datasets(), skip=['Proxy Data', 'Projected Sea Level Rise', 'Tide Gauge Data'])
+    print gcis.test_login()
+    sync_dataset_metadata(gcis, webform.get_aggregated_datasets(), skip=['Proxy Data', 'Projected Sea Level Rise', 'Tide Gauge Data'])
     sync(replace=False)
 
 

gcis_clients/__init__.py

 from gcis_client import GcisClient
 from webform_client import WebformClient
+from nca3_client import Nca3Client
 
 from os.path import expanduser, exists
-from os import makedirs
+from os import makedirs, getenv
 
 
 def default_image_dir():
@@ -11,3 +12,11 @@ def default_image_dir():
         makedirs(image_dir)
     return image_dir
 
+#Magic environment variables
+gcis_prod_auth = (getenv('GCIS_PROD_USER'), getenv('GCIS_PROD_KEY'))
+gcis_stage_auth = (getenv('GCIS_STAGE_USER'), getenv('GCIS_STAGE_KEY'))
+gcis_dev_auth = (getenv('GCIS_DEV_USER'), getenv('GCIS_DEV_KEY'))
+
+gcis_auth = (getenv('GCIS_USER'), getenv('GCIS_KEY'))
+
+webform_token = getenv('WEBFORM_TOKEN')
\ No newline at end of file

gcis_clients/gcis_client.py

-import urllib
 import json
-from os.path import exists, basename
+import os
+from os.path import basename, expanduser
 import re
 import requests
+import yaml
+import getpass
 
 from domain import Figure, Image, Dataset, Activity, Person, Organization
 
@@ -40,6 +42,34 @@ def http_resp(fn):
     return wrapped
 
 
+def get_credentials(url):
+    #First check our magic enviroment variables (GCIS_USER and GCIS_KEY)
+    from gcis_clients import gcis_auth
+    env_user, env_key = gcis_auth
+
+    if env_user is not None and env_key is not None:
+        return env_user, env_key
+
+    #Next, see if we can find Gcis.conf somewhere
+    conf_possibilities = [expanduser(f) for f in ['~/etc/Gcis.conf', '~/.gcis-py-client/Gcis.conf'] if
+                          os.path.exists(expanduser(f))]
+
+    for gcis_config in conf_possibilities:
+        print 'Using {gc} for credentials...'.format(gc=gcis_config)
+        all_creds = yaml.load(open(gcis_config, 'r'))
+        instance_creds = [c for c in all_creds if c['url'] == url][0]
+
+        return instance_creds['userinfo'].split(':')[0], instance_creds['key']
+
+    #Else prompt for credentials
+    #Wow, I managed to use it
+    else:
+        username = raw_input('Username: ')
+        api_key = getpass.getpass('API key: ')
+
+        return username, api_key
+
+
 class AssociationException(Exception):
     def __init__(self, value):
         self.value = value
@@ -49,10 +79,15 @@ class AssociationException(Exception):
 
 
 class GcisClient(object):
-    def __init__(self, url, username, password):
+    def __init__(self, url, username, api_key):
         self.base_url = url
+
+        #If credentials were not provided, obtain them
+        if username is None or api_key is None:
+            username, api_key = get_credentials(url)
+
         self.s = requests.Session()
-        self.s.auth = (username, password)
+        self.s.auth = (username, api_key)
         self.s.headers.update({'Accept': 'application/json'})
 
     @http_resp
@@ -152,7 +187,7 @@ class GcisClient(object):
         url = '{b}/image/files/{id}/{fn}'.format(b=self.base_url, id=image_id, fn=basename(local_path))
         # For future multi-part encoding support
         # return self.s.put(url, headers=headers, files={'file': (filename, open(filepath, 'rb'))})
-        if not exists(local_path):
+        if not os.path.exists(local_path):
             raise Exception('File not found: ' + local_path)
 
         return self.s.put(url, data=open(local_path, 'rb'), verify=False)
@@ -161,15 +196,13 @@ class GcisClient(object):
     def get_figure_listing(self, report_id, chapter_id=None):
         chapter_filter = '/chapter/' + chapter_id if chapter_id else ''
 
-        url = '{b}/report/{rpt}{chap}/figure?{p}'.format(
-            b=self.base_url, rpt=report_id, chap=chapter_filter, p=urllib.urlencode({'all': '1'})
-        )
-        resp = self.s.get(url, verify=False)
+        url = '{b}/report/{rpt}{chap}/figure'.format(b=self.base_url, rpt=report_id, chap=chapter_filter)
+        resp = self.s.get(url, params={'all': '1'}, verify=False)
 
         try:
             return [Figure(figure) for figure in resp.json()]
         except ValueError:
-            raise Exception('Add a better exception string here')
+            raise Exception(resp.text)
 
     def get_figure(self, report_id, figure_id, chapter_id=None):
         chapter_filter = '/chapter/' + chapter_id if chapter_id else ''
@@ -188,8 +221,8 @@ class GcisClient(object):
     def figure_exists(self, report_id, figure_id, chapter_id=None):
         chapter_filter = '/chapter/' + chapter_id if chapter_id else ''
 
-        url = '{b}/report/{rpt}{chap}/figure/{fig}?{p}'.format(
-            b=self.base_url, rpt=report_id, chap=chapter_filter, fig=figure_id, p=urllib.urlencode({'all': '1'})
+        url = '{b}/report/{rpt}{chap}/figure/{fig}'.format(
+            b=self.base_url, rpt=report_id, chap=chapter_filter, fig=figure_id
         )
         return self.s.head(url, verify=False)
 
@@ -229,8 +262,8 @@ class GcisClient(object):
         return resp.status_code, resp.text
 
     def get_keyword_listing(self):
-        url = '{b}/gcmd_keyword?{p}'.format(b=self.base_url, p=urllib.urlencode({'all': '1'}))
-        resp = self.s.get(url, verify=False)
+        url = '{b}/gcmd_keyword'.format(b=self.base_url)
+        resp = self.s.get(url, params={'all': '1'}, verify=False)
 
         return resp.json()
 

gcis_clients/nca3_client.py

+__author__ = 'abuddenberg'
+
+import requests
+
+
+def http_resp(fn):
+    def wrapped(*args, **kwargs):
+        resp = fn(*args, **kwargs)
+        if resp.status_code == 200:
+            return resp
+        else:
+            raise Exception('Status: {code} \n{txt}'.format(code=resp.status_code, txt=resp.text))
+    return wrapped
+
+
+class Nca3Client(object):
+    def __init__(self, url, username, password, http_basic_user=None, http_basic_pass=None):
+        self.base_url = url
+        self.s = requests.Session()
+        self.s.auth = (http_basic_user, http_basic_pass)
+
+        self.drupal_user = username
+        self.drupal_pass = password
+
+    def do_login(self):
+        url = '{b}/user'.format(b=self.base_url)
+        resp = self.s.post(
+            url,
+            data={
+                'name': self.drupal_user,
+                'pass': self.drupal_pass,
+                'form_id': 'user_login',
+                'op': 'Log in'
+            },
+            allow_redirects=False
+        )
+
+        return resp
+
+    @http_resp
+    def get_all_captions(self):
+        self.do_login()
+        url = '{b}/gcis/figure-table-captions'.format(b=self.base_url)
+
+        resp = self.s.get(url, verify=False)
+        return resp

gcis_clients/webform_client.py

@@ -3,7 +3,7 @@
 import urllib
 import re
 from os.path import join
-
+import getpass
 import requests
 from dateutil.parser import parse
 
@@ -16,7 +16,7 @@ def sanitized(pattern):
             if re.match(pattern, urllib.quote(args[1])):
                 return fn(*args, **kwargs)
             else:
-                print 'Shitlisted: ', args[1]
+                print 'Rejected: ', args[1]
         return wrapped
     return dec
 
@@ -36,10 +36,26 @@ def parse_creators(field):
     return contributor
 
 
+def get_credentials():
+    #First check our magic enviroment variable (WEBFORM_TOKEN)
+    from gcis_clients import webform_token
+
+    if webform_token is not None:
+        return webform_token
+
+    else:
+        return getpass.getpass('Webform token: ')
+
+
 class WebformClient:
 
     def __init__(self, url, token, local_image_dir=None, remote_dir='/system/files/'):
         self.base_url = url
+
+        #If token was not provided, obtain it
+        if token is None:
+            token = get_credentials()
+
         self.token = token
 
         if local_image_dir:

setup.py

@@ -19,7 +19,7 @@ class PyTest(TestCommand):
 
 setup(
     name='GcisPyClient',
-    version='0.68',
+    version='1.0',
     author='Andrew Buddenberg',
     author_email='andrew.buddenberg@noaa.gov',
     packages=find_packages(),
@@ -29,7 +29,8 @@ setup(
     long_description=open('README.txt').read(),
     install_requires=[
         "requests >= 2.1.0",
-        "python-dateutil >= 2.2"
+        "python-dateutil >= 2.2",
+        "PyYAML >= 3.11"
     ],
     tests_require=["pytest >= 2.5.2"],
     cmdclass={'test': PyTest},