Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Andrew Buddenberg
gcis-py-client
Commits
5379b488
Commit
5379b488
authored
Apr 09, 2015
by
abuddenberg
Browse files
snapshot Thu Apr 9 16:35:15 EDT 2015
parent
f4c738dd
Changes
3
Hide whitespace changes
Inline
Side-by-side
gcis_clients/domain.py
View file @
5379b488
...
...
@@ -50,7 +50,7 @@ class Gcisbase(object):
return
self
def
as_json
(
self
,
indent
=
0
,
omit_fields
=
[]
):
def
as_json
(
self
,
indent
=
0
,
omit_fields
=
()
):
out_fields
=
set
(
self
.
gcis_fields
)
-
(
set
([
'uri'
,
'href'
])
|
set
(
omit_fields
))
return
json
.
dumps
({
f
:
getattr
(
self
,
f
)
for
f
in
out_fields
},
indent
=
indent
)
...
...
@@ -80,7 +80,7 @@ class GcisObject(Gcisbase):
class
Figure
(
GcisObject
):
def
__init__
(
self
,
data
,
trans
=
()):
def
__init__
(
self
,
data
,
local_path
=
None
,
remote_path
=
None
,
trans
=
()):
self
.
gcis_fields
=
[
'usage_limits'
,
'kindred_figures'
,
'time_start'
,
'time_end'
,
'keywords'
,
'lat_min'
,
'create_dt'
,
'lat_max'
,
'title'
,
'ordinal'
,
'lon_min'
,
'report_identifier'
,
'chapter'
,
'submission_dt'
,
'uri'
,
'lon_max'
,
...
...
@@ -89,6 +89,9 @@ class Figure(GcisObject):
super
(
Figure
,
self
).
__init__
(
data
,
fields
=
self
.
gcis_fields
,
trans
=
trans
)
self
.
local_path
=
local_path
self
.
remote_path
=
remote_path
#Special case for chapter
chap_tree
=
data
.
pop
(
'chapter'
,
None
)
self
.
chapter
=
Chapter
(
chap_tree
)
if
chap_tree
else
self
.
chapter
...
...
@@ -126,8 +129,8 @@ class Figure(GcisObject):
else
:
self
.
chapter
=
chp
def
as_json
(
self
,
indent
=
0
):
return
super
(
Figure
,
self
).
as_json
(
omit_fields
=
[
'images'
,
'chapter'
,
'kindred_figures'
,
'keywords'
]
)
def
as_json
(
self
,
indent
=
0
,
omit_fields
=
(
'images'
,
'chapter'
,
'kindred_figures'
,
'keywords'
)
):
return
super
(
Figure
,
self
).
as_json
(
omit_fields
=
omit_fields
)
def
__str__
(
self
):
string
=
'{f_id}: Figure {f_num}: {f_name}
\n\t
Images: {imgs}'
.
format
(
...
...
@@ -153,12 +156,40 @@ class Figure(GcisObject):
return
super
(
Figure
,
self
).
merge
(
other
)
class
Report
(
GcisObject
):
def
__init__
(
self
,
data
,
trans
=
()):
self
.
gcis_fields
=
[
'doi'
,
'contact_note'
,
'title'
,
'publication_year'
,
'summary'
,
'url'
,
'contact_email'
,
'identifier'
,
'report_type_identifier'
]
super
(
Report
,
self
).
__init__
(
data
,
fields
=
self
.
gcis_fields
,
trans
=
trans
)
# if self.report_type_identifier not in ['report', 'assessment', 'technical_input', 'indicator']:
# raise ValueError("report_type_identifier must be one of 'report', 'assessment', 'technical_input', 'indicator'")
def
as_json
(
self
,
indent
=
0
,
omit_fields
=
()):
return
super
(
Report
,
self
).
as_json
(
omit_fields
=
omit_fields
)
def
__repr__
(
self
):
return
'Report: {id}'
.
format
(
id
=
self
.
identifier
)
def
__str__
(
self
):
return
self
.
__repr__
()
class
Chapter
(
GcisObject
):
def
__init__
(
self
,
data
):
self
.
gcis_fields
=
[
'report_identifier'
,
'identifier'
,
'number'
,
'url'
,
'title'
]
super
(
Chapter
,
self
).
__init__
(
data
,
fields
=
self
.
gcis_fields
)
def
as_json
(
self
,
indent
=
0
,
omit_fields
=
()):
return
super
(
Chapter
,
self
).
as_json
(
omit_fields
=
omit_fields
)
def
__repr__
(
self
):
return
'Chapter: {id}'
.
format
(
id
=
self
.
identifier
)
def
__str__
(
self
):
return
self
.
__repr__
()
class
Image
(
GcisObject
):
def
__init__
(
self
,
data
,
local_path
=
None
,
remote_path
=
None
,
trans
=
()):
...
...
@@ -271,8 +302,8 @@ class Dataset(GcisObject):
def
__str__
(
self
):
return
self
.
__repr__
()
def
as_json
(
self
,
indent
=
0
):
return
super
(
Dataset
,
self
).
as_json
(
omit_fields
=
[
'files'
,
'parents'
,
'contributors'
,
'references'
]
)
def
as_json
(
self
,
indent
=
0
,
omit_fields
=
(
'files'
,
'parents'
,
'contributors'
,
'references'
)
):
return
super
(
Dataset
,
self
).
as_json
(
omit_fields
=
omit_fields
)
def
merge
(
self
,
other
):
for
k
in
self
.
__dict__
:
...
...
@@ -326,8 +357,8 @@ class Activity(GcisObject):
super
(
Activity
,
self
).
__init__
(
data
,
fields
=
self
.
gcis_fields
,
trans
=
trans
)
def
as_json
(
self
,
indent
=
0
):
return
super
(
Activity
,
self
).
as_json
(
omit_fields
=
[
'metholodogies'
,
'publication_maps'
]
)
def
as_json
(
self
,
indent
=
0
,
omit_fields
=
(
'metholodogies'
,
'publication_maps'
)
):
return
super
(
Activity
,
self
).
as_json
(
omit_fields
=
omit_fields
)
def
__repr__
(
self
):
return
'Activity: {id}'
.
format
(
id
=
self
.
identifier
)
...
...
@@ -343,8 +374,8 @@ class Person(Gcisbase):
super
(
Person
,
self
).
__init__
(
data
,
fields
=
self
.
gcis_fields
,
trans
=
trans
)
def
as_json
(
self
,
indent
=
0
):
return
super
(
Person
,
self
).
as_json
(
omit_fields
=
[
'contributors'
]
)
def
as_json
(
self
,
indent
=
0
,
omit_fields
=
(
'contributors'
,)
):
return
super
(
Person
,
self
).
as_json
(
omit_fields
=
omit_fields
)
def
__repr__
(
self
):
return
'Person: {id}: {fn} {ln}'
.
format
(
id
=
self
.
id
,
fn
=
self
.
first_name
,
ln
=
self
.
last_name
)
...
...
@@ -613,4 +644,4 @@ class Parent(Gcisbase):
)
def
__str__
(
self
):
return
self
.
__repr__
()
\ No newline at end of file
return
self
.
__repr__
()
gcis_clients/gcis_client.py
View file @
5379b488
...
...
@@ -38,7 +38,7 @@ def http_resp(fn):
if
resp
.
status_code
==
200
:
return
resp
else
:
raise
Exception
(
resp
.
text
)
raise
Exception
(
'{url}
\n
{stat} {txt}'
.
format
(
url
=
resp
.
url
,
stat
=
resp
.
status_code
,
txt
=
resp
.
text
)
)
return
wrapped
...
...
@@ -283,6 +283,60 @@ class GcisClient(object):
resp
=
self
.
s
.
get
(
url
,
verify
=
False
)
return
resp
.
status_code
,
resp
.
text
@
http_resp
def
get_report
(
self
,
report_id
):
url
=
'{b}/report/{id}'
.
format
(
b
=
self
.
base_url
,
id
=
report_id
)
resp
=
self
.
s
.
get
(
url
,
verify
=
False
)
return
resp
.
json
()
@
exists
def
report_exists
(
self
,
report_id
):
url
=
'{b}/report/{id}'
.
format
(
b
=
self
.
base_url
,
id
=
report_id
)
return
self
.
s
.
head
(
url
,
verify
=
False
)
@
http_resp
def
create_report
(
self
,
report
):
url
=
'{b}/report/'
.
format
(
b
=
self
.
base_url
)
return
self
.
s
.
post
(
url
,
data
=
report
.
as_json
(),
verify
=
False
)
@
http_resp
def
update_report
(
self
,
report
,
old_id
=
None
):
url
=
'{b}/report/{id}'
.
format
(
b
=
self
.
base_url
,
id
=
old_id
or
report
.
identifier
)
return
self
.
s
.
post
(
url
,
data
=
report
.
as_json
(),
verify
=
False
)
@
http_resp
def
delete_report
(
self
,
report
):
url
=
'{b}/report/{ds}'
.
format
(
b
=
self
.
base_url
,
ds
=
report
.
identifier
)
return
self
.
s
.
delete
(
url
,
verify
=
False
)
@
http_resp
def
get_chapter
(
self
,
chapter_id
):
url
=
'{b}/chapter/{id}'
.
format
(
b
=
self
.
base_url
,
id
=
chapter_id
)
resp
=
self
.
s
.
get
(
url
,
verify
=
False
)
return
resp
.
json
()
@
exists
def
chapter_exists
(
self
,
report_id
,
chapter_id
):
url
=
'{b}/report/{rpt}/chapter/{id}'
.
format
(
b
=
self
.
base_url
,
rpt
=
report_id
,
id
=
chapter_id
)
return
self
.
s
.
head
(
url
,
verify
=
False
)
@
http_resp
def
create_chapter
(
self
,
report_id
,
chapter
):
url
=
'{b}/report/{rpt}/chapter/'
.
format
(
b
=
self
.
base_url
,
rpt
=
report_id
)
return
self
.
s
.
post
(
url
,
data
=
chapter
.
as_json
(),
verify
=
False
)
@
http_resp
def
update_chapter
(
self
,
chapter
,
old_id
=
None
):
url
=
'{b}/chapter/{id}'
.
format
(
b
=
self
.
base_url
,
id
=
old_id
or
chapter
.
identifier
)
return
self
.
s
.
post
(
url
,
data
=
chapter
.
as_json
(),
verify
=
False
)
@
http_resp
def
delete_chapter
(
self
,
chapter
):
url
=
'{b}/chapter/{ds}'
.
format
(
b
=
self
.
base_url
,
ds
=
chapter
.
identifier
)
return
self
.
s
.
delete
(
url
,
verify
=
False
)
def
get_keyword_listing
(
self
):
url
=
'{b}/gcmd_keyword'
.
format
(
b
=
self
.
base_url
)
resp
=
self
.
s
.
get
(
url
,
params
=
{
'all'
:
'1'
},
verify
=
False
)
...
...
gcis_clients/survey_client.py
View file @
5379b488
...
...
@@ -3,6 +3,7 @@ __author__ = 'abuddenberg'
import
getpass
import
requests
import
re
from
os.path
import
join
,
basename
from
gcis_clients.domain
import
Figure
,
Image
,
Dataset
,
Parent
...
...
@@ -34,8 +35,9 @@ def populate_figure(fig_json):
f
.
create_dt
=
fig_json
[
'graphics_create_date'
]
f
.
time_start
,
f
.
time_end
=
fig_json
[
'period_record'
]
f
.
lat_min
,
f
.
lat_max
,
f
.
lon_min
,
f
.
lon_max
=
fig_json
[
'spatial_extent'
]
f
.
remote_path
=
fig_json
[
'filepath'
]
except
Exception
,
e
:
print
e
print
'Exception: '
,
e
return
f
...
...
@@ -49,7 +51,7 @@ def populate_image(img_json):
img
.
time_start
,
img
.
time_end
=
img_json
[
'period_record'
]
img
.
lat_min
,
img
.
lat_max
,
img
.
lon_min
,
img
.
lon_max
=
img_json
[
'spatial_extent'
]
except
Exception
,
e
:
print
e
print
'Exception: '
,
e
return
img
...
...
@@ -60,7 +62,7 @@ def populate_dataset(ds_json):
ds
.
name
=
ds_json
[
'dataset_name'
]
ds
.
url
=
ds_json
[
'dataset_url'
]
except
Exception
,
e
:
print
e
print
'Exception: '
,
e
image_select
=
ds_json
[
'imageSelect'
]
if
'imageSelect'
in
ds_json
else
[]
associated_images
=
[
idx
for
idx
,
value
in
enumerate
(
image_select
)
if
value
==
'on'
]
...
...
@@ -76,13 +78,13 @@ def populate_parent(pub_json):
p
.
url
=
''
except
Exception
,
e
:
print
e
print
'Exception: '
,
e
return
p
class
SurveyClient
:
def
__init__
(
self
,
url
,
token
,
local_
image_dir
=
None
,
remote_dir
=
'/system/files/
'
):
def
__init__
(
self
,
url
,
token
,
local_
download_dir
=
'.
'
):
self
.
base_url
=
url
#If token was not provided, obtain it
...
...
@@ -91,29 +93,30 @@ class SurveyClient:
self
.
token
=
token
if
local_image_dir
:
self
.
images_dir
=
local_image_dir
else
:
from
gcis_clients
import
default_image_dir
self
.
images_dir
=
default_image_dir
()
self
.
remote_image_dir
=
remote_dir
self
.
local_download_dir
=
local_download_dir
def
get_list
(
self
):
url
=
'{b}/metadata/list?token={t}'
.
format
(
b
=
self
.
base_url
,
t
=
self
.
token
)
return
requests
.
get
(
url
).
json
()
def
get_survey
(
self
,
fig_url
,
download
_images
=
False
):
def
get_survey
(
self
,
fig_url
,
do_
download
=
False
):
full_url
=
'{b}{url}?token={t}'
.
format
(
b
=
self
.
base_url
,
url
=
fig_url
,
t
=
self
.
token
)
survey_json
=
requests
.
get
(
full_url
).
json
()
tier1_json
=
survey_json
[
0
][
't1'
]
fig_json
=
tier1_json
[
'figure'
]
tier1_json
=
survey_json
[
0
][
't1'
]
if
survey_json
[
0
][
't1'
]
is
not
None
else
[]
f
=
None
#It's not worth trying to translations on this data; it's too different
f
=
populate_figure
(
fig_json
)
if
'figure'
in
tier1_json
:
#It's not worth trying to translations on this data; it's too different
f
=
populate_figure
(
tier1_json
[
'figure'
])
f
.
local_path
=
join
(
self
.
local_download_dir
,
basename
(
f
.
remote_path
))
if
f
.
remote_path
else
None
if
'images'
in
tier1_json
:
images
=
[
populate_image
(
img
)
for
img
in
tier1_json
[
'images'
]]
f
.
images
.
extend
(
images
)
elif
'figure'
in
tier1_json
:
default_image
=
populate_image
(
tier1_json
[
'figure'
])
f
.
images
.
append
(
default_image
)
if
'datasets'
in
tier1_json
:
datasets
=
[
populate_dataset
(
ds
)
for
ds
in
tier1_json
[
'datasets'
]]
...
...
@@ -126,5 +129,24 @@ class SurveyClient:
if
'origination'
in
tier1_json
and
tier1_json
[
'origination'
]
not
in
(
'Original'
,):
f
.
parents
.
append
(
populate_parent
(
tier1_json
[
'publication'
]))
if
do_download
:
self
.
download_figure
(
f
)
return
f
def
download_figure
(
self
,
figure
):
url
=
'{b}/{path}?token={t}'
.
format
(
b
=
self
.
base_url
,
path
=
figure
.
remote_path
,
t
=
self
.
token
)
print
url
resp
=
requests
.
get
(
url
,
stream
=
True
)
if
resp
.
status_code
==
200
:
filepath
=
join
(
self
.
local_download_dir
,
figure
.
remote_path
.
split
(
'/'
)[
-
1
])
with
open
(
filepath
,
'wb'
)
as
fig_out
:
for
bytes
in
resp
.
iter_content
(
chunk_size
=
4096
):
fig_out
.
write
(
bytes
)
return
filepath
elif
resp
.
status_code
==
404
:
raise
Exception
(
'Image not found: {u}'
.
format
(
u
=
url
))
else
:
raise
Exception
(
resp
.
status_code
)
\ No newline at end of file
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment