Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Andrew Buddenberg
gcis-py-client
Commits
ee68a6e3
Commit
ee68a6e3
authored
May 05, 2014
by
abuddenberg
Browse files
Added support for syncing all available figures for NCA rollout
parent
fe0413dc
Changes
4
Hide whitespace changes
Inline
Side-by-side
bin/problems
View file @
ee68a6e3
...
...
@@ -11,8 +11,8 @@ gcis = GcisClient('http://data.gcis-dev-front.joss.ucar.edu', *gcis_dev_auth)
# gcis = GcisClient('http://data-stage.globalchange.gov', *gcis_stage_auth)
whitelisted_dupes
=
[
'e13cdf2e-0699-4c52-a642-56c8fc3317c4'
,
#33.1
1
#34.11
'03937439-cc5a-45c7-a0fd-78b863c49979'
,
#33.1
1
#34.11
'e13cdf2e-0699-4c52-a642-56c8fc3317c4'
,
#33.1 #34.11
'03937439-cc5a-45c7-a0fd-78b863c49979'
,
#33.1 #34.11
'0bcb7984-50c0-4f94-9946-c3fa635125a4'
,
#2.1 #34.2
...
...
bin/sync_figures
View file @
ee68a6e3
#!/usr/bin/env python
__author__
=
'abuddenberg'
from
gcis_clients
import
WebformClient
,
GcisClient
,
gcis_dev_auth
,
gcis_stage_auth
,
webform_token
from
gcis_clients
import
WebformClient
,
GcisClient
,
gcis_dev_auth
,
gcis_stage_auth
,
webform_token
,
AssociationException
from
gcis_clients.sync_utils
import
move_images_to_gcis
,
sync_dataset_metadata
,
realize_contributors
,
realize_parents
from
collections
import
OrderedDict
webform
=
WebformClient
(
'http://resources.assessment.globalchange.gov'
,
webform_token
)
gcis
=
GcisClient
(
'http://data.gcis-dev-front.joss.ucar.edu'
,
*
gcis_dev_auth
)
#
gcis = GcisClient('http://data-stage.globalchange.gov', *gcis_stage_auth)
#
gcis = GcisClient('http://data.gcis-dev-front.joss.ucar.edu', *gcis_dev_auth)
gcis
=
GcisClient
(
'http://data-stage.globalchange.gov'
,
*
gcis_stage_auth
)
sync_metadata_tree
=
{
#Reports
...
...
@@ -20,17 +20,25 @@ sync_metadata_tree = {
(
'/metadata/figures/3293'
,
'observed-increase-in-frostfree-season-length'
),
# 2.10
(
'/metadata/figures/3294'
,
'projected-changes-in-frostfree-season-length'
),
# 2.11
(
'/metadata/figures/2677'
,
'observed-us-precipitation-change'
),
# 2.12
(
'/metadata/figures/3296'
,
'ann-max-precip-cons-dry-day'
),
# 2.13
(
'/metadata/figures/3297'
,
'projected-precipitation-change-by-season'
),
# 2.14
(
'/metadata/figures/2955'
,
'proj-precip-change'
),
# 2.15
(
'/metadata/figures/3298'
,
'observed-us-trend-in-heavy-precipitation'
),
# 2.16
(
'/metadata/figures/2506'
,
'observed-change-in-very-heavy-precipitation'
),
# 2.17
(
'/metadata/figures/2997'
,
'observed-change-in-very-heavy-precipitation-2'
),
# 2.18
(
'/metadata/figures/3170'
,
'global-temperature-and-carbon-dioxide'
),
# 2.2
(
'/metadata/figures/2939'
,
'projected-changes-in-soil-moisture-for-the-western-us'
),
# 2.22
(
'/metadata/figures/3304'
,
'observed-trends-in-hurricane-power-dissipation'
),
# 2.23
(
'/metadata/figures/3305'
,
'variation-of-storm-frequency-and-intensity-during-the-cold-season-november--march'
),
# 2.24
(
'/metadata/figures/3067'
,
'ice-cover-in-the-great-lakes'
),
# 2.27
(
'/metadata/figures/3073'
,
'projected-arctic-sea-ice-decline'
),
# 2.29
(
'/metadata/figures/2940'
,
'as-oceans-absorb-co-they-become-more-acidic'
),
# 2.30
(
'/metadata/figures/2523'
,
'shells-dissolve-in-acidified-ocean-water'
),
# 2.31
(
'/metadata/figures/3078'
,
'emissions-levels-determine-temperature-rises'
),
# 2.4
(
'/metadata/figures/3172'
,
'projected-change-in-average-annual-temperature'
),
#2.5
(
'/metadata/figures/3176'
,
'projected-change-in-average-annual-precipitation'
),
# 2.6
(
'/metadata/figures/3175'
,
'observed-us-temperature-change'
),
# 2.7
(
'/metadata/figures/3001'
,
'projected-temperature-change'
),
# 2.8
(
'/metadata/figures/3000'
,
'projected-temperature-change-by-20712099-cmip5-models'
),
#2.9
]),
...
...
@@ -46,8 +54,8 @@ sync_metadata_tree = {
]),
#Chapter 5
(
'transportation'
,
[
#
('/metadata/figures/3088', 'possible-future-flood-depths-in-mobile-al-with-rising-sea-level'), # 5.1
#
('/metadata/figures/2905', 'gulf-coast-transportation-hubs-at-risk'), # 5.3
(
'/metadata/figures/3088'
,
'possible-future-flood-depths-in-mobile-al-with-rising-sea-level'
),
# 5.1
(
'/metadata/figures/2905'
,
'gulf-coast-transportation-hubs-at-risk'
),
# 5.3
(
'/metadata/figures/3568'
,
'hurricane-sandy-causes-flooding-in-new-york-city-subway-stations'
),
# 5.4
(
'/metadata/figures/2952'
,
'role-of-adaptive-strategies-and-tactics-in-reducing-impacts-and-consequences'
),
# 5.5
(
'/metadata/figures/3169'
,
'tropical-storm-impact-on-vermont-road'
),
# 5.6
...
...
@@ -69,6 +77,7 @@ sync_metadata_tree = {
]),
#Chapter 8
(
'ecosystems'
,
[
(
'/metadata/figures/2576'
,
'the-aftermath-of-hurricanes'
),
# 8.2
(
'/metadata/figures/2456'
,
'adaptation-planning-and-implementation-framework'
),
# 8.3
(
'/metadata/figures/3574'
,
'biological-responses-to-climate-change'
),
# 8.4
]),
...
...
@@ -99,6 +108,7 @@ sync_metadata_tree = {
(
'/metadata/figures/3131'
,
'indigenous-populations-extend-beyond-reservation-lands'
),
# 12.1
(
'/metadata/figures/2594'
,
'many-tribes-many-climate-change-initiatives'
),
# 12.2
(
'/metadata/figures/2909'
,
'sand-dune-expansion'
),
# 12.3
(
'/metadata/figures/3572'
,
'sea-ice-cover-reaches-record-low'
),
# 12.4
(
'/metadata/figures/2911'
,
'arctic-marine-food-web'
),
# 12.5
]),
#Chapter 13
...
...
@@ -115,6 +125,7 @@ sync_metadata_tree = {
# Chapter 15
(
'biogeochemical-cycles'
,
[
(
'/metadata/figures/2874'
,
'major-north-american-carbon-dioxide-sources-and-sinks'
),
# 15.1
(
'/metadata/figures/2574'
,
'nitrogen-emissions'
),
# 15.3
(
'/metadata/figures/2575'
,
'many-factors-combine-to-affect-biogeochemical-cycles'
),
# 15.4
]),
#Chapter 16
...
...
@@ -149,9 +160,11 @@ sync_metadata_tree = {
(
'/metadata/figures/2990'
,
'projected-change-in-number-of-warm-nights'
),
# 19.3
(
'/metadata/figures/2991'
,
'projected-change-in-number-of-heavy-precipitation-days'
),
# 19.4
(
'/metadata/figures/3002'
,
'projected-change-in-number-of-consecutive-dry-days'
),
# 19.5
(
'/metadata/figures/3280'
,
'historical-and-current-range-of-sage-grouse-habitat'
)
#19.7
]),
#Chapter 20
(
'southwest'
,
[
(
'/metadata/figures/2568'
,
'coastal-risks-posed-by-sea-level-rise-and-high-tides'
),
#20.5
(
'/metadata/figures/3484'
,
'urban-heat-and-public-health'
),
# 20.6
]),
#Chapter 21
...
...
@@ -166,6 +179,7 @@ sync_metadata_tree = {
#Chapter 23
(
'hawaii'
,
[
(
'/metadata/figures/2834'
,
'us-pacific-islands-region'
),
# 23.1
(
'/metadata/figures/3283'
,
'high-and-low-pacific-islands-face-different-threats'
),
#23.2
(
'/metadata/figures/3284'
,
'increased-acidification-decreases-suitable-coral-habitat'
),
# 23.3
(
'/metadata/figures/2837'
,
'observed-changes-in-annual-rainfall-in-the-western-north-pacific'
),
# 23.4
(
'/metadata/figures/2838'
,
'native-plants-at-risk'
),
# 23.5
...
...
@@ -173,6 +187,12 @@ sync_metadata_tree = {
(
'/metadata/figures/2840'
,
'residents-of-lowlying-islands-at-risk'
),
# 23.7
(
'/metadata/figures/3286'
,
'higher-sea-level-rise-in-western-pacific'
),
# 23.8
]),
#Chapter 24
(
'oceans-marine-resources'
,
[
(
'/metadata/figures/3106'
,
'observed-ocean-warming'
),
# 24.1
(
'/metadata/figures/2560'
,
'ocean-acidification-reduces-size-of-clams'
),
#24.3
(
'/metadata/figures/2853'
,
'warming-seas-are-a-doubleblow-to-corals'
)
# 24.4
]),
#Chapter 25
(
'coastal-zone'
,
[
(
'/metadata/figures/2540'
,
'adapting-coastal-infrastructure-to-sea-level-rise-and-land-loss'
),
# 25.5
...
...
@@ -202,6 +222,7 @@ sync_metadata_tree = {
#Chapter 33: Climate Science Appendix
(
'appendix-climate-science-supplement'
,
[
(
'/metadata/figures/3112'
,
'indicators-of-warming-from-multiple-data-sets'
),
# 33.10
(
'/metadata/figures/3097'
,
'human-influence-on-the-greenhouse-effect'
),
#33.1
(
'/metadata/figures/3138'
,
'warming-trend-and-effects-of-el-nino-la-nina'
),
# 33.14
(
'/metadata/figures/2798'
,
'detection-and-attribution-as-forensics'
),
# 33.16
(
'/metadata/figures/3301'
,
'human-influences-apparent-in-many-aspects-of-the-changing-climate'
),
# 33.17
...
...
@@ -213,6 +234,7 @@ sync_metadata_tree = {
(
'/metadata/figures/2806'
,
'ice-cover-on-lake-mendota'
),
# 33.29
(
'/metadata/figures/2534'
,
'app-extreme-precipitation'
),
# 33.32
(
'/metadata/figures/2809'
,
'extreme-drought-in-the-us-and-mexico-past-and-future'
),
# 33.35
(
'/metadata/figures/3148'
,
'arctic-sea-ice-decline'
),
# 33.37
(
'/metadata/figures/2393'
,
'permafrost-temperatures-rising'
),
# 33.38
(
'/metadata/figures/3144'
,
'melting-of-arctic-landbased-ice'
),
# 33.39
(
'/metadata/figures/3145'
,
'melting-glaciers-lead-to-sea-level-rise'
),
# 33.40
...
...
@@ -223,6 +245,7 @@ sync_metadata_tree = {
#Chapter 34: FAQ/CAQ
(
'appendix-faqs'
,
[
(
'/metadata/figures/2663'
,
'us-annual-temperature'
),
# 34.1
(
'/metadata/figures/3096'
,
'caq-human-influence-on-the-greenhouse-effect'
),
# 34.11
(
'/metadata/figures/3300'
,
'caq-human-influences-apparent-in-many-aspects-of-the-changing-climate'
),
# 34.13
(
'/metadata/figures/3091'
,
'published-climate-change-research-papers'
),
# 34.18
(
'/metadata/figures/3075'
,
'caq-ten-indicators-of-a-warming-world'
),
# 34.2
...
...
@@ -235,11 +258,22 @@ sync_metadata_tree = {
])
}
images_with_multiple_figures
=
{
'e13cdf2e-0699-4c52-a642-56c8fc3317c4'
:
[
'human-influence-on-the-greenhouse-effect'
,
'caq-human-influence-on-the-greenhouse-effect'
],
#33.1 #34.11
'03937439-cc5a-45c7-a0fd-78b863c49979'
:
[
'human-influence-on-the-greenhouse-effect'
,
'caq-human-influence-on-the-greenhouse-effect'
],
#33.1 #34.11
'0bcb7984-50c0-4f94-9946-c3fa635125a4'
:
[
'ten-indicators-of-a-warming-world'
,
'caq-ten-indicators-of-a-warming-world'
],
#2.1 #34.2
'eb9bd7dc-3e3a-4001-9a64-f2a0763b3f64'
:
[
'shells-dissolve-in-acidified-ocean-water'
,
'ocean-acidification-and-the-food-web'
],
#2.31 #34.21
'8c679de0-4a6c-4fc9-bbd1-59251ffe3d7f'
:
[
'shells-dissolve-in-acidified-ocean-water'
,
'ocean-acidification-and-the-food-web'
],
#2.31 #34.21
'a1bd40f2-424e-4af8-8d3d-f44ba7ba36b6'
:
[
'human-influences-apparent-in-many-aspects-of-the-changing-climate'
,
'caq-human-influences-apparent-in-many-aspects-of-the-changing-climate'
],
#33.17 #34.13
'834f7624-40de-4a09-9ced-faaca30d4d6d'
:
[
'ice-cover-in-the-great-lakes'
,
'midwest-ice-cover-in-the-great-lakes'
],
#2.27 #18.7
}
def
main
():
print
gcis
.
test_login
()
# sync_dataset_metadata(gcis, webform.get_aggregated_datasets())
sync
(
replace
=
False
)
# sync(replace=True)
cleanup_after_replace
()
def
sync
(
replace
=
False
):
...
...
@@ -265,8 +299,20 @@ def sync(replace=False):
#Hack: override webform's title in favor of gcis
figure_obj
.
title
=
gcis_fig
.
title
if
gcis_fig
.
title
not
in
(
None
,
''
)
else
figure_obj
.
title
if
len
(
gcis_fig
.
contributors
)
>
1
:
print
'*** Multiple contributors: '
,
gcis_id
if
replace
:
for
image
in
figure_obj
.
images
:
#Hack: removing existing contributors to address orphans
for
c
in
gcis_fig
.
contributors
:
gcis
.
delete_contributor_figure_assoc
(
c
,
report_id
,
chapter_id
,
gcis_id
)
# #Remove existing parents
for
p
in
gcis_fig
.
parents
:
gcis
.
delete_figure_pub_assoc
(
report_id
,
gcis_id
,
p
)
for
image
in
gcis_fig
.
images
:
#TODO: There are better ways to do this. Build File support.
print
'Deleting {img}'
.
format
(
img
=
image
.
identifier
)
gcis
.
delete_image
(
image
)
...
...
@@ -281,5 +327,12 @@ def sync(replace=False):
count
+=
1
print
count
#Hack
def
cleanup_after_replace
():
for
image_id
in
images_with_multiple_figures
:
for
figure_id
in
images_with_multiple_figures
[
image_id
]:
print
gcis
.
associate_image_with_figure
(
image_id
,
'nca3'
,
figure_id
)
if
__name__
==
'__main__'
:
main
()
gcis_clients/__init__.py
View file @
ee68a6e3
from
gcis_client
import
GcisClient
from
gcis_client
import
GcisClient
,
AssociationException
from
webform_client
import
WebformClient
from
nca3_client
import
Nca3Client
...
...
gcis_clients/domain.py
View file @
ee68a6e3
...
...
@@ -416,8 +416,12 @@ class Organization(Gcisbase):
'Institute of the Environment'
:
'university-arizona'
,
'USGS'
:
'us-geological-survey'
,
'University of Puerto Rico'
:
'university-puerto-rico'
,
'University of Alaska'
:
'university-alaska'
'University of Alaska'
:
'university-alaska'
,
'U.S. Department of Agriculture'
:
'us-department-agriculture'
,
'Kiksapa Consulting'
:
'kiksapa-consulting-llc'
,
'Centers for Disease Control and Prevention'
:
'centers-disease-control-and-prevention'
,
'Pacific Northwest Laboratories'
:
'pacific-northwest-national-laboratory'
,
'Susanne Moser Research & Consulting'
:
'susanne-moser-research-consulting'
}
...
...
@@ -435,7 +439,7 @@ class Organization(Gcisbase):
class
Contributor
(
Gcisbase
):
def
__init__
(
self
,
data
):
self
.
gcis_fields
=
[
'role_type_identifier'
,
'organization_uri'
,
'uri'
,
'href'
,
'person_uri'
]
self
.
gcis_fields
=
[
'role_type_identifier'
,
'organization_uri'
,
'uri'
,
'href'
,
'person_uri'
,
'person_id'
,
'id'
]
#Hack
self
.
people_role_map
=
{
...
...
@@ -461,7 +465,14 @@ class Contributor(Gcisbase):
'Emily Cloyd'
:
'contributing_author'
,
'F. Chapin'
:
'scientist'
,
' Chapin'
:
'scientist'
,
'Andrew Buddenberg'
:
'analyst'
'Andrew Buddenberg'
:
'analyst'
,
'Jerry Hatfield'
:
'author'
,
'George Luber'
:
'lead_author'
,
'Kathy Hibbard'
:
'lead_author'
,
'Susanne Moser'
:
'convening_lead_author'
,
'Bull Bennett'
:
'convening_lead_author'
,
'Ernesto Weil'
:
'scientist'
,
'William Elder'
:
'scientist'
}
self
.
_role
=
None
...
...
@@ -527,7 +538,7 @@ class Parent(Gcisbase):
self
.
search_hints
=
{
'report'
:
{
'The State of the Climate 2009 Highlights'
:
'
usgcrp-globclimchhighlights
-2009'
,
'The State of the Climate 2009 Highlights'
:
'
noaa-stateofclim
-2009'
,
'Global Climate Change Impacts in the United States'
:
'nca2'
,
'Impacts of Climate Change and Variability on Transportation Systems and Infrastructure: Gulf Study, Phase I.'
:
'ccsp-sap-4_7-2008'
,
'Climate and Energy-Water-Land System Interactions'
:
'pnnl-21185'
,
...
...
@@ -546,7 +557,7 @@ class Parent(Gcisbase):
'How to Avoid Dangerous Climate Change. A Target for U.S. Emissions Reductions'
:
'ucusa-howtoavoid-2007'
,
'Summary for Decision Makers. Assessment of Climate Change in the Southwest United States'
:
'swccar-assessment-climate-change-in-southwest-us'
,
'Climate Variability and Change in Mobile, Alabama: Task 2 Final Report. Impacts of Climate 25 Change and Variability on Transpo'
:
'fhwa-hep-12-053'
,
'Effects of Climatic Variability and Change on Forest Ecosystems: A Comprehensive Science Synthesis for the U.S. Forest Sector'
:
'usfs-
gtr-nrs
-87'
,
'Effects of Climatic Variability and Change on Forest Ecosystems: A Comprehensive Science Synthesis for the U.S. Forest Sector'
:
'usfs-
pnw-gtr
-87
0
'
,
'Future of America
\'
s Forests and Rangelands Forest Service. 2010 Resources Planning Act Assessment'
:
'usfs-gtr-wo-87'
...
...
@@ -566,7 +577,9 @@ class Parent(Gcisbase):
'Ecosystem Services Analysis of Climate Change and Urban Growth in the Upper Santa Cruz Watershed: SCWEPM'
:
'd4622f7e-aca7-42e6-95da-90579a187c30'
,
'State and Local Climate Adaptation'
:
'7de6bfc9-55fa-4d12-ae80-486561b3802c'
,
'Climate Change Response Framework - Northwoods'
:
'267378f7-278b-4201-8ffa-a820f5a694b8'
,
'NWHI Maps and Publications'
:
'e6438f11-85f4-4c29-abb5-b09efa3279b2'
'NWHI Maps and Publications'
:
'e6438f11-85f4-4c29-abb5-b09efa3279b2'
,
'The Cryosphere Today Compare Daily Sea Ice'
:
'e4a9eb6a-9421-42c3-94b1-47caf588d41d'
,
'NASA Earth Observatory Visualizing the Sea Ice Minimum'
:
'71b4c19e-42da-4f15-99d2-7c7746d8eaf2'
},
...
...
@@ -577,7 +590,9 @@ class Parent(Gcisbase):
'An update on Earth
\'
s energy balance in light of the latest global observations'
:
'10.1038/ngeo1580'
,
'About the Lack of Warming...'
:
(
'web_page'
,
'e2ec2d0f-430c-4032-a309-2514ca1f6572'
),
'The Myth of the 1970s Global Cooling Scientific Consensus'
:
'10.1175/2008BAMS2370.1'
,
'Hurricane Sandy devestates NY/NJ-area passenger rai systems'
:
(
'web_page'
,
'135ae7d9-56e3-4dcb-a81c-42a6f1e9b332'
)
'Hurricane Sandy devestates NY/NJ-area passenger rai systems'
:
(
'web_page'
,
'135ae7d9-56e3-4dcb-a81c-42a6f1e9b332'
),
'Climate change impacts of US reactive nitrogen'
:
'10.1073/pnas.1114243109'
,
'Range-wide patterns of greater sage-grouse persistence'
:
'10.1111/j.1472-4642.2008.00502.x'
},
'book'
:
{
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment