summaryrefslogtreecommitdiff
path: root/scraper/util.py
diff options
context:
space:
mode:
authorJules Laplace <julescarbon@gmail.com>2019-02-20 12:47:44 +0100
committerJules Laplace <julescarbon@gmail.com>2019-02-20 12:47:44 +0100
commit1bbee45bcb178fa47e60910c76bd914c43e7fd90 (patch)
tree3ba46b911d507cadaf53c9052ac9b6756ea7c3ec /scraper/util.py
parentbf7eecfc2c1879f237d67f6bedf556b51a034d6a (diff)
commenting
Diffstat (limited to 'scraper/util.py')
-rw-r--r--scraper/util.py9
1 files changed, 9 insertions, 0 deletions
diff --git a/scraper/util.py b/scraper/util.py
index 0c3e2169..fa9f6a22 100644
--- a/scraper/util.py
+++ b/scraper/util.py
@@ -400,6 +400,7 @@ def fetch_paper(s2, paper_id):
return paper
def fetch_spreadsheet():
+ """Open the Google Spreadsheet, which contains the individual worksheets"""
scope = ['https://spreadsheets.google.com/feeds','https://www.googleapis.com/auth/drive']
path = os.path.dirname(os.path.abspath(__file__))
credentials = ServiceAccountCredentials.from_json_keyfile_name(os.path.join(path, '.creds/Megapixels-ef28f91112a9.json'), scope)
@@ -409,16 +410,22 @@ def fetch_spreadsheet():
return spreadsheet
def fetch_worksheet(name="institutions"):
+ """Get a reference to a particular "worksheet" from the Google Spreadsheet"""
spreadsheet = fetch_spreadsheet()
return spreadsheet.worksheet(name)
def fetch_google_sheet(name="institutions"):
+ """Get all the values from a particular worksheet as a list of lists.
+ Returns:
+ :keys - the first row of the document
+ :lines - a list of lists with the rest of the rows"""
rows = fetch_worksheet(name).get_all_values()
keys = rows[0]
lines = rows[1:]
return keys, lines
def fetch_google_sheet_objects(name):
+ """Get all the values from a worksheet as a list of dictionaries"""
keys, rows = fetch_google_sheet(name)
recs = []
for row in rows:
@@ -429,6 +436,8 @@ def fetch_google_sheet_objects(name):
return recs
def fetch_google_lookup(name, item_key='key'):
+ """Get all the values from a worksheet as a dictionary of dictionaries.
+ Specify which field you want to use as the dictionary key."""
keys, rows = fetch_google_sheet(name)
lookup = {}
for row in rows: