From 110f3a34f1f36d0ea999d4aa34bbe66d5f2a01da Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Sun, 16 Dec 2018 15:02:59 +0100 Subject: skip empty, pull citations again --- scraper/util.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) (limited to 'scraper/util.py') diff --git a/scraper/util.py b/scraper/util.py index a435f91a..c02f018c 100644 --- a/scraper/util.py +++ b/scraper/util.py @@ -259,7 +259,7 @@ class AddressBook (object): def __init__(self): entities = {} lookup = {} - data = fetch_google_sheet() + keys, data = fetch_google_sheet() # keys, data = read_csv('reports/pdf_institutions_deduped.csv', keys=True) for index, line in enumerate(data): if line[0] == line[1] or line[0] not in entities: @@ -283,16 +283,20 @@ class AddressBook (object): return self.data[index] return None -def fetch_worksheet(): +def fetch_spreadsheet(): scope = ['https://spreadsheets.google.com/feeds','https://www.googleapis.com/auth/drive'] credentials = ServiceAccountCredentials.from_json_keyfile_name('./.creds/Megapixels-ef28f91112a9.json', scope) docid = "1denb7TjYsN9igHyvYah7fQ0daABW32Z30lwV7QrDJQc" client = gspread.authorize(credentials) spreadsheet = client.open_by_key(docid) - return spreadsheet.worksheet("institutions") + return spreadsheet -def fetch_google_sheet(): - rows = fetch_worksheet().get_all_values() +def fetch_worksheet(name="institutions"): + spreadsheet = fetch_spreadsheet() + return spreadsheet.worksheet(name) + +def fetch_google_sheet(name="institutions"): + rows = fetch_worksheet(name).get_all_values() keys = rows[0] lines = rows[1:] - return lines + return keys, lines -- cgit v1.2.3-70-g09d2