export totals and look for censorship

1 files changed, 7 insertions, 1 deletions

diff --git a/scraper/util.py b/scraper/util.py
index c7e18b44..7febf86f 100644
--- a/scraper/util.py
+++ b/scraper/util.py
@@ -467,15 +467,21 @@ def fetch_verified_paper_lookup():
   """Fetch a lookup keyed by dataset, where each dataset points to a hash of valid or invalid papers..."""
   keys, rows = fetch_google_sheet('verifications')
   verified_lookup = {}
+  verified_totals = {}
   for row in rows:
     rec = {}
     for index, key in enumerate(keys):
       rec[key] = row[index]
     if rec['dataset'] not in verified_lookup:
       verified_lookup[rec['dataset']] = {}
+      verified_totals[rec['dataset']] = { 'yes': 0, 'no': 0, 'total': 0 }
     if str(rec['uses_dataset']) == '1':
       verified_lookup[rec['dataset']][rec['paper_id']] = rec
-  return verified_lookup
+      verified_totals[rec['dataset']]['yes'] += 1
+    else:
+      verified_totals[rec['dataset']]['no'] += 1
+    verified_totals[rec['dataset']]['total'] += 1
+  return verified_lookup, verified_totals
 
 def update_or_append_worksheet(name, form):
   worksheet = fetch_worksheet(name)