Merge branch 'master' of github.com:adamhrv/megapixels_dev

1 files changed, 15 insertions, 1 deletions

diff --git a/scraper/util.py b/scraper/util.py
index 05b01fa7..7febf86f 100644
--- a/scraper/util.py
+++ b/scraper/util.py
@@ -67,12 +67,20 @@ def read_text(fn):
     return f.read()
 
 def read_json(fn):
+  #try:
   with open(fn, 'r') as json_file:
     return json.load(json_file)
+  #except:
+  #  print("ERROR READING: {}".format(fn))
+  #  return {}
 
 def write_json(fn, data):
+  #try:
   with open(fn, 'w') as outfile:
     json.dump(data, outfile)
+  #except:
+  #  print("ERROR WRITING: {}".format(fn))
+  #  return {}
 
 def write_report(fn, title=None, keys=None, rows=[]):
   with open(fn, 'w') as f:
@@ -459,15 +467,21 @@ def fetch_verified_paper_lookup():
   """Fetch a lookup keyed by dataset, where each dataset points to a hash of valid or invalid papers..."""
   keys, rows = fetch_google_sheet('verifications')
   verified_lookup = {}
+  verified_totals = {}
   for row in rows:
     rec = {}
     for index, key in enumerate(keys):
       rec[key] = row[index]
     if rec['dataset'] not in verified_lookup:
       verified_lookup[rec['dataset']] = {}
+      verified_totals[rec['dataset']] = { 'yes': 0, 'no': 0, 'total': 0 }
     if str(rec['uses_dataset']) == '1':
       verified_lookup[rec['dataset']][rec['paper_id']] = rec
-  return verified_lookup
+      verified_totals[rec['dataset']]['yes'] += 1
+    else:
+      verified_totals[rec['dataset']]['no'] += 1
+    verified_totals[rec['dataset']]['total'] += 1
+  return verified_lookup, verified_totals
 
 def update_or_append_worksheet(name, form):
   worksheet = fetch_worksheet(name)