fetching verified papers

1 files changed, 8 insertions, 4 deletions

diff --git a/scraper/s2-dump-missing-paper-ids.py b/scraper/s2-dump-missing-paper-ids.py
index 47dd4238..6f7eb8ba 100644
--- a/scraper/s2-dump-missing-paper-ids.py
+++ b/scraper/s2-dump-missing-paper-ids.py
@@ -27,7 +27,7 @@ def load_missing_ids(fn):
       lookup[paper_id] = True
       found_count += 1
     else:
-      print(">> {} {}".format(dataset paper_id))
+      # print(">> {} {}".format(dataset, paper_id))
       missing_lookup[paper_id] = True
       missing_count += 1
 
@@ -35,9 +35,13 @@ def load_missing_ids(fn):
   rows = []
   for dataset, lookup in verified_lookup.items():
     for paper_id in lookup.keys():
-      paper_path = data_path('raw_papers', paper_id):
-      if not os.path.exists(paper_path):
-        print(">> {} {}".format(dataset paper_id))
+      if dataset == 'brainwash':
+        print('>> {} {}'.format(dataset, paper_id))
+      paper_path = make_raw_paper_path(paper_id)
+      if not os.path.exists(paper_path) and paper_id not in missing_lookup:
+        print(">> {} {}".format(dataset, paper_id))
+        missing_count += 1
+        missing_lookup[paper_id] = True
 
   print("{} papers found, {} must be fetched".format(found_count, missing_count))
   return missing_lookup.keys()