fetching more raw papers maybe

1 files changed, 9 insertions, 3 deletions

diff --git a/scraper/s2-raw-papers.py b/scraper/s2-raw-papers.py
index 089055da..8941d60d 100644
--- a/scraper/s2-raw-papers.py
+++ b/scraper/s2-raw-papers.py
@@ -16,9 +16,7 @@ s2 = SemanticScholarAPI()
 @click.option('--fn', '-i', default='missing.csv', help='Filename of CSV (id,)')
 def fetch_raw_papers(fn):
   lines = read_csv(fn, keys=False)
-  for line in lines:
-    paper_id = line[0]
-    fetch_raw_paper(paper_id)
+  parallelize(fetch_raw_paper, lines)
 
 def fetch_raw_paper(paper_id):
   os.makedirs(make_raw_paper_path(paper_id), exist_ok=True)
@@ -27,10 +25,18 @@ def fetch_raw_paper(paper_id):
     return read_json(paper_fn)
   print(paper_id)
   paper = s2.raw_paper(paper_id)
+
   if paper is None:
     print("Got empty paper?? {}".format(paper_id))
     # time.sleep(random.randint(5, 10))
     return None  
+
+  if 'responseType' in paper and paper['responseType'] == 'CANONICAL':
+    write_json(paper_fn, paper)
+    paper = s2.raw_paper(data['canonicalId'])
+    paper_fn = make_raw_paper_fn(paper_id)
+    return paper
+
   write_json(paper_fn, paper)
   # time.sleep(random.randint(5, 10))
   return paper