Merge branch 'master' of asdf.us:megapixels_dev

1 files changed, 12 insertions, 6 deletions

diff --git a/scraper/s2-raw-papers.py b/scraper/s2-raw-papers.py
index 089055da..612c8099 100644
--- a/scraper/s2-raw-papers.py
+++ b/scraper/s2-raw-papers.py
@@ -16,21 +16,27 @@ s2 = SemanticScholarAPI()
 @click.option('--fn', '-i', default='missing.csv', help='Filename of CSV (id,)')
 def fetch_raw_papers(fn):
   lines = read_csv(fn, keys=False)
-  for line in lines:
-    paper_id = line[0]
-    fetch_raw_paper(paper_id)
+  parallelize(fetch_raw_paper, lines)
 
 def fetch_raw_paper(paper_id):
   os.makedirs(make_raw_paper_path(paper_id), exist_ok=True)
   paper_fn = make_raw_paper_fn(paper_id)
   if os.path.exists(paper_fn):
-    return read_json(paper_fn)
-  print(paper_id)
-  paper = s2.raw_paper(paper_id)
+    paper = read_json(paper_fn)
+  else:
+    paper = s2.raw_paper(paper_id)
+
   if paper is None:
     print("Got empty paper?? {}".format(paper_id))
     # time.sleep(random.randint(5, 10))
     return None  
+
+  if 'responseType' in paper and paper['responseType'] == 'CANONICAL':
+    write_json(paper_fn, paper)
+    paper = s2.raw_paper(data['canonicalId'])
+    paper_fn = make_raw_paper_fn(paper_id)
+    return paper
+
   write_json(paper_fn, paper)
   # time.sleep(random.randint(5, 10))
   return paper