From 7fe9d36385a2350c9c4fd5dec105e34d85fe637a Mon Sep 17 00:00:00 2001
From: Jules Laplace <julescarbon@gmail.com>
Date: Sun, 10 Feb 2019 17:15:58 +0100
Subject: parallelize doi

---
 scraper/s2-fetch-doi.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'scraper/s2-fetch-doi.py')

diff --git a/scraper/s2-fetch-doi.py b/scraper/s2-fetch-doi.py
index ae80036e..5e98d898 100644
--- a/scraper/s2-fetch-doi.py
+++ b/scraper/s2-fetch-doi.py
@@ -17,12 +17,12 @@ s2 = SemanticScholarAPI()
 @click.option('--fn', '-i', default='db_paper_doi.csv', help='Filename of CSV (id, url,)')
 def fetch_doi_list(fn):
   lines = read_csv(fn, keys=False)
-  domains = []
+  urls = []
   for line in lines:
     paper_id, url = line
     if url:
-      domain = fetch_doi(paper_id, url)
-      print(domain)
+      urls.append(line)
+  parallelize(fetch_doi, urls)
   print("{} papers processed".format(len(lines)))
 
 def fetch_doi(paper_id, url, replace=False):
-- 
cgit v1.2.3-70-g09d2