parallelize fetch pdf

1 files changed, 2 insertions, 5 deletions

diff --git a/scraper/s2-fetch-pdf.py b/scraper/s2-fetch-pdf.py
index 30bc5a40..72ca4ca8 100644
--- a/scraper/s2-fetch-pdf.py
+++ b/scraper/s2-fetch-pdf.py
@@ -7,6 +7,7 @@ import random
 import re
 import simplejson as json
 import click
+from multiprocessing import Pool
 from s2 import SemanticScholarAPI
 from util import *
 
@@ -16,9 +17,7 @@ s2 = SemanticScholarAPI()
 @click.option('--fn', '-i', default='db_paper_pdf.csv', help='Filename of CSV (id, url,)')
 def fetch_pdfs(fn):
   lines = read_csv(fn, keys=False)
-  for line in lines:
-    paper_id, url = line
-    fetch_pdf(paper_id, url)
+  parallelize(fetch_pdf, lines)
   print("{} papers processed".format(len(lines)))
 
 def fetch_pdf(paper_id, url):
@@ -33,8 +32,6 @@ def fetch_pdf(paper_id, url):
     print("{} empty?".format(paper_id))
     return None  
   print("{} {} kb".format(paper_id, int(size / 1024)))
-  return
-  # return paper
 
 def make_pdf_path(paper_id):
   return './datasets/s2/pdf/{}/{}'.format(paper_id[0:2], paper_id)