diff options
Diffstat (limited to 'scraper/s2-fetch-pdf.py')
| -rw-r--r-- | scraper/s2-fetch-pdf.py | 7 |
1 files changed, 2 insertions, 5 deletions
diff --git a/scraper/s2-fetch-pdf.py b/scraper/s2-fetch-pdf.py index 30bc5a40..72ca4ca8 100644 --- a/scraper/s2-fetch-pdf.py +++ b/scraper/s2-fetch-pdf.py @@ -7,6 +7,7 @@ import random import re import simplejson as json import click +from multiprocessing import Pool from s2 import SemanticScholarAPI from util import * @@ -16,9 +17,7 @@ s2 = SemanticScholarAPI() @click.option('--fn', '-i', default='db_paper_pdf.csv', help='Filename of CSV (id, url,)') def fetch_pdfs(fn): lines = read_csv(fn, keys=False) - for line in lines: - paper_id, url = line - fetch_pdf(paper_id, url) + parallelize(fetch_pdf, lines) print("{} papers processed".format(len(lines))) def fetch_pdf(paper_id, url): @@ -33,8 +32,6 @@ def fetch_pdf(paper_id, url): print("{} empty?".format(paper_id)) return None print("{} {} kb".format(paper_id, int(size / 1024))) - return - # return paper def make_pdf_path(paper_id): return './datasets/s2/pdf/{}/{}'.format(paper_id[0:2], paper_id) |