diff options
| -rw-r--r-- | scraper/s2-fetch-doi.py | 6 |
1 files changed, 3 insertions, 3 deletions
diff --git a/scraper/s2-fetch-doi.py b/scraper/s2-fetch-doi.py index ae80036e..5e98d898 100644 --- a/scraper/s2-fetch-doi.py +++ b/scraper/s2-fetch-doi.py @@ -17,12 +17,12 @@ s2 = SemanticScholarAPI() @click.option('--fn', '-i', default='db_paper_doi.csv', help='Filename of CSV (id, url,)') def fetch_doi_list(fn): lines = read_csv(fn, keys=False) - domains = [] + urls = [] for line in lines: paper_id, url = line if url: - domain = fetch_doi(paper_id, url) - print(domain) + urls.append(line) + parallelize(fetch_doi, urls) print("{} papers processed".format(len(lines))) def fetch_doi(paper_id, url, replace=False): |