diff options
| author | Jules Laplace <julescarbon@gmail.com> | 2019-02-10 17:15:58 +0100 |
|---|---|---|
| committer | Jules Laplace <julescarbon@gmail.com> | 2019-02-10 17:15:58 +0100 |
| commit | 7fe9d36385a2350c9c4fd5dec105e34d85fe637a (patch) | |
| tree | 313db438bed35e4763df66df6adc89bed85e6616 /scraper/s2-fetch-doi.py | |
| parent | 478903401f480f2284f8f534ac5ef9b99fdc0b1c (diff) | |
parallelize doi
Diffstat (limited to 'scraper/s2-fetch-doi.py')
| -rw-r--r-- | scraper/s2-fetch-doi.py | 6 |
1 files changed, 3 insertions, 3 deletions
diff --git a/scraper/s2-fetch-doi.py b/scraper/s2-fetch-doi.py index ae80036e..5e98d898 100644 --- a/scraper/s2-fetch-doi.py +++ b/scraper/s2-fetch-doi.py @@ -17,12 +17,12 @@ s2 = SemanticScholarAPI() @click.option('--fn', '-i', default='db_paper_doi.csv', help='Filename of CSV (id, url,)') def fetch_doi_list(fn): lines = read_csv(fn, keys=False) - domains = [] + urls = [] for line in lines: paper_id, url = line if url: - domain = fetch_doi(paper_id, url) - print(domain) + urls.append(line) + parallelize(fetch_doi, urls) print("{} papers processed".format(len(lines))) def fetch_doi(paper_id, url, replace=False): |