diff options
| -rw-r--r-- | scraper/s2-raw-papers.py | 18 |
1 files changed, 12 insertions, 6 deletions
diff --git a/scraper/s2-raw-papers.py b/scraper/s2-raw-papers.py index 089055da..612c8099 100644 --- a/scraper/s2-raw-papers.py +++ b/scraper/s2-raw-papers.py @@ -16,21 +16,27 @@ s2 = SemanticScholarAPI() @click.option('--fn', '-i', default='missing.csv', help='Filename of CSV (id,)') def fetch_raw_papers(fn): lines = read_csv(fn, keys=False) - for line in lines: - paper_id = line[0] - fetch_raw_paper(paper_id) + parallelize(fetch_raw_paper, lines) def fetch_raw_paper(paper_id): os.makedirs(make_raw_paper_path(paper_id), exist_ok=True) paper_fn = make_raw_paper_fn(paper_id) if os.path.exists(paper_fn): - return read_json(paper_fn) - print(paper_id) - paper = s2.raw_paper(paper_id) + paper = read_json(paper_fn) + else: + paper = s2.raw_paper(paper_id) + if paper is None: print("Got empty paper?? {}".format(paper_id)) # time.sleep(random.randint(5, 10)) return None + + if 'responseType' in paper and paper['responseType'] == 'CANONICAL': + write_json(paper_fn, paper) + paper = s2.raw_paper(data['canonicalId']) + paper_fn = make_raw_paper_fn(paper_id) + return paper + write_json(paper_fn, paper) # time.sleep(random.randint(5, 10)) return paper |