From 581bcd92f116a24e3aa67af4e0771f10b11440d4 Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Sat, 16 Feb 2019 14:33:29 +0100 Subject: fetching more raw papers maybe --- scraper/s2-raw-papers.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'scraper/s2-raw-papers.py') diff --git a/scraper/s2-raw-papers.py b/scraper/s2-raw-papers.py index 089055da..8941d60d 100644 --- a/scraper/s2-raw-papers.py +++ b/scraper/s2-raw-papers.py @@ -16,9 +16,7 @@ s2 = SemanticScholarAPI() @click.option('--fn', '-i', default='missing.csv', help='Filename of CSV (id,)') def fetch_raw_papers(fn): lines = read_csv(fn, keys=False) - for line in lines: - paper_id = line[0] - fetch_raw_paper(paper_id) + parallelize(fetch_raw_paper, lines) def fetch_raw_paper(paper_id): os.makedirs(make_raw_paper_path(paper_id), exist_ok=True) @@ -27,10 +25,18 @@ def fetch_raw_paper(paper_id): return read_json(paper_fn) print(paper_id) paper = s2.raw_paper(paper_id) + if paper is None: print("Got empty paper?? {}".format(paper_id)) # time.sleep(random.randint(5, 10)) return None + + if 'responseType' in paper and paper['responseType'] == 'CANONICAL': + write_json(paper_fn, paper) + paper = s2.raw_paper(data['canonicalId']) + paper_fn = make_raw_paper_fn(paper_id) + return paper + write_json(paper_fn, paper) # time.sleep(random.randint(5, 10)) return paper -- cgit v1.2.3-70-g09d2