From 8f0d59a5f44c71aeb4eecf60cb323d2fe0306a3e Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Fri, 8 Mar 2019 21:02:58 +0100 Subject: possibly freshen raw papers --- scraper/s2-geocode-spreadsheet.py | 4 ++++ scraper/s2-papers.py | 5 +++-- scraper/util.py | 4 ++-- 3 files changed, 9 insertions(+), 4 deletions(-) (limited to 'scraper') diff --git a/scraper/s2-geocode-spreadsheet.py b/scraper/s2-geocode-spreadsheet.py index 32d7c669..c36625a6 100644 --- a/scraper/s2-geocode-spreadsheet.py +++ b/scraper/s2-geocode-spreadsheet.py @@ -110,6 +110,8 @@ def update_country_from_address(address, i, countries, worksheet): country = None if possible_country in countries: country = countries[possible_country] + elif "CHINA" in address: + country = "China" elif "China" in address: country = "China" elif "Hong Kong" in address: @@ -118,6 +120,8 @@ def update_country_from_address(address, i, countries, worksheet): country = "Singapore" elif "Taiwan" in address: country = "Taiwan" + elif "Saudi Arabia" in address: + country = "Saudi Arabia" elif "Russia" in address: country = "Russia" elif "Ukraine" in address: diff --git a/scraper/s2-papers.py b/scraper/s2-papers.py index 782dc198..40489e75 100644 --- a/scraper/s2-papers.py +++ b/scraper/s2-papers.py @@ -13,7 +13,8 @@ from util import * s2 = SemanticScholarAPI() @click.command() -def fetch_papers(): +@click.option('--freshen/--no-freshen', '-f', help='Force it to query the paper API again') +def fetch_papers(freshen): addresses = AddressBook() lookup_keys, lines = fetch_google_sheet('citation_lookup') report_keys = [ @@ -30,7 +31,7 @@ def fetch_papers(): paper_id = line[3] if paper_id == '': continue - paper = fetch_paper(s2, paper_id) + paper = fetch_paper(s2, paper_id, freshen) if paper is None: continue db_paper = load_paper(paper_id) diff --git a/scraper/util.py b/scraper/util.py index fdbc0534..830dbe8b 100644 --- a/scraper/util.py +++ b/scraper/util.py @@ -386,10 +386,10 @@ def parallelize(func, rows): with Pool(processes=processCount) as pool: pool.starmap(func, rows, chunksize) -def fetch_paper(s2, paper_id): +def fetch_paper(s2, paper_id, freshen=False): os.makedirs('./datasets/s2/papers/{}/{}'.format(paper_id[0:2], paper_id), exist_ok=True) paper_fn = './datasets/s2/papers/{}/{}/paper.json'.format(paper_id[0:2], paper_id) - if os.path.exists(paper_fn): + if os.path.exists(paper_fn) and not freshen: return read_json(paper_fn) print(paper_id) paper = s2.paper(paper_id) -- cgit v1.2.3-70-g09d2 From faa59f192c00dadd84165fe14a90307c908e6ab2 Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Fri, 8 Mar 2019 21:05:03 +0100 Subject: possibly freshen raw papers --- scraper/s2-papers.py | 4 ++++ scraper/s2-raw-papers.py | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) (limited to 'scraper') diff --git a/scraper/s2-papers.py b/scraper/s2-papers.py index 40489e75..92bf8ebf 100644 --- a/scraper/s2-papers.py +++ b/scraper/s2-papers.py @@ -9,6 +9,8 @@ import operator import click from s2 import SemanticScholarAPI from util import * +from urllib.parse import unquote +raw_papers_api = import_module('s2-raw-papers') s2 = SemanticScholarAPI() @@ -34,6 +36,8 @@ def fetch_papers(freshen): paper = fetch_paper(s2, paper_id, freshen) if paper is None: continue + if freshen: + raw_papers_api.fetch_raw_paper(paper_id, freshen) db_paper = load_paper(paper_id) pdf_link = db_paper.pdf_link if db_paper else "" diff --git a/scraper/s2-raw-papers.py b/scraper/s2-raw-papers.py index 612c8099..8881cda0 100644 --- a/scraper/s2-raw-papers.py +++ b/scraper/s2-raw-papers.py @@ -18,10 +18,10 @@ def fetch_raw_papers(fn): lines = read_csv(fn, keys=False) parallelize(fetch_raw_paper, lines) -def fetch_raw_paper(paper_id): +def fetch_raw_paper(paper_id, freshen=False): os.makedirs(make_raw_paper_path(paper_id), exist_ok=True) paper_fn = make_raw_paper_fn(paper_id) - if os.path.exists(paper_fn): + if os.path.exists(paper_fn) and not freshen: paper = read_json(paper_fn) else: paper = s2.raw_paper(paper_id) -- cgit v1.2.3-70-g09d2 From 8396d4b3af07dc84c4243c3ee131750759ee5324 Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Fri, 8 Mar 2019 21:07:12 +0100 Subject: possibly freshen raw papers --- scraper/s2-doi-report.py | 2 +- scraper/s2-papers.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'scraper') diff --git a/scraper/s2-doi-report.py b/scraper/s2-doi-report.py index 1d7bf44a..ea708de2 100644 --- a/scraper/s2-doi-report.py +++ b/scraper/s2-doi-report.py @@ -7,8 +7,8 @@ import click import operator from util import * from bs4 import BeautifulSoup -from importlib import import_module from urllib.parse import unquote +from importlib import import_module doi = import_module('s2-fetch-doi') DOI_DIR = 'datasets/s2/doi' diff --git a/scraper/s2-papers.py b/scraper/s2-papers.py index 92bf8ebf..ef7d3d81 100644 --- a/scraper/s2-papers.py +++ b/scraper/s2-papers.py @@ -10,7 +10,8 @@ import click from s2 import SemanticScholarAPI from util import * from urllib.parse import unquote -raw_papers_api = import_module('s2-raw-papers') +import importlib +raw_papers_api = importlib.import_module('s2-raw-papers') s2 = SemanticScholarAPI() -- cgit v1.2.3-70-g09d2 From c0d78dc072bc9f9b6e373f49b74d98d393d0e96a Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Fri, 8 Mar 2019 21:08:13 +0100 Subject: possibly freshen raw papers --- scraper/s2-raw-papers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scraper') diff --git a/scraper/s2-raw-papers.py b/scraper/s2-raw-papers.py index 8881cda0..67ff0b65 100644 --- a/scraper/s2-raw-papers.py +++ b/scraper/s2-raw-papers.py @@ -33,7 +33,7 @@ def fetch_raw_paper(paper_id, freshen=False): if 'responseType' in paper and paper['responseType'] == 'CANONICAL': write_json(paper_fn, paper) - paper = s2.raw_paper(data['canonicalId']) + paper = s2.raw_paper(paper['canonicalId']) paper_fn = make_raw_paper_fn(paper_id) return paper -- cgit v1.2.3-70-g09d2