diff options
Diffstat (limited to 'scraper/s2-papers.py')
| -rw-r--r-- | scraper/s2-papers.py | 10 |
1 files changed, 8 insertions, 2 deletions
diff --git a/scraper/s2-papers.py b/scraper/s2-papers.py index 782dc198..ef7d3d81 100644 --- a/scraper/s2-papers.py +++ b/scraper/s2-papers.py @@ -9,11 +9,15 @@ import operator import click from s2 import SemanticScholarAPI from util import * +from urllib.parse import unquote +import importlib +raw_papers_api = importlib.import_module('s2-raw-papers') s2 = SemanticScholarAPI() @click.command() -def fetch_papers(): +@click.option('--freshen/--no-freshen', '-f', help='Force it to query the paper API again') +def fetch_papers(freshen): addresses = AddressBook() lookup_keys, lines = fetch_google_sheet('citation_lookup') report_keys = [ @@ -30,9 +34,11 @@ def fetch_papers(): paper_id = line[3] if paper_id == '': continue - paper = fetch_paper(s2, paper_id) + paper = fetch_paper(s2, paper_id, freshen) if paper is None: continue + if freshen: + raw_papers_api.fetch_raw_paper(paper_id, freshen) db_paper = load_paper(paper_id) pdf_link = db_paper.pdf_link if db_paper else "" |