Merge branch 'master' of github.com:adamhrv/megapixels_dev

1 files changed, 8 insertions, 2 deletions

diff --git a/scraper/s2-papers.py b/scraper/s2-papers.py
index 782dc198..ef7d3d81 100644
--- a/scraper/s2-papers.py
+++ b/scraper/s2-papers.py
@@ -9,11 +9,15 @@ import operator
 import click
 from s2 import SemanticScholarAPI
 from util import *
+from urllib.parse import unquote
+import importlib
+raw_papers_api = importlib.import_module('s2-raw-papers')
 
 s2 = SemanticScholarAPI()
 
 @click.command()
-def fetch_papers():
+@click.option('--freshen/--no-freshen', '-f', help='Force it to query the paper API again')
+def fetch_papers(freshen):
   addresses = AddressBook()
   lookup_keys, lines = fetch_google_sheet('citation_lookup')
   report_keys = [
@@ -30,9 +34,11 @@ def fetch_papers():
     paper_id = line[3]
     if paper_id == '':
       continue
-    paper = fetch_paper(s2, paper_id)
+    paper = fetch_paper(s2, paper_id, freshen)
     if paper is None:
       continue
+    if freshen:
+      raw_papers_api.fetch_raw_paper(paper_id, freshen)
     db_paper = load_paper(paper_id)
     pdf_link = db_paper.pdf_link if db_paper else ""