possibly freshen raw papers

3 files changed, 9 insertions, 4 deletions

diff --git a/scraper/s2-geocode-spreadsheet.py b/scraper/s2-geocode-spreadsheet.py
index 32d7c669..c36625a6 100644
--- a/scraper/s2-geocode-spreadsheet.py
+++ b/scraper/s2-geocode-spreadsheet.py
@@ -110,6 +110,8 @@ def update_country_from_address(address, i, countries, worksheet):
   country = None
   if possible_country in countries:
     country = countries[possible_country]
+  elif "CHINA" in address:
+    country = "China"
   elif "China" in address:
     country = "China"
   elif "Hong Kong" in address:
@@ -118,6 +120,8 @@ def update_country_from_address(address, i, countries, worksheet):
     country = "Singapore"
   elif "Taiwan" in address:
     country = "Taiwan"
+  elif "Saudi Arabia" in address:
+    country = "Saudi Arabia"
   elif "Russia" in address:
     country = "Russia"
   elif "Ukraine" in address:
diff --git a/scraper/s2-papers.py b/scraper/s2-papers.py
index 782dc198..40489e75 100644
--- a/scraper/s2-papers.py
+++ b/scraper/s2-papers.py
@@ -13,7 +13,8 @@ from util import *
 s2 = SemanticScholarAPI()
 
 @click.command()
-def fetch_papers():
+@click.option('--freshen/--no-freshen', '-f', help='Force it to query the paper API again')
+def fetch_papers(freshen):
   addresses = AddressBook()
   lookup_keys, lines = fetch_google_sheet('citation_lookup')
   report_keys = [
@@ -30,7 +31,7 @@ def fetch_papers():
     paper_id = line[3]
     if paper_id == '':
       continue
-    paper = fetch_paper(s2, paper_id)
+    paper = fetch_paper(s2, paper_id, freshen)
     if paper is None:
       continue
     db_paper = load_paper(paper_id)
diff --git a/scraper/util.py b/scraper/util.py
index fdbc0534..830dbe8b 100644
--- a/scraper/util.py
+++ b/scraper/util.py
@@ -386,10 +386,10 @@ def parallelize(func, rows):
   with Pool(processes=processCount) as pool:
     pool.starmap(func, rows, chunksize)
 
-def fetch_paper(s2, paper_id):
+def fetch_paper(s2, paper_id, freshen=False):
   os.makedirs('./datasets/s2/papers/{}/{}'.format(paper_id[0:2], paper_id), exist_ok=True)
   paper_fn = './datasets/s2/papers/{}/{}/paper.json'.format(paper_id[0:2], paper_id)
-  if os.path.exists(paper_fn):
+  if os.path.exists(paper_fn) and not freshen:
     return read_json(paper_fn)
   print(paper_id)
   paper = s2.paper(paper_id)