Merge branch 'master' of github.com:adamhrv/megapixels_dev

1 files changed, 13 insertions, 10 deletions

diff --git a/scraper/s2-search.py b/scraper/s2-search.py
index e943053a..d9b1beca 100644
--- a/scraper/s2-search.py
+++ b/scraper/s2-search.py
@@ -25,7 +25,8 @@ totalResults
 
 @click.command()
 @click.option('--index', '-n', default=0, help='Index of CSV (query,)')
-def fetch_entries(index):
+@click.option('--refresh/--no-refresh', '-f', help='Force it to query the paper API again')
+def fetch_entries(index, refresh):
   keys, lines = read_citation_list(index)
   citation_lookup = []
   s2 = SemanticScholarAPI()
@@ -37,22 +38,24 @@ def fetch_entries(index):
     if len(clean_title) < 2:
       continue
     dump_fn = './datasets/s2/dumps/{}.json'.format(key)
-    entry_fn = './datasets/s2/entries/{}.json'.format(key)
     result = None
-    if os.path.exists(entry_fn):
-      result = read_json(entry_fn)
+    if not refresh and os.path.exists(dump_fn):
+      results = read_json(dump_fn)
     else:
       results = s2.search(clean_title)
       write_json(dump_fn, results)
-      if len(results['results']) == 0:
-        print("- {}".format(title))
-      else:
-        print("+ {}".format(title))
-        result = results['results'][0]
-        write_json(entry_fn, result)
+
+    if len(results['results']) == 0:
+      print("- {}".format(title))
+    else:
+      print("+ {}".format(title))
+      result = results['results'][0]
+
     if result:
       paper_id = result['id']
       paper = fetch_paper(s2, paper_id)
+      entry_fn = './datasets/s2/entries/{}.json'.format(paper_id)
+      write_json(entry_fn, result)
       citation_lookup.append([key, name, title, paper_id])
   write_csv("datasets/citation_lookup.csv", keys=['key', 'name', 'title', 'paper_id'], rows=citation_lookup)