Merge branch 'master' of github.com:adamhrv/megapixels_dev

1 files changed, 46 insertions, 0 deletions

diff --git a/scraper/s2-papers.py b/scraper/s2-papers.py
new file mode 100644
index 00000000..bf77a734
--- /dev/null
+++ b/scraper/s2-papers.py
@@ -0,0 +1,46 @@
+import os
+import sys
+import csv
+import subprocess
+import time
+import random
+import re
+import simplejson as json
+import click
+from s2 import SemanticScholarAPI
+from util import *
+
+'''
+s2 search API format:
+results
+matchedAuthors
+matchedPresentations
+query
+querySuggestions
+results
+stats
+totalPages
+totalResults
+'''
+
+s2 = SemanticScholarAPI()
+
+@click.command()
+@click.option('--index', '-n', default=0, help='Index of CSV (query,)')
+@click.option('--depth', '-d', default=1, help='Depth to recurse (not implemented).')
+def fetch_papers(index, depth):
+  keys, lines = read_citation_list(index)
+  for line in lines:
+    label = line[0]
+    title = re.sub(r'[^-0-9a-zA-Z ]+', '', line[1])
+    entry_fn = './datasets/s2/entries/{}.json'.format(title)
+    if not os.path.exists(entry_fn):
+      print('not found: {}'.format(entry_fn))
+      continue
+    result = read_json(entry_fn)
+    paper_id = result['id']
+    paper = fetch_paper(paper_id)
+    # get all of the paper's citations
+
+if __name__ == '__main__':
+  fetch_papers()