diff options
Diffstat (limited to 'scraper/s2-search.py')
| -rw-r--r-- | scraper/s2-search.py | 6 |
1 files changed, 4 insertions, 2 deletions
diff --git a/scraper/s2-search.py b/scraper/s2-search.py index ddecf2f9..169a8d19 100644 --- a/scraper/s2-search.py +++ b/scraper/s2-search.py @@ -7,7 +7,7 @@ import random import re import simplejson as json import click -from s2 import SemanticScholarAPI +from s2 import SemanticScholarAPI, fetch_paper from util import * ''' @@ -33,6 +33,8 @@ def fetch_entries(index): key = line[0] name = line[1] title = re.sub(r'[^-0-9a-zA-Z ]+', '', line[2]) + if len(title) < 2: + continue dump_fn = './datasets/s2/dumps/{}.json'.format(key) entry_fn = './datasets/s2/entries/{}.json'.format(key) result = None @@ -49,7 +51,7 @@ def fetch_entries(index): write_json(entry_fn, result) if result: paper_id = result['id'] - paper = fetch_paper(paper_id) + paper = fetch_paper(s2, paper_id) citation_lookup.append([key, name, title, paper_id]) write_csv("datasets/citation_lookup.csv", keys=['key', 'name', 'title', 'paper_id'], rows=citation_lookup) |