diff options
Diffstat (limited to 'scraper/s2-search-deep.py')
| -rw-r--r-- | scraper/s2-search-deep.py | 10 |
1 files changed, 8 insertions, 2 deletions
diff --git a/scraper/s2-search-deep.py b/scraper/s2-search-deep.py index 93be3bea..b5dc640e 100644 --- a/scraper/s2-search-deep.py +++ b/scraper/s2-search-deep.py @@ -31,7 +31,10 @@ MAX_PAGES = 20 def fetch_query(query, since=None, refresh=False): clean_title = re.sub(r'[^-0-9a-zA-Z ]+', '', query) - yearFilter = {'min': since, 'max': 2020 } if since else None + if since: + yearFilter = {'min': since, 'max': 2020 } + else: + yearFilter = None results_path = './datasets/s2/search/{}'.format(clean_title) os.makedirs(results_path, exist_ok=True) page = 1 @@ -77,7 +80,10 @@ def search_deep(refresh): since = row['since'] if not since: continue - since = int(since) + if since == '#N/A': + since = None + else: + since = int(since) queries = [] row_paper_ids = {} for i in range(1, 6): |