diff options
| author | Jules Laplace <julescarbon@gmail.com> | 2019-03-08 21:05:03 +0100 |
|---|---|---|
| committer | Jules Laplace <julescarbon@gmail.com> | 2019-03-08 21:05:03 +0100 |
| commit | faa59f192c00dadd84165fe14a90307c908e6ab2 (patch) | |
| tree | 7454e60e4edbde0469aac8bd72c9d52490db3d6b /scraper/s2-raw-papers.py | |
| parent | 8f0d59a5f44c71aeb4eecf60cb323d2fe0306a3e (diff) | |
possibly freshen raw papers
Diffstat (limited to 'scraper/s2-raw-papers.py')
| -rw-r--r-- | scraper/s2-raw-papers.py | 4 |
1 files changed, 2 insertions, 2 deletions
diff --git a/scraper/s2-raw-papers.py b/scraper/s2-raw-papers.py index 612c8099..8881cda0 100644 --- a/scraper/s2-raw-papers.py +++ b/scraper/s2-raw-papers.py @@ -18,10 +18,10 @@ def fetch_raw_papers(fn): lines = read_csv(fn, keys=False) parallelize(fetch_raw_paper, lines) -def fetch_raw_paper(paper_id): +def fetch_raw_paper(paper_id, freshen=False): os.makedirs(make_raw_paper_path(paper_id), exist_ok=True) paper_fn = make_raw_paper_fn(paper_id) - if os.path.exists(paper_fn): + if os.path.exists(paper_fn) and not freshen: paper = read_json(paper_fn) else: paper = s2.raw_paper(paper_id) |