From d2cbf2f7cb64f36c04612e3a7d996ed1b8ce7228 Mon Sep 17 00:00:00 2001 From: "jules@lens" Date: Fri, 8 Feb 2019 23:24:07 +0100 Subject: nosleep --- scraper/s2-fetch-pdf.py | 2 -- scraper/s2-pdf-first-pages.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/scraper/s2-fetch-pdf.py b/scraper/s2-fetch-pdf.py index 5477cbd5..30bc5a40 100644 --- a/scraper/s2-fetch-pdf.py +++ b/scraper/s2-fetch-pdf.py @@ -31,10 +31,8 @@ def fetch_pdf(paper_id, url): size = s2.fetch_file(url, pdf_fn) if size is None: print("{} empty?".format(paper_id)) - time.sleep(random.randint(5, 10)) return None print("{} {} kb".format(paper_id, int(size / 1024))) - time.sleep(random.randint(5, 10)) return # return paper diff --git a/scraper/s2-pdf-first-pages.py b/scraper/s2-pdf-first-pages.py index 0a6b20bd..6f1d81e3 100644 --- a/scraper/s2-pdf-first-pages.py +++ b/scraper/s2-pdf-first-pages.py @@ -30,7 +30,7 @@ def report_first_pages(): write_report('reports/first_pages.html', title='First pages', keys=None, rows=rows) write_report('reports/institutions.html', title='Institutions', keys=None, rows=sorted(institutions, key=lambda x: x[1])) write_report('reports/institutions_missing.html', title='Institutions', keys=None, rows=no_institutions) - write_csv('reports/institution_names.csv', keys=None, rows=[(name,) for name in deduped_institutions]) + write_csv('reports/institution_names_extracted.csv', keys=None, rows=[(name,) for name in deduped_institutions]) print("{} deduped institutions".format(len(deduped_institutions))) def dedupe(a): -- cgit v1.2.3-70-g09d2