From fd1dff17a90afdd2478a98856ee3ad509c5c4900 Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Sat, 16 Feb 2019 16:09:48 +0100 Subject: store many pdfs --- megapixels/commands/datasets/citations_to_csv.py | 4 +++- scraper/client/paper/paper.address.js | 1 + scraper/s2-final-report.py | 4 ++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/megapixels/commands/datasets/citations_to_csv.py b/megapixels/commands/datasets/citations_to_csv.py index e54d0dac..253d15fc 100644 --- a/megapixels/commands/datasets/citations_to_csv.py +++ b/megapixels/commands/datasets/citations_to_csv.py @@ -94,6 +94,8 @@ def get_orig_paper(json_data): addresses = p.get('address','') if addresses: for a in addresses: + if type(a) == str or a is None: + continue paper = Paper(p['key'], p['name'], p['paper_id'], p['title'], d_type, year, p['pdf'], a['address'], a['type'], a['lat'], a['lng']) @@ -101,4 +103,4 @@ def get_orig_paper(json_data): else: paper = Paper(p['key'], p['name'], p['paper_id'], p['title'], d_type, year, p['pdf']) papers.append(paper) - return papers \ No newline at end of file + return papers diff --git a/scraper/client/paper/paper.address.js b/scraper/client/paper/paper.address.js index 9256d4ad..c571d8e2 100644 --- a/scraper/client/paper/paper.address.js +++ b/scraper/client/paper/paper.address.js @@ -142,6 +142,7 @@ class PaperAddress extends Component { if (!citation) { return
Citation not found in this paper
} + console.log(citation) return (

{citation.title}

diff --git a/scraper/s2-final-report.py b/scraper/s2-final-report.py index 451c1f78..f08bc748 100644 --- a/scraper/s2-final-report.py +++ b/scraper/s2-final-report.py @@ -152,14 +152,14 @@ def process_single_paper(row, paper_id, addresses, aggregate_citations, unknown_ 'title': citation.title, 'addresses': geocoded_addresses, 'year': citation.year, - 'pdf': citation.pdf_link, + 'pdf': citation.pdf_links(), } else: unknown_citations[citationId] = { 'id': citationId, 'title': citation.title, 'year': citation.year, - 'pdf': citation.pdf_link, + 'pdf': citation.pdf_links(), } return res -- cgit v1.2.3-70-g09d2