blob: 0fd84fd1564d87f3d42b8fe494db4998d6a9faba (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
|
import os
import glob
import simplejson as json
import click
from util import *
PAPER_JSON_DIR = 'datasets/s2/db_papers'
@click.command()
def s2_dump_pdf_urls():
# loop over all the papers in db_papers
# get all the PDF urls, pick the best one
# store it and the paper id
# another script will fetch the urls from this process
rows = []
for fn in glob.iglob('{}/**/paper.json'.format(PAPER_JSON_DIR), recursive=True):
row = process_paper(fn)
if row is not None:
rows.append(row)
print("Wrote {} rows".format(len(rows)))
write_csv('db_paper_pdf_list.csv', keys=['Paper ID', 'PDF URL', 'IEEE URL', 'Extra URL'], rows=rows)
def process_paper(fn):
print(fn)
paper = read_json(fn)
if paper is None:
return None
paper_id = paper['id']
pdf_url = None
ieee_url = None
extra_url = None
if paper['s2PdfUrl']:
pdf_url = paper['s2PdfUrl']
for url in paper['pdfUrls']:
if 'ieeexplore.ieee.org' in url:
ieee_url = url
elif pdf_url is None and 'pdf' in url:
pdf_url = url
else:
extra_url = url
return [paper_id, pdf_url, ieee_url, extra_url]
if __name__ == '__main__':
s2_dump_pdf_urls()
|
