blob: 473e90afb88b597059cdeece52bfbdfcc7982940 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
|
import os
import glob
import simplejson as json
import click
from util import *
PAPER_JSON_DIR = 'datasets/s2/db_papers'
@click.command()
def s2_dump_pdf_urls():
# loop over all the papers in db_papers
# get all the PDF urls, pick the best one
# store it and the paper id
# another script will fetch the urls from this process
rows = []
pdf_count = 0
ieee_count = 0
extra_count = 0
empty_count += 1
for fn in glob.iglob('{}/**/paper.json'.format(PAPER_JSON_DIR), recursive=True):
row = process_paper(fn)
if row is not None:
rows.append(row)
if row[1] is not None:
pdf_count += 1
if row[2] is not None:
ieee_count += 1
if row[3] is not None:
extra_count += 1
if row[1] is None and row[2] is None and row[3] is None:
empty_count += 1
print("Wrote {} rows".format(len(rows)))
print("pdf count: {}".format(pdf_count))
print("ieee count: {}".format(ieee_count))
print("url count: {}".format(url_count))
print("empty count: {}".format(empty_count))
write_csv('db_paper_pdf_list.csv', keys=['Paper ID', 'PDF URL', 'IEEE URL', 'Extra URL'], rows=rows)
def process_paper(fn):
print(fn)
paper = read_json(fn)
if paper is None:
return None
paper_id = paper['id']
pdf_url = None
ieee_url = None
extra_url = None
if paper['s2PdfUrl']:
pdf_url = paper['s2PdfUrl']
for url in paper['pdfUrls']:
if 'ieeexplore.ieee.org' in url:
ieee_url = url
elif pdf_url is None and 'pdf' in url:
pdf_url = url
else:
extra_url = url
return [paper_id, pdf_url, ieee_url, extra_url]
if __name__ == '__main__':
s2_dump_pdf_urls()
|
