1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
|
import re
import os
import gzip
import glob
import json
import click
from util import *
PDF_DIR = 'datasets/s2/pdf'
@click.command()
def pdf_report_first_pages():
rows = []
for fn in glob.iglob('{}/**/*.txt'.format(PDF_DIR), recursive=True):
row = process_paper(fn)
print(row)
rows.append(row)
write_report('reports/first_pages.html', title='First pages', keys=None, rows=rows)
print("Wrote {} rows".format(len(rows)))
def process_paper(fn):
paper_id = fn.replace(PDF_DIR, '').split('/')[2]
paper = load_paper(paper_id)
if paper is None:
print("{} no paper found!".format(paper_id))
return None
with open(fn, 'r') as f:
lines = []
emails = []
authors = [ (a[0], a[1], a[1].lower(),) for a in paper.authors ]
journal = paper.journal.lower()
found_authors = []
for line in f.readlines():
l = line.lower()
if 'abstract' in l:
break
if len(line) < 3:
continue
if journal and journal in l:
continue
if '@' in line:
# print('email {}'.format(line))
emails.append(line)
continue
names = [s.strip() for s in re.split(',| and ', l)]
was_found = False
for name in names:
found = find_authors(authors, name)
if found:
was_found = True
# print("found {}".format(found[1]))
if found[0]:
found_authors.append(found)
if was_found:
# lines.append(NameLine(line))
continue
if 'university' in l or 'universiteit' in l or 'research center' in l:
lines.append(BoldLine(line))
continue
lines.append(line)
return [
paper_id,
lines,
found_authors,
emails,
]
class NameLine(object):
def __init__(self, s):
self.s = s.strip()
def __str__(self):
return '<span class="name">' + self.s + '</span>'
class BoldLine(object):
def __init__(self, s):
self.s = s.strip()
def __str__(self):
return '<b>' + self.s + '</b>'
def find_authors(authors, line):
for a in authors:
if a[2] in line:
return a
return None
def paper_path(paper_id):
return '{}/{}/{}'.format(DATA_DIR, paper_id[0:2], paper_id)
if __name__ == '__main__':
pdf_report_first_pages()
|
