blob: ae080539b95a7ed6e097d23bc30deb1dd6a8a881 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
|
import os
import gzip
import glob
import json
import click
from util import *
PDF_DIR = 'datasets/s2/pdf'
FIRST_PAGES_KEYS = ''
@click.command()
def pdf_report_first_pages():
rows = []
for fn in glob.iglob('{}/**/*.txt'.format(PDF_DIR), recursive=True):
row = process_paper(fn)
rows.append(row)
write_report('reports/first_pages.html', title='First pages', keys=FIRST_PAGES_KEYS, rows=rows)
print("Wrote {} rows".format(len(rows)))
def process_paper(fn):
index = fn.replace(PDF_DIR, '').split('/')[2]
with open(fn, 'r') as f:
lines = ''
for line in f.readlines():
if 'abstract' in line.lower():
break
if len(line) < 3:
continue
lines += line + '<br>'
return [
index,
lines
]
def paper_path(paper_id):
return '{}/{}/{}'.format(DATA_DIR, paper_id[0:2], paper_id)
if __name__ == '__main__':
pdf_report_first_pages()
|
