import os
import re
import glob
import json
import math
import operator
import click
from util import *
@click.command()
def s2_citation_report():
addresses = AddressBook()
for fn in glob.iglob('datasets/s2/papers/**/*.json', recursive=True):
process_paper(fn, addresses)
def process_paper(fn, addresses):
res = {}
address_count = 0
geocode_count = 0
geocoded_citations = []
unknown_citations = []
display_geocoded_citations = []
with open(fn, 'r') as f:
data = json.load(f)
print('>> {}'.format(data['paperId']))
paper = load_paper(data['paperId'])
if paper.data is None:
print("Paper missing! {}".format(data['paperId']))
return
res['paperId'] = paper.paper_id
res['title'] = paper.title
res['journal'] = paper.journal
res['authors'] = paper.authors
res['citations'] = []
for cite in data['citations']:
citationId = cite['paperId']
citation = load_paper(citationId)
if citation.data is None:
print("Citation missing! {}".format(cite['paperId']))
continue
institutions = load_institutions(citationId)
geocoded_institutions = []
unknown_institutions = []
institution = ''
address = None
for inst in sorted(institutions, key=operator.itemgetter(1)):
# print(inst[1])
address_count += 1
institution = inst[1]
next_address = addresses.find(institution)
if next_address:
address = next_address
geocode_count += 1
geocoded_institutions.append(institution)
else:
unknown_institutions.append(institution)
if not address:
if os.path.exists(file_path('pdf', citationId, 'paper.txt')):
headings, found_abstract = read_headings(file_path('pdf', citationId, 'paper.txt'), citation)
heading_string = '\n'.join(headings[0:20])
found_addresses = []
if len(headings):
for heading in headings:
l = heading.lower().strip()
if l:
next_address = addresses.find(l)
if next_address:
address = next_address
geocode_count += 1
geocoded_institutions.append(heading)
else:
unknown_institutions.append(heading)
res['citations'].append({
'title': citation.title,
'journal': citation.journal,
'authors': citation.authors,
'institutions': [inst[1] for inst in institutions],
'geocoded': geocoded_institutions,
})
if address:
geocoded_citations.append([
citation.title,
institution,
] + address)
display_geocoded_citations.append([
citation.title,
] + address)
else:
unknown_citations.append([
citation.title,
'
'.join(unknown_institutions),
])
paper_institutions = load_institutions(paper.paper_id)
paper_address = None
for inst in sorted(paper_institutions, key=operator.itemgetter(1)):
# print(inst[1])
address_count += 1
institution = inst[1]
paper_address = addresses.find(institution)
if paper_address:
print(paper_address)
total_citations = len(geocoded_citations) + len(unknown_citations)
os.makedirs('reports/papers/', exist_ok=True)
with open('reports/papers/{}.html'.format(paper.paper_id), 'w') as f:
f.write("")
f.write("")
f.write("")
f.write('')
f.write("{}".format(paper.title))
f.write("")
f.write('')
f.write("")
f.write("")
f.write("")
f.write("{}
".format(paper.title))
f.write('')
if paper.journal:
f.write('- Journal: {}
'.format(paper.journal))
if paper_address:
f.write('- Research institution: {}
'.format(paper_address[0]))
f.write('- Address: {}
'.format(paper_address[3]))
f.write('- {}
'.format(paper.year))
f.write('- {} / {} citations were located ({} %).
'.format(len(geocoded_citations), total_citations, math.floor(len(geocoded_citations) / total_citations * 100)))
f.write('
')
f.write('{}
'.format('Geocoded Citations'))
write_table(f, keys=None, rows=sorted(display_geocoded_citations, key=operator.itemgetter(0)))
f.write('{}
'.format('Other Citations'))
write_table(f, keys=None, rows=sorted(unknown_citations, key=operator.itemgetter(0)))
f.write("")
f.write('')
f.write('')
f.write('')
f.write('')
f.write('')
f.write('')
f.write('')
f.write("")
return res
def load_institutions(paperId):
if os.path.exists(file_path('pdf', paperId, 'institutions.json')):
return read_json(file_path('pdf', paperId, 'institutions.json'))['institutions']
elif os.path.exists(file_path('doi', paperId, 'institutions.json')):
return read_json(file_path('doi', paperId, 'institutions.json'))['institutions']
else:
return []
def data_path(key, paper_id):
return 'datasets/s2/{}/{}/{}'.format(key, paper_id[0:2], paper_id)
def file_path(key, paper_id, fn):
return os.path.join(data_path(key, paper_id), fn)
if __name__ == '__main__':
s2_citation_report()