blob: 4c9846b14aefb188d5af746bc220fdd46b206e5a (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
|
import os
import gzip
import glob
import simplejson as json
import click
from util import *
PAPER_JSON_DIR = 'datasets/s2/papers'
@click.command()
def s2_dump_ids():
ids = {}
for fn in glob.iglob('{}/**/*.json'.format(PAPER_JSON_DIR), recursive=True):
process_paper(fn, ids)
id_list = list(ids.keys())
print("Wrote {} ids".format(len(id_list)))
write_json('ids.json', id_list)
def process_paper(fn, ids):
with open(fn, 'r') as f:
data = json.load(f)
# print(data['paperId'])
ids[data['paperId']] = True
for cite in data['citations']:
ids[cite['paperId']] = True
def paper_path(paper_id):
return '{}/{}/{}'.format(DATA_DIR, paper_id[0:2], paper_id)
if __name__ == '__main__':
s2_dump_ids()
|
