blob: 8941d60d2dee94f3fa50d9e4dd62b0c6c2a833af (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
|
import os
import sys
import csv
import subprocess
import time
import random
import re
import simplejson as json
import click
from s2 import SemanticScholarAPI
from util import *
s2 = SemanticScholarAPI()
@click.command()
@click.option('--fn', '-i', default='missing.csv', help='Filename of CSV (id,)')
def fetch_raw_papers(fn):
lines = read_csv(fn, keys=False)
parallelize(fetch_raw_paper, lines)
def fetch_raw_paper(paper_id):
os.makedirs(make_raw_paper_path(paper_id), exist_ok=True)
paper_fn = make_raw_paper_fn(paper_id)
if os.path.exists(paper_fn):
return read_json(paper_fn)
print(paper_id)
paper = s2.raw_paper(paper_id)
if paper is None:
print("Got empty paper?? {}".format(paper_id))
# time.sleep(random.randint(5, 10))
return None
if 'responseType' in paper and paper['responseType'] == 'CANONICAL':
write_json(paper_fn, paper)
paper = s2.raw_paper(data['canonicalId'])
paper_fn = make_raw_paper_fn(paper_id)
return paper
write_json(paper_fn, paper)
# time.sleep(random.randint(5, 10))
return paper
def make_raw_paper_path(paper_id):
return './datasets/s2/raw_papers/{}/{}'.format(paper_id[0:2], paper_id)
def make_raw_paper_fn(paper_id):
return './datasets/s2/raw_papers/{}/{}/paper.json'.format(paper_id[0:2], paper_id)
if __name__ == '__main__':
fetch_raw_papers()
|
