From ee3d0d98e19f1d8177d85af1866fd0ee431fe9ea Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Sun, 25 Nov 2018 22:19:15 +0100 Subject: moving stuff --- s2-raw-papers.py | 44 -------------------------------------------- 1 file changed, 44 deletions(-) delete mode 100644 s2-raw-papers.py (limited to 's2-raw-papers.py') diff --git a/s2-raw-papers.py b/s2-raw-papers.py deleted file mode 100644 index 2d61f450..00000000 --- a/s2-raw-papers.py +++ /dev/null @@ -1,44 +0,0 @@ -import os -import sys -import csv -import subprocess -import time -import random -import re -import json -import click -from s2 import SemanticScholarAPI -from util import * - -s2 = SemanticScholarAPI() - -@click.command() -@click.option('--fn', '-i', default='missing.csv', help='Filename of CSV (id,)') -def fetch_raw_papers(fn): - lines = read_csv(fn, keys=False) - for line in lines: - paper_id = line[0] - fetch_raw_paper(paper_id) - -def fetch_raw_paper(paper_id): - os.makedirs(make_raw_paper_path(paper_id), exist_ok=True) - paper_fn = make_raw_paper_fn(paper_id) - if os.path.exists(paper_fn): - return read_json(paper_fn) - print(paper_id) - paper = s2.raw_paper(paper_id) - if paper is None: - print("Got empty paper?? {}".format(paper_id)) - time.sleep(random.randint(5, 10)) - return None - write_json(paper_fn, paper) - time.sleep(random.randint(5, 10)) - return paper - -def make_raw_paper_path(paper_id): - return './datasets/s2/raw_papers/{}/{}'.format(paper_id[0:2], paper_id) -def make_raw_paper_fn(paper_id): - return './datasets/s2/raw_papers/{}/{}/paper.json'.format(paper_id[0:2], paper_id) - -if __name__ == '__main__': - fetch_raw_papers() -- cgit v1.2.3-70-g09d2