s2-dump-pdf-urls.py


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

import os
import glob
import simplejson as json
import click
from util import *

PAPER_JSON_DIR = 'datasets/s2/db_papers'

@click.command()
def s2_dump_pdf_urls():
  # loop over all the papers in db_papers
  # get all the PDF urls, pick the best one
  # store it and the paper id
  # another script will fetch the urls from this process
  ids = {}
  for fn in glob.iglob('{}/**/paper.json'.format(PAPER_JSON_DIR), recursive=True):
    process_paper(fn, ids)
  id_list = list(ids.keys())
  print("Wrote {} ids".format(len(id_list)))
  write_csv('pdf_list.csv', id_list)

  for line in lines:
    label = line[0]
    title = re.sub(r'[^-0-9a-zA-Z ]+', '', line[1])
    entry_fn = './datasets/s2/entries/{}.json'.format(title)
    if not os.path.exists(entry_fn):
      results = s2.search(title)
      write_json(dump_fn, results)
      if len(results['results']) == 0:
        print("No results for {}".format(title))
      else:
        print(title)
        write_json(entry_fn, results['results'][0])
      time.sleep(random.randint(10, 20))

if __name__ == '__main__':
  s2_dump_pdf_urls()