import re
import os
import gzip
import glob
import json
import click
import math
import string
from util import *

PDF_DIR = 'datasets/s2/pdf'

@click.command()
def s2_pdf_report():
  rows = []
  empty_papers = []
  no_separator_papers = []
  geocoded_papers = [] 
  unknown_papers = []
  found_count = 0
  total_count = 0
  addresses = AddressBook()
  for fn in glob.iglob('{}/**/*.txt'.format(PDF_DIR), recursive=True):
    paper_id = fn.replace(PDF_DIR, '').split('/')[2]
    total_count += 1
    # print(paper_id)
    headings, found_abstract = read_headings(fn)
    heading_string = '\n'.join(headings[0:20])
    found_addresses = []
    if not found_abstract:
      if len(headings) == 0:
        empty_papers.append(paper.record())
        continue
      if len(headings) > 20:
        no_separator_papers.append(paper.record())
        # continue
    for heading in headings:
      l = heading.lower().strip()
      address = addresses.find(l)
      if address:
        found_addresses.append(address)

    # MAYBE try checking the entire string against everything?
    # if not len(found_addresses):
    #   l = heading_string.lower().strip()
    #   address = addresses.find(l)
    #   if address:
    #     found_addresses.append(address)

    if len(found_addresses):
      found_count += 1
      for address in found_addresses:
        geocoded_papers.append([paper.paper_id, paper.title] + address)
    else:
      unknown_papers.append([paper.paper_id, paper.title, heading_string])
  write_csv('reports/stats/empty_papers.csv', keys=None, rows=empty_papers)
  write_csv('reports/stats/no_separator_papers.csv', keys=None, rows=no_separator_papers)
  write_csv('reports/stats/geocoded_papers.csv', keys=None, rows=geocoded_papers)
  write_csv('reports/stats/unknown_papers.csv', keys=None, rows=unknown_papers)
  print("{} {} ({}%)".format('empty', len(empty_papers), percent(len(empty_papers), total_count)))
  print("{} {} ({}%)".format('no separator', len(no_separator_papers), percent(len(no_separator_papers), total_count)))
  print("{} {} ({}%)".format('found', found_count, percent(found_count, total_count)))
  print("{} {} ({}%)".format('unknown', len(unknown_papers), percent(len(unknown_papers), total_count)))
  print("{} {} entities".format('geocoded', len(geocoded_papers)))

def percent(a,b):
  return round(100 * a / b)

def paper_path(paper_id):
  return '{}/{}/{}'.format(PDF_DIR, paper_id[0:2], paper_id)
  
if __name__ == '__main__':
  s2_pdf_report()