arcs on dark maps

1 files changed, 31 insertions, 49 deletions

diff --git a/s2-doi-report.py b/s2-doi-report.py
index d2facb92..611b6391 100644
--- a/s2-doi-report.py
+++ b/s2-doi-report.py
@@ -14,27 +14,45 @@ def doi_report():
   rows = []
   domains = {}
   institutions = {}
-  geocode_lookup = load_geocode_lookup()
+  # geocode_lookup = load_geocode_lookup()
+  addresses = AddressBook()
+
+  geocoded_papers = []
+  unknown_papers = []
+  unattributed_papers = []
 
   for fn in glob.iglob('{}/**/*.url'.format(DOI_DIR), recursive=True):
     url_info = read_json(fn)
     domain = url_info['domain']
     paper_id = url_info['paper_id']
+    paper = load_paper(paper_id)
     doi_fn = fn.replace('.url', '.doi')
-    institutions_fn = fn.replace('paper.url', 'institutions.json')
+    # institutions_fn = fn.replace('paper.url', 'institutions.json')
+    address = None
     if domain in domains:
       domains[domain] += 1
     else:
       domains[domain] = 1
-    if os.path.exists(institutions_fn):
-      continue
+    # if not os.path.exists(institutions_fn):
+      # continue
+    paper_affiliation_count = 0
     if 'ieee' in domain:
       affiliations = load_ieee(paper_id,  doi_fn)
       for affiliation in affiliations:
-        if affiliation in institutions:
-          institutions[affiliation] += 1
-        else:
-          institutions[affiliation] = 1
+        if affiliation:
+          paper_affiliation_count += 1
+          if affiliation in institutions:
+            institutions[affiliation] += 1
+          else:
+            institutions[affiliation] = 1
+            address = addresses.find(affiliation)
+            if not address:
+              unknown_papers.append([paper.paper_id, paper.title, affiliation])
+      if paper_affiliation_count == 0:
+        unattributed_papers.append([paper.paper_id, paper.title])
+    if address:
+      geocoded_papers.append([paper.paper_id, paper.title] + address)
+
   domain_list = reversed(sorted(domains.items(), key=operator.itemgetter(1)))
   # for domain, count in domain_list:
     # print('{}\t{}'.format(count, domain))
@@ -44,43 +62,16 @@ def doi_report():
   display_institution_list = []
   raw_institution_list = []
   for inst in institution_list:
-    raw_institution_list.append(inst)
-    if inst[0] in geocode_lookup:
-      display_institution_list.append((BoldLine(inst[0]), inst[1],))
-      continue
-    inst_parts = inst[0].split(',')
-    if inst_parts[0] in geocode_lookup:
-      display_institution_list.append((BoldLine(inst[0]), inst[1],))
-    elif len(inst_parts) > 1 and inst_parts[1] in geocode_lookup:
+    addr = addresses.find(inst[0])
+    if addr:
       display_institution_list.append((BoldLine(inst[0]), inst[1],))
     else:
       display_institution_list.append(inst)
   write_report('reports/doi_domains.html', title='DOI Domains', keys=None, rows=domain_list)
   write_report('reports/doi_institutions.html', title='Institutions from IEEE', keys=None, rows=display_institution_list)
-  write_csv('reports/doi_institutions.csv', keys=None, rows=raw_institution_list)
-
-  # print(domain_list)
-    # rows.append(data['first_pages'])
-    # if data['institutions']:
-    #   for institution in data['institutions']:
-    #     institutions.append(institution)
-    #     institution_names.append(institution[1])
-    # if data['no_institutions']:
-    #   no_institutions.append(data['no_institutions'])
-  # deduped_institutions = dedupe(institution_names)
-
-  # write_report('reports/first_pages.html', title='First pages', keys=None, rows=rows)
-  # write_report('reports/institutions.html', title='Institutions', keys=None, rows=sorted(institutions, key=lambda x: x[1]))
-  # write_report('reports/institutions_missing.html', title='Institutions', keys=None, rows=no_institutions)
-  # write_csv('reports/institution_names.csv', keys=None, rows=[(name,) for name in deduped_institutions])
-  # print("{} deduped institutions".format(len(deduped_institutions)))
-
-def dedupe(a):
-  p = {}
-  for s in a:
-    p[s] = None
-  ss = sorted(p.keys())
-  return ss
+  write_csv('reports/doi_institutions_geocoded.csv', keys=None, rows=geocoded_papers)
+  write_csv('reports/doi_institutions_unknown.csv', keys=None, rows=unknown_papers)
+  write_csv('reports/doi_institutions_unattributed.csv', keys=None, rows=unattributed_papers)
 
 def load_ieee(paper_id, fn):
   paper = load_paper(paper_id)
@@ -103,15 +94,6 @@ def load_ieee(paper_id, fn):
     write_json('{}/{}'.format(paper_path(paper_id), 'institutions.json'), { 'institutions': institutions })
     return affiliations
 
-def load_geocode_lookup():
-  insts = read_csv('reports/institutions_found.csv', keys=None)
-  lookup = {}
-  for inst in insts:
-    # print(inst)
-    lookup[inst[0]] = True
-    lookup[inst[3]] = True
-  return lookup
-
 def find_authors(authors, line):
   for a in authors:
     if a[2] in line: