fix unicode woes

1 files changed, 22 insertions, 11 deletions

diff --git a/util.py b/util.py
index b38cfec1..a1575db7 100644
--- a/util.py
+++ b/util.py
@@ -1,5 +1,6 @@
 import os
 import csv
+import codecs
 import simplejson as json
 
 def read_citation_list(index=0):
@@ -14,11 +15,21 @@ def read_citation_list(index=0):
     lines = lines[1:]
     return keys, lines
 
+def unfussy_reader(reader):
+  while True:
+    try:
+      yield next(reader)
+    except csv.Error:
+      print(csv.Error)
+      # log the problem or whatever
+      continue
+
 def read_csv(fn, keys=True, create=False):
   try:
-    with open(fn, 'r') as f:
-      reader = csv.reader( (line.replace('\0','') for line in f) )
-      lines = list(reader)
+    with open(fn, 'r', newline='', encoding='utf-8') as f:
+      # reader = csv.reader( (line.replace('\0','') for line in f) )
+      reader = csv.reader(f)
+      lines = list(unfussy_reader(reader))
       if keys:
         keys = lines[0]
         lines = lines[1:]
@@ -29,6 +40,14 @@ def read_csv(fn, keys=True, create=False):
       return []
     raise
 
+def write_csv(fn, keys, rows):
+  with open(fn, 'w', newline='', encoding='utf-8') as f:
+    writer = csv.writer(f)
+    if keys is not None:
+      writer.writerow(keys)
+    for row in rows:
+      writer.writerow(row)
+
 def read_json(fn):
   with open(fn, 'r') as json_file:
     return json.load(json_file)
@@ -37,14 +56,6 @@ def write_json(fn, data):
   with open(fn, 'w') as outfile:
     json.dump(data, outfile)
 
-def write_csv(fn, keys, rows):
-  with open(fn, 'w') as f:
-    writer = csv.writer(f)
-    if keys is not None:
-      writer.writerow(keys)
-    for row in rows:
-      writer.writerow(row)
-
 def write_report(fn, title=None, keys=None, rows=[]):
   count = 0
   with open(fn, 'w') as f: