Merge branch 'master' of github.com:adamhrv/megapixels_dev

1 files changed, 28 insertions, 0 deletions

diff --git a/scraper/s2-merge-csv.py b/scraper/s2-merge-csv.py
new file mode 100644
index 00000000..301f47ae
--- /dev/null
+++ b/scraper/s2-merge-csv.py
@@ -0,0 +1,28 @@
+import os
+import glob
+import time
+import simplejson as json
+import click
+import operator
+from util import *
+import random
+
+@click.command()
+@click.option('--path', '-d', default='report/institutions_geocoded', help='Path to CSVs')
+def s2_merge_csv(path):
+  print(path)
+  lookup = {}
+  for fn in glob.iglob('{}/*.csv'.format(path)):
+    rows = read_csv(fn, keys=False)
+    for row in rows:
+      key = row[0]
+      if key not in lookup:
+        lookup[key] = row
+  keys = sorted(lookup.keys())
+  deduped = []
+  for key in keys:
+    deduped.append(lookup[key])
+  write_csv('{}.csv'.format(path), keys=None, rows=deduped)
+
+if __name__ == '__main__':
+  s2_merge_csv()