From ee3d0d98e19f1d8177d85af1866fd0ee431fe9ea Mon Sep 17 00:00:00 2001
From: Jules Laplace <julescarbon@gmail.com>
Date: Sun, 25 Nov 2018 22:19:15 +0100
Subject: moving stuff

---
 scraper/s2-merge-csv.py | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)
 create mode 100644 scraper/s2-merge-csv.py

(limited to 'scraper/s2-merge-csv.py')

diff --git a/scraper/s2-merge-csv.py b/scraper/s2-merge-csv.py
new file mode 100644
index 00000000..301f47ae
--- /dev/null
+++ b/scraper/s2-merge-csv.py
@@ -0,0 +1,28 @@
+import os
+import glob
+import time
+import simplejson as json
+import click
+import operator
+from util import *
+import random
+
+@click.command()
+@click.option('--path', '-d', default='report/institutions_geocoded', help='Path to CSVs')
+def s2_merge_csv(path):
+  print(path)
+  lookup = {}
+  for fn in glob.iglob('{}/*.csv'.format(path)):
+    rows = read_csv(fn, keys=False)
+    for row in rows:
+      key = row[0]
+      if key not in lookup:
+        lookup[key] = row
+  keys = sorted(lookup.keys())
+  deduped = []
+  for key in keys:
+    deduped.append(lookup[key])
+  write_csv('{}.csv'.format(path), keys=None, rows=deduped)
+
+if __name__ == '__main__':
+  s2_merge_csv()
-- 
cgit v1.2.3-70-g09d2