moving stuff

1 files changed, 50 insertions, 0 deletions

diff --git a/scraper/split-csv.py b/scraper/split-csv.py
new file mode 100644
index 00000000..122d2ddc
--- /dev/null
+++ b/scraper/split-csv.py
@@ -0,0 +1,50 @@
+import os
+import sys
+import csv
+from math import ceil
+import subprocess
+import random
+from util import *
+
+import click
+
+@click.command()
+@click.option('--count', '-c', default=2, help='Number of subdivisions.')
+@click.option('--has_keys/--no_keys', '-k', default=False, help='Whether to split off the keys.')
+@click.option('--shuffle/--no_shuffle', default=False, help='Whether to shuffle.')
+@click.argument('filename')
+def split_csv(count, has_keys, shuffle, filename):
+  """Split a CSV into groups."""
+  with open(filename, 'r') as f:
+    reader = csv.reader(f)
+    lines = list(unfussy_reader(reader))
+    if has_keys:
+      keys = lines[0]
+      lines = lines[1:]
+    else:
+      keys = None
+
+    fn, ext = os.path.splitext(filename)
+    if shuffle:
+      random.shuffle(lines)
+    n = max(1, ceil(len(lines) / count))
+    for index in range(count):
+      m = index * n
+      chunk = lines[m:m+n]
+      print(chunk[0])
+      out_fn = fn + '-' + str(index+1) + ext
+      write_csv(out_fn, keys, chunk)
+
+# Write a CSV
+def write_csv(fn, keys, chunk):
+  print(fn)
+  with open(fn, 'w') as f:
+    writer = csv.writer(f)
+    if keys is not None:
+      writer.writerow(keys)
+    for row in chunk:
+      writer.writerow(row)
+
+
+if __name__ == '__main__':
+  split_csv()