14 files changed, 1743 insertions, 0 deletions
diff --git a/animism-align/cli/app/utils/__pycache__/click_factory.cpython-37.pyc b/animism-align/cli/app/utils/__pycache__/click_factory.cpython-37.pyc
new file mode 100644
index 0000000..e44ccff
--- /dev/null
+++ b/animism-align/cli/app/utils/__pycache__/click_factory.cpython-37.pyc
diff --git a/animism-align/cli/app/utils/__pycache__/click_utils.cpython-37.pyc b/animism-align/cli/app/utils/__pycache__/click_utils.cpython-37.pyc
new file mode 100644
index 0000000..8aa694e
--- /dev/null
+++ b/animism-align/cli/app/utils/__pycache__/click_utils.cpython-37.pyc
diff --git a/animism-align/cli/app/utils/__pycache__/file_utils.cpython-37.pyc b/animism-align/cli/app/utils/__pycache__/file_utils.cpython-37.pyc
new file mode 100644
index 0000000..55b8f6a
--- /dev/null
+++ b/animism-align/cli/app/utils/__pycache__/file_utils.cpython-37.pyc
diff --git a/animism-align/cli/app/utils/__pycache__/log_utils.cpython-37.pyc b/animism-align/cli/app/utils/__pycache__/log_utils.cpython-37.pyc
new file mode 100644
index 0000000..2aaddb6
--- /dev/null
+++ b/animism-align/cli/app/utils/__pycache__/log_utils.cpython-37.pyc
diff --git a/animism-align/cli/app/utils/click_factory.py b/animism-align/cli/app/utils/click_factory.py
new file mode 100644
index 0000000..40b0c82
--- /dev/null
+++ b/animism-align/cli/app/utils/click_factory.py
@@ -0,0 +1,145 @@
+"""
+Click processor factory
+- Inspired by and used code from @wiretapped's HTSLAM codebase
+- In particular the very useful 
+"""
+
+import os
+import sys
+from os.path import join
+from pathlib import Path
+import os
+from os.path import join
+import sys
+from functools import update_wrapper, wraps
+import itertools
+from pathlib import Path
+from glob import glob
+import importlib
+import logging
+
+import click
+from app.settings import app_cfg as cfg
+
+
+# --------------------------------------------------------
+# Click Group Class
+# --------------------------------------------------------
+
+# set global variable during parent class create
+dir_plugins = None  # set in create
+
+class ClickComplex:
+  """Wrapper generator for custom Click CLI's based on LR's coroutine"""
+
+  def __init__(self):
+    pass
+
+
+  class CustomGroup(click.Group):
+    #global dir_plugins  # from CliGenerator init
+
+    # lists commands in plugin directory
+    def list_commands(self, ctx):
+      global dir_plugins  # from CliGenerator init
+      rv = list(self.commands.keys())
+      fp_cmds = [Path(x) for x in Path(dir_plugins).iterdir() \
+        if str(x).endswith('.py') \
+        and '__init__' not in str(x)]
+      for fp_cmd in fp_cmds:
+        try:
+          assert fp_cmd.name not in rv, "[-] Error: {} can't exist in cli.py and {}".format(fp_cmd.name)
+        except Exception as ex:
+          logging.getLogger('app').error('{}'.format(ex))
+        rv.append(fp_cmd.stem)
+      rv.sort()
+      return rv
+
+    # Complex version: gets commands in directory and in this file
+    # Based on code from @wiretapped + HTSLAM
+    def get_command(self, ctx, cmd_name):
+      global dir_plugins 
+      if cmd_name in self.commands:
+       return self.commands[cmd_name]
+      ns = {}
+      fpp_cmd = Path(dir_plugins, cmd_name + '.py')
+      fp_cmd = fpp_cmd.as_posix()
+      if not fpp_cmd.exists():
+        sys.exit('[-] {} file does not exist'.format(fpp_cmd))
+      code = compile(fpp_cmd.read_bytes(), fp_cmd, 'exec')
+      try:
+        eval(code, ns, ns)
+      except Exception as ex:
+        logging.getLogger('swimmer').error('exception: {}'.format(ex))
+        @click.command()
+        def _fail():
+          raise Exception('while loading {}'.format(fpp_cmd.name))
+        _fail.short_help = repr(ex)
+        _fail.help = repr(ex)
+        return _fail
+      if 'cli' not in ns:
+        sys.exit('[-] Error: {} does not contain a cli function'.format(fp_cmd))
+      return ns['cli']
+
+  @classmethod
+  def create(self, dir_plugins_local):
+    global dir_plugins
+    dir_plugins = dir_plugins_local
+    return self.CustomGroup
+
+
+
+class ClickSimple:
+  """Wrapper generator for custom Click CLI's"""
+
+  def __init__(self):
+    pass
+
+
+  class CustomGroup(click.Group):
+    #global dir_plugins  # from CliGenerator init
+
+    # lists commands in plugin directory
+    def list_commands(self, ctx):
+      global dir_plugins  # from CliGenerator init
+      rv = list(self.commands.keys())
+      fp_cmds = [Path(x) for x in Path(dir_plugins).iterdir() \
+        if str(x).endswith('.py') \
+        and '__init__' not in str(x)]
+      for fp_cmd in fp_cmds:
+        assert fp_cmd.name not in rv, "[-] Error: {} can't exist in cli.py and {}".format(fp_cmd.name)
+        rv.append(fp_cmd.stem)
+      rv.sort()
+      return rv
+
+    # Complex version: gets commands in directory and in this file
+    # from HTSLAM
+    def get_command(self, ctx, cmd_name):
+      global dir_plugins  # from CliGenerator init
+      if cmd_name in self.commands:
+       return self.commands[cmd_name]
+      ns = {}
+      fpp_cmd = Path(dir_plugins, cmd_name + '.py')
+      fp_cmd = fpp_cmd.as_posix()
+      if not fpp_cmd.exists():
+        sys.exit('[-] {} file does not exist'.format(fpp_cmd))
+      code = compile(fpp_cmd.read_bytes(), fp_cmd, 'exec')
+      try:
+        eval(code, ns, ns)
+      except Exception as ex:
+        logging.getLogger('swimmer').error('exception: {}'.format(ex))
+        @click.command()
+        def _fail():
+          raise Exception('while loading {}'.format(fpp_cmd.name))
+        _fail.short_help = repr(ex)
+        _fail.help = repr(ex)
+        return _fail
+      if 'cli' not in ns:
+        sys.exit('[-] Error: {} does not contain a cli function'.format(fp_cmd))
+      return ns['cli']
+
+  @classmethod
+  def create(self, dir_plugins_local):
+    global dir_plugins
+    dir_plugins = dir_plugins_local
+    return self.CustomGroup
diff --git a/animism-align/cli/app/utils/click_utils.py b/animism-align/cli/app/utils/click_utils.py
new file mode 100644
index 0000000..83e4f75
--- /dev/null
+++ b/animism-align/cli/app/utils/click_utils.py
@@ -0,0 +1,40 @@
+"""
+Custom Click parameter types
+"""
+import click
+
+from app.settings import app_cfg
+from app.models import types
+
+# --------------------------------------------------------
+# Click command helpers
+# --------------------------------------------------------
+
+def enum_to_names(enum_type):
+  return {x.name.lower(): x for x in enum_type}
+  
+def show_help(enum_type):
+  names = enum_to_names(enum_type)
+  return 'Options: "{}"'.format(', '.join(list(names.keys())))
+
+def get_default(opt):
+  return opt.name.lower()
+
+
+# --------------------------------------------------------
+# Custom Click parameter class
+# --------------------------------------------------------
+
+class ParamVar(click.ParamType):
+
+  name = 'default_type'
+
+  def __init__(self, param_type):
+    self.opts = {x.name.lower(): x for x in param_type}
+  
+  def convert(self, value, param, ctx):
+    """converts (str) repr to Enum hash"""
+    try:
+      return self.opts[value.lower()]
+    except:
+      self.fail('{} is not a valid option'.format(value, param, ctx))
diff --git a/animism-align/cli/app/utils/display_utils.py b/animism-align/cli/app/utils/display_utils.py
new file mode 100644
index 0000000..7bc1782
--- /dev/null
+++ b/animism-align/cli/app/utils/display_utils.py
@@ -0,0 +1,28 @@
+import sys
+import logging
+
+import cv2 as cv
+
+
+
+log = logging.getLogger('swimmer')
+
+def handle_keyboard(delay_amt=1):
+  '''Used with cv.imshow('title', image) to wait for keyboard press
+  '''
+  while True:
+    k = cv.waitKey(delay_amt) & 0xFF
+    if k == 27 or k == ord('q'):  # ESC
+      cv.destroyAllWindows()
+      sys.exit()
+    elif k == 32 or k == 83:  # 83 = right arrow
+      break
+    elif k != 255:
+      log.debug(f'k: {k}')
+
+def handle_keyboard_video(delay_amt=1):
+  key = cv.waitKey(1) & 0xFF
+  # if the `q` key was pressed, break from the loop
+  if key == ord("q"):
+    cv.destroyAllWindows()
+    sys.exit()
diff --git a/animism-align/cli/app/utils/draw_utils.py b/animism-align/cli/app/utils/draw_utils.py
new file mode 100644
index 0000000..8ae47fe
--- /dev/null
+++ b/animism-align/cli/app/utils/draw_utils.py
@@ -0,0 +1,166 @@
+import sys
+from math import sqrt
+import logging
+
+import numpy as np
+import cv2 as cv
+
+log = logging.getLogger('swimmer')
+
+
+# ---------------------------------------------------------------------------
+# 3D landmark drawing utilities
+# ---------------------------------------------------------------------------
+
+end_list = np.array([17, 22, 27, 42, 48, 31, 36, 68], dtype=np.int32) - 1
+
+def plot_keypoints(im, kpts):
+  '''Draw 68 key points
+  :param im: the input im
+  :param kpts: (68, 3). flattened list
+  '''
+  im = im.copy()
+  kpts = np.round(kpts).astype(np.int32)
+  for i in range(kpts.shape[0]):
+    st = kpts[i, :2]
+    im = cv.circle(im, (st[0], st[1]), 1, (0, 0, 255), 2)
+    if i in end_list:
+      continue
+    ed = kpts[i + 1, :2]
+    im = cv.line(im, (st[0], st[1]), (ed[0], ed[1]), (255, 255, 255), 1)
+  return im
+
+
+def calc_hypotenuse(pts):
+  bbox = [min(pts[0, :]), min(pts[1, :]), max(pts[0, :]), max(pts[1, :])]
+  center = [(bbox[0] + bbox[2]) / 2, (bbox[1] + bbox[3]) / 2]
+  radius = max(bbox[2] - bbox[0], bbox[3] - bbox[1]) / 2
+  bbox = [center[0] - radius, center[1] - radius, center[0] + radius, center[1] + radius]
+  llength = sqrt((bbox[2] - bbox[0]) ** 2 + (bbox[3] - bbox[1]) ** 2)
+  return llength / 3
+
+def build_camera_box(rear_size=90):
+  point_3d = []
+  rear_depth = 0
+  point_3d.append((-rear_size, -rear_size, rear_depth))
+  point_3d.append((-rear_size, rear_size, rear_depth))
+  point_3d.append((rear_size, rear_size, rear_depth))
+  point_3d.append((rear_size, -rear_size, rear_depth))
+  point_3d.append((-rear_size, -rear_size, rear_depth))
+
+  front_size = int(4 / 3 * rear_size)
+  front_depth = int(4 / 3 * rear_size)
+  point_3d.append((-front_size, -front_size, front_depth))
+  point_3d.append((-front_size, front_size, front_depth))
+  point_3d.append((front_size, front_size, front_depth))
+  point_3d.append((front_size, -front_size, front_depth))
+  point_3d.append((-front_size, -front_size, front_depth))
+  point_3d = np.array(point_3d, dtype=np.float).reshape(-1, 3)
+
+  return point_3d
+
+
+def plot_pose_box(im, Ps, pts68s, color=(40, 255, 0), line_width=2):
+  '''Draw a 3D box as annotation of pose. 
+    ref: https://github.com/yinguobing/head-pose-estimation/blob/master/pose_estimator.py
+  :param image: the input image
+  :param P: (3, 4). Affine Camera Matrix.
+  :param kpts: (2, 68) or (3, 68)
+  '''
+  im_draw = im.copy()
+  if not isinstance(pts68s, list):
+    pts68s = [pts68s]
+  
+  if not isinstance(Ps, list):
+    Ps = [Ps]
+  
+  for i in range(len(pts68s)):
+    pts68 = pts68s[i]
+    llength = calc_hypotenuse(pts68)
+    point_3d = build_camera_box(llength)
+    P = Ps[i]
+
+    # Map to 2d im points
+    point_3d_homo = np.hstack((point_3d, np.ones([point_3d.shape[0], 1])))  # n x 4
+    point_2d = point_3d_homo.dot(P.T)[:, :2]
+
+    point_2d[:, 1] = - point_2d[:, 1]
+    point_2d[:, :2] = point_2d[:, :2] - np.mean(point_2d[:4, :2], 0) + np.mean(pts68[:2, :27], 1)
+    point_2d = np.int32(point_2d.reshape(-1, 2))
+
+    # Draw all the lines
+    cv.polylines(im_draw, [point_2d], True, color, line_width, cv.LINE_AA)
+    cv.line(im_draw, tuple(point_2d[1]), tuple(point_2d[6]), color, line_width, cv.LINE_AA)
+    cv.line(im_draw, tuple(point_2d[2]), tuple(point_2d[7]), color, line_width, cv.LINE_AA)
+    cv.line(im_draw, tuple(point_2d[3]), tuple(point_2d[8]), color, line_width, cv.LINE_AA)
+
+    return im_draw
+
+
+
+# ---------------------------------------------------------------------------
+#
+# OpenCV drawing functions
+#
+# ---------------------------------------------------------------------------
+
+pose_types = {'pitch': (0,0,255), 'roll': (255,0,0), 'yaw': (0,255,0)}
+
+def draw_landmarks2D(im, points_norm, radius=3, color=(0,255,0)):
+  '''Draws facial landmarks, either 5pt or 68pt
+  '''
+  im_dst = im.copy()
+  dim = im.shape[:2][::-1]
+  for x,y in points_norm:
+    pt = (int(x*dim[0]), int(y*dim[1]))
+    cv.circle(im_dst, pt, radius, color, -1, cv.LINE_AA)
+  return im_dst
+
+def draw_landmarks3D(im, points, radius=3, color=(0,255,0)):
+  '''Draws 3D facial landmarks
+  '''
+  im_dst = im.copy()
+  for x,y,z in points:
+    cv.circle(im_dst, (x,y), radius, color, -1, cv.LINE_AA)
+  return im_dst
+
+def draw_bbox(im, bbox_norm, color=(0,255,0), stroke_weight=2):
+  '''Draws BBox onto cv image
+  '''
+  im_dst = im.copy()
+  bbox_dim = bbox_norm.to_bbox_dim(im.shape[:2][::-1])
+  cv.rectangle(im_dst, bbox_dim.p1.xy, bbox_dim.p2.xy, color, stroke_weight, cv.LINE_AA)
+  return im_dst
+
+def draw_pose(im, pt_nose, image_pts):
+  '''Draws 3-axis pose over image
+  TODO: normalize point data
+  '''
+  im_dst = im.copy()
+  log.debug(f'pt_nose: {pt_nose}')
+  log.debug(f'image_pts pitch: {image_pts["pitch"]}')
+  cv.line(im_dst, pt_nose, tuple(image_pts['pitch']), pose_types['pitch'], 3)
+  cv.line(im_dst, pt_nose, tuple(image_pts['yaw']), pose_types['yaw'], 3)
+  cv.line(im_dst, pt_nose, tuple(image_pts['roll']), pose_types['roll'], 3)
+  return im_dst
+
+def draw_text(im, pt_norm, text, size=1.0, color=(0,255,0)):
+  '''Draws degrees as text over image
+  '''
+  im_dst = im.copy()
+  dim = im.shape[:2][::-1]
+  pt = tuple(map(int, (pt_norm[0]*dim[0], pt_norm[1]*dim[1])))
+  cv.putText(im_dst, text, pt, cv.FONT_HERSHEY_SIMPLEX, size, color, thickness=1, lineType=cv.LINE_AA)
+  return im_dst
+
+def draw_degrees(im, pose_data, color=(0,255,0)):
+  '''Draws degrees as text over image
+  '''
+  im_dst = im.copy()
+  for i, pose_type in enumerate(pose_types.items()):
+    k, clr = pose_type
+    v = pose_data[k]
+    t = '{}: {:.2f}'.format(k, v)
+    origin = (10, 30 + (25 * i))
+    cv.putText(im_dst, t, origin, cv.FONT_HERSHEY_SIMPLEX, 0.5, clr, thickness=2, lineType=2)
+  return im_dst
+\ No newline at end of file
diff --git a/animism-align/cli/app/utils/file_utils.py b/animism-align/cli/app/utils/file_utils.py
new file mode 100644
index 0000000..7f1f417
--- /dev/null
+++ b/animism-align/cli/app/utils/file_utils.py
@@ -0,0 +1,519 @@
+"""
+File utility scripts
+- load and save files in csv, json, yaml, pkl formats
+- shutil helper scripts to move, copy, symlink files
+"""
+
+import sys
+import os
+from os.path import join
+import stat
+
+from glob import glob
+from pprint import pprint
+import shutil
+import distutils
+import pathlib
+from pathlib import Path
+import json
+import csv
+import pickle
+import threading
+from queue import Queue
+import time
+import logging
+import itertools
+import collections
+
+import yaml
+import hashlib
+import click
+from tqdm import tqdm
+
+
+
+# -----------------------------------------------------------------------------
+# Filesystem settings
+# -----------------------------------------------------------------------------
+
+ZERO_PADDING = 6  # padding for enumerated image filenames
+HASH_TREE_DEPTH = 2
+HASH_BRANCH_SIZE = 2
+VALID_IMAGE_EXTS = ['jpg', 'jpeg', 'png']
+VALID_VIDEO_EXTS = ['mp4', 'mov']
+
+# ------------------------------------------
+# Logger: use local app logging
+# ------------------------------------------
+
+log = logging.getLogger('swimmer')
+
+
+# ------------------------------------------
+# File I/O read/write little helpers
+# ------------------------------------------
+
+def glob_exts(dir_in, exts, recursive=False):
+  """Globs folders for multiple extensions
+  :param dir_in: input directory
+  :param exts: extensions as jpg, png, mp4
+  :param recursive: use recursive globbing
+  """
+  if not (type(exts) == list or type(exts) == tuple):
+    exts = [exts]
+  files = []
+  for ext in exts:
+    if recursive:
+      fp_glob = join(dir_in, '**/*.{}'.format(ext))
+      log.info(f'glob {fp_glob}')
+      files +=  glob(fp_glob, recursive=True)
+    else:
+      fp_glob = join(dir_in, '*.{}'.format(ext))
+      files += glob(fp_glob)
+  return files
+
+
+def zpad(x, zeros=ZERO_PADDING):
+  return str(x).zfill(zeros)
+
+def get_ext(fpp, lower=True):
+  """Retuns the file extension w/o dot
+  :param fpp: (Pathlib.path) filepath
+  :param lower: (bool) force lowercase
+  :returns: (str) file extension (ie 'jpg')
+  """
+  fpp = ensure_posixpath(fpp)
+  ext = fpp.suffix.replace('.', '')
+  return ext.lower() if lower else ext
+
+
+def convert(fp_in, fp_out):
+  """Converts between JSON and Pickle formats
+  Pickle files are about 30-40% smaller filesize
+  """
+  if get_ext(fp_in) == get_ext(fp_out):
+    log.error('Input: {} and output: {} are the same. Use this to convert.')
+
+  lazywrite(lazyload(fp_in), fp_out)
+
+
+def load_csv(fp_in, as_list=True):
+  """Loads CSV and retuns list of items
+  :param fp_in: string filepath to CSV
+  :returns: list of all CSV data
+  """ 
+  if not Path(fp_in).exists():
+    log.info('not found: {}'.format(fp_in))
+  log.info('loading: {}'.format(fp_in))
+  with open(fp_in, 'r') as fp:
+    items = csv.DictReader(fp)
+    if as_list:
+      items = [x for x in items]
+    log.info('returning {:,} items'.format(len(items)))
+    return items
+
+def unfussy_csv_reader(reader):
+  """Loads a CSV while ignoring possible data errors
+  :param reader: Special reader for load_csv_safe which ignores CSV parse errors
+  """
+  while True:
+    try:
+      yield next(reader)
+    except StopIteration:
+      return
+    except csv.Error:
+      print(csv.Error)
+      # log the problem or whatever
+      continue
+
+def load_csv_safe(fp_in, keys=True, create=False):
+  """Loads a CSV while ignoring possible data errors
+  :param fp_in: string filepath to JSON file
+  :param keys: boolean set to false if the first line is not headers (for some reason)
+  :param create: boolean set to true to return an empty keys/values if the CSV does not exist
+  """ 
+  try:
+    with open(fp_in, 'r', newline='', encoding='utf-8') as f:
+      # reader = csv.reader( (line.replace('\0','') for line in f) )
+      reader = csv.reader(f)
+      lines = list(unfussy_csv_reader(reader))
+      if keys:
+        keys = lines[0]
+        lines = lines[1:]
+        return keys, lines
+      return lines
+  except:
+    if create:
+      if keys:
+        return {}, []
+      return []
+    raise
+
+def load_recipe(fp_in):
+  """Loads a JSON file as an object with properties accessible with dot syntax
+  :param fp_in: string filepath to JSON file
+  """ 
+  with open(path) as fh:
+    return json.load(fh, object_hook=lambda d: collections.namedtuple('X', d.keys())(*d.values()))
+
+
+def lazywrite(data, fp_out, sort_keys=True):
+  """Writes JSON or Pickle data"""
+  ext = get_ext(fp_out)
+  if ext == 'json':
+    return write_json(data, fp_out, sort_keys=sort_keys)
+  elif ext == 'pkl':
+    return write_pickle(data, fp_out)
+  else:
+    raise NotImplementedError('[!] {} is not yet supported. Use .pkl or .json'.format(ext))
+
+
+def lazyload(fp_in, ordered=True):
+  """Loads JSON or Pickle serialized data"""
+  if not Path(fp_in).exists():
+    log.error('file does not exist: {}'.format(fp_in))
+    return {}
+  ext = get_ext(fp_in)
+  if ext == 'json':
+    items = load_json(fp_in)
+  elif ext == 'pkl':
+    items = load_pickle(fp_in)
+  else:
+    raise NotImplementedError('[!] {} is not yet supported. Use .pkl or .json'.format(ext))
+
+  if ordered:
+    return collections.OrderedDict(sorted(items.items(), key=lambda t: t[0]))
+  else:
+    return items
+
+
+def load_yaml(fp_in):
+  """Loads YAML file and returns (dict)
+  :param fp_in: (str) filepath
+  """
+  with open(fp_in, 'r') as fp:
+    cfg = yaml.load(fp, Loader=yaml.Loader)
+  return cfg
+
+def load_text(fp_in):
+  """Load a text file into an array
+  :param fp_in: (str) filepath
+  """
+  with open(fp_in, 'rt') as fp:
+    lines = fp.read().rstrip('\n').split('\n')
+  return lines
+
+def load_line_lookup(fp_in):
+  """Load a text file into a lookup of lines to line numbers
+  :param fp_in: (str) filepath
+  """
+  with open(fp_in, 'rt') as fp:
+    lines = fp.read().rstrip('\n').split('\n')
+  return { line: i for i, line in enumerate(lines) }
+
+def load_json(fp_in):
+  """Loads JSON and returns items
+  :param fp_in: (str) filepath
+  :returns: data from JSON
+  """
+  if not Path(fp_in).exists():
+    log.error('file does not exist: {}'.format(fp_in))
+    return {}
+  with open(str(fp_in), 'r') as fp:
+    data = json.load(fp)
+  return data
+
+
+def load_pickle(fp_in):
+  """Loads Pickle and returns items
+  :param fp_in: (str) filepath
+  :returns: data from JSON
+  """
+  if not Path(fp_in).exists():
+    log.error('file does not exist: {}'.format(fp_in))
+    return {}
+  with open(str(fp_in), 'rb') as fp:
+    data = pickle.load(fp)
+  return data
+
+
+def order_items(records):
+  """Orders records by ASC SHA256"""
+  return collections.OrderedDict(sorted(records.items(), key=lambda t: t[0]))
+
+def write_text(data, fp_out, ensure_path=True):
+  if not data:
+    log.error('no data')
+    return
+    
+  if ensure_path:
+    mkdirs(fp_out)
+  with open(fp_out, 'w') as fp:
+    if type(data) == list:
+      fp.write('\n'.join(data))
+    else:
+      fp.write(data)
+
+
+def write_pickle(data, fp_out, ensure_path=True):
+  """
+  """
+  if ensure_path:
+    mkdirs(fp_out) # mkdir
+  with open(fp_out, 'wb') as fp:
+    pickle.dump(data, fp)
+
+
+def write_json(data, fp_out, minify=True, ensure_path=True, sort_keys=True, verbose=False):
+  """
+  """
+  if ensure_path:
+    mkdirs(fp_out)
+  with open(fp_out, 'w') as fp:
+    if minify:
+      json.dump(data, fp, separators=(',',':'), sort_keys=sort_keys)
+    else:
+      json.dump(data, fp, indent=2, sort_keys=sort_keys)
+  if verbose:
+    log.info('Wrote JSON: {}'.format(fp_out))
+
+def write_csv(data, fp_out, header=None):
+  """ """
+  with open(fp_out, 'w') as fp:
+    writer = csv.DictWriter(fp, fieldnames=header)
+    writer.writeheader()
+    if type(data) is dict:
+      for k, v in data.items():
+        fp.writerow('{},{}'.format(k, v))
+
+def write_serialized_items(items, fp_out, ensure_path=True, minify=True, sort_keys=True):
+  """Writes serialized data
+  :param items: (dict) a sha256 dict of MappingItems
+  :param serialize: (bool) serialize the data
+  :param ensure_path: ensure the parent directories exist
+  :param minify: reduces JSON file size
+  """
+  log.info('Writing serialized data...')
+  fpp_out = ensure_posixpath(fp_out)
+  serialized_items = {k: v.serialize() for k, v in tqdm(items.items()) }
+  # write data
+  ext = get_ext(fpp_out)
+  if ext == 'json':
+    write_json(serialized_items, fp_out, ensure_path=ensure_path, minify=minify, sort_keys=sort_keys)
+  elif ext == 'pkl':
+    write_pickle(serialized_items, fp_out)
+  else:
+    raise NotImplementedError('[!] {} is not yet supported. Use .pkl or .json'.format(ext))
+  log.info('Wrote {:,} items to {}'.format(len(items), fp_out))
+
+
+def write_modeled_data(data, fp_out, ensure_path=False):
+  """
+  """
+  fpp_out = ensure_posixpath(fp_out)
+  if ensure_path:
+    mkdirs(fpp_out)
+  ext = get_ext(fpp_out)
+  if ext == 'pkl':
+    write_pickle(data, str(fp_out))
+  else:
+    raise NotImplementedError('[!] {} is not yet supported. Use .pkl or .json'.format(ext))
+
+
+# ---------------------------------------------------------------------
+# Filepath utilities
+# ---------------------------------------------------------------------
+
+def ensure_posixpath(fp):
+  """Ensures filepath is pathlib.Path
+  :param fp: a (str, LazyFile, PosixPath)
+  :returns: a PosixPath filepath object
+  """
+  if type(fp) == str:
+    fpp = Path(fp)
+  elif type(fp) == click.utils.LazyFile:
+    fpp = Path(fp.name)
+  elif type(fp) == pathlib.PosixPath:
+    fpp = fp
+  else:
+    raise TypeError('{} is not a valid filepath type'.format(type(fp)))
+  return fpp
+
+
+def mkdirs(fp):
+  """Ensure parent directories exist for a filepath
+  :param fp: string, Path, or click.File
+  """
+  fpp = ensure_posixpath(fp)
+  fpp = fpp.parent if fpp.suffix else fpp
+  fpp.mkdir(parents=True, exist_ok=True)
+
+def ensure_posixpath(fp):
+  """Ensures filepath is pathlib.Path
+  :param fp: a (str, LazyFile, PosixPath)
+  :returns: a PosixPath filepath object
+  """
+  if type(fp) == str:
+    fpp = Path(fp)
+  elif type(fp) == click.utils.LazyFile:
+    fpp = Path(fp.name)
+  elif type(fp) == pathlib.PosixPath:
+    fpp = fp
+  else:
+    raise TypeError('{} is not a valid filepath type'.format(type(fp)))
+  return fpp
+
+def ensure_dir(fp):
+  if not Path(fp).is_dir():
+    mkdirs(fp)
+
+
+def ext_media_format(ext):
+  """Converts file extension into Enum MediaType
+  param ext: str of file extension"
+  """
+  for media_format, exts in VALID_MEDIA_EXTS.items():
+    if ext in exts:
+      return media_format
+  raise ValueError('{} is not a valid option'.format(ext))
+
+
+def sha256(fp_in, block_size=65536):
+  """Generates SHA256 hash for a file
+  :param fp_in: (str) filepath
+  :param block_size: (int) byte size of block
+  :returns: (str) hash
+  """
+  sha256 = hashlib.sha256()
+  with open(fp_in, 'rb') as fp:
+    for block in iter(lambda: fp.read(block_size), b''):
+      sha256.update(block)
+  return sha256.hexdigest()
+
+def sha256_stream(stream, block_size=65536):
+  """Generates SHA256 hash for a file stream (from Flask)
+  :param fp_in: (FileStream) stream object
+  :param block_size: (int) byte size of block
+  :returns: (str) hash
+  """
+  sha256 = hashlib.sha256()
+  for block in iter(lambda: stream.read(block_size), b''):
+    sha256.update(block)
+  return sha256.hexdigest()
+
+def sha256_tree(sha256):
+  """Split hash into branches with tree-depth for faster file indexing
+  :param sha256: str of a sha256 hash
+  :returns: str with sha256 tree with '/' delimeter
+  """
+  branch_size = HASH_BRANCH_SIZE
+  tree_size = HASH_TREE_DEPTH * branch_size
+  sha256_tree = [sha256[i:(i+branch_size)] for i in range(0, tree_size, branch_size)]
+  return '/'.join(sha256_tree)
+
+
+def migrate(fmaps, threads=1, action='copy', force=False):
+  """Copy/move/symlink files form src to dst directory
+  :param fmaps: (dict) with 'src' and 'dst' filepaths
+  :param threads: (int) number of threads
+  :param action: (str) copy/move/symlink
+  :param force: (bool) force overwrite existing files
+  """
+  log = log
+  num_items = len(fmaps)
+  
+  def copytree(src, dst, symlinks = False, ignore = None):
+    # ozxyqk: https://stackoverflow.com/questions/22588225/how-do-you-merge-two-directories-or-move-with-replace-from-the-windows-command
+    if not os.path.exists(dst):
+      mkdirs(dst)
+      # os.makedirs(dst)
+      shutil.copystat(src, dst)
+    lst = os.listdir(src)
+    if ignore:
+      excl = ignore(src, lst)
+      lst = [x for x in lst if x not in excl]
+    for item in lst:
+      s = os.path.join(src, item)
+      d = os.path.join(dst, item)
+      if symlinks and os.path.islink(s):
+        if os.path.exists(d):
+          os.remove(d)
+        os.symlink(os.readlink(s), d)
+        try:
+          st = os.lstat(s)
+          mode = stat.S_IMODE(st.st_mode)
+          os.lchmod(d, mode)
+        except:
+          pass # lchmod not available
+      elif os.path.isdir(s):
+        copytree(s, d, symlinks, ignore)
+      else:
+        shutil.copy(s, d)
+
+  assert(action in ['copy','move','symlink'])
+
+  if threads > 1:
+    # threaded
+    task_queue = Queue()
+    print_lock = threading.Lock()
+
+    def migrate_action(fmap):
+      data_local = threading.local()
+      data_local.src, data_local.dst = (fmap['src'], fmap['dst'])
+      data_local.src_path = Path(data_local.src)
+      data_local.dst_path = Path(data_local.dst)
+      
+      if force or not data_local.dst_path.exists():
+        if action == 'copy':
+          shutil.copy(data_local.src, data_local.dst)
+          #if data_local.src_path.is_dir():
+          #  copytree(data_local.src, data_local.dst)
+          #else:
+        elif action == 'move':
+          shutil.move(data_local.src, data_local.dst)
+        elif action == 'symlink':
+          if force:
+            data_local.dst_path.unlink()
+          Path(data_local.src).symlink_to(data_local.dst)
+
+    def process_queue(num_items):
+      # TODO: progress bar
+      while True:
+        fmap = task_queue.get()
+        migrate_action(fmap)
+        log.info('migrate: {:.2f} {:,}/{:,}'.format( 
+          (task_queue.qsize() / num_items)*100, task_queue.qsize(), num_items))
+        task_queue.task_done()
+
+    # avoid race conditions by creating dir structure here
+    log.info('create directory structure')
+    for fmap in tqdm(fmaps):
+      mkdirs(fmap['dst'])
+
+    # init threads
+    for i in range(threads):
+      t = threading.Thread(target=process_queue, args=(num_items,))
+      t.daemon = True
+      t.start()
+
+    # process threads
+    start = time.time()
+    for fmap in fmaps:
+      task_queue.put(fmap)
+
+    task_queue.join()
+
+  else:
+    # non-threaded
+    for fmap in tqdm(fmaps):
+      mkdirs(fmap['dst'])
+      if action == 'copy':
+        shutil.copy(fmap['src'], fmap['dst'])
+      elif action == 'move':
+        shutil.move(fmap['src'], fmap['dst'])
+      elif action == 'symlink':
+        if force:
+          Path(fmap['dst'].unlink())
+        Path(fp_src).symlink_to(fp_dst)
+  return
+
diff --git a/animism-align/cli/app/utils/im_utils.py b/animism-align/cli/app/utils/im_utils.py
new file mode 100644
index 0000000..839c268
--- /dev/null
+++ b/animism-align/cli/app/utils/im_utils.py
@@ -0,0 +1,579 @@
+import sys
+import os
+from os.path import join
+import cv2 as cv
+import imagehash
+from PIL import Image, ImageDraw, ImageFilter, ImageOps
+from skimage.filters.rank import entropy
+from skimage.morphology import disk
+from skimage import feature
+# import matplotlib.pyplot as plt
+import imutils
+import time
+import numpy as np
+import struct
+from sklearn.metrics.pairwise import cosine_similarity
+import datetime
+
+def ensure_pil(im):
+    """Ensure image is Pillow format"""
+    try:
+        im.verify()
+        return im
+    except:
+        return Image.fromarray(im.astype('uint8'), 'RGB')
+
+def ensure_np(im):
+    """Ensure image is numpy array"""
+    if type(im) == np.ndarray:
+        return im
+    return np.asarray(im, np.uint8)
+
+def np2pil(im, swap=True):
+  """Ensure image is Pillow format
+    :param im: image in numpy or PIL.Image format
+    :returns: image in Pillow RGB format
+  """
+  try:
+    im.verify()
+    return im
+  except:
+    if swap:
+      im = bgr2rgb(im)
+    return Image.fromarray(im.astype('uint8'), 'RGB')
+
+def pil2np(im, swap=True):
+  """Ensure image is Numpy.ndarry format
+    :param im: image in numpy or PIL.Image format
+    :returns: image in Numpy uint8 format
+  """
+  if type(im) == np.ndarray:
+    return im
+  im = np.asarray(im, np.uint8)
+  if swap:
+    im = rgb2bgr(im)
+  return im
+
+def num_channels(im):
+  '''Returns number of channels in numpy.ndarray image'''
+  if len(im.shape) > 2:
+    return im.shape[2]
+  else:
+    return 1
+
+def is_grayscale(im, threshold=5):
+  """Returns True if image is grayscale
+  :param im: (numpy.array) image
+  :return (bool) of if image is grayscale"""
+  b = im[:,:,0]
+  g = im[:,:,1]
+  mean = np.mean(np.abs(g - b))
+  return mean < threshold
+
+
+def compute_features(fe,frames,phashes,phash_thresh=1):
+  """
+  Get vector embedding using FeatureExtractor
+  :param fe: FeatureExtractor class
+  :param frames: list of frame images as numpy.ndarray
+  :param phash_thresh: perceptual hash threshold
+  :returns: list of feature vectors
+  """
+  vals = []
+  phash_pre = phashes[0]
+  for i,im in enumerate(frames):
+    if i == 0 or (phashes[i] - phashes[i-1]) > phash_thresh:
+      vals.append(fe.extract(im))
+    else:
+      vals.append(vals[i-1])
+  return vals
+
+
+# def np2pil(im, swap=True):
+#   """Ensure image is Pillow format
+#     :param im: image in numpy or PIL.Image format
+#     :returns: image in Pillow RGB format
+#   """
+#   try:
+#       im.verify()
+#       return im
+#   except:
+#     if swap:
+#       im = cv.cvtColor(im,cv.COLOR_BGR2RGB)
+#     return Image.fromarray(im.astype('uint8'), 'RGB')
+
+# def pil2np(im, swap=True):
+#   """Ensure image is Numpy.ndarry format
+#     :param im: image in numpy or PIL.Image format
+#     :returns: image in Numpy uint8 format
+#   """
+#   if type(im) == np.ndarray:
+#     return im
+#   im = np.asarray(im, np.uint8)
+#   if swap:
+#     im = cv.cvtColor(im, cv.COLOR_RGB2BGR)
+#   return im
+
+
+def resize(im, width=0, height=0):
+  """resize image using imutils. Use w/h=[0 || None] to prioritize other edge size
+    :param im: a Numpy.ndarray image
+    :param wh: a tuple of (width, height)
+  """
+  # TODO change to cv.resize and add algorithm choices
+  w = width
+  h = height
+  if w is 0 and h is 0:
+    return im
+  elif w > 0 and h > 0:
+    ws = im.shape[1] / w
+    hs = im.shape[0] / h
+    if ws > hs:
+      return imutils.resize(im, width=w)
+    else:
+      return imutils.resize(im, height=h)
+  elif w > 0 and h is 0:
+    return imutils.resize(im, width=w)
+  elif w is 0 and h > 0:
+    return imutils.resize(im, height=h)
+  else:
+    return im
+
+def filter_pixellate(im,num_cells):
+  """Pixellate image by downsample then upsample
+    :param im: PIL.Image
+    :returns: PIL.Image
+  """
+  w,h = im.size
+  im = im.resize((num_cells,num_cells), Image.NEAREST)
+  im = im.resize((w,h), Image.NEAREST)
+  return im
+
+# Plot images inline using Matplotlib
+# def pltimg(im,title=None,mode='rgb',figsize=(8,12),dpi=160,output=None):
+#   plt.figure(figsize=figsize)
+#   plt.xticks([]),plt.yticks([])
+#   if title is not None:
+#     plt.title(title)
+#   if mode.lower() == 'bgr':
+#     im = cv.cvtColor(im,cv.COLOR_BGR2RGB)
+
+#   f = plt.gcf()
+#   if mode.lower() =='grey' or mode.lower() == 'gray':
+#     plt.imshow(im,cmap='gray')
+#   else:
+#     plt.imshow(im)
+#   plt.show()
+#   plt.draw()
+#   if output is not None:
+#     bbox_inches='tight'
+#     ext=osp.splitext(output)[1].replace('.','')
+#     f.savefig(output,dpi=dpi,format=ext)
+#     print('Image saved to: {}'.format(output))
+
+
+
+# Utilities for analyzing frames
+
+# def compute_gray(im):
+#   im = cv.cvtColor(im,cv.COLOR_BGR2GRAY)
+#   n_vals = float(im.shape[0] * im.shape[1])
+#   avg = np.sum(im[:]) / n_vals
+#   return avg
+
+# def compute_rgb(im):
+#   im = cv.cvtColor(im,cv.COLOR_BGR2RGB)
+#   n_vals = float(im.shape[0] * im.shape[1])
+#   avg_r = np.sum(im[:,:,0]) / n_vals 
+#   avg_g = np.sum(im[:,:,1]) / n_vals
+#   avg_b = np.sum(im[:,:,2]) / n_vals
+#   avg_rgb = np.sum(im[:,:,:]) / (n_vals * 3.0)
+#   return avg_r, avg_b, avg_g, avg_rgb
+
+# def compute_hsv(im):
+#   im = cv.cvtColor(im,cv.COLOR_BGR2HSV)
+#   n_vals = float(im.shape[0] * im.shape[1])
+#   avg_h = np.sum(frame[:,:,0]) / n_vals
+#   avg_s = np.sum(frame[:,:,1]) / n_vals
+#   avg_v = np.sum(frame[:,:,2]) / n_vals
+#   avg_hsv = np.sum(frame[:,:,:]) / (n_vals * 3.0)
+#   return avg_h, avg_s, avg_v, avg_hsv
+
+# def pys_dhash(im, hashSize=8):
+#   # resize the input image, adding a single column (width) so we
+#   # can compute the horizontal gradient
+#   resized = cv.resize(im, (hashSize + 1, hashSize))
+#   # compute the (relative) horizontal gradient between adjacent
+#   # column pixels
+#   diff = resized[:, 1:] > resized[:, :-1]
+#   # convert the difference image to a hash
+#   return sum([2 ** i for (i, v) in enumerate(diff.flatten()) if v])
+
+
+############################################
+# ImageHash 
+# pip install imagehash
+############################################
+
+def compute_ahash(im):
+  """Compute average hash using ImageHash library
+    :param im: Numpy.ndarray
+    :returns: Imagehash.ImageHash
+  """
+  return imagehash.average_hash(ensure_pil(im_pil))
+
+def compute_phash(im):
+  """Compute perceptual hash using ImageHash library
+    :param im: Numpy.ndarray
+    :returns: Imagehash.ImageHash
+  """
+  return imagehash.phash(ensure_pil(im))
+
+def phash2int(phash):
+  """Compute perceptual hash using ImageHash library and convert to binary
+    :param phash: Imagehash.ImageHash
+    :returns: binary-encoded bigint
+  """
+  phash.hash[-1] = False
+  phash_as_bigint = struct.unpack('Q', np.packbits(phash.hash))[0]
+  return phash_as_bigint
+
+def compute_phash_int(im):
+  """Compute perceptual hash using ImageHash library and convert to binary
+    :param im: Numpy.ndarray
+    :returns: binary-encoded bigint
+  """
+  return phash2int(compute_phash(im))
+
+def compute_dhash(im):
+  """Compute difference hash using ImageHash library
+    :param im: Numpy.ndarray
+    :returns: Imagehash.ImageHash
+  """
+  return imagehash.dhash(ensure_pil(im))
+
+def compute_whash(im):
+  """Compute wavelet hash using ImageHash library
+    :param im: Numpy.ndarray
+    :returns: Imagehash.ImageHash
+  """
+  return imagehash.whash(ensure_pil(im))
+
+def compute_whash_b64(im):
+  """Compute wavelest hash base64 using ImageHash library
+    :param im: Numpy.ndarray
+    :returns: Imagehash.ImageHash
+  """
+  return lambda im: imagehash.whash(ensure_pil(im), mode='db4')
+
+
+############################################
+# Pillow 
+############################################
+
+def sharpen(im):
+  """Sharpen image using PIL.ImageFilter
+  param: im: PIL.Image
+  returns: PIL.Image
+  """
+  im = ensure_pil(im)
+  im.filter(ImageFilter.SHARPEN)
+  return ensure_np(im)
+
+def fit_image(im,targ_size):
+  """Force fit image by cropping
+  param: im: PIL.Image
+  param: targ_size: a tuple of target (width, height)
+  returns: PIL.Image
+  """
+  im_pil = ensure_pil(im)
+  frame_pil = ImageOps.fit(im_pil, targ_size, 
+    method=Image.BICUBIC, centering=(0.5, 0.5))
+  return ensure_np(frame_pil)
+
+
+def compute_entropy(im):
+  entr_img = entropy(im, disk(10))
+
+
+############################################
+# scikit-learn 
+############################################
+
+def compute_entropy(im):
+  # im is grayscale numpy
+  return entropy(im, disk(10))
+
+############################################
+# OpenCV 
+############################################
+
+def bgr2gray(im):
+  """Wrapper for cv2.cvtColor transform
+    :param im: Numpy.ndarray (BGR)
+    :returns: Numpy.ndarray (Gray)
+  """
+  return cv.cvtColor(im,cv.COLOR_BGR2GRAY)
+
+def gray2bgr(im):
+  """Wrapper for cv2.cvtColor transform
+    :param im: Numpy.ndarray (Gray)
+    :returns: Numpy.ndarray (BGR)
+  """
+  return cv.cvtColor(im,cv.COLOR_GRAY2BGR)
+
+def bgr2rgb(im):
+  """Wrapper for cv2.cvtColor transform
+    :param im: Numpy.ndarray (BGR)
+    :returns: Numpy.ndarray (RGB)
+  """
+  return cv.cvtColor(im,cv.COLOR_BGR2RGB)
+
+def compute_laplacian(im):
+  # below 100 is usually blurry
+  return cv.Laplacian(im, cv.CV_64F).var()
+
+
+# # http://radjkarl.github.io/imgProcessor/index.html#
+
+# def modifiedLaplacian(img):
+#     ''''LAPM' algorithm (Nayar89)'''
+#     M = np.array([-1, 2, -1])
+#     G = cv.getGaussianKernel(ksize=3, sigma=-1)
+#     Lx = cv.sepFilter2D(src=img, ddepth=cv.CV_64F, kernelX=M, kernelY=G)
+#     Ly = cv.sepFilter2D(src=img, ddepth=cv.CV_64F, kernelX=G, kernelY=M)
+#     FM = np.abs(Lx) + np.abs(Ly)
+#     return cv.mean(FM)[0]
+
+# def varianceOfLaplacian(img):
+#     ''''LAPV' algorithm (Pech2000)'''
+#     lap = cv.Laplacian(img, ddepth=-1)#cv.cv.CV_64F)
+#     stdev = cv.meanStdDev(lap)[1]
+#     s = stdev[0]**2
+#     return s[0]
+
+# def tenengrad(img, ksize=3):
+#     ''''TENG' algorithm (Krotkov86)'''
+#     Gx = cv.Sobel(img, ddepth=cv.CV_64F, dx=1, dy=0, ksize=ksize)
+#     Gy = cv.Sobel(img, ddepth=cv.CV_64F, dx=0, dy=1, ksize=ksize)
+#     FM = Gx**2 + Gy**2
+#     return cv.mean(FM)[0]
+
+# def normalizedGraylevelVariance(img):
+#     ''''GLVN' algorithm (Santos97)'''
+#     mean, stdev = cv.meanStdDev(img)
+#     s = stdev[0]**2 / mean[0]
+#     return s[0]
+
+def compute_if_blank(im,width=100,sigma=0,thresh_canny=.1,thresh_mean=4,mask=None):
+  # im is graysacale np
+  #im = imutils.resize(im,width=width)
+  #mask = imutils.resize(mask,width=width)
+  if mask is not None:
+    im_canny = feature.canny(im,sigma=sigma,mask=mask)
+    total = len(np.where(mask > 0)[0])
+  else:
+    im_canny = feature.canny(im,sigma=sigma)
+    total = (im.shape[0]*im.shape[1])
+  n_white = len(np.where(im_canny > 0)[0])
+  per = n_white/total
+  if np.mean(im) < thresh_mean or per < thresh_canny:
+    return 1
+  else:
+    return 0
+
+
+def print_timing(t,n):
+    t = time.time()-t
+    print('Elapsed time: {:.2f}'.format(t))
+    print('FPS: {:.2f}'.format(n/t))
+
+# def vid2frames(fpath, limit=5000, width=None, idxs=None):
+#   """Convert a video file into list of frames
+#     :param fpath: filepath to the video file
+#     :param limit: maximum number of frames to read
+#     :param fpath: the indices of frames to keep (rest are skipped)
+#     :returns: (fps, number of frames, list of Numpy.ndarray frames)
+#   """
+#   frames = []
+#   try:
+#     cap = cv.VideoCapture(fpath)
+#   except:
+#     print('[-] Error. Could not read video file: {}'.format(fpath))
+#     try:
+#       cap.release()
+#     except:
+#       pass
+#     return frames
+
+#   fps = cap.get(cv.CAP_PROP_FPS)
+#   nframes = int(cap.get(cv.CAP_PROP_FRAME_COUNT))
+
+#   if idxs is not None:
+#     # read sample indices by seeking to frame index
+#     for idx in idxs:
+#       cap.set(cv.CAP_PROP_POS_FRAMES, idx)
+#       res, frame = cap.read()
+#       if width is not None:
+#         frame = imutils.resize(frame, width=width)
+#       frames.append(frame)
+#   else:
+#     while(True and len(frames) < limit):
+#       res, frame = cap.read()
+#       if not res:
+#         break
+#       if width is not None:
+#         frame = imutils.resize(frame, width=width)
+#       frames.append(frame)
+
+#   cap.release()
+#   del cap
+#   #return fps,nframes,frames
+#   return frames
+
+def convolve_filter(vals,filters=[1]):
+  for k in filters:
+    vals_tmp = np.zeros_like(vals)
+    t = len(vals_tmp)
+    for i,v in enumerate(vals):
+      sum_vals = vals[max(0,i-k):min(t-1,i+k)]
+      vals_tmp[i] = np.mean(sum_vals)
+    vals = vals_tmp.copy()
+  return vals
+
+def cosine_delta(v1,v2):
+  return 1.0 - cosine_similarity(v1.reshape((1, -1)), v2.reshape((1, -1)))[0][0]
+
+
+
+def compute_edges(vals):
+  # find edges (1 = rising, -1 = falling)
+  edges = np.zeros_like(vals)
+  for i in range(len(vals[1:])):
+    delta = vals[i] - vals[i-1]
+    if delta == -1:
+      edges[i] = 1 # rising edge 0 --> 1
+    elif delta == 1:
+      edges[i+1] = 2 # falling edge 1 --> 0
+  # get index for rise fall
+  rising = np.where(np.array(edges) == 1)[0]
+  falling = np.where(np.array(edges) == 2)[0]
+  return rising, falling 
+
+
+############################################
+# Point, Rect
+############################################
+
+class Point(object):
+  def __init__(self, x, y):
+    self.x = x
+    self.y = y
+
+class Rect(object):
+  def __init__(self, p1, p2):
+    '''Store the top, bottom, left and right values for points 
+           p1 and p2 are the (corners) in either order
+    '''
+    self.left   = min(p1.x, p2.x)
+    self.right  = max(p1.x, p2.x)
+    self.top    = min(p1.y, p2.y)
+    self.bottom = max(p1.y, p2.y)
+    
+def overlap(r1, r2):
+    '''Overlapping rectangles overlap both horizontally & vertically
+    '''
+    return range_overlap(r1.left, r1.right, r2.left, r2.right) and \
+      range_overlap(r1.top, r1.bottom, r2.top, r2.bottom)
+
+def range_overlap(a_min, a_max, b_min, b_max):
+  '''Neither range is completely greater than the other
+  '''
+  return (a_min <= b_max) and (b_min <= a_max)
+
+def merge_rects(r1,r2):
+  p1 = Point(min(r1.left,r2.left),min(r1.top,r2.top))
+  p2 = Point(max(r1.right,r2.right),max(r1.bottom,r2.bottom))
+  return Rect(p1,p2)
+
+def is_overlapping(r1,r2):
+  """r1,r2 as [x1,y1,x2,y2] list"""
+  r1x = Rect(Point(r1[0],r1[1]),Point(r1[2],r1[3]))
+  r2x = Rect(Point(r2[0],r2[1]),Point(r2[2],r2[3]))
+  return overlap(r1x,r2x)
+
+def get_rects_merged(rects,bounds,expand=0):
+  """rects: list of points in [x1,y1,x2,y2] format"""
+  rects_expanded = []
+  bx,by = bounds
+  # expand
+  for x1,y1,x2,y2 in rects:
+    x1 = max(0,x1-expand)
+    y1 = max(0,y1-expand)
+    x2 = min(bx,x2+expand)
+    y2 = min(by,y2+expand)
+    rects_expanded.append(Rect(Point(x1,y1),Point(x2,y2)))
+
+  #rects_expanded = [Rect(Point(x1,y1),Point(x2,y2)) for x1,y1,x2,y2 in rects_expanded]
+  rects_merged = []
+  for i,r in enumerate(rects_expanded):
+    found = False
+    for j,rm in enumerate(rects_merged):
+      if overlap(r,rm):
+        rects_merged[j] = merge_rects(r,rm) #expand
+        found = True
+    if not found:
+      rects_merged.append(r)
+  # convert back to [x1,y1,x2,y2] format
+  rects_merged = [(r.left,r.top,r.right,r.bottom) for r in rects_merged]
+  # contract
+  rects_contracted = []
+  for x1,y1,x2,y2 in rects_merged:
+    x1 = min(bx,x1+expand)
+    y1 = min(by,y1+expand)
+    x2 = max(0,x2-expand)
+    y2 = max(0,y2-expand)
+    rects_contracted.append((x1,y1,x2,y2))
+
+  return rects_contracted
+
+
+############################################
+# Image display
+############################################
+
+
+def montage(frames,ncols=4,nrows=None,width=None):
+  """Convert list of frames into a grid montage
+  param: frames: list of frames as Numpy.ndarray
+  param: ncols: number of columns
+  param: width: resize images to this width before adding to grid
+  returns: Numpy.ndarray grid of all images
+  """
+
+  # expand image size if not enough frames
+  if nrows is not None and len(frames) < ncols * nrows:
+    blank = np.zeros_like(frames[0])
+    n = ncols * nrows - len(frames)
+    for i in range(n): frames.append(blank) 
+
+  rows = []
+  for i,im in enumerate(frames):
+    if width is not None:
+      im = imutils.resize(im,width=width)
+    h,w = im.shape[:2]
+    if i % ncols == 0:
+      if i > 0:
+        rows.append(ims)
+      ims = []
+    ims.append(im)
+  if len(ims) > 0:
+    for j in range(ncols-len(ims)):
+      ims.append(np.zeros_like(im))
+    rows.append(ims)
+  row_ims = []
+  for row in rows:
+    row_im = np.hstack(np.array(row))
+    row_ims.append(row_im)
+  contact_sheet = np.vstack(np.array(row_ims))
+  return contact_sheet
diff --git a/animism-align/cli/app/utils/log_utils.py b/animism-align/cli/app/utils/log_utils.py
new file mode 100644
index 0000000..1e0f22f
--- /dev/null
+++ b/animism-align/cli/app/utils/log_utils.py
@@ -0,0 +1,69 @@
+"""
+Logger instantiator for use with Click utlity scripts
+"""
+import sys
+import os
+import logging
+
+import colorlog
+
+class Logger:
+  
+  LOGGER_NAME = 'swimmer'
+  LOGFILE_FORMAT = "%(log_color)s%(levelname)-8s%(reset)s %(cyan)s%(filename)s:%(lineno)s:%(bold_cyan)s%(funcName)s() %(reset)s%(message)s"
+
+  def __init__(self):
+    pass
+
+  @staticmethod
+  def create(verbosity=4, logfile=None):
+    """Configures a logger from click params
+    :param verbosity: (int) between 0 and 5
+    :param logfile: (str) path to logfile
+    :returns: logging root object
+    """
+    
+    loglevel = (5 - (max(0, min(verbosity, 5)))) * 10 # where logging.DEBUG = 10
+    date_format = '%Y-%m-%d %H:%M:%S'
+    if 'colorlog' in sys.modules and os.isatty(2):
+      cformat = '%(log_color)s' + Logger.LOGFILE_FORMAT
+      f = colorlog.ColoredFormatter(cformat, date_format,
+        log_colors = { 'DEBUG'   : 'yellow',       'INFO' : 'white',
+        'WARNING' : 'bold_yellow', 'ERROR': 'bold_red',
+        'CRITICAL': 'bold_red' })
+    else:
+      f = logging.Formatter(Logger.LOGFILE_FORMAT, date_format)
+    
+    logger = logging.getLogger(Logger.LOGGER_NAME)
+    logger.setLevel(loglevel)
+    
+    # remove existing handlers
+    for handler in logger.handlers:
+      logger.removeHandler(handler)
+
+    if logfile:
+      # create file handler which logs even debug messages
+      fh = logging.FileHandler(logfile)
+      fh.setLevel(loglevel)
+      logger.addHandler(fh)
+
+    # add colored handler
+    ch = logging.StreamHandler()
+    ch.setFormatter(f)
+    logger.addHandler(ch)
+
+    if verbosity == 0:
+      logger.disabled = True
+
+    # test
+    # logger.debug('Hello Debug')
+    # logger.info('Hello Info')
+    # logger.warn('Hello Warn')
+    # logger.error('Hello Error')
+    # logger.critical('Hello Critical')
+
+    return logger
+
+  @staticmethod
+  def getLogger():
+    return logging.getLogger(Logger.LOGGER_NAME)
+\ No newline at end of file
diff --git a/animism-align/cli/app/utils/process_utils.py b/animism-align/cli/app/utils/process_utils.py
new file mode 100644
index 0000000..7f243ae
--- /dev/null
+++ b/animism-align/cli/app/utils/process_utils.py
@@ -0,0 +1,60 @@
+import os
+import pathos.pools as pp
+from tqdm import tqdm
+from concurrent.futures import ProcessPoolExecutor, as_completed
+
+def parallel_process(array, function, n_jobs=16, use_kwargs=False, front_num=3):
+    """
+        A parallel version of the map function with a progress bar. 
+
+        Args:
+            array (array-like): An array to iterate over.
+            function (function): A python function to apply to the elements of array
+            n_jobs (int, default=16): The number of cores to use
+            use_kwargs (boolean, default=False): Whether to consider the elements of array as dictionaries of 
+                keyword arguments to function 
+            front_num (int, default=3): The number of iterations to run serially before kicking off the parallel job. 
+                Useful for catching bugs
+        Returns:
+            [function(array[0]), function(array[1]), ...]
+    """
+    #We run the first few iterations serially to catch bugs
+    if front_num > 0:
+        front = [function(**a) if use_kwargs else function(a) for a in array[:front_num]]
+    #If we set n_jobs to 1, just run a list comprehension. This is useful for benchmarking and debugging.
+    if n_jobs==1:
+        return front + [function(**a) if use_kwargs else function(a) for a in tqdm(array[front_num:])]
+    #Assemble the workers
+    with ProcessPoolExecutor(max_workers=n_jobs) as pool:
+        #Pass the elements of array into function
+        if use_kwargs:
+            futures = [pool.submit(function, **a) for a in array[front_num:]]
+        else:
+            futures = [pool.submit(function, a) for a in array[front_num:]]
+        kwargs = {
+            'total': len(futures),
+            'unit': 'it',
+            'unit_scale': True,
+            'leave': True
+        }
+        #Print out the progress as tasks complete
+        for f in tqdm(as_completed(futures), **kwargs):
+            pass
+    out = []
+    #Get the results from the futures. 
+    for i, future in tqdm(enumerate(futures)):
+        try:
+            out.append(future.result())
+        except Exception as e:
+            out.append(e)
+    return front + out
+
+def parallelize(rows, func):
+  print("Processing {} items".format(len(rows)))
+  if hasattr(os, 'sched_getaffinity'):
+    processCount = len(os.sched_getaffinity(0))
+  else:
+    processCount = 4
+  print('processes {}'.format(processCount))
+  with pp.ProcessPool(processes=processCount) as pool:
+    pool.map(func, rows)
+\ No newline at end of file
diff --git a/animism-align/cli/app/utils/s3_utils.py b/animism-align/cli/app/utils/s3_utils.py
new file mode 100644
index 0000000..d8cff79
--- /dev/null
+++ b/animism-align/cli/app/utils/s3_utils.py
@@ -0,0 +1,114 @@
+import os
+import glob
+from os.path import join
+from dataclasses import dataclass
+from glob import glob
+from pathlib import Path
+import logging
+
+import dacite  # dataclass helper util
+import boto3
+
+
+@dataclass
+class S3Config:
+  S3_BUCKET: str
+  S3_KEY: str
+  S3_SECRET: str
+  S3_ENDPOINT: str
+  S3_REGION: str
+
+
+class RemoteStorageS3:
+
+  def __init__(self):
+
+    self.log = logging.getLogger('swimmer')
+
+    self.s3_cfg = dacite.from_dict(data_class=S3Config, data=os.environ)
+
+    self.session = boto3.session.Session()
+
+    self.s3_client = self.session.client(
+      service_name='s3',
+      aws_access_key_id=self.s3_cfg.S3_KEY,
+      aws_secret_access_key=self.s3_cfg.S3_SECRET,
+      endpoint_url=self.s3_cfg.S3_ENDPOINT,
+      region_name=self.s3_cfg.S3_REGION,
+    )
+
+
+  def list_dir(self, fp_dir_remote):
+    '''Sync local directory to remote directory
+    '''
+    
+    obj_list_remote = self.s3_client.list_objects(
+      Bucket=self.s3_cfg.S3_BUCKET, 
+      Prefix=fp_dir_remote)
+
+
+    for obj in obj_list_remote.get('Contents', []):
+      s3_fn = obj['Key']
+      self.log.debug(s3_fn)
+
+
+  def sync_dir(self, fp_dir_local, fp_dir_remote):
+    '''Sync local directory to remote directory
+    '''
+    
+    # get list of local files
+    fps_local = glob(join(fp_dir_local, '*'))
+    fp_local_lkup = {}
+    for fp in fps_local:
+      fp_local_lkup[Path(fp).name] = fp
+
+    # get list of remote files
+    obj_list_remote = self.s3_client.list_objects(Bucket=self.s3_cfg.S3_BUCKET, Prefix=fp_dir_remote)
+    # check if remove files exist locally
+    if 'Contents' in obj_list_remote:
+      for obj in obj_list_remote['Contents']:
+        s3_fn = obj['Key']
+        fn_remote = Path(s3_fn).name
+        if fn_remote in fp_local_lkup.keys():
+          # remove from queue
+          # compare timestamps
+          fp_local = fp_local_lkup[fn_remote]
+          del fp_local_lkup[fn_remote]
+          if obj['LastModified'].timestamp() < os.path.getmtime(fp_local):
+            self.log.debug("Update s3 with newer local file: {}".format(s3_fn))
+            self.s3_client.upload_file(
+              fp_local,
+              self.s3_cfg.S3_BUCKET,
+              s3_fn,
+              ExtraArgs={'ACL': 'public-read' })
+          else:
+            self.log.debug(f'Skipping same file: {s3_fn}')
+        else:
+          self.log.debug(f'Orphan remote file: {s3_fn}')
+          self.log.debug("s3 delete {}".format(s3_fn))
+          response = self.s3_client.delete_object(
+            Bucket=self.s3_cfg.S3_BUCKET,
+            Key=s3_fn,
+          )
+    else:
+      self.log.debug(f'No "Contents" in {obj_list_remote.keys()}')
+
+    # put the remaining files to S3
+    for fn_local, fp_local in fp_local_lkup.items():
+      s3_fn = join(fp_dir_remote, fn_local)
+      self.log.debug("s3 create {}".format(s3_fn))
+      self.s3_client.upload_file(
+        fp_local,
+        os.getenv('S3_BUCKET'),
+        s3_fn,
+        ExtraArgs={ 'ACL': 'public-read' })
+
+
+  def sync_file(self, fp_local, fp_remote):
+    '''Sync local file to remove file
+    '''
+    self.log.warn('Not yet implemented')
+
+
+  #def make_s3_path(s3_dir, metadata_path):
+  #  return "{}/{}/{}{}".format(os.getenv('S3_ENDPOINT'), os.getenv('S3_BUCKET'), s3_dir, metadata_path)
diff --git a/animism-align/cli/app/utils/video_utils.py b/animism-align/cli/app/utils/video_utils.py
new file mode 100644
index 0000000..992f0f8
--- /dev/null
+++ b/animism-align/cli/app/utils/video_utils.py
@@ -0,0 +1,23 @@
+
+from pymediainfo import MediaInfo
+
+def mediainfo(fp_in):
+  """Returns abbreviated video/audio metadata for video files
+  :param fp_in: filepath"""
+  
+  result = {}
+  media_info_raw = MediaInfo.parse(fp_in).to_data()
+
+  for d in media_info_raw['tracks']:
+    if d['track_type'] == 'Video':
+      result = {
+        'codec_cc': d['codec_cc'],
+        'duration': int(d['duration']),
+        'display_aspect_ratio': float(d['display_aspect_ratio']),
+        'width': int(d['width']),
+        'height': int(d['height']),
+        'frame_rate': float(d['frame_rate']),
+        'frame_count': int(d['frame_count']),
+        }
+  
+  return result
+\ No newline at end of file