.

15 files changed, 1604 insertions, 0 deletions

diff --git a/megapixels/commands/admin/rsync.py b/megapixels/commands/admin/rsync.py
new file mode 100644
index 00000000..a821b460
--- /dev/null
+++ b/megapixels/commands/admin/rsync.py
@@ -0,0 +1,106 @@
+"""
+Parallel rsync media_records between drives
+For parallel rsync with media records, use vframe/commands/rsync
+"""
+
+import click
+
+from app.settings import types
+from app.utils import click_utils
+from app.settings import app_cfg as cfg
+
+@click.command()
+@click.option('-i', '--input', 'dir_in', required=True,
+  help='Input directory')
+@click.option('-o', '--output', 'dir_out', required=True,
+  help='Output directory')
+@click.option('-t', '--threads', 'opt_threads', default=8,
+  help='Number of threads')
+@click.option('--validate/--no-validate', 'opt_validate', is_flag=True, default=False,
+  help='Validate files after copy')
+@click.option('--extract/--no-extract', 'opt_extract', is_flag=True, default=False,
+  help='Extract files after copy')
+@click.pass_context
+def cli(ctx, dir_in, dir_out, opt_threads, opt_validate, opt_extract):
+  """rsync folders"""
+  
+  import os
+  from os.path import join
+  from pathlib import Path
+
+  # NB deactivate logger in imported module
+  import logging
+  logging.getLogger().addHandler(logging.NullHandler())
+  from parallel_sync import rsync
+  
+  from app.settings.paths import Paths
+  from app.utils import logger_utils, file_utils
+  
+  # -------------------------------------------------
+  # process here
+
+  log = logger_utils.Logger.getLogger()
+  log.info('RSYNC from {} to {}'.format(dir_in, dir_out))  
+  log.info('opt_extract: {}'.format(opt_extract))
+  log.info('opt_validate: {}'.format(opt_validate))
+  log.info('opt_threads: {}'.format(opt_validate))
+  
+  file_utils.mkdirs(dir_out)
+
+  rsync.copy(dir_in, dir_out, parallelism=opt_threads, 
+    validate=opt_validate, extract=opt_extract)
+
+  log.info('done rsyncing')
+
+
+  # ---------------------------------------------------------------
+
+
+
+  # if dir_in:
+  #   # use input filepath as source
+  #   if not Path(dir_in).is_dir():
+  #     log.error('{} is not a directory'.format(dir_in))
+  #     ctx.exit()
+  #   if not Path(dir_out).is_dir():
+  #     ctx.log.error('{} is not a directory'.format(dir_out))
+  #     return
+
+  #   log.info('RSYNC from {} to {}'.format(dir_in, dir_out))
+  #   log.debug('opt_validate: {}'.format(opt_validate))
+  #   log.debug('opt_extract: {}'.format(opt_extract))
+  #   #  local_copy(paths, parallelism=10, extract=False, validate=False):
+  #   file_utils.mkdirs(dir_out)
+  #   rsync.copy(dir_in, dir_out, parallelism=opt_threads, 
+  #     validate=opt_validate, extract=opt_extract)
+  # else:
+  #   log.debug('get paths')
+  #   # use source mappings as rsync source
+  #   if not opt_media_format:
+  #     ctx.log.error('--media format not supplied for source mappings')
+  #     return
+
+  #   # ensure FILEPATH metadata exists
+  #   #  parallel-rsync accepts a list of tupes (src, dst)
+  #   file_routes = []
+  #   for chair_item in chair_items:
+  #     item = chair_item.item
+  #     sha256 = chair_item.item.sha256
+  #     filepath_metadata = item.get_metadata(types.Metadata.FILEPATH)
+  #     if not filepath_metadata:
+  #       ctx.log.error('no FILEPATH metadata')
+  #       return
+  #     fp_media = 
+  #     src = join('')
+  #     dir_media = Paths.media_dir(opt_media_format, data_store=opt_disk, verified=ctx.opts['verified'])
+  #     dst = join('')
+  #     file_routes.append((src, dst))
+
+  #   ctx.log.debug('dir_media: {}'.format(dir_media))
+  #   return
+    
+  # # -------------------------------------------------
+
+  # # send back to sink
+  # for chair_item in chair_items:
+  #   sink.send(chair_item)
diff --git a/megapixels/commands/cv/crop.py b/megapixels/commands/cv/crop.py
new file mode 100644
index 00000000..778be0c4
--- /dev/null
+++ b/megapixels/commands/cv/crop.py
@@ -0,0 +1,104 @@
+"""
+Crop images to prepare for training
+"""
+
+import click
+from PIL import Image, ImageOps, ImageFilter, ImageDraw
+
+from app.settings import types
+from app.utils import click_utils
+from app.settings import app_cfg as cfg
+
+@click.command()
+@click.option('-i', '--input', 'opt_dir_in', required=True,
+  help='Input directory')
+@click.option('-o', '--output', 'opt_dir_out', required=True,
+  help='Output directory')
+@click.option('-e', '--ext', 'opt_ext',
+  default='jpg', type=click.Choice(['jpg', 'png']),
+  help='File glob ext')
+@click.option('--size', 'opt_size', 
+  type=(int, int), default=(256, 256),
+  help='Output image size')
+@click.option('-t', '--crop-type', 'opt_crop_type',
+  default='center', type=click.Choice(['center', 'mirror', 'face', 'person', 'none']),
+  help='Force fit image center location')
+@click.pass_context
+def cli(ctx, opt_dir_in, opt_dir_out, opt_ext, opt_size, opt_crop_type):
+  """Crop, mirror images"""
+  
+  import os
+  from os.path import join
+  from pathlib import Path
+  from glob import glob
+  from tqdm import tqdm
+  
+
+  from app.utils import logger_utils, file_utils, im_utils
+  
+  # -------------------------------------------------
+  # process here
+
+  log = logger_utils.Logger.getLogger()
+  log.info('crop images')
+
+  # get list of files to process
+  fp_ims = glob(join(opt_dir_in, '*.{}'.format(opt_ext)))
+  log.debug('files: {}'.format(len(fp_ims)))
+  
+  # ensure output dir exists
+  file_utils.mkdirs(opt_dir_out)
+
+  for fp_im in tqdm(fp_ims):
+    im = process_crop(fp_im, opt_size, opt_crop_type)
+    fp_out = join(opt_dir_out, Path(fp_im).name)
+    im.save(fp_out)
+
+  
+def process_crop(fp_im, opt_size, crop_type):
+  im  = Image.open(fp_im)
+  if crop_type == 'center':
+    im = crop_square_fit(im, opt_size)
+  elif crop_type == 'mirror':
+    im = mirror_crop_square(im, opt_size)
+  return im
+
+def crop_square_fit(im, size, center=(0.5, 0.5)):
+  return ImageOps.fit(im, size, method=Image.BICUBIC, centering=center)
+
+def mirror_crop_square(im, size):
+  # force to even dims
+  if im.size[0] % 2 or im.size[1] % 2:
+    im = ImageOps.fit(im, ((im.size[0] // 2) * 2, (im.size[1] // 2) * 2))
+
+  # create new square image
+  min_size, max_size = (min(im.size), max(im.size))
+  orig_w, orig_h = im.size
+  margin = (max_size - min_size) // 2
+  w, h = (max_size, max_size)
+  im_new = Image.new('RGB', (w, h), color=(0, 0, 0))
+  
+  #crop (l, t, r, b)
+  if orig_w > orig_h:
+    # landscape, mirror expand T/B
+    im_top = ImageOps.mirror(im.crop((0, 0, margin, w)))
+    im_bot = ImageOps.mirror(im.crop((orig_h - margin, 0, orig_h, w)))
+    im_new.paste(im_top, (0, 0))
+    im_new.paste(im, (margin, 0, orig_h + margin, w))
+    im_new.paste(im_bot, (h - margin, 0))
+  elif orig_h > orig_w:
+    # portrait, mirror expand L/R
+    im_left = ImageOps.mirror(im.crop((0, 0, margin, h)))
+    im_right = ImageOps.mirror(im.crop((orig_w - margin, 0, orig_w, h)))
+    im_new.paste(im_left, (0, 0))
+    im_new.paste(im, (margin, 0, orig_w + margin, h))
+    im_new.paste(im_right, (w - margin, 0))
+
+  return im_new.resize(size)
+
+  
+def center_crop_face():
+  pass
+
+def center_crop_person():
+  pass
\ No newline at end of file
diff --git a/megapixels/commands/cv/csv_to_faces.py b/megapixels/commands/cv/csv_to_faces.py
new file mode 100644
index 00000000..64c8b965
--- /dev/null
+++ b/megapixels/commands/cv/csv_to_faces.py
@@ -0,0 +1,105 @@
+"""
+Reads in CSV of ROIs and extracts facial regions with padding
+"""
+
+import click
+
+from app.settings import types
+from app.utils import click_utils
+from app.settings import app_cfg as cfg
+
+@click.command()
+@click.option('-i', '--input', 'opt_fp_in', required=True,
+  help='Input CSV')
+@click.option('-m', '--media', 'opt_dir_media', required=True,
+  help='Input image/video directory')
+@click.option('-o', '--output', 'opt_dir_out', required=True,
+  help='Output directory for extracted ROI images')
+@click.option('--slice', 'opt_slice', type=(int, int), default=(None, None),
+  help='Slice list of files')
+@click.option('--padding', 'opt_padding', default=0.25,
+  help='Facial padding as percentage of face width')
+@click.option('--ext', 'opt_ext_out', default='png', type=click.Choice(['jpg', 'png']),
+  help='Output image type')
+@click.option('--min', 'opt_min', default=(60, 60),
+  help='Minimum original face size')
+@click.pass_context
+def cli(ctx, opt_fp_in, opt_dir_media, opt_dir_out, opt_slice, 
+  opt_padding, opt_ext_out, opt_min):
+  """Converts ROIs to images"""
+  
+  import os
+  from os.path import join
+  from pathlib import Path
+  from glob import glob
+  
+  from tqdm import tqdm
+  import numpy as np
+  from PIL import Image, ImageOps, ImageFilter, ImageDraw
+  import cv2 as cv
+  import pandas as pd
+
+  from app.utils import logger_utils, file_utils, im_utils
+  from app.models.bbox import BBox
+
+  # -------------------------------------------------
+  # process here
+  log = logger_utils.Logger.getLogger()
+  
+  df_rois = pd.read_csv(opt_fp_in, dtype={'subdir': str, 'fn': str})
+  if opt_slice:
+    df_rois = df_rois[opt_slice[0]:opt_slice[1]]
+  
+  log.info('Processing {:,} rows'.format(len(df_rois)))
+
+  file_utils.mkdirs(opt_dir_out)
+
+  df_rois_grouped = df_rois.groupby(['fn'])  # group by fn/filename
+  groups = df_rois_grouped.groups
+  skipped = []
+
+  for group in tqdm(groups):
+    # get image
+    group_rows = df_rois_grouped.get_group(group)
+
+    row = group_rows.iloc[0]
+    fp_im = join(opt_dir_media, str(row['subdir']), '{fn}.{ext}'.format(**row))  # TODO change to ext
+    try:
+      im = Image.open(fp_im).convert('RGB')
+      im.verify()
+    except Exception as e:
+      log.warn('Could not open: {}'.format(fp_im))
+      log.error(e)
+      continue
+
+    for idx, roi in group_rows.iterrows():
+      # get bbox to im dimensions
+      xywh = [roi['x'], roi['y'], roi['w'] , roi['h']]
+      bbox = BBox.from_xywh(*xywh)
+      dim = im.size
+      bbox_dim = bbox.to_dim(dim)
+      # expand
+      opt_padding_px = int(opt_padding * bbox_dim.width)
+      bbox_dim_exp = bbox_dim.expand_dim(opt_padding_px, dim)
+      # crop
+      x1y2 = bbox_dim_exp.pt_tl + bbox_dim_exp.pt_br
+      im_crop = im.crop(box=x1y2)
+
+      # strip exif, create new image and paste data
+      im_crop_data = list(im_crop.getdata())
+      im_crop_no_exif = Image.new(im_crop.mode, im_crop.size)
+      im_crop_no_exif.putdata(im_crop_data)
+
+      # save
+      idx_zpad = file_utils.zpad(idx, zeros=3)
+      subdir = '' if roi['subdir'] == '.' else '{}_'.format(roi['subdir'])
+      subdir = subdir.replace('/', '_')
+      fp_im_out = join(opt_dir_out, '{}{}{}.{}'.format(subdir, roi['fn'], idx_zpad, opt_ext_out))
+      # threshold size and save
+      if im_crop_no_exif.size[0] < opt_min[0] or im_crop_no_exif.size[1] < opt_min[1]:
+        skipped.append(fp_im_out)
+        log.info('Face too small: {}, idx: {}'.format(fp_im, idx))
+      else:
+        im_crop_no_exif.save(fp_im_out)
+
+  log.info('Skipped {:,} images'.format(len(skipped)))
diff --git a/megapixels/commands/cv/face_frames.py b/megapixels/commands/cv/face_frames.py
new file mode 100644
index 00000000..76f23af1
--- /dev/null
+++ b/megapixels/commands/cv/face_frames.py
@@ -0,0 +1,82 @@
+from glob import glob
+import os
+from os.path import join
+from pathlib import Path
+
+import click
+
+
+
+
+@click.command()
+@click.option('-i', '--input', 'opt_fp_in', required=True,
+  help='Input directory to glob')
+@click.option('-o', '--output', 'opt_fp_out', required=True,
+  help='Output directory for face frames')
+@click.option('--size', 'opt_size', 
+  type=(int, int), default=(300, 300),
+  help='Output image size')
+@click.option('--slice', 'opt_slice', type=(int, int), default=(None, None),
+  help='Slice list of files')
+@click.pass_context
+def cli(ctx, opt_fp_in, opt_fp_out, opt_size, opt_slice):
+  """Split video to face frames"""
+
+  from tqdm import tqdm
+  import dlib
+  import pandas as pd
+  from PIL import Image, ImageOps, ImageFilter
+  import cv2 as cv
+  import numpy as np
+
+  from app.processors import face_detector
+  from app.utils import logger_utils, file_utils, im_utils
+  from app.settings import types
+  from app.utils import click_utils
+  from app.settings import app_cfg as cfg
+  from app.models.bbox import BBox
+
+  log = logger_utils.Logger.getLogger()
+
+  # -------------------------------------------------
+  # process  
+   
+  detector = face_detector.DetectorDLIBCNN()
+
+  # get file list
+  fp_videos = glob(join(opt_fp_in, '*.mp4'))
+  fp_videos += glob(join(opt_fp_in, '*.webm'))
+  fp_videos += glob(join(opt_fp_in, '*.mkv'))
+
+  min_distance_per = .025  # minimum distance percentage to save new face image
+  face_interval = 5
+  frame_interval_count = 0
+  frame_count = 0
+  bbox_prev = BBox(0,0,0,0)
+  file_utils.mkdirs(opt_fp_out)
+  dnn_size = opt_size
+  max_dim = max(dnn_size)
+  px_thresh = int(max_dim * min_distance_per)
+
+  for fp_video in tqdm(fp_videos):
+    # load video  
+    video = cv.VideoCapture(fp_video)
+    # iterate through frames
+    while video.isOpened():
+      res, frame = video.read()
+      if not res:
+        break
+      # increment frames, save frame if interval has passed
+      frame_count += 1  # for naming
+      frame_interval_count += 1  # for interval
+      bboxes = detector.detect(frame, opt_size=dnn_size, opt_pyramids=0)
+      if len(bboxes) > 0 and frame_interval_count >= face_interval:
+        dim = frame.shape[:2][::-1]
+        d = bboxes[0].to_dim(dim).distance(bbox_prev)
+        if d > px_thresh:
+          # save frame
+          zfc = file_utils.zpad(frame_count)
+          fp_frame = join(opt_fp_out, '{}_{}.jpg'.format(Path(fp_video).stem, zfc))
+          cv.imwrite(fp_frame, frame)
+          frame_interval_count = 0
+          bbox_prev = bboxes[0]
diff --git a/megapixels/commands/cv/faces_to_3dlm.py b/megapixels/commands/cv/faces_to_3dlm.py
new file mode 100644
index 00000000..658d4484
--- /dev/null
+++ b/megapixels/commands/cv/faces_to_3dlm.py
@@ -0,0 +1,96 @@
+"""
+Crop images to prepare for training
+"""
+
+import click
+# from PIL import Image, ImageOps, ImageFilter, ImageDraw
+
+from app.settings import types
+from app.utils import click_utils
+from app.settings import app_cfg as cfg
+
+color_filters = {'color': 1, 'gray': 2, 'all': 3}
+
+@click.command()
+@click.option('-i', '--input', 'opt_dirs_in', required=True, multiple=True,
+  help='Input directory')
+@click.option('-o', '--output', 'opt_fp_out', required=True,
+  help='Output CSV')
+@click.option('-e', '--ext', 'opt_ext',
+  default='jpg', type=click.Choice(['jpg', 'png']),
+  help='File glob ext')
+@click.option('--size', 'opt_size', 
+  type=(int, int), default=(300, 300),
+  help='Output image size')
+@click.option('-g', '--gpu', 'opt_gpu', default=0,
+  help='GPU index')
+@click.option('--slice', 'opt_slice', type=(int, int), default=(None, None),
+  help='Slice list of files')
+@click.option('--recursive/--no-recursive', 'opt_recursive', is_flag=True, default=False,
+  help='Use glob recursion (slower)')
+@click.option('-f', '--force', 'opt_force', is_flag=True,
+  help='Force overwrite file')
+@click.pass_context
+def cli(ctx, opt_dirs_in, opt_fp_out, opt_ext, opt_size, opt_gpu, opt_slice, 
+  opt_recursive, opt_force):
+  """Converts face imges to 3D landmarks"""
+  
+  import sys
+  import os
+  from os.path import join
+  from pathlib import Path
+  from glob import glob
+  
+  from tqdm import tqdm
+  import numpy as np
+  import dlib  # must keep a local reference for dlib
+  import cv2 as cv
+  import pandas as pd
+  from face_alignment import FaceAlignment, LandmarksType
+  from skimage import io
+
+  from app.utils import logger_utils, file_utils
+  from app.processors import face_detector 
+
+  # -------------------------------------------------
+  # init here
+
+  log = logger_utils.Logger.getLogger()
+
+  if not opt_force and Path(opt_fp_out).exists():
+    log.error('File exists. Use "-f / --force" to overwite')
+    return
+  
+  device = 'cuda' if opt_gpu > -1 else 'cpu'
+  fa = FaceAlignment(LandmarksType._3D, flip_input=False,  device=device)
+ 
+  # get list of files to process
+  fp_ims = []
+  for opt_dir_in in opt_dirs_in:
+    if opt_recursive:
+      fp_glob = join(opt_dir_in, '**/*.{}'.format(opt_ext))
+      fp_ims += glob(fp_glob, recursive=True)
+    else:
+      fp_glob = join(opt_dir_in, '*.{}'.format(opt_ext))
+      fp_ims += glob(fp_glob)
+    log.debug(fp_glob)
+
+
+  if opt_slice:
+    fp_ims = fp_ims[opt_slice[0]:opt_slice[1]]
+  log.debug('processing {:,} files'.format(len(fp_ims)))
+
+
+  data = {}
+
+  for fp_im in tqdm(fp_ims):
+    fpp_im = Path(fp_im)
+    im = io.imread(fp_im)
+    preds = fa.get_landmarks(im)
+    if preds and len(preds) > 0:
+      data[fpp_im.name]  = preds[0].tolist()
+    
+  # save date
+  file_utils.mkdirs(opt_fp_out)
+
+  file_utils.write_json(data, opt_fp_out, verbose=True)
\ No newline at end of file
diff --git a/megapixels/commands/cv/faces_to_csv.py b/megapixels/commands/cv/faces_to_csv.py
new file mode 100644
index 00000000..07226c31
--- /dev/null
+++ b/megapixels/commands/cv/faces_to_csv.py
@@ -0,0 +1,164 @@
+"""
+Crop images to prepare for training
+"""
+
+import click
+# from PIL import Image, ImageOps, ImageFilter, ImageDraw
+
+from app.settings import types
+from app.utils import click_utils
+from app.settings import app_cfg as cfg
+
+color_filters = {'color': 1, 'gray': 2, 'all': 3}
+
+@click.command()
+@click.option('-i', '--input', 'opt_dirs_in', required=True, multiple=True,
+  help='Input directory')
+@click.option('-o', '--output', 'opt_fp_out', required=True,
+  help='Output CSV')
+@click.option('-e', '--ext', 'opt_ext',
+  default='jpg', type=click.Choice(['jpg', 'png']),
+  help='File glob ext')
+@click.option('--size', 'opt_size', 
+  type=(int, int), default=(300, 300),
+  help='Output image size')
+@click.option('-t', '--detector-type', 'opt_detector_type',
+  type=cfg.FaceDetectNetVar,
+  default=click_utils.get_default(types.FaceDetectNet.DLIB_CNN),
+  help=click_utils.show_help(types.FaceDetectNet))
+@click.option('-g', '--gpu', 'opt_gpu', default=0,
+  help='GPU index')
+@click.option('--conf', 'opt_conf_thresh', default=0.85, type=click.FloatRange(0,1),
+  help='Confidence minimum threshold')
+@click.option('--pyramids', 'opt_pyramids', default=0, type=click.IntRange(0,4),
+  help='Number pyramids to upscale for DLIB detectors')
+@click.option('--slice', 'opt_slice', type=(int, int), default=(None, None),
+  help='Slice list of files')
+@click.option('--display/--no-display', 'opt_display', is_flag=True, default=False,
+  help='Display detections to debug')
+@click.option('--recursive/--no-recursive', 'opt_recursive', is_flag=True, default=False,
+  help='Use glob recursion (slower)')
+@click.option('-f', '--force', 'opt_force', is_flag=True,
+  help='Force overwrite file')
+@click.option('--color', 'opt_color_filter', 
+  type=click.Choice(color_filters.keys()), default='color',
+  help='Filter to keep color or grayscale images (color = keep color')
+@click.pass_context
+def cli(ctx, opt_dirs_in, opt_fp_out, opt_ext, opt_size, opt_detector_type, 
+  opt_gpu, opt_conf_thresh, opt_pyramids, opt_slice, opt_display, opt_recursive, opt_force, opt_color_filter):
+  """Converts frames with faces to CSV of ROIs"""
+  
+  import sys
+  import os
+  from os.path import join
+  from pathlib import Path
+  from glob import glob
+  
+  from tqdm import tqdm
+  import numpy as np
+  import dlib  # must keep a local reference for dlib
+  import cv2 as cv
+  import pandas as pd
+
+  from app.utils import logger_utils, file_utils, im_utils
+  from app.processors import face_detector 
+
+  # -------------------------------------------------
+  # init here
+
+  log = logger_utils.Logger.getLogger()
+
+  if not opt_force and Path(opt_fp_out).exists():
+    log.error('File exists. Use "-f / --force" to overwite')
+    return
+  
+  if opt_detector_type == types.FaceDetectNet.CVDNN:
+    detector = face_detector.DetectorCVDNN()
+  elif opt_detector_type == types.FaceDetectNet.DLIB_CNN:
+    detector = face_detector.DetectorDLIBCNN(opt_gpu)
+  elif opt_detector_type == types.FaceDetectNet.DLIB_HOG:
+    detector = face_detector.DetectorDLIBHOG()
+  elif opt_detector_type == types.FaceDetectNet.HAAR:
+    log.error('{} not yet implemented'.format(opt_detector_type.name))
+    return
+
+  
+  # -------------------------------------------------
+  # process here
+  color_filter = color_filters[opt_color_filter]
+  
+  # get list of files to process
+  fp_ims = []
+  for opt_dir_in in opt_dirs_in:
+    if opt_recursive:
+      fp_glob = join(opt_dir_in, '**/*.{}'.format(opt_ext))
+      fp_ims += glob(fp_glob, recursive=True)
+    else:
+      fp_glob = join(opt_dir_in, '*.{}'.format(opt_ext))
+      fp_ims += glob(fp_glob)
+    log.debug(fp_glob)
+
+
+  if opt_slice:
+    fp_ims = fp_ims[opt_slice[0]:opt_slice[1]]
+  log.debug('processing {:,} files'.format(len(fp_ims)))
+
+
+  data = []
+
+  for fp_im in tqdm(fp_ims):
+    im = cv.imread(fp_im)
+
+    # filter out color or grayscale iamges
+    if color_filter != color_filters['all']:
+      try:
+        is_gray = im_utils.is_grayscale(im)
+        if is_gray and color_filter != color_filters['gray']:
+          log.debug('Skipping grayscale image: {}'.format(fp_im))
+          continue
+      except Exception as e:
+        log.error('Could not check grayscale: {}'.format(fp_im))
+        continue
+        
+    try:
+      bboxes = detector.detect(im, opt_size=opt_size, opt_pyramids=opt_pyramids)
+    except Exception as e:
+      log.error('could not detect: {}'.format(fp_im))
+      log.error('{}'.format(e))
+    fpp_im = Path(fp_im)
+    subdir = str(fpp_im.parent.relative_to(opt_dir_in))
+
+    for bbox in bboxes:
+      roi = {
+        'fn': fpp_im.stem, 
+        'ext': fpp_im.suffix.replace('.',''), 
+        'x': bbox.x, 
+        'y': bbox.y, 
+        'w': bbox.w, 
+        'h': bbox.h,
+        'image_height': im.shape[0],
+        'image_width': im.shape[1],
+        'subdir': subdir}
+      bbox_dim = bbox.to_dim(im.shape[:2][::-1])  # w,h
+      data.append(roi)
+    
+    # debug display
+    if opt_display and len(bboxes):
+      im_md = im_utils.resize(im, width=min(1200, opt_size[0]))
+      for bbox in bboxes:
+        bbox_dim = bbox.to_dim(im_md.shape[:2][::-1])
+        cv.rectangle(im_md, bbox_dim.pt_tl, bbox_dim.pt_br, (0,255,0), 3)
+      cv.imshow('', im_md)
+      while True:
+        k = cv.waitKey(1) & 0xFF
+        if k == 27 or k == ord('q'):  # ESC
+          cv.destroyAllWindows()
+          sys.exit()
+        elif k != 255:
+          # any key to continue
+          break
+
+  # save date
+  file_utils.mkdirs(opt_fp_out)
+  df = pd.DataFrame.from_dict(data)
+  df.to_csv(opt_fp_out, index=False)
\ No newline at end of file
diff --git a/megapixels/commands/cv/mirror.py b/megapixels/commands/cv/mirror.py
new file mode 100644
index 00000000..9ca1cac7
--- /dev/null
+++ b/megapixels/commands/cv/mirror.py
@@ -0,0 +1,57 @@
+"""
+Crop images to prepare for training
+"""
+
+import click
+import cv2 as cv
+from PIL import Image, ImageOps, ImageFilter
+
+from app.settings import types
+from app.utils import click_utils
+from app.settings import app_cfg as cfg
+
+
+@click.command()
+@click.option('-i', '--input', 'opt_dir_in', required=True,
+  help='Input directory')
+@click.option('-o', '--output', 'opt_dir_out', required=True,
+  help='Output directory')
+@click.option('--slice', 'opt_slice', type=(int, int), default=(None, None),
+  help='Slice the input list')
+@click.pass_context
+def cli(ctx, opt_dir_in, opt_dir_out,  opt_slice):
+  """Mirror augment image directory"""
+  
+  import os
+  from os.path import join
+  from pathlib import Path
+  from glob import glob
+  from tqdm import tqdm
+
+  from app.utils import logger_utils, file_utils, im_utils
+  
+  # -------------------------------------------------
+  # init
+
+  log = logger_utils.Logger.getLogger()
+  
+  # -------------------------------------------------
+  # process here
+
+  # get list of files to process
+  fp_ims = glob(join(opt_dir_in, '*.jpg'))
+  fp_ims += glob(join(opt_dir_in, '*.png'))
+  
+  if opt_slice:
+    fp_ims = fp_ims[opt_slice[0]:opt_slice[1]]
+  log.info('processing {:,} files'.format(len(fp_ims)))
+  
+  # ensure output dir exists
+  file_utils.mkdirs(opt_dir_out)
+
+  # resize and save images
+  for fp_im in tqdm(fp_ims):
+    im = Image.open(fp_im)
+    fpp_im = Path(fp_im)
+    fp_out = join(opt_dir_out, '{}_mirror{}'.format(fpp_im.stem, fpp_im.suffix))
+    im.save(fp_out)
\ No newline at end of file
diff --git a/megapixels/commands/cv/resize.py b/megapixels/commands/cv/resize.py
new file mode 100644
index 00000000..f535c8b6
--- /dev/null
+++ b/megapixels/commands/cv/resize.py
@@ -0,0 +1,128 @@
+"""
+Crop images to prepare for training
+"""
+
+import click
+import cv2 as cv
+from PIL import Image, ImageOps, ImageFilter
+
+from app.settings import types
+from app.utils import click_utils
+from app.settings import app_cfg as cfg
+
+"""
+Filter    Q-Down Q-Up   Speed
+NEAREST              ⭐⭐⭐⭐⭐
+BOX       ⭐          ⭐⭐⭐⭐
+BILINEAR  ⭐      ⭐   ⭐⭐⭐
+HAMMING   ⭐⭐         ⭐⭐⭐
+BICUBIC   ⭐⭐⭐   ⭐⭐⭐  ⭐⭐
+LANCZOS   ⭐⭐⭐⭐  ⭐⭐⭐⭐  ⭐
+"""
+methods = {
+  'lanczos': Image.LANCZOS,
+  'bicubic': Image.BICUBIC,
+  'hamming': Image.HAMMING,
+  'bileaner': Image.BILINEAR,
+  'box': Image.BOX,
+  'nearest': Image.NEAREST
+  }
+centerings = {
+  'tl': (0.0, 0.0),
+  'tc': (0.5, 0.0),
+  'tr': (0.0, 0.0),
+  'lc': (0.0, 0.5),
+  'cc': (0.5, 0.5),
+  'rc': (1.0, 0.5),
+  'bl': (0.0, 1.0),
+  'bc': (1.0, 0.5),
+  'br': (1.0, 1.0)
+}
+
+@click.command()
+@click.option('-i', '--input', 'opt_dir_in', required=True,
+  help='Input directory')
+@click.option('-o', '--output', 'opt_dir_out', required=True,
+  help='Output directory')
+@click.option('-e', '--ext', 'opt_glob_ext',
+  default='png', type=click.Choice(['jpg', 'png']),
+  help='File glob ext')
+@click.option('--size', 'opt_size',  
+  type=(int, int), default=(256, 256),
+  help='Output image size (square)')
+@click.option('--method', 'opt_scale_method', 
+  type=click.Choice(methods.keys()),
+  default='lanczos',
+  help='Scaling method to use')
+@click.option('--equalize', 'opt_equalize', is_flag=True,
+  help='Equalize historgram')
+@click.option('--sharpen', 'opt_sharpen', is_flag=True,
+  help='Unsharp mask')
+@click.option('--center', 'opt_center', default='cc', type=click.Choice(centerings.keys()),
+  help='Crop focal point')
+@click.option('--slice', 'opt_slice', type=(int, int), default=(None, None),
+  help='Slice the input list')
+@click.pass_context
+def cli(ctx, opt_dir_in, opt_dir_out, opt_glob_ext, opt_size, opt_scale_method,
+ opt_equalize, opt_sharpen, opt_center, opt_slice):
+  """Crop, mirror images"""
+  
+  import os
+  from os.path import join
+  from pathlib import Path
+  from glob import glob
+  from tqdm import tqdm
+
+  from app.utils import logger_utils, file_utils, im_utils
+  
+  # -------------------------------------------------
+  # init
+
+  log = logger_utils.Logger.getLogger()
+
+  centering = centerings[opt_center]
+
+  # -------------------------------------------------
+  # process here
+
+  # get list of files to process
+  fp_ims = glob(join(opt_dir_in, '*.{}'.format(opt_glob_ext)))
+  if opt_slice:
+    fp_ims = fp_ims[opt_slice[0]:opt_slice[1]]
+  log.info('processing {:,} files'.format(len(fp_ims)))
+  
+  # set scale method
+  scale_method = methods[opt_scale_method]
+
+  # ensure output dir exists
+  file_utils.mkdirs(opt_dir_out)
+
+  # resize and save images
+  for fp_im in tqdm(fp_ims):
+    try:
+      im = Image.open(fp_im).convert('RGB')
+      im.verify()
+    except Exception as e:
+      log.warn('Could not open: {}'.format(fp_im))
+      log.error(e)
+      continue
+
+    im = ImageOps.fit(im, opt_size, method=scale_method, centering=centering)
+
+    if opt_equalize:
+      im_np = im_utils.pil2np(im)
+      im_np_eq = eq_hist_yuv(im_np)
+      im_np = cv.addWeighted(im_np_eq, 0.35, im_np, 0.65, 0)
+      im = im_utils.np2pil(im_np)
+
+    if opt_sharpen:
+      im = im.filter(ImageFilter.UnsharpMask)
+      
+    fp_out = join(opt_dir_out, Path(fp_im).name)
+    im.save(fp_out)
+
+
+def eq_hist_yuv(im):
+  im_yuv = cv.cvtColor(im, cv.COLOR_BGR2YUV)
+  im_yuv[:,:,0] = cv.equalizeHist(im_yuv[:,:,0])
+  return cv.cvtColor(im_yuv, cv.COLOR_YUV2BGR)
diff --git a/megapixels/commands/cv/videos_to_frames.py b/megapixels/commands/cv/videos_to_frames.py
new file mode 100644
index 00000000..0b56c46a
--- /dev/null
+++ b/megapixels/commands/cv/videos_to_frames.py
@@ -0,0 +1,73 @@
+from glob import glob
+import os
+from os.path import join
+from pathlib import Path
+
+import click
+
+from app.settings import types
+from app.utils import click_utils
+from app.settings import app_cfg as cfg
+from app.utils import logger_utils
+
+import dlib
+import pandas as pd
+from PIL import Image, ImageOps, ImageFilter
+from app.utils import file_utils, im_utils
+
+
+log = logger_utils.Logger.getLogger()
+
+@click.command()
+@click.option('-i', '--input', 'opt_fp_in', required=True,
+  help='Input directory')
+@click.option('-o', '--output', 'opt_fp_out', required=True,
+  help='Output directory')
+@click.option('--size', 'opt_size', default=(320, 240),
+  help='Inference size for face detection' )
+@click.option('--interval', 'opt_frame_interval', default=20,
+  help='Number of frames before saving next face')
+@click.pass_context
+def cli(ctx, opt_fp_in, opt_fp_out, opt_size, opt_frame_interval):
+  """Converts videos to frames with faces"""
+
+  # -------------------------------------------------
+  # process  
+  
+  from tqdm import tqdm
+  import cv2 as cv
+  from tqdm import tqdm
+  from app.processors import face_detector
+  
+  detector = face_detector.DetectorDLIBCNN()
+
+  # get file list
+  fp_videos = glob(join(opt_fp_in, '*.mp4'))
+  fp_videos += glob(join(opt_fp_in, '*.webm'))
+  fp_videos += glob(join(opt_fp_in, '*.mkv'))
+  
+  frame_interval_count = 0
+  frame_count = 0
+
+  file_utils.mkdirs(opt_fp_out)
+
+  for fp_video in tqdm(fp_videos):
+
+    video = cv.VideoCapture(fp_video)
+    
+    while video.isOpened():
+      res, frame = video.read()
+      if not res:
+        break
+
+      frame_count += 1  # for naming
+      frame_interval_count += 1  # for interval
+      
+      bboxes = detector.detect(frame, opt_size=opt_size, opt_pyramids=0)
+      if len(bboxes) > 0 and frame_interval_count >= opt_frame_interval:
+        # save frame
+        fname = file_utils.zpad(frame_count)
+        fp_frame = join(opt_fp_out, '{}_{}.jpg'.format(Path(fp_video).stem, fname))
+        cv.imwrite(fp_frame, frame)
+        frame_interval_count = 0
+      
diff --git a/megapixels/commands/datasets/50people.py b/megapixels/commands/datasets/50people.py
new file mode 100644
index 00000000..fb35b2fe
--- /dev/null
+++ b/megapixels/commands/datasets/50people.py
@@ -0,0 +1,129 @@
+from glob import glob
+import os
+from os.path import join
+from pathlib import Path
+
+import click
+
+from app.settings import types
+from app.utils import click_utils
+from app.settings import app_cfg as cfg
+from app.utils import logger_utils
+
+import dlib
+import pandas as pd
+from PIL import Image, ImageOps, ImageFilter
+from app.utils import file_utils, im_utils
+
+
+log = logger_utils.Logger.getLogger()
+
+@click.command()
+@click.option('-i', '--input', 'opt_fp_in', required=True,
+  help='Input directory')
+@click.option('-o', '--output', 'opt_fp_out',
+  help='Output directory')
+@click.option('--media', 'opt_dir_media',
+  help='Output directory')
+@click.option('--action', 'opt_action', 
+  type=click.Choice(['download']),
+  default='info',
+  help='Command action')
+@click.option('--slice', 'opt_slice', type=(int, int), default=(None, None),
+  help='Slice list of files')
+@click.pass_context
+def cli(ctx, opt_fp_in, opt_fp_out, opt_dir_media, opt_action, opt_slice):
+  """YTMU utils"""
+
+  
+  from tqdm import tqdm
+
+  # -------------------------------------------------
+  # process  
+ 
+  if opt_action == 'download':
+    # downloads video files with ytdl
+    handle_download(opt_fp_in, opt_fp_out, opt_slice)
+  elif opt_action == 'face_frames':
+    handle_face_frames(opt_fp_in, opt_fp_out, dir_media, opt_slice)
+
+
+
+
+
+def handle_face_frames(fp_in, dir_out, dir_videos):
+  if not dir_out or not dir_videos:
+    log.error('-o/--output and --videos required')
+    return
+  
+  import cv2 as cv
+  from tqdm import tqdm
+  from app.processors import face_detector
+  detector = face_detector.DetectorDLIBCNN()
+
+  # get file list
+  fp_videos = glob(join(dir_videos, '*.mp4'))
+  fp_videos += glob(join(dir_videos, '*.webm'))
+  fp_videos += glob(join(dir_videos, '*.mkv'))
+
+  face_interval = 30
+  frame_interval_count = 0
+  frame_count = 0
+
+  file_utils.mkdirs(dir_out)
+
+  for fp_video in tqdm(fp_videos):
+    # log.debug('opening: {}'.format(fp_video))
+    video = cv.VideoCapture(fp_video)
+    while video.isOpened():
+      res, frame = video.read()
+      if not res:
+        break
+
+      frame_count += 1  # for naming
+      frame_interval_count += 1  # for interval
+      bboxes = detector.detect(frame, opt_size=(320, 240), opt_pyramids=0)
+      if len(bboxes) > 0 and frame_interval_count >= face_interval:
+        # save frame
+        fp_frame = join(dir_out, '{}_{}.jpg'.format(Path(fp_video).stem, file_utils.zpad(frame_count)))
+        cv.imwrite(fp_frame, frame)
+        frame_interval_count = 0
+
+
+def handle_download(fp_in, dir_out, opt_slice):
+  import youtube_dl
+  df = pd.read_csv(fp_in)
+  if opt_slice:
+    df = df[opt_slice[0]:opt_slice[1]]
+  df = df.fillna('')
+  fp_videos = glob(join(dir_out, '*.mp4'))
+  fp_videos += glob(join(dir_out, '*.webm'))
+  fp_videos += glob(join(dir_out, '*.mkv'))
+  
+  ydl = youtube_dl.YoutubeDL({'outtmpl': join(dir_out, '') + '%(id)s.%(ext)s'})
+
+  for i, row in df.iterrows():
+    vid = str(row['youtube_id'])
+    if not vid:
+      vid = row['vimeo_id']
+      if vid:
+        vid = str(int(vid))
+        url = 'https://vimeo.com/{}'.format(vid)
+    else:
+      url = 'https://youtube.com/watch?v={}'.format(vid)
+    if not vid:
+      log.warn('no video id: {} for {}'.format(vid, row['city']))
+      continue
+    
+    found = False
+    for fp_video in fp_videos:
+      if vid in fp_video:
+        #log.debug('skip: {}'.format(vid))
+        found = True
+    
+    if not found:
+      try:
+        with ydl:
+          ydl.download([url])
+      except:
+        log.error('could not dl: {}'.format(vid))
diff --git a/megapixels/commands/datasets/megaface_flickr_api.py b/megapixels/commands/datasets/megaface_flickr_api.py
new file mode 100644
index 00000000..62232ab8
--- /dev/null
+++ b/megapixels/commands/datasets/megaface_flickr_api.py
@@ -0,0 +1,141 @@
+from glob import glob
+import os
+from os.path import join
+from pathlib import Path
+
+import click
+
+from app.settings import types
+from app.utils import click_utils
+from app.settings import app_cfg as cfg
+from app.utils import logger_utils
+
+import dlib
+import pandas as pd
+from PIL import Image, ImageOps, ImageFilter
+from app.utils import file_utils, im_utils
+
+
+log = logger_utils.Logger.getLogger()
+
+@click.command()
+@click.option('-i', '--input', 'opt_fp_in', required=True,
+  help='Input directory')
+@click.option('-o', '--output', 'opt_fp_out',
+  help='Output directory')
+@click.option('--slice', 'opt_slice', type=(int, int), default=(None, None),
+  help='Slice list of files')
+@click.option('-d', '--delay', 'opt_delay', default=None, type=int,
+  help='Delay between API calls to prevent rate-limiting')
+@click.option('--checkpoints', 'opt_checkpoints', is_flag=True,
+  help='Save checkpoints')
+@click.option('--api_key', 'opt_api_key', envvar='FLICKR_API_KEY')
+@click.option('--api_secret', 'opt_api_secret', envvar='FLICKR_API_SECRET')
+@click.option('--checkpoint_interval', 'opt_ckpt_interval', default=10000,
+  help='Save checkpoint interval')
+@click.pass_context
+def cli(ctx, opt_fp_in, opt_fp_out, opt_slice, opt_api_key, opt_api_secret, 
+  opt_delay, opt_checkpoints, opt_ckpt_interval):
+  """Appends Flickr API info to CSV"""
+  
+  from tqdm import tqdm
+  from glob import glob
+  import time
+  import flickr_api  # pip install flickr_api
+  from flickr_api.flickrerrors import FlickrAPIError
+  
+  # -------------------------------------------------
+  # process  
+  
+  if not opt_api_key or not opt_api_secret:
+    log.error('source .env vars for Flickr API and try again')
+    return
+
+  # init Flickr API
+  flickr_api.set_keys(api_key=opt_api_key, api_secret=opt_api_secret)
+
+  # reqd in CSV
+  df_ids = pd.read_csv(opt_fp_in)
+  if opt_slice:
+    df_ids = df_ids[opt_slice[0]:opt_slice[1]]
+  
+  log.info('Processing: {:,} items'.format(len(df_ids)))
+
+  # iterate MegaFace IDs
+  identities = []
+  
+  tqdm.pandas()
+
+  for idx, df_id in tqdm(df_ids.iterrows(), total=len(df_ids)):
+    # a = flickr_api.Person(id='123456789@N01')
+    df_id_dict = dict(df_id)
+    
+    # append relevant data
+    try:
+      person = flickr_api.Person(id=df_id['nsid'])
+      info = person.getInfo()
+      df_id_dict.update( {
+        'user_name': info.get('username', ''), 
+        'location': info.get('location', ''),
+        'real_name': info.get('realname', ''),
+        'time_zone': info.get('timezone', {}).get('timezone_id', ''),
+        'time_first_photo': info.get('photos_info', {}).get('firstdatetaken'),
+        'photos_count': info.get('photos_info', {}).get('count'),
+        'description': info.get('description', ''),
+        'id': info.get('id'),
+        'path_alias': info.get('path_alias', ''),
+        'is_pro': info.get('ispro', ''),
+        'url_photos': info.get('photosurl', ''),
+        'url_profile': info.get('photosurl', ''),
+        'url_mobile': info.get('mobileurl', ''),
+        })
+      identities.append(df_id_dict)
+
+    except FlickrAPIError as e:
+      log.error(e)
+      
+
+    if opt_checkpoints:
+      if (idx + 1) % opt_ckpt_interval == 0:
+        df = pd.DataFrame.from_dict(identities)
+        fpp_out = Path(opt_fp_out)
+        opt_fp_out_ckpt = join(fpp_out.parent, '{}_ckpt_{}.csv'.format(fpp_out.stem, file_utils.zpad(idx + 1)))
+        log.info('Saving checkpoint {:,} to {}'.format(idx + 1, opt_fp_out_ckpt))
+        df.to_csv(opt_fp_out_ckpt, index=False)
+
+    if opt_delay:
+      time.sleep(opt_delay)
+
+
+  df = pd.DataFrame.from_dict(identities)
+  df.to_csv(opt_fp_out, index=False)
+  
+  log.info('Wrote: {:,} lines to {}'.format(len(df), opt_fp_out))
+
+
+"""
+Example API data:
+{'id': '7124086@N07',
+ 'nsid': '7124086@N07',
+ 'ispro': 1,
+ 'can_buy_pro': 0,
+ 'iconserver': '2325',
+ 'iconfarm': 3,
+ 'path_alias': 'shirleylin',
+ 'has_stats': '1',
+ 'pro_badge': 'standard',
+ 'expire': '0',
+ 'username': 'ShirleyLin',
+ 'realname': 'Shirley Lin',
+ 'location': 'Fremont, California, US',
+ 'timezone': {'label': 'Pacific Time (US & Canada); Tijuana',
+  'offset': '-08:00',
+  'timezone_id': 'PST8PDT'},
+ 'description': '',
+ 'photosurl': 'https://www.flickr.com/photos/shirleylin/',
+ 'profileurl': 'https://www.flickr.com/people/shirleylin/',
+ 'mobileurl': 'https://m.flickr.com/photostream.gne?id=7102756',
+ 'photos_info': {'firstdatetaken': '2004-05-24 12:12:15',
+  'firstdate': '1172556588',
+  'count': 9665}}
+"""
\ No newline at end of file
diff --git a/megapixels/commands/datasets/megaface_names.py b/megapixels/commands/datasets/megaface_names.py
new file mode 100644
index 00000000..01e93e2d
--- /dev/null
+++ b/megapixels/commands/datasets/megaface_names.py
@@ -0,0 +1,65 @@
+from glob import glob
+import os
+from os.path import join
+from pathlib import Path
+
+import click
+
+from app.settings import types
+from app.utils import click_utils
+from app.settings import app_cfg as cfg
+from app.utils import logger_utils
+
+import dlib
+import pandas as pd
+from PIL import Image, ImageOps, ImageFilter
+from app.utils import file_utils, im_utils
+
+
+log = logger_utils.Logger.getLogger()
+
+@click.command()
+@click.option('-i', '--input', 'opt_fp_in', required=True,
+  help='Input directory')
+@click.option('-o', '--output', 'opt_fp_out',
+  help='Output directory')
+@click.pass_context
+def cli(ctx, opt_fp_in, opt_fp_out):
+  """Creates CSV of NSIDs from MegaFace"""
+  
+  from tqdm import tqdm
+  from glob import glob
+
+  # -------------------------------------------------
+  # process  
+  fp_im_dirs = glob(join(opt_fp_in, '**/'), recursive=True)
+  
+  log.info('Found {} directories'.format(len(fp_im_dirs)))
+
+  identities = {}
+
+  for fp_im_dir in tqdm(fp_im_dirs):
+    # 1234567@N05_identity_1
+    try:
+      dir_id_name = Path(fp_im_dir).name
+      nsid = dir_id_name.split('_')[0]
+      identity_num = dir_id_name.split('_')[2]
+      id_key = '{}_{}'.format(nsid, identity_num)
+      num_images = len(glob(join(fp_im_dir, '*.jpg')))
+      if not id_key in identities.keys():
+        identities[id_key] = {'nsid': nsid, 'identity': identity_num, 'images': num_images}
+      else:
+        identities[id_key]['images'] += num_images
+    except Exception as e:
+      continue  
+
+  # convert to dict 
+  identities_list = [v for k, v in identities.items()]
+  df = pd.DataFrame.from_dict(identities_list)
+
+  file_utils.mkdirs(opt_fp_out)
+
+  log.info('Wrote {} lines to {}'.format(len(df), opt_fp_out))
+  df.to_csv(opt_fp_out, index=False)
+
+
diff --git a/megapixels/commands/datasets/sha256.py b/megapixels/commands/datasets/sha256.py
new file mode 100644
index 00000000..c04fb504
--- /dev/null
+++ b/megapixels/commands/datasets/sha256.py
@@ -0,0 +1,90 @@
+import click
+
+from app.settings import types
+from app.utils import click_utils
+from app.settings import app_cfg as cfg
+from app.utils.logger_utils import Logger
+
+log = Logger.getLogger()
+
+@click.command()
+@click.option('-i', '--input', 'opt_fp_in', required=True,
+  help='Input directory')
+@click.option('-o', '--output', 'opt_fp_out',
+  help='Output directory')
+@click.option('--slice', 'opt_slice', type=(int, int), default=(None, None),
+  help='Slice list of files')
+@click.option('--recursive/--no-recursive', 'opt_recursive', is_flag=True, default=False,
+  help='Use glob recursion (slower)')
+@click.option('-t', '--threads', 'opt_threads', default=4,
+  help='Number of threads')
+@click.option('-f', '--force', 'opt_force', is_flag=True,
+  help='Force overwrite file')
+@click.pass_context
+def cli(ctx, opt_fp_in, opt_fp_out, opt_slice, opt_recursive, opt_threads, opt_force):
+  """Multithreading test"""
+  
+  from glob import glob
+  from os.path import join
+  from pathlib import Path
+  import time
+  from multiprocessing.dummy import Pool as ThreadPool 
+  import random
+
+  import pandas as pd
+  from tqdm import tqdm
+  from glob import glob
+  
+  from app.utils import file_utils, im_utils
+
+
+  if not opt_force and Path(opt_fp_out).exists():
+    log.error('File exists. Use "-f / --force" to overwite')
+    return
+
+  fp_ims = []
+  for ext in ['jpg', 'png']:
+    if opt_recursive:
+      fp_glob = join(opt_fp_in, '**/*.{}'.format(ext))
+      fp_ims += glob(fp_glob, recursive=True)
+    else:
+      fp_glob = join(opt_fp_in, '*.{}'.format(ext))
+      fp_ims += glob(fp_glob)
+
+  if opt_slice:
+    fp_ims = fp_ims[opt_slice[0]:opt_slice[1]]
+
+  log.info('Processing {:,} images'.format(len(fp_ims)))
+
+  pbar = tqdm(total=100)
+
+  def as_sha256(fp_im):
+    pbar.update(1)
+    return file_utils.sha256(fp_im)
+
+  # multithread pool
+  st = time.time()
+  pool = ThreadPool(opt_threads) 
+  with tqdm(total=len(fp_ims)) as pbar:
+    sha256s = pool.map(as_sha256, fp_ims)
+  pbar.close()
+
+  # convert data to dict
+  data = []
+  for i, fp_im in enumerate(fp_ims):
+    fpp_im = Path(fp_im)
+    subdir = str(fpp_im.parent.relative_to(opt_fp_in))
+    sha256 = sha256s[i]
+    data.append( {
+      'sha256': sha256, 
+      'subdir': subdir,
+      'fn': fpp_im.stem,
+      'ext': fpp_im.suffix.replace('.','')
+      })
+
+  # save to CSV
+  df = pd.DataFrame.from_dict(data)
+  df.to_csv(opt_fp_out, index=False)
+
+  # timing
+  log.info('time: {:.2f}, theads: {}'.format(time.time() - st, opt_threads))
\ No newline at end of file
diff --git a/megapixels/commands/datasets/ytmu.py b/megapixels/commands/datasets/ytmu.py
new file mode 100644
index 00000000..66680ed0
--- /dev/null
+++ b/megapixels/commands/datasets/ytmu.py
@@ -0,0 +1,205 @@
+from glob import glob
+import os
+from os.path import join
+from pathlib import Path
+
+import click
+
+from app.settings import types
+from app.utils import click_utils
+from app.settings import app_cfg as cfg
+from app.utils import logger_utils
+
+import dlib
+import pandas as pd
+from PIL import Image, ImageOps, ImageFilter
+from app.utils import file_utils, im_utils
+
+
+log = logger_utils.Logger.getLogger()
+
+@click.command()
+@click.option('-i', '--input', 'opt_fp_in', required=True,
+  help='Input directory')
+@click.option('-o', '--output', 'opt_fp_out',
+  help='Output directory')
+@click.option('--videos', 'opt_dir_videos',
+  help='Output directory')
+@click.option('--action', 'opt_action', 
+  type=click.Choice(['info', 'faces', 'rename', 'download', 'metadata', 'split_frames']),
+  default='info',
+  help='Command action')
+@click.pass_context
+def cli(ctx, opt_fp_in, opt_fp_out, opt_dir_videos, opt_action):
+  """YTMU utils"""
+
+  
+  from tqdm import tqdm
+
+  # -------------------------------------------------
+  # process  
+  
+  if opt_action == 'metadata':
+    # downloads video metadata with ytdl
+    handle_metadata(opt_fp_in, opt_fp_out)
+  elif opt_action == 'download':
+    # downloads video files with ytdl
+    handle_download(opt_fp_in, opt_fp_out)
+  elif opt_action == 'info':
+    # converts original data file to clean CSV
+    handle_info()
+  elif opt_action == 'rename':
+    # rename the videos to video ID
+    handle_rename(opt_fp_in, opt_fp_out, opt_dir_videos)
+  elif opt_action == 'split_frames':
+    # rename the videos to video ID
+    handle_split_frames(opt_fp_in, opt_fp_out, opt_dir_videos)
+  
+
+
+
+# ----------------------------------------------------
+# handlers
+
+def handle_split_frames(fp_in, dir_out, dir_videos):
+  if not dir_out or not dir_videos:
+    log.error('-o/--output and --videos required')
+    return
+  import cv2 as cv
+  from tqdm import tqdm
+  from app.processors import face_detector
+  detector = face_detector.DetectorDLIBCNN()
+
+  # get file list
+  fp_videos = glob(join(dir_videos, '*.mp4'))
+  fp_videos += glob(join(dir_videos, '*.webm'))
+  fp_videos += glob(join(dir_videos, '*.mkv'))
+  face_interval = 30
+  frame_interval_count = 0
+  frame_count = 0
+
+  file_utils.mkdirs(dir_out)
+
+  for fp_video in tqdm(fp_videos):
+    # log.debug('opening: {}'.format(fp_video))
+    video = cv.VideoCapture(fp_video)
+    while video.isOpened():
+      res, frame = video.read()
+      if not res:
+        break
+
+      frame_count += 1  # for naming
+      frame_interval_count += 1  # for interval
+      bboxes = detector.detect(frame, opt_size=(320, 240), opt_pyramids=0)
+      if len(bboxes) > 0 and frame_interval_count >= face_interval:
+        # save frame
+        fp_frame = join(dir_out, '{}_{}.jpg'.format(Path(fp_video).stem, file_utils.zpad(frame_count)))
+        cv.imwrite(fp_frame, frame)
+        frame_interval_count = 0
+
+
+def handle_metadata(fp_in, fp_out):
+  
+  keys = ['description', 'average_rating', 'dislike_count', 'categories', 
+  'thumbnail', 'title', 'upload_date', 'uploader_url', 'uploader_id',
+  'fps', 'height', 'width', 'like_count', 'license', 'tags']
+
+  import youtube_dl
+
+  ydl = youtube_dl.YoutubeDL({'outtmpl': '%(id)s%(ext)s'})
+
+  df = pd.read_csv(fp_in)
+  data_exp = []
+
+  for i, row in df.iterrows():
+    video_data = {'url': row['url'], 'id': row['id']}
+    try:
+      with ydl:
+        url = 'http://www.youtube.com/watch?v={}'.format(row['id'])
+        result = ydl.extract_info(url, download=False)
+      video = result['entries'][0] if 'entries' in result else result
+      for k in keys:
+        val = video[k]
+        if k == 'title':
+          log.debug(val)
+        if type(val) == list:
+          val = '; '.join(val)
+        if type(val) == str:
+          video_data[k] = str(val).replace(',',';')
+      # log.debug('video_data: {}'.format(video_data))
+    except Exception as e:
+      log.warn('video unavilable: {}'.format(row['url']))
+      log.error(e)
+      continue
+    data_exp.append(video_data)
+
+  df_exp = pd.DataFrame.from_dict(data_exp)
+  df_exp.to_csv(fp_out)
+
+
+def handle_download(fp_in, dir_out):
+  import youtube_dl
+  df = pd.read_csv(fp_in)
+  fp_videos = glob(join(dir_out, '*.mp4'))
+  fp_videos += glob(join(dir_out, '*.webm'))
+  fp_videos += glob(join(dir_out, '*.mkv'))
+  
+  ydl = youtube_dl.YoutubeDL({'outtmpl': '%(id)s%(ext)s'})
+
+  for i, row in df.iterrows():
+    vid = row['id']
+    found = False
+    for fp_video in fp_videos:
+      if vid in fp_video:
+        log.debug('skip: {}'.format(vid))
+        found = True
+    if not found:
+      try:
+        with ydl:
+          ydl.download(['http://www.youtube.com/watch?v={}'.format(vid)])
+      except:
+        log.error('could not dl: {}'.format(vid))
+
+
+def handle_info(fp_in, fp_out):
+  if not fp_out:
+    log.error('--output required')
+    return
+  urls = file_utils.load_text(fp_in)
+  videos  = []
+  for url in urls:
+    splits = url.split('v=')
+    try:
+      vid = splits[1]
+      vid = vid.split('&')[0]
+      videos.append({'url': url, 'id': vid})
+    except:
+      log.warn('no video id for {}'.format(url))
+  # convert to df
+  df = pd.DataFrame.from_dict(videos)
+  df.to_csv(opt_fp_out)
+
+  
+def handle_rename(fp_in, fp_out, dir_videos):
+  import shutil
+  
+  if not dir_videos:
+    log.error('--videos required')
+    return
+
+  fp_videos = glob(join(dir_videos, '*.mp4'))
+  fp_videos += glob(join(dir_videos, '*.webm'))
+  fp_videos += glob(join(dir_videos, '*.mkv'))
+
+  df = pd.read_csv(fp_in)
+
+  for i, row in df.iterrows():
+    vid = row['id']
+    fp_videos_copy = fp_videos.copy()
+    for fp_video in fp_videos:
+      if vid in fp_video:
+        dst = join(dir_videos, '{}{}'.format(vid, Path(fp_video).suffix))
+        shutil.move(fp_video, dst)
+        log.debug('move {} to {}'.format(fp_video, dst))
+        fp_videos.remove(fp_video)
+        break
\ No newline at end of file
diff --git a/megapixels/commands/misc/compare_sres.py b/megapixels/commands/misc/compare_sres.py
new file mode 100644
index 00000000..b96570fe
--- /dev/null
+++ b/megapixels/commands/misc/compare_sres.py
@@ -0,0 +1,59 @@
+import click
+
+
+@click.command()
+@click.option('-i', '--orig', 'opt_dir_in_orig', required=True,
+  help='Input directory')
+@click.option('-n', '--new', 'opt_dir_in_new', required=True,
+  help='Input directory files to compare to')
+@click.pass_context
+def cli(ctx, opt_dir_in_orig, opt_dir_in_new):
+  """Compare quality of super resolution images"""
+
+  import os
+
+  import sys
+  from os.path import join
+  from pathlib import Path
+  from glob import glob
+
+  from random import randint
+  from PIL import Image, ImageOps, ImageFilter
+  from pathlib import Path
+  import cv2 as cv
+
+  from app.settings import types
+  from app.utils import click_utils
+  from app.settings import app_cfg as cfg
+  from app.utils import file_utils, im_utils, logger_utils
+  
+  log = logger_utils.Logger.getLogger()
+
+  fp_ims = glob(join(opt_dir_in_new, '*.jpg'))
+  fp_ims += glob(join(opt_dir_in_new, '*.png'))
+
+  log.info('{}'.format(len(fp_ims)))
+
+  while True:
+    rn = randint(0, len(fp_ims) - 1)
+    fp_im_new = fp_ims[rn]
+    fp_im_orig = fp_im_new.replace(opt_dir_in_new, opt_dir_in_orig)
+    log.info('new: {}'.format(fp_im_new))
+    log.info('orig: {}'.format(fp_im_orig))
+
+    im_new = cv.imread(fp_im_new)
+    im_orig = cv.imread(fp_im_orig)
+
+    # show
+    cv.imshow('new', im_new)
+    cv.imshow('orig', im_orig)
+
+    # handle key io
+    k = cv.waitKey(0) & 0xFF
+    if k == 27 or k == ord('q'):  # ESC
+      # exits the app
+      cv.destroyAllWindows()
+      sys.exit('Exiting because Q or ESC was pressed')
+    elif k == ord(' ') or k == 81 or k == 83:
+      continue
+