add pose, indexing

19 files changed, 1221 insertions, 59 deletions

diff --git a/megapixels/app/models/bbox.py b/megapixels/app/models/bbox.py
index ccdf229e..e6da960e 100644
--- a/megapixels/app/models/bbox.py
+++ b/megapixels/app/models/bbox.py
@@ -45,8 +45,12 @@ class BBox:
     self._tl = (x1, y1)
     self._br = (x2, y2)
     self._rect = (self._x1, self._y1, self._x2, self._y2)
+    self._area = self._width * self._height  # as percentage
 
-
+  @property
+  def area(self):
+    return self._area
+  
   @property
   def pt_tl(self):
     return self._tl
@@ -178,23 +182,23 @@ class BBox:
   # -----------------------------------------------------------------
   # Format as
 
-  def as_xyxy(self):
+  def to_xyxy(self):
     """Converts BBox back to x1, y1, x2, y2 rect"""
     return (self._x1, self._y1, self._x2, self._y2)
 
-  def as_xywh(self):
+  def to_xywh(self):
     """Converts BBox back to haar type"""
     return (self._x1, self._y1, self._width, self._height)
 
-  def as_trbl(self):
+  def to_trbl(self):
     """Converts BBox to CSS (top, right, bottom, left)""" 
     return (self._y1, self._x2, self._y2, self._x1)
 
-  def as_dlib(self):
+  def to_dlib(self):
     """Converts BBox to dlib rect type"""
-    return dlib.rectangle(self._x1, self._y1, self._x2, self._y2)
+    return dlib_rectangle(self._x1, self._y1, self._x2, self._y2)
 
-  def as_yolo(self):
+  def to_yolo(self):
     """Converts BBox to normalized center x, center y, w, h"""
     return (self._cx, self._cy, self._width, self._height)
 
@@ -256,8 +260,7 @@ class BBox:
     """ 
     rect = (rect.left(), rect.top(), rect.right(), rect.bottom())
     rect = cls.normalize(cls, rect, dim)
-    return cls(*rect)
-
+    return cls(*rect) 
 
   def str(self):
     """Return BBox as a string "x1, y1, x2, y2" """
diff --git a/megapixels/app/processors/face_detector.py b/megapixels/app/processors/face_detector.py
index 747e057b..593e9feb 100644
--- a/megapixels/app/processors/face_detector.py
+++ b/megapixels/app/processors/face_detector.py
@@ -4,12 +4,51 @@ from pathlib import Path
 
 import cv2 as cv
 import numpy as np
-import dlib
-# import imutils
+import imutils
+import operator
 
 from app.utils import im_utils, logger_utils
 from app.models.bbox import BBox
 from app.settings import app_cfg as cfg
+from app.settings import types
+
+
+class DetectorMTCNN:
+  
+  # https://github.com/ipazc/mtcnn
+  # pip install mtcnn
+
+  dnn_size = (300, 300)
+
+  def __init__(self, size=(400,400)):
+    from mtcnn.mtcnn import MTCNN
+    self.detector = MTCNN()
+
+  def detect(self, im, opt_size=(400,400), opt_conf_thresh=None, opt_pyramids=None, opt_largest=False):
+    '''Detects face using MTCNN and returns (list) of BBox
+    :param im: (numpy.ndarray) image
+    :returns list of BBox
+    '''
+    bboxes = []
+    #conf_thresh = self.conf_thresh if opt_conf_thresh is None else opt_conf_thresh
+    #pyramids = self.pyramids if opt_pyramids is None else opt_pyramids
+    dnn_size = self.dnn_size if opt_size is None else opt_size
+
+    im = im_utils.resize(im, width=dnn_size[0], height=dnn_size[1])
+    dim = im.shape[:2][::-1]
+    dets = self.detector.detect_faces(im)
+    for det in dets:
+      rect = det['box']
+      #keypoints = det['keypoints']  # not using here. see 'face_landmarks.py'
+      bbox = BBox.from_xywh_dim(*rect, dim)
+      bboxes.append(bbox)
+
+    if opt_largest and len(bboxes) > 1:
+      # only keep largest
+      bboxes.sort(key=operator.attrgetter('area'), reverse=True)
+      bboxes = [bboxes[0]]
+
+    return bboxes
 
 
 class DetectorHaar:
@@ -21,16 +60,18 @@ class DetectorHaar:
     self.log = logger_utils.Logger.getLogger()
 
   def detect(self, im, scale_factor=1.05, overlaps=5):
-    return
+    pass
     
 
 class DetectorDLIBCNN:
+  
 
   dnn_size = (300, 300)
   pyramids = 0
   conf_thresh = 0.85
 
   def __init__(self, opt_gpu=0):
+    import dlib
     self.log = logger_utils.Logger.getLogger()
     cuda_visible_devices = os.getenv('CUDA_VISIBLE_DEVICES', '')
     os.environ['CUDA_VISIBLE_DEVICES'] = str(opt_gpu)
@@ -38,8 +79,8 @@ class DetectorDLIBCNN:
     self.detector = dlib.cnn_face_detection_model_v1(cfg.DIR_MODELS_DLIB_CNN)
     os.environ['CUDA_VISIBLE_DEVICES'] = cuda_visible_devices  # reset
 
-  def detect(self, im, opt_size=None, opt_conf_thresh=None, opt_pyramids=None):
-    rois = []
+  def detect(self, im, opt_size=None, opt_conf_thresh=None, opt_pyramids=None, opt_largest=False):
+    bboxes = []
     conf_thresh = self.conf_thresh if opt_conf_thresh is None else opt_conf_thresh
     pyramids = self.pyramids if opt_pyramids is None else opt_pyramids
     dnn_size = self.dnn_size if opt_size is None else opt_size
@@ -48,24 +89,34 @@ class DetectorDLIBCNN:
     dim = im.shape[:2][::-1]
     im = im_utils.bgr2rgb(im)  # convert to RGB for dlib
     # run detector
-    mmod_rects = self.detector(im, 1)
+    mmod_rects = self.detector(im, opt_pyramids)
     # sort results
     for mmod_rect in mmod_rects:
+      self.log.debug('conf: {}, this: {}'.format(conf_thresh, mmod_rect.confidence))
       if mmod_rect.confidence > conf_thresh:
         bbox = BBox.from_dlib_dim(mmod_rect.rect, dim)
-        rois.append(bbox)
-    return rois
+        bboxes.append(bbox)
+
+    if opt_largest and len(bboxes) > 1:
+      # only keep largest
+      bboxes.sort(key=operator.attrgetter('area'), reverse=True)
+      bboxes = [bboxes[0]]
+
+    return bboxes
 
 
 class DetectorDLIBHOG:
 
   size = (320, 240)
   pyramids = 0
+  conf_thresh = 0.85
 
   def __init__(self):
+    import dlib
+    self.log = logger_utils.Logger.getLogger()
     self.detector = dlib.get_frontal_face_detector()
 
-  def detect(self, im, opt_size=None, opt_conf_thresh=None, opt_pyramids=0):
+  def detect(self, im, opt_size=None, opt_conf_thresh=None, opt_pyramids=0, opt_largest=False):
     conf_thresh = self.conf_thresh if opt_conf_thresh is None else opt_conf_thresh
     dnn_size = self.size if opt_size is None else opt_size
     pyramids = self.pyramids if opt_pyramids is None else opt_pyramids
@@ -75,13 +126,20 @@ class DetectorDLIBHOG:
     im = im_utils.bgr2rgb(im)  # ?
     hog_results = self.detector.run(im, pyramids)
     
-    rois = []
+    bboxes = []
     if len(hog_results[0]) > 0:
+      self.log.debug(hog_results)
       for rect, score, direction in zip(*hog_results):
-        if score > opt_conf_thresh:
+        if score > conf_thresh:
           bbox = BBox.from_dlib_dim(rect, dim)
-          rois.append(bbox)
-    return rois
+          bboxes.append(bbox)
+    
+    if opt_largest and len(bboxes) > 1:
+      # only keep largest
+      bboxes.sort(key=operator.attrgetter('area'), reverse=True)
+      bboxes = [bboxes[0]]
+
+    return bboxes
 
 class DetectorCVDNN:
 
@@ -92,13 +150,14 @@ class DetectorCVDNN:
   conf_thresh = 0.85
   
   def __init__(self):
+    import dlib
     fp_prototxt = join(cfg.DIR_MODELS_CAFFE, 'face_detect', 'opencv_face_detector.prototxt')
     fp_model = join(cfg.DIR_MODELS_CAFFE, 'face_detect', 'opencv_face_detector.caffemodel')
     self.net = cv.dnn.readNet(fp_prototxt, fp_model)
     self.net.setPreferableBackend(cv.dnn.DNN_BACKEND_OPENCV)
     self.net.setPreferableTarget(cv.dnn.DNN_TARGET_CPU)
 
-  def detect(self, im, opt_size=None, opt_conf_thresh=None):
+  def detect(self, im, opt_size=None, opt_conf_thresh=None, opt_largest=False, opt_pyramids=None):
     """Detects faces and returns (list) of (BBox)"""
     conf_thresh = self.conf_thresh if opt_conf_thresh is None else opt_conf_thresh
     dnn_size = self.size if opt_size is None else opt_size
@@ -107,10 +166,16 @@ class DetectorCVDNN:
     self.net.setInput(blob)
     net_outputs = self.net.forward()
 
-    rois = []
+    bboxes = []
     for i in range(0, net_outputs.shape[2]):
       conf = net_outputs[0, 0, i, 2]
-      if conf > opt_conf_thresh:
+      if conf > conf_thresh:
         rect_norm = net_outputs[0, 0, i, 3:7]
-        rois.append(BBox(*rect_norm))
-    return rois
\ No newline at end of file
+        bboxes.append(BBox(*rect_norm))
+    
+    if opt_largest and len(bboxes) > 1:
+      # only keep largest
+      bboxes.sort(key=operator.attrgetter('area'), reverse=True)
+      bboxes = [bboxes[0]]
+
+    return bboxes
\ No newline at end of file
diff --git a/megapixels/app/processors/face_landmarks.py b/megapixels/app/processors/face_landmarks.py
new file mode 100644
index 00000000..dfcb9ee8
--- /dev/null
+++ b/megapixels/app/processors/face_landmarks.py
@@ -0,0 +1,60 @@
+import os
+from os.path import join
+from pathlib import Path
+
+import cv2 as cv
+import numpy as np
+import imutils
+from app.utils import im_utils, logger_utils
+from app.models.bbox import BBox
+from app.settings import app_cfg as cfg
+from app.settings import types
+from app.models.bbox import BBox
+
+class LandmarksDLIB:
+
+  def __init__(self):
+    # init dlib
+    import dlib
+    self.log = logger_utils.Logger.getLogger()
+    self.predictor = dlib.shape_predictor(cfg.DIR_MODELS_DLIB_68PT)
+
+  def landmarks(self, im, bbox):
+    # Draw high-confidence faces
+    dim = im.shape[:2][::-1]
+    bbox = bbox.to_dlib()
+    im_gray = cv.cvtColor(im, cv.COLOR_BGR2GRAY)
+    landmarks = [[p.x, p.y] for p in self.predictor(im_gray, bbox).parts()]
+    return landmarks
+
+
+class LandmarksMTCNN:
+  
+  # https://github.com/ipazc/mtcnn
+  # pip install mtcnn
+
+  dnn_size = (400, 400)
+
+  def __init__(self, size=(400,400)):
+    from mtcnn.mtcnn import MTCNN
+    self.detector = MTCNN()
+
+  def detect(self, im, opt_size=None, opt_conf_thresh=None, opt_pyramids=None):
+    '''Detects face using MTCNN and returns (list) of BBox
+    :param im: (numpy.ndarray) image
+    :returns list of BBox
+    '''
+    rois = []
+    dnn_size = self.dnn_size if opt_size is None else opt_size
+    im = im_utils.resize(im, width=dnn_size[0], height=dnn_size[1])
+    dim = im.shape[:2][::-1]
+    
+    # run MTCNN
+    dets = self.detector.detect_faces(im)
+
+    for det in dets:
+      rect = det['box']
+      keypoints = det['keypoints']  # not using here. see 'face_landmarks.py'
+      bbox = BBox.from_xywh_dim(*rect, dim)
+      rois.append(bbox)
+    return rois
\ No newline at end of file
diff --git a/megapixels/app/processors/face_pose.py b/megapixels/app/processors/face_pose.py
new file mode 100644
index 00000000..67ac685d
--- /dev/null
+++ b/megapixels/app/processors/face_pose.py
@@ -0,0 +1,110 @@
+import os
+from os.path import join
+from pathlib import Path
+import math
+
+import cv2 as cv
+import numpy as np
+import imutils
+
+from app.utils import im_utils, logger_utils
+from app.models.bbox import BBox
+from app.settings import app_cfg as cfg
+from app.settings import types
+
+
+
+class FacePoseDLIB:
+
+
+  dnn_size = (400, 400)
+
+  def __init__(self):
+    pass
+
+  def pose(self, landmarks, dim): 
+    '''Calculates pose
+    '''
+    degrees = compute_pose_degrees(landmarks, dim)    
+    return degrees
+
+
+# -----------------------------------------------------------
+# utilities
+# -----------------------------------------------------------
+
+def compute_pose_degrees(landmarks, dim):
+  # computes pose using 6 / 68 points from dlib face landmarks
+  # based on learnopencv.com and 
+  # https://github.com/jerryhouuu/Face-Yaw-Roll-Pitch-from-Pose-Estimation-using-OpenCV/
+  # NB: not as accurate as MTCNN, see @jerryhouuu for ideas
+  
+  pose_points_idx = (30, 8, 36, 45, 48, 54)
+  axis = np.float32([[500,0,0], [0,500,0], [0,0,500]])
+  
+  # 3D model points.
+  model_points = np.array([
+      (0.0, 0.0, 0.0),             # Nose tip
+      (0.0, -330.0, -65.0),        # Chin
+      (-225.0, 170.0, -135.0),     # Left eye left corner
+      (225.0, 170.0, -135.0),      # Right eye right corne
+      (-150.0, -150.0, -125.0),    # Left Mouth corner
+      (150.0, -150.0, -125.0)      # Right mouth corner
+  ])
+  
+  # Assuming no lens distortion
+  dist_coeffs = np.zeros((4,1))
+
+  # find 6 pose points
+  pose_points = []
+  for j, idx in enumerate(pose_points_idx):
+    pt = landmarks[idx]
+    pose_points.append((pt[0], pt[1]))
+  pose_points = np.array(pose_points, dtype='double')  # convert to double
+  
+  # create camera matrix
+  focal_length = dim[0]
+  center = (dim[0]/2, dim[1]/2)
+  cam_mat = np.array(
+      [[focal_length, 0, center[0]],
+      [0, focal_length, center[1]],
+      [0, 1, 1]], dtype = "double")
+  
+  # solve PnP for rotation and translation
+  (success, rot_vec, tran_vec) = cv.solvePnP(model_points, pose_points, 
+                                             cam_mat, dist_coeffs, 
+                                             flags=cv.SOLVEPNP_ITERATIVE)
+
+  # project points
+  #pts_im, jac = cv.projectPoints(axis, rot_vec, tran_vec, cam_mat, dist_coeffs)
+  #pts_model, jac2 = cv.projectPoints(model_points, rot_vec, tran_vec, cam_mat, dist_coeffs)
+  rvec_matrix = cv.Rodrigues(rot_vec)[0]
+  
+  # convert to degrees
+  proj_matrix = np.hstack((rvec_matrix, tran_vec))
+  eulerAngles = cv.decomposeProjectionMatrix(proj_matrix)[6] 
+  pitch, yaw, roll = [math.radians(x) for x in eulerAngles]
+  pitch = math.degrees(math.asin(math.sin(pitch)))
+  roll = -math.degrees(math.asin(math.sin(roll)))
+  yaw = math.degrees(math.asin(math.sin(yaw)))
+  degrees = {'pitch': pitch, 'roll': roll, 'yaw': yaw}
+  
+  # add nose point
+  #pt_nose = tuple(landmarks[pose_points_idx[0]])
+  return degrees
+  #return pts_im, pts_model, degrees, pt_nose
+
+
+def draw_pose(im, pts_im, pts_model, pt_nose):
+  cv.line(im, pt_nose, tuple(pts_im[1].ravel()), (0,255,0), 3) #GREEN
+  cv.line(im, pt_nose, tuple(pts_im[0].ravel()), (255,0,), 3) #BLUE
+  cv.line(im, pt_nose, tuple(pts_im[2].ravel()), (0,0,255), 3) #RED
+  return im
+
+
+def draw_degrees(im, degrees, color=(0,255,0)):
+  for i, item in enumerate(degrees.items()):
+    k, v = item
+    t = '{}: {:.2f}'.format(k, v)
+    origin = (10, 30 + (25 * i))
+    cv.putText(im, t, origin, cv.FONT_HERSHEY_SIMPLEX, 0.5, color, thickness=2, lineType=2)
\ No newline at end of file
diff --git a/megapixels/app/processors/face_recognition.py b/megapixels/app/processors/face_recognition.py
new file mode 100644
index 00000000..9c3a301d
--- /dev/null
+++ b/megapixels/app/processors/face_recognition.py
@@ -0,0 +1,43 @@
+import os
+from os.path import join
+from pathlib import Path
+
+import cv2 as cv
+import numpy as np
+import dlib
+import imutils
+
+from app.utils import im_utils, logger_utils
+from app.models.bbox import BBox
+from app.settings import app_cfg as cfg
+from app.settings import types
+    
+class RecognitionDLIB:
+
+  # https://github.com/davisking/dlib/blob/master/python_examples/face_recognition.py
+  # facerec.compute_face_descriptor(img, shape, 100, 0.25)
+
+  def __init__(self, opt_gpu=0):
+    self.log = logger_utils.Logger.getLogger()
+    if opt_gpu > 0:
+      cuda_visible_devices = os.getenv('CUDA_VISIBLE_DEVICES', '')
+      os.environ['CUDA_VISIBLE_DEVICES'] = str(opt_gpu)
+      self.predictor = dlib.shape_predictor(cfg.DIR_MODELS_DLIB_5PT)
+      self.facerec = dlib.face_recognition_model_v1(cfg.DIR_MODELS_DLIB_FACEREC_RESNET)
+      os.environ['CUDA_VISIBLE_DEVICES'] = cuda_visible_devices  # reset GPU env
+
+  def vec(self, im, bbox, width=100, 
+    jitters=cfg.DLIB_FACEREC_JITTERS, padding=cfg.DLIB_FACEREC_PADDING):
+    # Converts image and bbox into 128d vector
+    # scale the image so the face is always 100x100 pixels
+
+    scale = width / bbox.width
+    im = cv.resize(im, (scale, scale), interploation=cv.INTER_LANCZOS4)
+    bbox_dlib = bbox.to_dlib()
+    face_shape = self.predictor(im, bbox_dlib)
+    vec = self.facerec.compute_face_descriptor(im, face_shape, jitters, padding)
+    return vec
+
+
+  def similarity(self, query_enc, known_enc):
+    return np.linalg.norm(query_enc - known_enc, axis=1)
diff --git a/megapixels/app/settings/app_cfg.py b/megapixels/app/settings/app_cfg.py
index 3e60a9b4..4c540231 100644
--- a/megapixels/app/settings/app_cfg.py
+++ b/megapixels/app/settings/app_cfg.py
@@ -39,9 +39,7 @@ DIR_MODELS_DLIB = join(DIR_MODELS,'dlib')
 DIR_MODELS_DLIB_CNN = join(DIR_MODELS_DLIB, 'mmod_human_face_detector.dat')
 DIR_MODELS_DLIB_5PT = join(DIR_MODELS_DLIB, 'shape_predictor_5_face_landmarks.dat')
 DIR_MODELS_DLIB_68PT = join(DIR_MODELS_DLIB, 'shape_predictor_68_face_landmarks.dat')
-
-
-types.HaarCascade.FRONTAL
+DIR_MODELS_DLIB_FACEREC_RESNET = join(DIR_MODELS_DLIB, 'dlib_face_recognition_resnet_model_v1.dat')
 
 # Test images
 DIR_TEST_IMAGES = join(DIR_APP, 'test', 'images')
@@ -71,6 +69,10 @@ CKPT_ZERO_PADDING = 9
 HASH_TREE_DEPTH = 3
 HASH_BRANCH_SIZE = 3
 
+
+DLIB_FACEREC_JITTERS = 5  # number of face recognition jitters
+DLIB_FACEREC_PADDING = 0.25  # default dlib
+
 # -----------------------------------------------------------------------------
 # Logging options exposed for custom click Params
 # -----------------------------------------------------------------------------
@@ -94,4 +96,11 @@ black, red, green, yellow, blue, purple, cyan and white.
 bold, bold_{color}, fg_bold_{color}, bg_bold_{color}: Bold/bright colors.
 reset: Clear all formatting (both foreground and background colors).
 """
-LOGFILE_FORMAT = "%(log_color)s%(levelname)-8s%(reset)s %(cyan)s%(filename)s:%(lineno)s:%(bold_cyan)s%(funcName)s() %(reset)s%(message)s"
\ No newline at end of file
+LOGFILE_FORMAT = "%(log_color)s%(levelname)-8s%(reset)s %(cyan)s%(filename)s:%(lineno)s:%(bold_cyan)s%(funcName)s() %(reset)s%(message)s"
+
+
+# -----------------------------------------------------------------------------
+# S3 storage
+# -----------------------------------------------------------------------------
+S3_MEDIA_ROOT = 's3://megapixels/v1/media/'
+S3_METADATA_ROOT = 's3://megapixels/v1/metadata/'
diff --git a/megapixels/app/settings/types.py b/megapixels/app/settings/types.py
index 52470034..e9107803 100644
--- a/megapixels/app/settings/types.py
+++ b/megapixels/app/settings/types.py
@@ -10,7 +10,7 @@ def find_type(name, enum_type):
 
 class FaceDetectNet(Enum):
   """Scene text detector networks"""
-  HAAR, DLIB_CNN, DLIB_HOG, CVDNN = range(4)
+  HAAR, DLIB_CNN, DLIB_HOG, CVDNN, MTCNN = range(5)
 
 class CVBackend(Enum):
   """OpenCV 3.4.2+ DNN target type"""
diff --git a/megapixels/commands/cv/csv_to_faces_mt.py b/megapixels/commands/cv/csv_to_faces_mt.py
new file mode 100644
index 00000000..64c8b965
--- /dev/null
+++ b/megapixels/commands/cv/csv_to_faces_mt.py
@@ -0,0 +1,105 @@
+"""
+Reads in CSV of ROIs and extracts facial regions with padding
+"""
+
+import click
+
+from app.settings import types
+from app.utils import click_utils
+from app.settings import app_cfg as cfg
+
+@click.command()
+@click.option('-i', '--input', 'opt_fp_in', required=True,
+  help='Input CSV')
+@click.option('-m', '--media', 'opt_dir_media', required=True,
+  help='Input image/video directory')
+@click.option('-o', '--output', 'opt_dir_out', required=True,
+  help='Output directory for extracted ROI images')
+@click.option('--slice', 'opt_slice', type=(int, int), default=(None, None),
+  help='Slice list of files')
+@click.option('--padding', 'opt_padding', default=0.25,
+  help='Facial padding as percentage of face width')
+@click.option('--ext', 'opt_ext_out', default='png', type=click.Choice(['jpg', 'png']),
+  help='Output image type')
+@click.option('--min', 'opt_min', default=(60, 60),
+  help='Minimum original face size')
+@click.pass_context
+def cli(ctx, opt_fp_in, opt_dir_media, opt_dir_out, opt_slice, 
+  opt_padding, opt_ext_out, opt_min):
+  """Converts ROIs to images"""
+  
+  import os
+  from os.path import join
+  from pathlib import Path
+  from glob import glob
+  
+  from tqdm import tqdm
+  import numpy as np
+  from PIL import Image, ImageOps, ImageFilter, ImageDraw
+  import cv2 as cv
+  import pandas as pd
+
+  from app.utils import logger_utils, file_utils, im_utils
+  from app.models.bbox import BBox
+
+  # -------------------------------------------------
+  # process here
+  log = logger_utils.Logger.getLogger()
+  
+  df_rois = pd.read_csv(opt_fp_in, dtype={'subdir': str, 'fn': str})
+  if opt_slice:
+    df_rois = df_rois[opt_slice[0]:opt_slice[1]]
+  
+  log.info('Processing {:,} rows'.format(len(df_rois)))
+
+  file_utils.mkdirs(opt_dir_out)
+
+  df_rois_grouped = df_rois.groupby(['fn'])  # group by fn/filename
+  groups = df_rois_grouped.groups
+  skipped = []
+
+  for group in tqdm(groups):
+    # get image
+    group_rows = df_rois_grouped.get_group(group)
+
+    row = group_rows.iloc[0]
+    fp_im = join(opt_dir_media, str(row['subdir']), '{fn}.{ext}'.format(**row))  # TODO change to ext
+    try:
+      im = Image.open(fp_im).convert('RGB')
+      im.verify()
+    except Exception as e:
+      log.warn('Could not open: {}'.format(fp_im))
+      log.error(e)
+      continue
+
+    for idx, roi in group_rows.iterrows():
+      # get bbox to im dimensions
+      xywh = [roi['x'], roi['y'], roi['w'] , roi['h']]
+      bbox = BBox.from_xywh(*xywh)
+      dim = im.size
+      bbox_dim = bbox.to_dim(dim)
+      # expand
+      opt_padding_px = int(opt_padding * bbox_dim.width)
+      bbox_dim_exp = bbox_dim.expand_dim(opt_padding_px, dim)
+      # crop
+      x1y2 = bbox_dim_exp.pt_tl + bbox_dim_exp.pt_br
+      im_crop = im.crop(box=x1y2)
+
+      # strip exif, create new image and paste data
+      im_crop_data = list(im_crop.getdata())
+      im_crop_no_exif = Image.new(im_crop.mode, im_crop.size)
+      im_crop_no_exif.putdata(im_crop_data)
+
+      # save
+      idx_zpad = file_utils.zpad(idx, zeros=3)
+      subdir = '' if roi['subdir'] == '.' else '{}_'.format(roi['subdir'])
+      subdir = subdir.replace('/', '_')
+      fp_im_out = join(opt_dir_out, '{}{}{}.{}'.format(subdir, roi['fn'], idx_zpad, opt_ext_out))
+      # threshold size and save
+      if im_crop_no_exif.size[0] < opt_min[0] or im_crop_no_exif.size[1] < opt_min[1]:
+        skipped.append(fp_im_out)
+        log.info('Face too small: {}, idx: {}'.format(fp_im, idx))
+      else:
+        im_crop_no_exif.save(fp_im_out)
+
+  log.info('Skipped {:,} images'.format(len(skipped)))
diff --git a/megapixels/commands/cv/embeddings.py b/megapixels/commands/cv/embeddings.py
new file mode 100644
index 00000000..9cb26ae7
--- /dev/null
+++ b/megapixels/commands/cv/embeddings.py
@@ -0,0 +1,100 @@
+"""
+Crop images to prepare for training
+"""
+
+import click
+
+from app.settings import types
+from app.utils import click_utils
+from app.settings import app_cfg as cfg
+
+@click.command()
+@click.option('-i', '--input', 'opt_fp_in', required=True,
+  help='Input directory')
+@click.option('-r', '--records', 'opt_fp_records', required=True,
+  help='Input directory')
+@click.option('-m', '--media', 'opt_fp_media', required=True,
+  help='Image directory')
+@click.option('-o', '--output', 'opt_fp_out', required=True,
+  help='Output CSV')
+@click.option('--size', 'opt_size', 
+  type=(int, int), default=(300, 300),
+  help='Output image size')
+@click.option('-g', '--gpu', 'opt_gpu', default=0,
+  help='GPU index')
+@click.option('--slice', 'opt_slice', type=(int, int), default=(None, None),
+  help='Slice list of files')
+@click.option('-f', '--force', 'opt_force', is_flag=True,
+  help='Force overwrite file')
+@click.option('-j', '--jitters', 'opt_jitters', default=cfg.DLIB_FACEREC_JITTERS,
+  help='Number of jitters')
+@click.option('-p', '--padding', 'opt_padding', default=cfg.DLIB_FACEREC_PADDING,
+  help='Percentage padding')
+@click.pass_context
+def cli(ctx, opt_fp_in, opt_fp_records, opt_fp_out, opt_fp_media, opt_size, opt_gpu,
+  opt_slice, opt_jitters, opt_padding, opt_force):
+  """Converts frames with faces to CSV of rows"""
+  
+  import sys
+  import os
+  from os.path import join
+  from pathlib import Path
+  
+  from tqdm import tqdm
+  import numpy as np
+  import dlib  # must keep a local reference for dlib
+  import cv2 as cv
+  import dlib
+  import pandas as pd
+
+  from app.utils import logger_utils, file_utils, im_utils
+  from app.models.bbox import BBox
+  from app.processors import face_recognition
+
+  # -------------------------------------------------
+  # init here
+
+  log = logger_utils.Logger.getLogger()
+
+  if not opt_force and Path(opt_fp_out).exists():
+    log.error('File exists. Use "-f / --force" to overwite')
+    return
+  
+  # init dlib FR
+  facerec = face_recognition.RecognitionDLIB()
+
+  # load data
+  df_rois = pd.read_csv(opt_fp_in)
+  df_records = pd.read_csv(opt_fp_records)
+
+  if opt_slice:
+    df_rois = df_rois[opt_slice[0]:opt_slice[1]]
+  log.info('Processing {:,} rows'.format(len(df_rois)))
+  nrows = len(df_rois)
+
+  # face vecs
+  vecs = []
+
+  for roi_idx, row in tqdm(df_rois.iterrows(), total=nrows):
+    # make image path
+    record_id = int(row['id'])
+    df = df_records.iloc[record_id]
+    fp_im = join(opt_fp_media, df['subdir'], '{}.{}'.format(df['fn'], df['ext'])) 
+    # load image
+    im = cv.imread(fp_im)
+    # make bbox
+    xywh = [row['x'], row['y'], row['w'] , row['h']]
+    bbox = BBox.from_xywh(*xywh)
+    # scale to actual image size
+    dim = (row['image_width'], row['image_height'])
+    bbox_dim = bbox.to_dim(dim)
+    # compute vec
+    vec = facerec.vec(im, bbox_dim, jitters=opt_jitters, padding=opt_padding)
+    vec_str = ','.join([repr(x) for x in vec])
+    vecs.append( {'id': row['id'], 'vec': vec_str})
+  
+  # save data
+  file_utils.mkdirs(opt_fp_out)
+  df_vecs = pd.DataFrame.from_dict(vecs)
+  df_vecs.to_csv(opt_fp_out, index=False)
+  log.info('saved {:,} lines to {}'.format(len(df_vecs), opt_fp_out))
\ No newline at end of file
diff --git a/megapixels/commands/cv/face_pose_to_csv.py b/megapixels/commands/cv/face_pose_to_csv.py
new file mode 100644
index 00000000..ca7489de
--- /dev/null
+++ b/megapixels/commands/cv/face_pose_to_csv.py
@@ -0,0 +1,105 @@
+"""
+Crop images to prepare for training
+"""
+
+import click
+# from PIL import Image, ImageOps, ImageFilter, ImageDraw
+
+from app.settings import types
+from app.utils import click_utils
+from app.settings import app_cfg as cfg
+
+color_filters = {'color': 1, 'gray': 2, 'all': 3}
+
+@click.command()
+@click.option('-f', '--files', 'opt_fp_files', required=True,
+  help='Input ROI CSV')
+@click.option('-r', '--rois', 'opt_fp_rois', required=True,
+  help='Input ROI CSV')
+@click.option('-m', '--media', 'opt_dir_media', required=True,
+  help='Input media directory')
+@click.option('-o', '--output', 'opt_fp_out', required=True,
+  help='Output CSV')
+@click.option('--size', 'opt_size', 
+  type=(int, int), default=(300, 300),
+  help='Output image size')
+@click.option('--slice', 'opt_slice', type=(int, int), default=(None, None),
+  help='Slice list of files')
+@click.option('-f', '--force', 'opt_force', is_flag=True,
+  help='Force overwrite file')
+@click.pass_context
+def cli(ctx, opt_fp_files, opt_fp_rois, opt_dir_media, opt_fp_out, opt_size, 
+  opt_slice, opt_force):
+  """Converts ROIs to pose: roll, yaw, pitch"""
+  
+  import sys
+  import os
+  from os.path import join
+  from pathlib import Path
+  from glob import glob
+  
+  from tqdm import tqdm
+  import numpy as np
+  import dlib  # must keep a local reference for dlib
+  import cv2 as cv
+  import pandas as pd
+
+  from app.models.bbox import BBox
+  from app.utils import logger_utils, file_utils, im_utils
+  from app.processors.face_landmarks import LandmarksDLIB
+  from app.processors.face_pose import FacePoseDLIB
+
+  # -------------------------------------------------
+  # init here
+
+  log = logger_utils.Logger.getLogger()
+
+  # init face processors
+  face_pose = FacePoseDLIB()
+  face_landmarks = LandmarksDLIB()
+
+  df_files = pd.read_csv(opt_fp_files)
+  df_rois = pd.read_csv(opt_fp_rois)
+
+  if not opt_force and Path(opt_fp_out).exists():
+    log.error('File exists. Use "-f / --force" to overwite')
+    return
+  
+  if opt_slice:
+    df_rois = df_rois[opt_slice[0]:opt_slice[1]]
+  
+  # -------------------------------------------------
+  # process here
+
+  df_roi_groups = df_rois.groupby('index')
+  log.debug('processing {:,} groups'.format(len(df_roi_groups)))
+
+
+  poses = []
+
+  #for df_roi_group in tqdm(df_roi_groups.itertuples(), total=len(df_roi_groups)):
+  for df_roi_group_idx, df_roi_group in tqdm(df_roi_groups):
+    # make fp
+    image_index = df_roi_group.image_index.values[0]
+    pds_file = df_files.iloc[image_index]
+    fp_im = join(opt_dir_media, pds_file.subdir, '{}.{}'.format(pds_file.fn, pds_file.ext))
+    im = cv.imread(fp_im)
+    # get bbox
+    x = df_roi_group.x.values[0]
+    y = df_roi_group.y.values[0]
+    w = df_roi_group.w.values[0]
+    h = df_roi_group.h.values[0]
+    dim = im.shape[:2][::-1]
+    bbox = BBox.from_xywh(x, y, w, h).to_dim(dim)
+    # get pose
+    landmarks = face_landmarks.landmarks(im, bbox)
+    pose = face_pose.pose(landmarks, dim)
+    pose['image_index'] = image_index
+    poses.append(pose)
+
+
+  # save date
+  file_utils.mkdirs(opt_fp_out)
+  df = pd.DataFrame.from_dict(poses)
+  df.index.name = 'index'
+  df.to_csv(opt_fp_out)
\ No newline at end of file
diff --git a/megapixels/commands/cv/faces_to_csv.py b/megapixels/commands/cv/faces_to_csv.py
index 07226c31..1fd47571 100644
--- a/megapixels/commands/cv/faces_to_csv.py
+++ b/megapixels/commands/cv/faces_to_csv.py
@@ -30,7 +30,7 @@ color_filters = {'color': 1, 'gray': 2, 'all': 3}
   help='GPU index')
 @click.option('--conf', 'opt_conf_thresh', default=0.85, type=click.FloatRange(0,1),
   help='Confidence minimum threshold')
-@click.option('--pyramids', 'opt_pyramids', default=0, type=click.IntRange(0,4),
+@click.option('-p', '--pyramids', 'opt_pyramids', default=0, type=click.IntRange(0,4),
   help='Number pyramids to upscale for DLIB detectors')
 @click.option('--slice', 'opt_slice', type=(int, int), default=(None, None),
   help='Slice list of files')
@@ -78,6 +78,8 @@ def cli(ctx, opt_dirs_in, opt_fp_out, opt_ext, opt_size, opt_detector_type,
     detector = face_detector.DetectorDLIBCNN(opt_gpu)
   elif opt_detector_type == types.FaceDetectNet.DLIB_HOG:
     detector = face_detector.DetectorDLIBHOG()
+  elif opt_detector_type == types.FaceDetectNet.MTCNN:
+    detector = face_detector.DetectorMTCNN()
   elif opt_detector_type == types.FaceDetectNet.HAAR:
     log.error('{} not yet implemented'.format(opt_detector_type.name))
     return
@@ -129,6 +131,8 @@ def cli(ctx, opt_dirs_in, opt_fp_out, opt_ext, opt_size, opt_detector_type,
     subdir = str(fpp_im.parent.relative_to(opt_dir_in))
 
     for bbox in bboxes:
+      # log.debug('is square: {}'.format(bbox.w == bbox.h))
+      nw,nh = int(bbox.w * im.shape[1]),  int(bbox.h * im.shape[0])
       roi = {
         'fn': fpp_im.stem, 
         'ext': fpp_im.suffix.replace('.',''), 
diff --git a/megapixels/commands/cv/faces_to_csv_indexed.py b/megapixels/commands/cv/faces_to_csv_indexed.py
new file mode 100644
index 00000000..ef958f89
--- /dev/null
+++ b/megapixels/commands/cv/faces_to_csv_indexed.py
@@ -0,0 +1,156 @@
+"""
+Crop images to prepare for training
+"""
+
+import click
+# from PIL import Image, ImageOps, ImageFilter, ImageDraw
+
+from app.settings import types
+from app.utils import click_utils
+from app.settings import app_cfg as cfg
+
+color_filters = {'color': 1, 'gray': 2, 'all': 3}
+
+@click.command()
+@click.option('-i', '--input', 'opt_fp_in', required=True,
+  help='Input CSV (eg image_files.csv)')
+@click.option('-m', '--media', 'opt_dir_media', required=True,
+  help='Input media directory')
+@click.option('-o', '--output', 'opt_fp_out', required=True,
+  help='Output CSV')
+@click.option('--size', 'opt_size', 
+  type=(int, int), default=(300, 300),
+  help='Output image size')
+@click.option('-t', '--detector-type', 'opt_detector_type',
+  type=cfg.FaceDetectNetVar,
+  default=click_utils.get_default(types.FaceDetectNet.DLIB_CNN),
+  help=click_utils.show_help(types.FaceDetectNet))
+@click.option('-g', '--gpu', 'opt_gpu', default=0,
+  help='GPU index')
+@click.option('--conf', 'opt_conf_thresh', default=0.85, type=click.FloatRange(0,1),
+  help='Confidence minimum threshold')
+@click.option('-p', '--pyramids', 'opt_pyramids', default=0, type=click.IntRange(0,4),
+  help='Number pyramids to upscale for DLIB detectors')
+@click.option('--slice', 'opt_slice', type=(int, int), default=(None, None),
+  help='Slice list of files')
+@click.option('--display/--no-display', 'opt_display', is_flag=True, default=False,
+  help='Display detections to debug')
+@click.option('-f', '--force', 'opt_force', is_flag=True,
+  help='Force overwrite file')
+@click.option('--color', 'opt_color_filter', 
+  type=click.Choice(color_filters.keys()), default='all',
+  help='Filter to keep color or grayscale images (color = keep color')
+@click.option('--largest', 'opt_largest', is_flag=True, 
+  help='Only keep largest face')
+@click.pass_context
+def cli(ctx, opt_fp_in, opt_dir_media, opt_fp_out, opt_size, opt_detector_type, 
+  opt_gpu, opt_conf_thresh, opt_pyramids, opt_slice, opt_display, opt_force, opt_color_filter,
+  opt_largest):
+  """Converts frames with faces to CSV of ROIs"""
+  
+  import sys
+  import os
+  from os.path import join
+  from pathlib import Path
+  from glob import glob
+  
+  from tqdm import tqdm
+  import numpy as np
+  import dlib  # must keep a local reference for dlib
+  import cv2 as cv
+  import pandas as pd
+
+  from app.utils import logger_utils, file_utils, im_utils
+  from app.processors import face_detector 
+
+  # -------------------------------------------------
+  # init here
+
+  log = logger_utils.Logger.getLogger()
+
+  if not opt_force and Path(opt_fp_out).exists():
+    log.error('File exists. Use "-f / --force" to overwite')
+    return
+  
+  if opt_detector_type == types.FaceDetectNet.CVDNN:
+    detector = face_detector.DetectorCVDNN()
+  elif opt_detector_type == types.FaceDetectNet.DLIB_CNN:
+    detector = face_detector.DetectorDLIBCNN(opt_gpu)
+  elif opt_detector_type == types.FaceDetectNet.DLIB_HOG:
+    detector = face_detector.DetectorDLIBHOG()
+  elif opt_detector_type == types.FaceDetectNet.MTCNN:
+    detector = face_detector.DetectorMTCNN()
+  elif opt_detector_type == types.FaceDetectNet.HAAR:
+    log.error('{} not yet implemented'.format(opt_detector_type.name))
+    return
+
+  
+  # -------------------------------------------------
+  # process here
+  color_filter = color_filters[opt_color_filter]
+  
+  # get list of files to process
+  df_files = pd.read_csv(opt_fp_in).set_index('index')
+
+  if opt_slice:
+    df_files = df_files[opt_slice[0]:opt_slice[1]]
+  log.debug('processing {:,} files'.format(len(df_files)))
+
+
+  data = []
+
+  for df_file in tqdm(df_files.itertuples(), total=len(df_files)):
+    fp_im = join(opt_dir_media, df_file.subdir, '{}.{}'.format(df_file.fn, df_file.ext))
+    im = cv.imread(fp_im)
+
+    # filter out color or grayscale iamges
+    if color_filter != color_filters['all']:
+      try:
+        is_gray = im_utils.is_grayscale(im)
+        if is_gray and color_filter != color_filters['gray']:
+          log.debug('Skipping grayscale image: {}'.format(fp_im))
+          continue
+      except Exception as e:
+        log.error('Could not check grayscale: {}'.format(fp_im))
+        continue
+        
+    try:
+      bboxes = detector.detect(im, opt_size=opt_size, opt_pyramids=opt_pyramids, opt_largest=opt_largest)
+    except Exception as e:
+      log.error('could not detect: {}'.format(fp_im))
+      log.error('{}'.format(e))
+      continue
+
+    for bbox in bboxes:
+      roi = {
+        'image_index': int(df_file.Index),
+        'x': bbox.x, 
+        'y': bbox.y, 
+        'w': bbox.w, 
+        'h': bbox.h,
+        'image_width': im.shape[1],
+        'image_height': im.shape[0]}
+      data.append(roi)
+    
+    # debug display
+    if opt_display and len(bboxes):
+      bbox_dim = bbox.to_dim(im.shape[:2][::-1])  # w,h
+      im_md = im_utils.resize(im, width=min(1200, opt_size[0]))
+      for bbox in bboxes:
+        bbox_dim = bbox.to_dim(im_md.shape[:2][::-1])
+        cv.rectangle(im_md, bbox_dim.pt_tl, bbox_dim.pt_br, (0,255,0), 3)
+      cv.imshow('', im_md)
+      while True:
+        k = cv.waitKey(1) & 0xFF
+        if k == 27 or k == ord('q'):  # ESC
+          cv.destroyAllWindows()
+          sys.exit()
+        elif k != 255:
+          # any key to continue
+          break
+
+  # save date
+  file_utils.mkdirs(opt_fp_out)
+  df = pd.DataFrame.from_dict(data)
+  df.index.name = 'index'
+  df.to_csv(opt_fp_out)
\ No newline at end of file
diff --git a/megapixels/commands/cv/resize.py b/megapixels/commands/cv/resize.py
index f535c8b6..dcd621b3 100644
--- a/megapixels/commands/cv/resize.py
+++ b/megapixels/commands/cv/resize.py
@@ -62,9 +62,11 @@ centerings = {
   help='Crop focal point')
 @click.option('--slice', 'opt_slice', type=(int, int), default=(None, None),
   help='Slice the input list')
+@click.option('-t', '--threads', 'opt_threads', default=8,
+  help='Number of threads')
 @click.pass_context
 def cli(ctx, opt_dir_in, opt_dir_out, opt_glob_ext, opt_size, opt_scale_method,
- opt_equalize, opt_sharpen, opt_center, opt_slice):
+ opt_equalize, opt_sharpen, opt_center, opt_slice, opt_threads):
   """Crop, mirror images"""
   
   import os
@@ -72,6 +74,8 @@ def cli(ctx, opt_dir_in, opt_dir_out, opt_glob_ext, opt_size, opt_scale_method,
   from pathlib import Path
   from glob import glob
   from tqdm import tqdm
+  from multiprocessing.dummy import Pool as ThreadPool
+  from functools import partial
 
   from app.utils import logger_utils, file_utils, im_utils
   
@@ -80,46 +84,63 @@ def cli(ctx, opt_dir_in, opt_dir_out, opt_glob_ext, opt_size, opt_scale_method,
 
   log = logger_utils.Logger.getLogger()
 
-  centering = centerings[opt_center]
 
   # -------------------------------------------------
   # process here
 
+  def pool_resize(fp_im, opt_size, scale_method, centering):
+    # Threaded image resize function
+    try:
+      pbar.update(1)
+      try:
+        im = Image.open(fp_im).convert('RGB')
+        im.verify()
+      except Exception as e:
+        log.warn('Could not open: {}'.format(fp_im))
+        log.error(e)
+        return False
+
+      im = ImageOps.fit(im, opt_size, method=scale_method, centering=centering)
+
+      if opt_equalize:
+        im_np = im_utils.pil2np(im)
+        im_np_eq = eq_hist_yuv(im_np)
+        im_np = cv.addWeighted(im_np_eq, 0.35, im_np, 0.65, 0)
+        im = im_utils.np2pil(im_np)
+
+      if opt_sharpen:
+        im = im.filter(ImageFilter.UnsharpMask)
+        
+      fp_out = join(opt_dir_out, Path(fp_im).name)
+      im.save(fp_out)
+      return True
+    except:
+      return False
+
+  centering = centerings[opt_center]
+  scale_method = methods[opt_scale_method]
+
   # get list of files to process
   fp_ims = glob(join(opt_dir_in, '*.{}'.format(opt_glob_ext)))
   if opt_slice:
     fp_ims = fp_ims[opt_slice[0]:opt_slice[1]]
   log.info('processing {:,} files'.format(len(fp_ims)))
   
-  # set scale method
-  scale_method = methods[opt_scale_method]
-
+  
   # ensure output dir exists
   file_utils.mkdirs(opt_dir_out)
 
-  # resize and save images
-  for fp_im in tqdm(fp_ims):
-    try:
-      im = Image.open(fp_im).convert('RGB')
-      im.verify()
-    except Exception as e:
-      log.warn('Could not open: {}'.format(fp_im))
-      log.error(e)
-      continue
-
-    im = ImageOps.fit(im, opt_size, method=scale_method, centering=centering)
+  # setup multithreading
+  pbar = tqdm(total=len(fp_ims))
+  pool_resize = partial(pool_resize, opt_size=opt_size, scale_method=scale_method, centering=centering)
+  #result_list = pool.map(prod_x, data_list)
+  pool = ThreadPool(opt_threads) 
+  with tqdm(total=len(fp_ims)) as pbar:
+    results = pool.map(pool_resize, fp_ims)
+  pbar.close()
 
-    if opt_equalize:
-      im_np = im_utils.pil2np(im)
-      im_np_eq = eq_hist_yuv(im_np)
-      im_np = cv.addWeighted(im_np_eq, 0.35, im_np, 0.65, 0)
-      im = im_utils.np2pil(im_np)
+  log.info('Resized: {} / {} images'.format(results.count(True), len(fp_ims)))
 
-    if opt_sharpen:
-      im = im.filter(ImageFilter.UnsharpMask)
-      
-    fp_out = join(opt_dir_out, Path(fp_im).name)
-    im.save(fp_out)
 
 
 def eq_hist_yuv(im):
diff --git a/megapixels/commands/datasets/add_uuid.py b/megapixels/commands/datasets/add_uuid.py
new file mode 100644
index 00000000..9c14c0e3
--- /dev/null
+++ b/megapixels/commands/datasets/add_uuid.py
@@ -0,0 +1,44 @@
+import click
+
+from app.settings import types
+from app.utils import click_utils
+from app.settings import app_cfg as cfg
+from app.utils.logger_utils import Logger
+
+log = Logger.getLogger()
+
+@click.command()
+@click.option('-i', '--input', 'opt_fp_in', required=True,
+  help='Input directory')
+@click.option('-o', '--output', 'opt_fp_out',
+  help='Output directory')
+@click.option('-f', '--force', 'opt_force', is_flag=True,
+  help='Force overwrite file')
+@click.pass_context
+def cli(ctx, opt_fp_in, opt_fp_out, opt_force):
+  """Appends UUID to records CSV"""
+  
+  from glob import glob
+  from os.path import join
+  from pathlib import Path
+  import base64
+  import uuid
+
+  from tqdm import tqdm
+  import pandas as pd
+  
+  if not opt_force and Path(opt_fp_out).exists():
+    log.error('File exists. Use "-f / --force" to overwite')
+    return
+
+  # load names
+  df_records = pd.read_csv(opt_fp_in)
+  records = df_records.to_dict('index')
+  # append a UUID to every entry
+  for idx, item in records.items():
+    records[idx]['uuid'] = uuid.uuid4()
+  # save to csv
+  df_uuid = pd.DataFrame.from_dict(list(records.values()))  # ignore the indices
+  df_uuid.to_csv(opt_fp_out, index=False)
+
+  log.info('done')
\ No newline at end of file
diff --git a/megapixels/commands/datasets/feret.py b/megapixels/commands/datasets/feret.py
new file mode 100644
index 00000000..906b4e37
--- /dev/null
+++ b/megapixels/commands/datasets/feret.py
@@ -0,0 +1,139 @@
+import bz2
+import io
+
+import click
+from PIL import Image
+
+from app.settings import types
+from app.utils import click_utils
+from app.settings import app_cfg as cfg
+from app.utils.logger_utils import Logger
+
+log = Logger.getLogger()
+
+pose_choices = {
+'fa':0, 'fb':0, 'hl':67.5, 'hr':-67.5, 'pl':90, 'pr':-90, 
+'ql':22.5, 'qr':-22.5, 'ra':45, 'rb':15, 'rc':-15, 'rd':-45, 're':-75}
+
+poses_left = ['hl', 'ql', 'pl', 'ra', 'rb']
+poses_right = ['hr', 'qr', 'pr', 'rc', 're', 're']
+
+@click.command()
+@click.option('-i', '--input', 'opt_fp_in', required=True,
+  help='Input directory')
+@click.option('-o', '--output', 'opt_fp_out', required=True,
+  help='Output directory')
+@click.option('-a', '--angle', 'opt_angle', type=(float, float), default=(0,0),
+  help='Min/max face angles')
+@click.option('-t', '--threads', 'opt_threads', default=8,
+  help='Number of threads')
+@click.option('--flip', 'opt_flip', type=click.Choice(['r', 'l']), 
+  help='Flip profile images to the R or L')
+@click.pass_context
+def cli(ctx, opt_fp_in, opt_fp_out, opt_angle, opt_threads, opt_flip):
+  """Extracts FERET images"""
+  
+  from glob import glob
+  from os.path import join
+  from pathlib import Path
+  import time
+  from tqdm import tqdm
+  from multiprocessing.dummy import Pool as ThreadPool
+  from functools import partial
+
+  from PIL import ImageOps
+  from app.utils import file_utils
+
+  # filter angles
+  poses = [k for k, v in pose_choices.items() if \
+    abs(v) >= opt_angle[0] and abs(v) <= opt_angle[1]]
+  
+  # glob images dir for all *ppm.bz2
+  fp_ims = []
+  for pose in poses:
+    log.info('globbing pose: {}'.format(pose))
+    fp_ims += glob(join(opt_fp_in, '**/*_{}.ppm.bz2').format(pose))
+  log.info('Processing: {:,} files'.format(len(fp_ims)))
+  
+  # convert bz2 to png
+  def pool_func(fp_im, opt_fp_out, opt_flip):
+    try:
+      pbar.update(1)
+      im_pil = bz2_to_pil(fp_im)
+      fpp_im = Path(fp_im)
+      fp_out = join(opt_fp_out, '{}.png'.format(fpp_im.stem))
+      fp_out = fp_out.replace('.ppm','')  # remove ppm
+      if opt_flip:
+        pose_code = fpp_im.stem.split('_')[-1][:2]
+        # log.debug('opt_flip: {}, found: {}'.format(opt_flip, pose_code))
+        if opt_flip == 'r' and pose_code in poses_right \
+          or opt_flip == 'l' and pose_code in poses_left:
+            im_pil = ImageOps.mirror(im_pil)
+      im_pil.save(fp_out)
+      return True
+    except Exception as e:
+      log.error('Error processing: {}, error: {}'.format(fp_im, e))
+      return False
+
+  # make output directory
+  file_utils.mkdirs(opt_fp_out)
+
+  # setup multithreading
+  pbar = tqdm(total=len(fp_ims))
+  pool_resize = partial(pool_func, opt_fp_out=opt_fp_out, opt_flip=opt_flip)
+  pool = ThreadPool(opt_threads) 
+  with tqdm(total=len(fp_ims)) as pbar:
+    results = pool.map(pool_resize, fp_ims)
+  pbar.close()
+
+  # results
+  log.info('Converted: {} / {} images'.format(results.count(True), len(fp_ims)))
+
+  
+# ------------------------------------------------------------------
+# local utils
+
+def bz2_to_pil(fp_src):
+  with open(fp_src, 'rb') as fp:
+    im_raw = bz2.decompress(fp.read())
+  im_pil = Image.open(io.BytesIO(im_raw))
+  return im_pil
+
+
+
+"""
+
+A breakdown of the images by pose is:
+  Pose Angle   Images Subjects
+  fa       0     1364      994
+  fb       0     1358      993
+  hl   +67.5     1267      917
+  hr   -67.5     1320      953
+  pl     +90     1312      960
+  pr     -90     1363      994
+  ql   +22.5      761      501
+  qr   -22.5      761      501
+  ra     +45      321      261
+  rb     +15      321      261
+  rc     -15      610      423
+  rd     -45      290      236
+  re     -75      290      236
+
+  There are 13 different poses. (The orientation "right" means
+facing the photographer's right.)
+  fa  regular frontal image
+  fb  alternative frontal image, taken shortly after the
+      corresponding fa image
+  pl  profile left
+  hl  half left - head turned about 67.5 degrees left
+  ql  quarter left - head turned about 22.5 degrees left
+  pr  profile right
+  hr  half right - head turned about 67.5 degrees right
+  qr  quarter right - head turned about 22.5 degrees right
+  ra  random image - head turned about 45 degree left
+  rb  random image - head turned about 15 degree left
+  rc  random image - head turned about 15 degree right
+  rd  random image - head turned about 45 degree right
+  re  random image - head turned about 75 degree right
+
+"""
\ No newline at end of file
diff --git a/megapixels/commands/datasets/s3.py b/megapixels/commands/datasets/s3.py
new file mode 100644
index 00000000..7769896b
--- /dev/null
+++ b/megapixels/commands/datasets/s3.py
@@ -0,0 +1,47 @@
+import click
+
+from app.settings import types
+from app.utils import click_utils
+from app.settings import app_cfg as cfg
+
+s3_dirs = {'media': cfg.S3_MEDIA_ROOT, 'metadata': cfg.S3_METADATA_ROOT}
+
+@click.command()
+@click.option('-i', '--input', 'opt_fps_in', required=True, multiple=True,
+  help='Input directory')
+@click.option('--name', 'opt_dataset_name', required=True,
+  help='Dataset key (eg "lfw"')
+@click.option('-a', '--action', 'opt_action', type=click.Choice(['sync', 'put']), default='sync',
+  help='S3 action')
+@click.option('-t', '--type', 'opt_type', type=click.Choice(s3_dirs.keys()), required=True,
+  help='S3 location')
+@click.option('--dry-run', 'opt_dryrun', is_flag=True, default=False)
+@click.pass_context
+def cli(ctx, opt_fps_in, opt_dataset_name, opt_action, opt_type, opt_dryrun):
+  """Syncs files with S3/spaces server"""
+  
+  from os.path import join
+  from pathlib import Path
+  
+  from tqdm import tqdm
+  import pandas as pd
+  import subprocess
+
+  from app.utils import logger_utils, file_utils
+  
+  # -------------------------------------------------
+  # init here
+
+  log = logger_utils.Logger.getLogger()
+  for opt_fp_in in opt_fps_in:
+    dir_dst = join(s3_dirs[opt_type], opt_dataset_name, '')
+    if Path(opt_fp_in).is_dir():
+      fp_src = join(opt_fp_in, '')  # add trailing slashes
+    else:
+      fp_src = join(opt_fp_in) 
+    cmd = ['s3cmd', opt_action, fp_src, dir_dst, '-P', '--follow-symlinks']
+    log.info(' '.join(cmd))
+    if not opt_dryrun:
+      subprocess.call(cmd)
+
+  
\ No newline at end of file
diff --git a/megapixels/commands/datasets/symlink.py b/megapixels/commands/datasets/symlink.py
new file mode 100644
index 00000000..70ec6c46
--- /dev/null
+++ b/megapixels/commands/datasets/symlink.py
@@ -0,0 +1,45 @@
+import click
+
+from app.settings import types
+from app.utils import click_utils
+from app.settings import app_cfg as cfg
+
+@click.command()
+@click.option('-i', '--input', 'opt_fp_in', required=True,
+  help='Input records CSV')
+@click.option('-m', '--media', 'opt_fp_media', required=True,
+  help='Input media directory')
+@click.option('-o', '--output', 'opt_fp_out', required=True,
+  help='Output directory')
+@click.pass_context
+def cli(ctx, opt_fp_in, opt_fp_media, opt_fp_out):
+  """Symlinks images to new directory for S3"""
+  
+  import sys
+  import os
+  from os.path import join
+  from pathlib import Path
+  
+  from tqdm import tqdm
+  import pandas as pd
+
+  from app.utils import logger_utils, file_utils
+  
+  # -------------------------------------------------
+  # init here
+
+  log = logger_utils.Logger.getLogger()
+
+  df_records = pd.read_csv(opt_fp_in)
+  nrows = len(df_records)
+
+  file_utils.mkdirs(opt_fp_out)
+
+  for record_id, row in tqdm(df_records.iterrows(), total=nrows):
+    # make image path
+    df = df_records.iloc[record_id]
+    fpp_src = Path(join(opt_fp_media, df['subdir'], '{}.{}'.format(df['fn'], df['ext'])))
+    fpp_dst = Path(join(opt_fp_out, '{}.{}'.format(df['uuid'], df['ext'])))
+    fpp_dst.symlink_to(fpp_src)
+
+  log.info('symlinked {:,} files'.format(nrows))
\ No newline at end of file
diff --git a/megapixels/commands/datasets/vecs_to_id.py b/megapixels/commands/datasets/vecs_to_id.py
new file mode 100644
index 00000000..07c7389e
--- /dev/null
+++ b/megapixels/commands/datasets/vecs_to_id.py
@@ -0,0 +1,50 @@
+import click
+
+from app.settings import types
+from app.utils import click_utils
+from app.settings import app_cfg as cfg
+
+@click.command()
+@click.option('-i', '--input', 'opt_fp_in', required=True,
+  help='Input directory')
+@click.option('-r', '--records', 'opt_fp_records', required=True,
+  help='Input directory')
+@click.option('-o', '--output', 'opt_fp_out', required=True,
+  help='Output JSON')
+@click.option('-f', '--force', 'opt_force', is_flag=True,
+  help='Force overwrite file')
+@click.pass_context
+def cli(ctx, opt_fp_in, opt_fp_records, opt_fp_out,opt_force):
+  """Merges ID with face vectors"""
+  
+  import sys
+  import os
+  from os.path import join
+  from pathlib import Path
+  
+  from tqdm import tqdm
+  import pandas as pd
+
+  from app.utils import logger_utils, file_utils
+  
+  # -------------------------------------------------
+  # init here
+
+  log = logger_utils.Logger.getLogger()
+
+  df_vecs = pd.read_csv(opt_fp_in)
+  df_records = pd.read_csv(opt_fp_records)
+  nrows = len(df_vecs)
+
+  # face vecs
+  id_vecs = {}
+
+  for roi_idx, row in tqdm(df_vecs.iterrows(), total=nrows):
+    record_id = int(row['id'])
+    vec = row['vec'].split(',')
+    id_vecs[record_id] = vec
+
+  # save as JSON
+  file_utils.write_json(id_vecs, opt_fp_out, verbose=True)
+
+  
\ No newline at end of file
diff --git a/megapixels/commands/datasets/vecs_to_uuid.py b/megapixels/commands/datasets/vecs_to_uuid.py
new file mode 100644
index 00000000..7bb82083
--- /dev/null
+++ b/megapixels/commands/datasets/vecs_to_uuid.py
@@ -0,0 +1,56 @@
+"""
+Crop images to prepare for training
+"""
+
+import click
+
+from app.settings import types
+from app.utils import click_utils
+from app.settings import app_cfg as cfg
+
+@click.command()
+@click.option('-i', '--input', 'opt_fp_in', required=True,
+  help='Input directory')
+@click.option('-r', '--records', 'opt_fp_records', required=True,
+  help='Input directory')
+@click.option('-o', '--output', 'opt_fp_out', required=True,
+  help='Output JSON')
+@click.option('-f', '--force', 'opt_force', is_flag=True,
+  help='Force overwrite file')
+@click.pass_context
+def cli(ctx, opt_fp_in, opt_fp_records, opt_fp_out,opt_force):
+  """Merges UUID with face vectors"""
+  
+  import sys
+  import os
+  from os.path import join
+  from pathlib import Path
+  
+  from tqdm import tqdm
+  import pandas as pd
+
+  from app.utils import logger_utils, file_utils
+  
+  # -------------------------------------------------
+  # init here
+
+  log = logger_utils.Logger.getLogger()
+
+  df_vecs = pd.read_csv(opt_fp_in)
+  df_records = pd.read_csv(opt_fp_records)
+  nrows = len(df_vecs)
+
+  # face vecs
+  uuid_vecs = {}
+
+  for roi_idx, row in tqdm(df_vecs.iterrows(), total=nrows):
+    # make image path
+    record_id = int(row['id'])
+    uuid = df_records.iloc[record_id]['uuid']
+    vec = row['vec'].split(',')
+    uuid_vecs[uuid] = vec
+
+  # save as JSON
+  file_utils.write_json(uuid_vecs, opt_fp_out)
+
+  
\ No newline at end of file