add demos, in progress

9 files changed, 175 insertions, 29 deletions

diff --git a/megapixels/app/processors/face_detector.py b/megapixels/app/processors/face_detector.py
index 6bf27576..c0762564 100644
--- a/megapixels/app/processors/face_detector.py
+++ b/megapixels/app/processors/face_detector.py
@@ -1,3 +1,4 @@
+import sys
 import os
 from os.path import join
 from pathlib import Path
@@ -30,8 +31,6 @@ class DetectorMTCNN:
     :returns list of BBox
     '''
     bboxes = []
-    #conf_thresh = self.conf_thresh if conf_thresh is None else conf_thresh
-    #pyramids = self.pyramids if pyramids is None else pyramids
     dnn_size = self.dnn_size if size is None else size
 
     im = im_utils.resize(im, width=dnn_size[0], height=dnn_size[1])
@@ -72,6 +71,9 @@ class DetectorDLIBCNN:
     import dlib
     self.log = logger_utils.Logger.getLogger()
     cuda_visible_devices = os.getenv('CUDA_VISIBLE_DEVICES', '')
+    if dlib.DLIB_USE_CUDA and gpu < 0:
+      self.log.error('dlib was compiled with CUDA but you selected CPU. Use GPU >= 0 if dlib.DLIB_USE_CUDA')
+      sys.exit()
     os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu)
     self.log.info('load model: {}'.format(cfg.DIR_MODELS_DLIB_CNN))
     self.detector = dlib.cnn_face_detection_model_v1(cfg.DIR_MODELS_DLIB_CNN)
diff --git a/megapixels/app/processors/face_landmarks.py b/megapixels/app/processors/face_landmarks.py
index 8086ba1e..171fc666 100644
--- a/megapixels/app/processors/face_landmarks.py
+++ b/megapixels/app/processors/face_landmarks.py
@@ -83,8 +83,11 @@ class Dlib2D(Landmarks2D):
     self.log.info(f'loaded predictor model: {model}')
 
   def landmarks(self, im, bbox):
-    # Draw high-confidence faces
-    dim_wh = im.shape[:2][::-1]
+    '''Generates 68-pt landmarks using dlib predictor
+    :param im: (numpy.ndarray) BGR image
+    :param bbox: (app.models.BBox) dimensioned
+    :returns (list) of (int, int) for x,y values
+    '''
     bbox = bbox.to_dlib()
     im_gray = cv.cvtColor(im, cv.COLOR_BGR2GRAY)
     points = [[p.x, p.y] for p in self.predictor(im_gray, bbox).parts()]
@@ -168,8 +171,8 @@ class Landmarks3D:
         points_formatted[f'{d}{idx}'] = pt[j]
     return points_formatted
 
-  def normalize(self, points, dim):
-    return [np.array(p)/dim for p in points]  # divides each point by w,h dim
+  # def normalize(self, points):
+  #   '''TODO'''
 
     
 class FaceAlignment3D_68(Landmarks3D):
@@ -182,13 +185,14 @@ class FaceAlignment3D_68(Landmarks3D):
     device = f'cuda:{gpu}' if gpu > -1 else 'cpu'
     self.fa = face_alignment.FaceAlignment(face_alignment.LandmarksType._3D, device=device, flip_input=flip_input)
 
-  def landmarks(self, im, as_type=str):
+  def landmarks(self, im, rect):
     '''Calculates the 3D facial landmarks
     :param im: (numpy.ndarray) BGR image
+    :param rect: (list) of face (x1, y1, x2, y2)
     :returns (list) of 68 (int) (tuples) as (x,y, z) 
     '''
     # predict landmarks
-    points = self.fa.get_landmarks(im)  # returns array of arrays of 68 3D pts/face
+    points = self.fa.get_landmarks(im, [rect])  # returns array of arrays of 68 3D pts/face
     # convert to data type
     points = [list(map(int, p)) for p in points[0]]
     return points
\ No newline at end of file
diff --git a/megapixels/app/processors/face_landmarks_3d.py b/megapixels/app/processors/face_landmarks_3d.py
index 470d263c..5a0d6097 100644
--- a/megapixels/app/processors/face_landmarks_3d.py
+++ b/megapixels/app/processors/face_landmarks_3d.py
@@ -26,14 +26,15 @@ class FaceAlignment3D(Landmarks3D):
   # Estimates 3D facial landmarks
   import face_alignment
 
-  def __init__(self, gpu=0, flip_input=False):
+  def __init__(self, gpu=0, flip_input=True):
     super().__init__()
     device = f'cuda:{gpu}' if gpu > -1 else 'cpu'
     self.fa = face_alignment.FaceAlignment(face_alignment.LandmarksType._3D, device=device, flip_input=flip_input)
 
-  def landmarks(self, im, as_type=str):
+  def landmarks(self, im, bbox, as_type=str):
     '''Calculates the 3D facial landmarks
     :param im: (numpy.ndarray) image
+    :param bbox: (BBox) dimensioned to real (int) sizes
     :param as_type: (str) or (list) type to return data
     '''
     preds = self.fa.get_landmarks(im)
diff --git a/megapixels/app/processors/face_pose.py b/megapixels/app/processors/face_pose.py
index 8bc95f8d..5ac510ec 100644
--- a/megapixels/app/processors/face_pose.py
+++ b/megapixels/app/processors/face_pose.py
@@ -25,6 +25,12 @@ class FacePoseDLIB:
 
 
   def pose(self, landmarks, dim):
+    '''Returns face pose information
+    :param landmarks: (list) of 68 (int, int) xy tuples
+    :param dim: (tuple|list) of image (width, height)
+    :returns (dict) of pose attributes
+    '''
+
     # computes pose using 6 / 68 points from dlib face landmarks
     # based on learnopencv.com and 
     # https://github.com/jerryhouuu/Face-Yaw-Roll-Pitch-from-Pose-Estimation-using-OpenCV/
diff --git a/megapixels/app/settings/app_cfg.py b/megapixels/app/settings/app_cfg.py
index b13ff8ec..d206f40b 100644
--- a/megapixels/app/settings/app_cfg.py
+++ b/megapixels/app/settings/app_cfg.py
@@ -76,6 +76,7 @@ FP_FONT = join(DIR_ASSETS, 'font')
 # click chair settings
 # -----------------------------------------------------------------------------
 DIR_COMMANDS_CV = 'commands/cv'
+DIR_COMMANDS_VIZ = 'commands/visualize'
 DIR_COMMANDS_ADMIN = 'commands/admin'
 DIR_COMMANDS_DATASETS = 'commands/datasets'
 DIR_COMMANDS_FAISS = 'commands/faiss'
diff --git a/megapixels/app/settings/types.py b/megapixels/app/settings/types.py
index 50e395e0..1d77fdbd 100644
--- a/megapixels/app/settings/types.py
+++ b/megapixels/app/settings/types.py
@@ -64,9 +64,6 @@ class FaceLandmark2D_68(Enum):
 
 class FaceLandmark3D_68(Enum):
   FACE_ALIGNMENT = range(1)
-
-class FaceLandmark3D(Enum):
-  FACE_ALIGNMENT = range(1)
   
 class FaceEmotion(Enum):
   # Map these to text strings for web display
diff --git a/megapixels/app/utils/display_utils.py b/megapixels/app/utils/display_utils.py
index 58e2feec..7b74aa46 100644
--- a/megapixels/app/utils/display_utils.py
+++ b/megapixels/app/utils/display_utils.py
@@ -3,11 +3,11 @@ import sys
 import cv2 as cv
 
 
-def handle_keyboard():
+def handle_keyboard(delay_amt=1):
   '''Used with cv.imshow('title', image) to wait for keyboard press
   '''
   while True:
-    k = cv.waitKey(1) & 0xFF
+    k = cv.waitKey(delay_amt) & 0xFF
     if k == 27 or k == ord('q'):  # ESC
       cv.destroyAllWindows()
       sys.exit()
diff --git a/megapixels/app/utils/draw_utils.py b/megapixels/app/utils/draw_utils.py
index f6d53609..47bb7978 100644
--- a/megapixels/app/utils/draw_utils.py
+++ b/megapixels/app/utils/draw_utils.py
@@ -49,17 +49,3 @@ def draw_degrees(im, pose_data, color=(0,255,0)):
     t = '{}: {:.2f}'.format(k, v)
     origin = (10, 30 + (25 * i))
     cv.putText(im, t, origin, cv.FONT_HERSHEY_SIMPLEX, 0.5, clr, thickness=2, lineType=2)
-
-
-# ---------------------------------------------------------------------------
-#
-# Matplotlib drawing functions
-#
-# ---------------------------------------------------------------------------
-
-def plot_landmarks3D(im, points, radius=3, color=(0,255,0), stroke_weight=2):
-  '''Draws facial landmarks, either 5pt or 68pt
-  '''
-  for pt in points:
-    cv.circle(im, tuple(pt), radius, color, -1, cv.LINE_AA)
-
diff --git a/megapixels/app/utils/plot_utils.py b/megapixels/app/utils/plot_utils.py
new file mode 100644
index 00000000..5bbb8ac2
--- /dev/null
+++ b/megapixels/app/utils/plot_utils.py
@@ -0,0 +1,149 @@
+import sys
+from os.path import join
+import time
+import random
+from pathlib import Path
+
+import numpy as np
+
+import matplotlib.pyplot as plt
+import matplotlib.animation
+from mpl_toolkits.mplot3d import Axes3D
+from matplotlib import cbook
+from matplotlib import cm
+from matplotlib import animation
+
+
+
+# ---------------------------------------------------------------------------
+#
+# Matplotlib drawing functions
+#
+# ---------------------------------------------------------------------------
+
+# Generate random hex colors
+def rhex():
+  r = lambda: random.randint(0,255)
+  return '#%02X%02X%02X' % (r(), r(), r())
+
+ # line weight
+def generate_3d_landmark_anim(lm, fp_out, num_frames=30, fps=12, dpi=72, size=(480,480),
+  stroke_weight=2, mark_size=10, mark_type='.', bg_clr=(0,0,0), transparent=False):
+  '''Generates animated 3D plot of face landmarks
+  '''
+
+  # convert opencv BGR numpy image to RGB
+  bg_clr_hex = '#%02x%02x%02x' % bg_clr
+  #mark_clr = '#%02x%02x%02x' % mark_clr
+  
+  # center x,y,z
+  xmm = (np.min(lm[:,0]),np.max(lm[:,0]))
+  ymm = (np.min(lm[:,1]),np.max(lm[:,1]))
+  zmm = (np.min(lm[:,2]),np.max(lm[:,2]))
+  
+  # make copy of landmarks
+  lm_orig = lm.copy()
+  xmm = (np.min(lm_orig[:,0]),np.max(lm_orig[:,0]))
+  ymm = (np.min(lm_orig[:,1]),np.max(lm_orig[:,1]))
+  zmm = (np.min(lm_orig[:,2]),np.max(lm_orig[:,2]))
+  
+  # swap the y and z components to improve 3d rotation angles for matplotlib
+  lm = np.zeros_like(lm_orig).astype(np.uint8)
+  for i,p in enumerate(lm_orig):
+    x,y,z = p
+    lm[i] = np.array([x - xmm[0], z - zmm[0], y - ymm[0]])
+  
+  # Create plot
+  figsize = (size[0]/dpi, size[1]/dpi )
+  fig = plt.figure(figsize=figsize, dpi=dpi)  # frameon=False
+  fig.tight_layout()
+  # remove whitespace in matplotlib
+  fig.subplots_adjust(left=0, bottom=0, right=1, top=1, wspace=None, hspace=None)
+  ax = fig.add_subplot(111, projection='3d')
+  ax.set_facecolor(bg_clr_hex) # background color
+  
+  xscale, yscale, zscale = (1.2, 1.0, 1.0)
+  
+  # scatter plot the dots
+
+  # jaw line
+  mark_clr = '#%02x%02x%02x' % (0,255,0)  # green
+  ax.plot3D(lm[:17,0]*1.2,lm[:17,1], lm[:17,2],
+            marker=mark_type, markersize=mark_size, color=mark_clr,linewidth=stroke_weight)
+
+  # stage-right eyebrow
+  mark_clr = '#%02x%02x%02x' % (255,0,0)  # green
+  ax.plot3D(lm[17:22,0]*1.2,lm[17:22,1],lm[17:22,2],
+            marker=mark_type, markersize=mark_size, color=mark_clr,linewidth=stroke_weight)
+  
+  # stage-left eyebrow
+  mark_clr = '#%02x%02x%02x' % (255,255,0)  # yellow
+  ax.plot3D(lm[22:27,0]*1.2,lm[22:27,1],lm[22:27,2], 
+            marker=mark_type, markersize=mark_size, color=mark_clr,linewidth=stroke_weight)
+  
+  # nose ridge
+  mark_clr = '#%02x%02x%02x' % (0,0,255)  # blue
+  ax.plot3D(lm[27:31,0]*1.2,lm[27:31,1],lm[27:31,2],
+            marker=mark_type, markersize=mark_size, color=mark_clr,linewidth=stroke_weight)
+  
+  # nose-bottom
+  mark_clr = '#%02x%02x%02x' % (255,0,255)  # magenta
+  ax.plot3D(lm[31:36,0]*1.2,lm[31:36,1],lm[31:36,2],
+            marker=mark_type, markersize=mark_size, color=mark_clr,linewidth=stroke_weight)
+  
+  # stage-left eye
+  mark_clr = '#%02x%02x%02x' % (0,255,255)  # cyan
+  px, py, pz = lm[36:42,0]*1.2,lm[36:42,1],lm[36:42,2]
+  px = np.append(px, lm[36,0]*1.2)
+  py = np.append(py, lm[36,1])
+  pz = np.append(pz, lm[36,2])
+  ax.plot3D(px, py, pz, marker=mark_type, markersize=mark_size, color=mark_clr,linewidth=stroke_weight)
+  
+  # stage-right eye
+  mark_clr = '#%02x%02x%02x' % (255,255,255)  # white
+  px, py, pz = lm[42:48,0]*1.2,lm[42:48,1],lm[42:48,2]
+  px = np.append(px, lm[42,0]*1.2)
+  py = np.append(py, lm[42,1])
+  pz = np.append(pz, lm[42,2])
+  ax.plot3D(px, py, pz, marker=mark_type, markersize=mark_size, color=mark_clr,linewidth=stroke_weight)
+  
+  # mouth
+  mark_clr = '#%02x%02x%02x' % (255,125,0)  # orange?
+  px, py, pz = lm[48:,0]*1.2,lm[48:,1],lm[48:,2]
+  px = np.append(px, lm[48,0]*1.2)
+  py = np.append(py, lm[48,1])
+  pz = np.append(pz, lm[48,2])
+  ax.plot3D(px, py, pz, marker=mark_type, markersize=mark_size, color=mark_clr, linewidth=stroke_weight)
+  
+  #rh = '#00ff00'  # edge color
+  #ax.scatter(lm[:,0]*xscale,lm[:,1]*yscale,lm[:,2]*zscale, c=rh, alpha=1.0, s=35, edgecolor=rh)
+  #ax.scatter(lm[:,0]*xscale,lm[:,1]*yscale,lm[:,2]*zscale, c=rh, alpha=1.0, s=1)
+  
+  # center center x,y,z points
+  cx = ((xmm[0] - xmm[1]) // 2) + xmm[1]
+  cy = ((ymm[1] - ymm[0]) // 2) + ymm[0]
+  cz = ((zmm[1] - zmm[0]) // 2) + zmm[0]
+  
+  # remove ticks
+  ax.set_xticks([])
+  ax.set_yticks([])
+  ax.set_zticks([])
+    
+  # remove axis
+  ax.set_frame_on(False)
+  ax.set_axis_off()
+
+  # set initial plot view
+  ax.view_init(elev=120., azim=70.)
+
+  # rotation increments: from 0 to 360 in num_frames
+  phi = np.linspace(0, 2*np.pi, num_frames)
+
+  # animation instruction
+  def update(phi):
+    ax.view_init(180,phi*180./np.pi)
+  
+  ani = matplotlib.animation.FuncAnimation(fig, update, frames=phi)
+  
+  savefig_kwargs = {'pad_inches': 0, 'transparent': transparent}
+  ani.save(fp_out, writer='imagemagick', fps=fps, savefig_kwargs=savefig_kwargs)
\ No newline at end of file