add scut-fbp beauty predictor

2 files changed, 186 insertions, 4 deletions

diff --git a/megapixels/app/processors/face_beauty.py b/megapixels/app/processors/face_beauty.py
index a1ddd9f8..2e8221b7 100644
--- a/megapixels/app/processors/face_beauty.py
+++ b/megapixels/app/processors/face_beauty.py
@@ -1,3 +1,4 @@
+import sys
 import os
 from os.path import join
 from pathlib import Path
@@ -6,6 +7,20 @@ import math
 import cv2 as cv
 import numpy as np
 import imutils
+import pickle
+
+os.environ['CUDA_VISIBLE_DEVICES'] = ''
+import keras
+from keras.layers import Conv2D, Input, MaxPool2D,Flatten, Dense, Permute, GlobalAveragePooling2D
+from keras.models import Model
+from keras.optimizers import adam
+import os.path
+from keras.models import Sequential
+from keras.applications.resnet50 import ResNet50
+#from keras.applications.resnet50 import Dense
+from keras.layers import Dense
+from keras.optimizers import Adam
+from keras.layers import Dropout
 
 from app.utils import im_utils, logger_utils
 from app.models.bbox import BBox
@@ -18,10 +33,56 @@ class FaceBeauty:
 
   # Estimates beauty using CNN
 
-  def __init__(self):
+  def __init__(self, gpu=-1):
+    # don't really need GPU, CPU is quick enough
     self.log = logger_utils.Logger.getLogger()
-    pass
+    resnet = ResNet50(include_top=False, pooling='avg')
+    self.model = Sequential()
+    self.model.add(resnet)
+    self.model.add(Dense(5, activation='softmax'))
+    self.model.layers[0].trainable = False
+    fp_model = join(cfg.DIR_MODELS_KERAS, 'model-ldl-resnet.h5')
+    self.model.load_weights(fp_model)
+
+
+  def beauty(self, im, bbox_dim):
+    '''Predicts facial "beauty" score based on SCUT-FBP attractiveness labels
+    :param im: (numpy.ndarray) BGR image
+    :param bbox_dim: (BBox) dimensioned BBox
+    :returns (float) 0.0-1.0 with 1 being most attractive
+    '''
+
+    face = bbox_dim.to_xyxy()
+    self.log.debug(f'face: {face}')
+
+    cropped_im = im[face[1]:face[3], face[0]:face[2]]
+
+    im_resized = cv.resize(cropped_im, (224, 224))  # force size
+    im_norm = np.array([(im_resized - 127.5) / 127.5])  # subtract mean
+
+    # forward pass
+    pred = self.model.predict(im_norm)
+
+    # combines score to make final estimate?
+    ldList = pred[0]
+    score = 1 * ldList[0] + 2 * ldList[1] + 3 * ldList[2] + 4 * ldList[3] + 5 * ldList[4]
+    return score/5.0
+
+
+  def score_mapping(self, score):
+    '''(deprecated)
+    '''
 
+    # if score <= 1.9:
+    #   score_mapped = ((4 - 2.5) / (1.9 - 1.0)) * (score-1.0) + 2.5
+    # elif score <= 2.8:
+    #   score_mapped = ((5.5 - 4) / (2.8 - 1.9)) * (score-1.9) + 4
+    # elif score <= 3.4:
+    #   score_mapped = ((6.5 - 5.5) / (3.4 - 2.8)) * (score-2.8) + 5.5
+    # elif score <= 4:
+    #   score_mapped = ((8 - 6.5) / (4 - 3.4)) * (score-3.4) + 6.5
+    # elif score < 5:
+    #   score_mapped = ((9 - 8) / (5 - 4)) * (score-4) + 8
 
-  def beauty(self):
-    return 0.5
\ No newline at end of file
+    # return score_mapped
+    return False
diff --git a/megapixels/commands/demo/face_beauty.py b/megapixels/commands/demo/face_beauty.py
new file mode 100644
index 00000000..b1612f7c
--- /dev/null
+++ b/megapixels/commands/demo/face_beauty.py
@@ -0,0 +1,121 @@
+"""
+"""
+
+import click
+
+from app.settings import types
+from app.utils import click_utils
+from app.settings import app_cfg as cfg
+
+
+@click.command()
+@click.option('-i', '--input', 'opt_fp_in', default=None, required=True,
+  help='Image filepath')
+@click.option('-o', '--output', 'opt_fp_out', default=None,
+  help='GIF output path')
+@click.option('--size', 'opt_size', 
+  type=(int, int), default=(300, 300),
+  help='Output image size')
+@click.option('-g', '--gpu', 'opt_gpu', default=0,
+  help='GPU index')
+@click.option('-f', '--force', 'opt_force', is_flag=True,
+  help='Force overwrite file')
+@click.option('--display/--no-display', 'opt_display', is_flag=True, default=False,
+  help='Display detections to debug')
+@click.pass_context
+def cli(ctx, opt_fp_in, opt_fp_out, opt_gpu, opt_size, opt_force, opt_display):
+  """Face detector demo"""
+  
+  import sys
+  import os
+  from os.path import join
+  from pathlib import Path
+  import time
+  
+  from tqdm import tqdm
+  import numpy as np
+  import pandas as pd
+  import cv2 as cv
+  import dlib
+
+  from app.utils import logger_utils, file_utils, im_utils, display_utils, draw_utils
+  from app.utils import plot_utils
+  from app.processors import face_detector, face_beauty
+  from app.models.data_store import DataStore
+  
+
+  log = logger_utils.Logger.getLogger()
+
+  
+  # -------------------------------------------------
+  # load image
+
+  im = cv.imread(opt_fp_in)
+  # im = cv.cvtColor(im, cv.COLOR_BGR2RGB)
+  if im.shape[0] > 1280:
+    new_shape = (1280, im.shape[1] * 1280 / im.shape[0])
+  elif im.shape[1] > 1280:
+    new_shape = (im.shape[0] * 1280 / im.shape[1], 1280)
+  elif im.shape[0] < 640 or im.shape[1] < 640:
+    new_shape = (im.shape[0] * 2, im.shape[1] * 2)
+  else:
+    new_shape = im.shape[0:2]
+
+  im_resized = cv.resize(im, (int(new_shape[1]), int(new_shape[0])))
+
+  #im_resized = im_utils.resize(im, width=opt_size[0], height=opt_size[1])
+
+
+  # ----------------------------------------------------------------------------
+  # detect face
+
+  face_detector = face_detector.DetectorDLIBCNN()  # -1 for CPU
+  bboxes = face_detector.detect(im_resized, largest=True)
+  bbox = bboxes[0]
+  dim = im_resized.shape[:2][::-1]
+  bbox_dim = bbox.to_dim(dim)
+  if not bbox:
+    log.error('no face detected')
+    return
+  else:
+    log.info(f'face detected: {bbox_dim.to_xyxy()}')
+
+  # ----------------------------------------------------------------------------
+  # beauty
+  
+  beauty_predictor = face_beauty.FaceBeauty()
+  beauty_score = beauty_predictor.beauty(im_resized, bbox_dim)
+
+
+  # ----------------------------------------------------------------------------
+  # output
+  
+  log.info(f'Face coords: {bbox_dim} face')
+  log.info(f'beauty score: {(100*beauty_score):.2f}')
+
+
+  # ----------------------------------------------------------------------------
+  # draw
+
+  # draw 2d landmarks
+  im_beauty = im_resized.copy()
+  draw_utils.draw_bbox(im_beauty, bbox_dim)
+  txt = f'Beauty score: {(100*beauty_score):.2f}'
+  draw_utils.draw_text(im_beauty, bbox_dim.pt_tl, txt)
+
+
+  # ----------------------------------------------------------------------------
+  # save
+  
+  if opt_fp_out:
+    # save pose only
+    cv.imwrite(opt_fp_out, im_beauty)
+
+
+  # ----------------------------------------------------------------------------
+  # display
+
+  if opt_display:
+    # show all images here
+    cv.imshow('Beauty', im_beauty)
+    display_utils.handle_keyboard()
\ No newline at end of file