From 5e5a7d09774bde195fe31ae143704eb124a764ac Mon Sep 17 00:00:00 2001 From: adamhrv Date: Mon, 7 Jan 2019 02:26:34 +0100 Subject: add demos, in progress --- megapixels/app/processors/face_detector.py | 6 +- megapixels/app/processors/face_landmarks.py | 16 +- megapixels/app/processors/face_landmarks_3d.py | 5 +- megapixels/app/processors/face_pose.py | 6 + megapixels/app/settings/app_cfg.py | 1 + megapixels/app/settings/types.py | 3 - megapixels/app/utils/display_utils.py | 4 +- megapixels/app/utils/draw_utils.py | 14 - megapixels/app/utils/plot_utils.py | 149 +++ megapixels/cli_visualize.py | 36 + megapixels/commands/cv/face_landmark_3d_68.py | 11 +- megapixels/commands/demo/3d_landmark_anim.py | 219 +++++ megapixels/commands/demo/face_detection.py | 128 +++ megapixels/commands/demo/face_landmarks_2d.py | 219 +++++ megapixels/commands/demo/face_landmarks_3d.py | 219 +++++ megapixels/commands/demo/face_pose.py | 128 +++ megapixels/commands/demo/face_vector.py | 79 ++ megapixels/commands/visualize/plot_3d_landmarks.py | 89 ++ .../notebooks/face_analysis/3d_face_plot.ipynb | 1015 ++++---------------- 19 files changed, 1470 insertions(+), 877 deletions(-) create mode 100644 megapixels/app/utils/plot_utils.py create mode 100644 megapixels/cli_visualize.py create mode 100644 megapixels/commands/demo/3d_landmark_anim.py create mode 100644 megapixels/commands/demo/face_detection.py create mode 100644 megapixels/commands/demo/face_landmarks_2d.py create mode 100644 megapixels/commands/demo/face_landmarks_3d.py create mode 100644 megapixels/commands/demo/face_pose.py create mode 100644 megapixels/commands/demo/face_vector.py create mode 100644 megapixels/commands/visualize/plot_3d_landmarks.py diff --git a/megapixels/app/processors/face_detector.py b/megapixels/app/processors/face_detector.py index 6bf27576..c0762564 100644 --- a/megapixels/app/processors/face_detector.py +++ b/megapixels/app/processors/face_detector.py @@ -1,3 +1,4 @@ +import sys import os from os.path import join from pathlib import Path @@ -30,8 +31,6 @@ class DetectorMTCNN: :returns list of BBox ''' bboxes = [] - #conf_thresh = self.conf_thresh if conf_thresh is None else conf_thresh - #pyramids = self.pyramids if pyramids is None else pyramids dnn_size = self.dnn_size if size is None else size im = im_utils.resize(im, width=dnn_size[0], height=dnn_size[1]) @@ -72,6 +71,9 @@ class DetectorDLIBCNN: import dlib self.log = logger_utils.Logger.getLogger() cuda_visible_devices = os.getenv('CUDA_VISIBLE_DEVICES', '') + if dlib.DLIB_USE_CUDA and gpu < 0: + self.log.error('dlib was compiled with CUDA but you selected CPU. Use GPU >= 0 if dlib.DLIB_USE_CUDA') + sys.exit() os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu) self.log.info('load model: {}'.format(cfg.DIR_MODELS_DLIB_CNN)) self.detector = dlib.cnn_face_detection_model_v1(cfg.DIR_MODELS_DLIB_CNN) diff --git a/megapixels/app/processors/face_landmarks.py b/megapixels/app/processors/face_landmarks.py index 8086ba1e..171fc666 100644 --- a/megapixels/app/processors/face_landmarks.py +++ b/megapixels/app/processors/face_landmarks.py @@ -83,8 +83,11 @@ class Dlib2D(Landmarks2D): self.log.info(f'loaded predictor model: {model}') def landmarks(self, im, bbox): - # Draw high-confidence faces - dim_wh = im.shape[:2][::-1] + '''Generates 68-pt landmarks using dlib predictor + :param im: (numpy.ndarray) BGR image + :param bbox: (app.models.BBox) dimensioned + :returns (list) of (int, int) for x,y values + ''' bbox = bbox.to_dlib() im_gray = cv.cvtColor(im, cv.COLOR_BGR2GRAY) points = [[p.x, p.y] for p in self.predictor(im_gray, bbox).parts()] @@ -168,8 +171,8 @@ class Landmarks3D: points_formatted[f'{d}{idx}'] = pt[j] return points_formatted - def normalize(self, points, dim): - return [np.array(p)/dim for p in points] # divides each point by w,h dim + # def normalize(self, points): + # '''TODO''' class FaceAlignment3D_68(Landmarks3D): @@ -182,13 +185,14 @@ class FaceAlignment3D_68(Landmarks3D): device = f'cuda:{gpu}' if gpu > -1 else 'cpu' self.fa = face_alignment.FaceAlignment(face_alignment.LandmarksType._3D, device=device, flip_input=flip_input) - def landmarks(self, im, as_type=str): + def landmarks(self, im, rect): '''Calculates the 3D facial landmarks :param im: (numpy.ndarray) BGR image + :param rect: (list) of face (x1, y1, x2, y2) :returns (list) of 68 (int) (tuples) as (x,y, z) ''' # predict landmarks - points = self.fa.get_landmarks(im) # returns array of arrays of 68 3D pts/face + points = self.fa.get_landmarks(im, [rect]) # returns array of arrays of 68 3D pts/face # convert to data type points = [list(map(int, p)) for p in points[0]] return points \ No newline at end of file diff --git a/megapixels/app/processors/face_landmarks_3d.py b/megapixels/app/processors/face_landmarks_3d.py index 470d263c..5a0d6097 100644 --- a/megapixels/app/processors/face_landmarks_3d.py +++ b/megapixels/app/processors/face_landmarks_3d.py @@ -26,14 +26,15 @@ class FaceAlignment3D(Landmarks3D): # Estimates 3D facial landmarks import face_alignment - def __init__(self, gpu=0, flip_input=False): + def __init__(self, gpu=0, flip_input=True): super().__init__() device = f'cuda:{gpu}' if gpu > -1 else 'cpu' self.fa = face_alignment.FaceAlignment(face_alignment.LandmarksType._3D, device=device, flip_input=flip_input) - def landmarks(self, im, as_type=str): + def landmarks(self, im, bbox, as_type=str): '''Calculates the 3D facial landmarks :param im: (numpy.ndarray) image + :param bbox: (BBox) dimensioned to real (int) sizes :param as_type: (str) or (list) type to return data ''' preds = self.fa.get_landmarks(im) diff --git a/megapixels/app/processors/face_pose.py b/megapixels/app/processors/face_pose.py index 8bc95f8d..5ac510ec 100644 --- a/megapixels/app/processors/face_pose.py +++ b/megapixels/app/processors/face_pose.py @@ -25,6 +25,12 @@ class FacePoseDLIB: def pose(self, landmarks, dim): + '''Returns face pose information + :param landmarks: (list) of 68 (int, int) xy tuples + :param dim: (tuple|list) of image (width, height) + :returns (dict) of pose attributes + ''' + # computes pose using 6 / 68 points from dlib face landmarks # based on learnopencv.com and # https://github.com/jerryhouuu/Face-Yaw-Roll-Pitch-from-Pose-Estimation-using-OpenCV/ diff --git a/megapixels/app/settings/app_cfg.py b/megapixels/app/settings/app_cfg.py index b13ff8ec..d206f40b 100644 --- a/megapixels/app/settings/app_cfg.py +++ b/megapixels/app/settings/app_cfg.py @@ -76,6 +76,7 @@ FP_FONT = join(DIR_ASSETS, 'font') # click chair settings # ----------------------------------------------------------------------------- DIR_COMMANDS_CV = 'commands/cv' +DIR_COMMANDS_VIZ = 'commands/visualize' DIR_COMMANDS_ADMIN = 'commands/admin' DIR_COMMANDS_DATASETS = 'commands/datasets' DIR_COMMANDS_FAISS = 'commands/faiss' diff --git a/megapixels/app/settings/types.py b/megapixels/app/settings/types.py index 50e395e0..1d77fdbd 100644 --- a/megapixels/app/settings/types.py +++ b/megapixels/app/settings/types.py @@ -64,9 +64,6 @@ class FaceLandmark2D_68(Enum): class FaceLandmark3D_68(Enum): FACE_ALIGNMENT = range(1) - -class FaceLandmark3D(Enum): - FACE_ALIGNMENT = range(1) class FaceEmotion(Enum): # Map these to text strings for web display diff --git a/megapixels/app/utils/display_utils.py b/megapixels/app/utils/display_utils.py index 58e2feec..7b74aa46 100644 --- a/megapixels/app/utils/display_utils.py +++ b/megapixels/app/utils/display_utils.py @@ -3,11 +3,11 @@ import sys import cv2 as cv -def handle_keyboard(): +def handle_keyboard(delay_amt=1): '''Used with cv.imshow('title', image) to wait for keyboard press ''' while True: - k = cv.waitKey(1) & 0xFF + k = cv.waitKey(delay_amt) & 0xFF if k == 27 or k == ord('q'): # ESC cv.destroyAllWindows() sys.exit() diff --git a/megapixels/app/utils/draw_utils.py b/megapixels/app/utils/draw_utils.py index f6d53609..47bb7978 100644 --- a/megapixels/app/utils/draw_utils.py +++ b/megapixels/app/utils/draw_utils.py @@ -49,17 +49,3 @@ def draw_degrees(im, pose_data, color=(0,255,0)): t = '{}: {:.2f}'.format(k, v) origin = (10, 30 + (25 * i)) cv.putText(im, t, origin, cv.FONT_HERSHEY_SIMPLEX, 0.5, clr, thickness=2, lineType=2) - - -# --------------------------------------------------------------------------- -# -# Matplotlib drawing functions -# -# --------------------------------------------------------------------------- - -def plot_landmarks3D(im, points, radius=3, color=(0,255,0), stroke_weight=2): - '''Draws facial landmarks, either 5pt or 68pt - ''' - for pt in points: - cv.circle(im, tuple(pt), radius, color, -1, cv.LINE_AA) - diff --git a/megapixels/app/utils/plot_utils.py b/megapixels/app/utils/plot_utils.py new file mode 100644 index 00000000..5bbb8ac2 --- /dev/null +++ b/megapixels/app/utils/plot_utils.py @@ -0,0 +1,149 @@ +import sys +from os.path import join +import time +import random +from pathlib import Path + +import numpy as np + +import matplotlib.pyplot as plt +import matplotlib.animation +from mpl_toolkits.mplot3d import Axes3D +from matplotlib import cbook +from matplotlib import cm +from matplotlib import animation + + + +# --------------------------------------------------------------------------- +# +# Matplotlib drawing functions +# +# --------------------------------------------------------------------------- + +# Generate random hex colors +def rhex(): + r = lambda: random.randint(0,255) + return '#%02X%02X%02X' % (r(), r(), r()) + + # line weight +def generate_3d_landmark_anim(lm, fp_out, num_frames=30, fps=12, dpi=72, size=(480,480), + stroke_weight=2, mark_size=10, mark_type='.', bg_clr=(0,0,0), transparent=False): + '''Generates animated 3D plot of face landmarks + ''' + + # convert opencv BGR numpy image to RGB + bg_clr_hex = '#%02x%02x%02x' % bg_clr + #mark_clr = '#%02x%02x%02x' % mark_clr + + # center x,y,z + xmm = (np.min(lm[:,0]),np.max(lm[:,0])) + ymm = (np.min(lm[:,1]),np.max(lm[:,1])) + zmm = (np.min(lm[:,2]),np.max(lm[:,2])) + + # make copy of landmarks + lm_orig = lm.copy() + xmm = (np.min(lm_orig[:,0]),np.max(lm_orig[:,0])) + ymm = (np.min(lm_orig[:,1]),np.max(lm_orig[:,1])) + zmm = (np.min(lm_orig[:,2]),np.max(lm_orig[:,2])) + + # swap the y and z components to improve 3d rotation angles for matplotlib + lm = np.zeros_like(lm_orig).astype(np.uint8) + for i,p in enumerate(lm_orig): + x,y,z = p + lm[i] = np.array([x - xmm[0], z - zmm[0], y - ymm[0]]) + + # Create plot + figsize = (size[0]/dpi, size[1]/dpi ) + fig = plt.figure(figsize=figsize, dpi=dpi) # frameon=False + fig.tight_layout() + # remove whitespace in matplotlib + fig.subplots_adjust(left=0, bottom=0, right=1, top=1, wspace=None, hspace=None) + ax = fig.add_subplot(111, projection='3d') + ax.set_facecolor(bg_clr_hex) # background color + + xscale, yscale, zscale = (1.2, 1.0, 1.0) + + # scatter plot the dots + + # jaw line + mark_clr = '#%02x%02x%02x' % (0,255,0) # green + ax.plot3D(lm[:17,0]*1.2,lm[:17,1], lm[:17,2], + marker=mark_type, markersize=mark_size, color=mark_clr,linewidth=stroke_weight) + + # stage-right eyebrow + mark_clr = '#%02x%02x%02x' % (255,0,0) # green + ax.plot3D(lm[17:22,0]*1.2,lm[17:22,1],lm[17:22,2], + marker=mark_type, markersize=mark_size, color=mark_clr,linewidth=stroke_weight) + + # stage-left eyebrow + mark_clr = '#%02x%02x%02x' % (255,255,0) # yellow + ax.plot3D(lm[22:27,0]*1.2,lm[22:27,1],lm[22:27,2], + marker=mark_type, markersize=mark_size, color=mark_clr,linewidth=stroke_weight) + + # nose ridge + mark_clr = '#%02x%02x%02x' % (0,0,255) # blue + ax.plot3D(lm[27:31,0]*1.2,lm[27:31,1],lm[27:31,2], + marker=mark_type, markersize=mark_size, color=mark_clr,linewidth=stroke_weight) + + # nose-bottom + mark_clr = '#%02x%02x%02x' % (255,0,255) # magenta + ax.plot3D(lm[31:36,0]*1.2,lm[31:36,1],lm[31:36,2], + marker=mark_type, markersize=mark_size, color=mark_clr,linewidth=stroke_weight) + + # stage-left eye + mark_clr = '#%02x%02x%02x' % (0,255,255) # cyan + px, py, pz = lm[36:42,0]*1.2,lm[36:42,1],lm[36:42,2] + px = np.append(px, lm[36,0]*1.2) + py = np.append(py, lm[36,1]) + pz = np.append(pz, lm[36,2]) + ax.plot3D(px, py, pz, marker=mark_type, markersize=mark_size, color=mark_clr,linewidth=stroke_weight) + + # stage-right eye + mark_clr = '#%02x%02x%02x' % (255,255,255) # white + px, py, pz = lm[42:48,0]*1.2,lm[42:48,1],lm[42:48,2] + px = np.append(px, lm[42,0]*1.2) + py = np.append(py, lm[42,1]) + pz = np.append(pz, lm[42,2]) + ax.plot3D(px, py, pz, marker=mark_type, markersize=mark_size, color=mark_clr,linewidth=stroke_weight) + + # mouth + mark_clr = '#%02x%02x%02x' % (255,125,0) # orange? + px, py, pz = lm[48:,0]*1.2,lm[48:,1],lm[48:,2] + px = np.append(px, lm[48,0]*1.2) + py = np.append(py, lm[48,1]) + pz = np.append(pz, lm[48,2]) + ax.plot3D(px, py, pz, marker=mark_type, markersize=mark_size, color=mark_clr, linewidth=stroke_weight) + + #rh = '#00ff00' # edge color + #ax.scatter(lm[:,0]*xscale,lm[:,1]*yscale,lm[:,2]*zscale, c=rh, alpha=1.0, s=35, edgecolor=rh) + #ax.scatter(lm[:,0]*xscale,lm[:,1]*yscale,lm[:,2]*zscale, c=rh, alpha=1.0, s=1) + + # center center x,y,z points + cx = ((xmm[0] - xmm[1]) // 2) + xmm[1] + cy = ((ymm[1] - ymm[0]) // 2) + ymm[0] + cz = ((zmm[1] - zmm[0]) // 2) + zmm[0] + + # remove ticks + ax.set_xticks([]) + ax.set_yticks([]) + ax.set_zticks([]) + + # remove axis + ax.set_frame_on(False) + ax.set_axis_off() + + # set initial plot view + ax.view_init(elev=120., azim=70.) + + # rotation increments: from 0 to 360 in num_frames + phi = np.linspace(0, 2*np.pi, num_frames) + + # animation instruction + def update(phi): + ax.view_init(180,phi*180./np.pi) + + ani = matplotlib.animation.FuncAnimation(fig, update, frames=phi) + + savefig_kwargs = {'pad_inches': 0, 'transparent': transparent} + ani.save(fp_out, writer='imagemagick', fps=fps, savefig_kwargs=savefig_kwargs) \ No newline at end of file diff --git a/megapixels/cli_visualize.py b/megapixels/cli_visualize.py new file mode 100644 index 00000000..0e80af53 --- /dev/null +++ b/megapixels/cli_visualize.py @@ -0,0 +1,36 @@ +# -------------------------------------------------------- +# add/edit commands in commands/datasets directory +# -------------------------------------------------------- + +import click + +from app.settings import app_cfg as cfg +from app.utils import logger_utils +from app.models.click_factory import ClickSimple + +# click cli factory +cc = ClickSimple.create(cfg.DIR_COMMANDS_VIZ) + +# -------------------------------------------------------- +# CLI +# -------------------------------------------------------- +@click.group(cls=cc, chain=False) +@click.option('-v', '--verbose', 'verbosity', count=True, default=4, + show_default=True, + help='Verbosity: -v DEBUG, -vv INFO, -vvv WARN, -vvvv ERROR, -vvvvv CRITICAL') +@click.pass_context +def cli(ctx, **kwargs): + """\033[1m\033[94mMegaPixels: Dataset Image Scripts\033[0m + """ + ctx.opts = {} + # init logger + logger_utils.Logger.create(verbosity=kwargs['verbosity']) + + + +# -------------------------------------------------------- +# Entrypoint +# -------------------------------------------------------- +if __name__ == '__main__': + cli() + diff --git a/megapixels/commands/cv/face_landmark_3d_68.py b/megapixels/commands/cv/face_landmark_3d_68.py index 56e60cda..a2d14d72 100644 --- a/megapixels/commands/cv/face_landmark_3d_68.py +++ b/megapixels/commands/cv/face_landmark_3d_68.py @@ -57,6 +57,7 @@ def cli(ctx, opt_fp_in, opt_fp_out, opt_dir_media, opt_data_store, opt_dataset, import pandas as pd from app.utils import logger_utils, file_utils, im_utils, display_utils, draw_utils + from app.utils import plot_utils from app.processors import face_landmarks from app.models.data_store import DataStore from app.models.bbox import BBox @@ -65,7 +66,7 @@ def cli(ctx, opt_fp_in, opt_fp_out, opt_dir_media, opt_data_store, opt_dataset, # init here log = logger_utils.Logger.getLogger() - log.warn('3D landmark points are normalized in a (200, 200, 200) XYZ space') + log.warn('not normalizing points') # init filepaths data_store = DataStore(opt_data_store, opt_dataset) # set file output path @@ -76,7 +77,7 @@ def cli(ctx, opt_fp_in, opt_fp_out, opt_dir_media, opt_data_store, opt_dataset, return # init face landmark processors - if opt_detector_type == types.FaceLandmark2D_5.FACE_ALIGNMENT: + if opt_detector_type == types.FaceLandmark3D_68.FACE_ALIGNMENT: # use FaceAlignment 68 point 3D detector landmark_detector = face_landmarks.FaceAlignment3D_68() else: @@ -122,16 +123,18 @@ def cli(ctx, opt_fp_in, opt_fp_out, opt_dir_media, opt_data_store, opt_dataset, # get landmark points points = landmark_detector.landmarks(im_resized, bbox) # NB can't really normalize these points, but are normalized against 3D space - points_norm = landmark_detector.normalize(points, dim) # normalized using 200 + #points_norm = landmark_detector.normalize(points, dim) # normalized using 200 points_flattenend = landmark_detector.flatten(points) # display to screen if optioned if opt_display: - draw_utils.draw_landmarks2D(im_resized, points) + draw_utils.draw_landmarks3D(im_resized, points) draw_utils.draw_bbox(im_resized, bbox) cv.imshow('', im_resized) display_utils.handle_keyboard() + #plot_utils.generate_3d_landmark_anim(points, '/home/adam/Downloads/3d.gif') + results.append(points_flattenend) # create DataFrame and save to CSV diff --git a/megapixels/commands/demo/3d_landmark_anim.py b/megapixels/commands/demo/3d_landmark_anim.py new file mode 100644 index 00000000..22e09297 --- /dev/null +++ b/megapixels/commands/demo/3d_landmark_anim.py @@ -0,0 +1,219 @@ +""" +Crop images to prepare for training +""" + +import click +# from PIL import Image, ImageOps, ImageFilter, ImageDraw + +from app.settings import types +from app.utils import click_utils +from app.settings import app_cfg as cfg + + +@click.command() +@click.option('-i', '--input', 'opt_fp_in', default=None, required=True, + help='Image filepath') +@click.option('-o', '--output', 'opt_fp_out', default=None, + help='GIF output path') +@click.option('--size', 'opt_size', + type=(int, int), default=(300, 300), + help='Output image size') +@click.option('--gif-size', 'opt_gif_size', + type=(int, int), default=(480, 480), + help='GIF output size') +@click.option('--gif-frames', 'opt_gif_frames', default=15, + help='GIF frames') +@click.option('-g', '--gpu', 'opt_gpu', default=0, + help='GPU index') +@click.option('-f', '--force', 'opt_force', is_flag=True, + help='Force overwrite file') +@click.option('--display/--no-display', 'opt_display', is_flag=True, default=False, + help='Display detections to debug') +@click.pass_context +def cli(ctx, opt_fp_in, opt_fp_out, opt_gpu, opt_gif_frames, + opt_size, opt_gif_size, opt_force, opt_display): + """Generates 3D landmark animations from CSV files""" + + import sys + import os + from os.path import join + from pathlib import Path + import time + + from tqdm import tqdm + import numpy as np + import pandas as pd + import cv2 as cv + import dlib + from PIL import Image + import matplotlib.pyplot as plt + + from app.utils import logger_utils, file_utils, im_utils, display_utils, draw_utils + from app.utils import plot_utils + from app.processors import face_detector, face_landmarks + from app.models.data_store import DataStore + + # TOOD add selective testing + opt_run_pose = True + opt_run_2d_68 = True + opt_run_3d_68 = True + opt_run_3d_68 = True + + + # ------------------------------------------------- + # init here + + + log = logger_utils.Logger.getLogger() + + # load image + im = cv.imread(opt_fp_in) + im_resized = im_utils.resize(im, width=opt_size[0], height=opt_size[1]) + + + # ---------------------------------------------------------------------------- + # detect face + + face_detector = face_detector.DetectorDLIBCNN(gpu=opt_gpu) # -1 for CPU + log.info('detecting face...') + st = time.time() + bboxes = face_detector.detect(im_resized, largest=True) + bbox = bboxes[0] + dim = im_resized.shape[:2][::-1] + bbox_dim = bbox.to_dim(dim) + if not bbox: + log.error('no face detected') + return + else: + log.info(f'Detected face in {(time.time() - st):.2f}s') + log.info('') + + + # ---------------------------------------------------------------------------- + # detect 3D landmarks + + log.info('loading 3D landmark generator files...') + landmark_detector_3d_68 = face_landmarks.FaceAlignment3D_68(gpu=opt_gpu) # -1 for CPU + log.info('generating 3D landmarks...') + st = time.time() + points_3d_68 = landmark_detector_3d_68.landmarks(im_resized, bbox_dim.to_xyxy()) + log.info(f'generated 3D landmarks in {(time.time() - st):.2f}s') + log.info('') + + + # ---------------------------------------------------------------------------- + # generate 3D GIF animation + + log.info('generating 3D animation...') + if not opt_fp_out: + fpp_im = Path(opt_fp_in) + fp_out = join(fpp_im.parent, f'{fpp_im.stem}_anim.gif') + else: + fp_out = opt_fp_out + st = time.time() + plot_utils.generate_3d_landmark_anim(np.array(points_3d_68), fp_out, + size=opt_gif_size, num_frames=opt_gif_frames) + log.info(f'Generated animation in {(time.time() - st):.2f}s') + log.info(f'Saved to: {fp_out}') + log.info('') + + + # ---------------------------------------------------------------------------- + # generate face vectors, only to test if feature extraction works + + log.info('initialize face recognition model...') + from app.processors import face_recognition + face_rec = face_recognition.RecognitionDLIB() + st = time.time() + log.info('generating face vector...') + vec = face_rec.vec(im_resized, bbox_dim) + log.info(f'generated face vector in {(time.time() - st):.2f}s') + log.info('') + + + # ---------------------------------------------------------------------------- + # generate 68 point landmarks using dlib + + log.info('initializing face landmarks 68 dlib...') + from app.processors import face_landmarks + landmark_detector_2d_68 = face_landmarks.Dlib2D_68() + log.info('generating 2D 68PT landmarks...') + st = time.time() + points_2d_68 = landmark_detector_2d_68.landmarks(im_resized, bbox_dim) + log.info(f'generated 2D 68PT face landmarks in {(time.time() - st):.2f}s') + log.info('') + + + # ---------------------------------------------------------------------------- + # generate pose from 68 point 2D landmarks + + if opt_run_pose: + log.info('initialize pose...') + from app.processors import face_pose + pose_detector = face_pose.FacePoseDLIB() + log.info('generating pose...') + st = time.time() + pose_data = pose_detector.pose(points_2d_68, dim) + log.info(f'generated pose {(time.time() - st):.2f}s') + log.info('') + + + # x + + + + # display + if opt_display: + + # draw bbox + + # draw 3d landmarks + im_landmarks_3d_68 = im_resized.copy() + draw_utils.draw_landmarks3D(im_landmarks_3d_68, points_3d_68) + draw_utils.draw_bbox(im_landmarks_3d_68, bbox_dim) + + # draw 2d landmarks + im_landmarks_2d_68 = im_resized.copy() + draw_utils.draw_landmarks2D(im_landmarks_2d_68, points_2d_68) + draw_utils.draw_bbox(im_landmarks_2d_68, bbox_dim) + + # draw pose + if opt_run_pose: + im_pose = im_resized.copy() + draw_utils.draw_pose(im_pose, pose_data['point_nose'], pose_data['points']) + draw_utils.draw_degrees(im_pose, pose_data) + + # draw animated GIF + im = Image.open(opt_fp_out) + im_frames = [] + duration = im.info['duration'] + try: + while True: + im.seek(len(im_frames)) + mypalette = im.getpalette() + im.putpalette(mypalette) + im_jpg = Image.new("RGB", im.size) + im_jpg.paste(im) + im_np = im_utils.pil2np(im_jpg.copy()) + im_frames.append(im_np) + except EOFError: + pass # end of GIF sequence + + n_frames = len(im_frames) + frame_number = 0 + + while True: + # show all images here + cv.imshow('Original', im_resized) + cv.imshow('2D 68PT Landmarks', im_landmarks_2d_68) + cv.imshow('3D 68PT Landmarks', im_landmarks_3d_68) + cv.imshow('Pose', im_pose) + cv.imshow('3D 68pt GIF', im_frames[frame_number]) + frame_number = (frame_number + 1) % n_frames + k = cv.waitKey(duration) & 0xFF + if k == 27 or k == ord('q'): # ESC + cv.destroyAllWindows() + sys.exit() + elif k != 255: + # any key to continue + break \ No newline at end of file diff --git a/megapixels/commands/demo/face_detection.py b/megapixels/commands/demo/face_detection.py new file mode 100644 index 00000000..fb23704b --- /dev/null +++ b/megapixels/commands/demo/face_detection.py @@ -0,0 +1,128 @@ +""" +Crop images to prepare for training +""" + +import click +# from PIL import Image, ImageOps, ImageFilter, ImageDraw + +from app.settings import types +from app.utils import click_utils +from app.settings import app_cfg as cfg + + +@click.command() +@click.option('-i', '--input', 'opt_fp_in', default=None, required=True, + help='Image filepath') +@click.option('-o', '--output', 'opt_fp_out', default=None, + help='GIF output path') +@click.option('--size', 'opt_size', + type=(int, int), default=(300, 300), + help='Output image size') +@click.option('-g', '--gpu', 'opt_gpu', default=0, + help='GPU index') +@click.option('-f', '--force', 'opt_force', is_flag=True, + help='Force overwrite file') +@click.option('--display/--no-display', 'opt_display', is_flag=True, default=False, + help='Display detections to debug') +@click.pass_context +def cli(ctx, opt_fp_in, opt_fp_out, opt_gpu, opt_size, opt_force, opt_display): + """Face detector demo""" + + import sys + import os + from os.path import join + from pathlib import Path + import time + + from tqdm import tqdm + import numpy as np + import pandas as pd + import cv2 as cv + import dlib + from PIL import Image + import matplotlib.pyplot as plt + + from app.utils import logger_utils, file_utils, im_utils, display_utils, draw_utils + from app.utils import plot_utils + from app.processors import face_detector, face_landmarks + from app.models.data_store import DataStore + + + log = logger_utils.Logger.getLogger() + + + # ------------------------------------------------- + # load image + + im = cv.imread(opt_fp_in) + im_resized = im_utils.resize(im, width=opt_size[0], height=opt_size[1]) + + + # ---------------------------------------------------------------------------- + # detect face + + face_detector = face_detector.DetectorDLIBCNN(gpu=opt_gpu) # -1 for CPU + bboxes = face_detector.detect(im_resized, largest=True) + bbox = bboxes[0] + dim = im_resized.shape[:2][::-1] + bbox_dim = bbox.to_dim(dim) + if not bbox: + log.error('no face detected') + return + + + # ---------------------------------------------------------------------------- + # generate 68 point landmarks using dlib + + from app.processors import face_landmarks + landmark_detector_2d_68 = face_landmarks.Dlib2D_68() + points_2d_68 = landmark_detector_2d_68.landmarks(im_resized, bbox_dim) + + + # ---------------------------------------------------------------------------- + # generate pose from 68 point 2D landmarks + + from app.processors import face_pose + pose_detector = face_pose.FacePoseDLIB() + pose_data = pose_detector.pose(points_2d_68, dim) + + # ---------------------------------------------------------------------------- + # output + + log.info(f'Face coords: {bbox_dim} face') + log.info(f'pitch: {pose_data["pitch"]}, roll: {pose_data["roll"]}, yaw: {pose_data["yaw"]}') + + + # ---------------------------------------------------------------------------- + # draw + + # draw 2d landmarks + im_landmarks_2d_68 = im_resized.copy() + draw_utils.draw_landmarks2D(im_landmarks_2d_68, points_2d_68) + draw_utils.draw_bbox(im_landmarks_2d_68, bbox_dim) + + # draw pose + im_pose = im_resized.copy() + draw_utils.draw_pose(im_pose, pose_data['point_nose'], pose_data['points']) + draw_utils.draw_degrees(im_pose, pose_data) + + + # ---------------------------------------------------------------------------- + # save + + if opt_fp_out: + # save pose only + cv.imwrite(opt_fp_out, im_pose) + + + # ---------------------------------------------------------------------------- + # display + + if opt_display: + + + # show all images here + cv.imshow('Original', im_resized) + cv.imshow('2D 68PT Landmarks', im_landmarks_2d_68) + cv.imshow('Pose', im_pose) + display_utils.handle_keyboard() \ No newline at end of file diff --git a/megapixels/commands/demo/face_landmarks_2d.py b/megapixels/commands/demo/face_landmarks_2d.py new file mode 100644 index 00000000..22e09297 --- /dev/null +++ b/megapixels/commands/demo/face_landmarks_2d.py @@ -0,0 +1,219 @@ +""" +Crop images to prepare for training +""" + +import click +# from PIL import Image, ImageOps, ImageFilter, ImageDraw + +from app.settings import types +from app.utils import click_utils +from app.settings import app_cfg as cfg + + +@click.command() +@click.option('-i', '--input', 'opt_fp_in', default=None, required=True, + help='Image filepath') +@click.option('-o', '--output', 'opt_fp_out', default=None, + help='GIF output path') +@click.option('--size', 'opt_size', + type=(int, int), default=(300, 300), + help='Output image size') +@click.option('--gif-size', 'opt_gif_size', + type=(int, int), default=(480, 480), + help='GIF output size') +@click.option('--gif-frames', 'opt_gif_frames', default=15, + help='GIF frames') +@click.option('-g', '--gpu', 'opt_gpu', default=0, + help='GPU index') +@click.option('-f', '--force', 'opt_force', is_flag=True, + help='Force overwrite file') +@click.option('--display/--no-display', 'opt_display', is_flag=True, default=False, + help='Display detections to debug') +@click.pass_context +def cli(ctx, opt_fp_in, opt_fp_out, opt_gpu, opt_gif_frames, + opt_size, opt_gif_size, opt_force, opt_display): + """Generates 3D landmark animations from CSV files""" + + import sys + import os + from os.path import join + from pathlib import Path + import time + + from tqdm import tqdm + import numpy as np + import pandas as pd + import cv2 as cv + import dlib + from PIL import Image + import matplotlib.pyplot as plt + + from app.utils import logger_utils, file_utils, im_utils, display_utils, draw_utils + from app.utils import plot_utils + from app.processors import face_detector, face_landmarks + from app.models.data_store import DataStore + + # TOOD add selective testing + opt_run_pose = True + opt_run_2d_68 = True + opt_run_3d_68 = True + opt_run_3d_68 = True + + + # ------------------------------------------------- + # init here + + + log = logger_utils.Logger.getLogger() + + # load image + im = cv.imread(opt_fp_in) + im_resized = im_utils.resize(im, width=opt_size[0], height=opt_size[1]) + + + # ---------------------------------------------------------------------------- + # detect face + + face_detector = face_detector.DetectorDLIBCNN(gpu=opt_gpu) # -1 for CPU + log.info('detecting face...') + st = time.time() + bboxes = face_detector.detect(im_resized, largest=True) + bbox = bboxes[0] + dim = im_resized.shape[:2][::-1] + bbox_dim = bbox.to_dim(dim) + if not bbox: + log.error('no face detected') + return + else: + log.info(f'Detected face in {(time.time() - st):.2f}s') + log.info('') + + + # ---------------------------------------------------------------------------- + # detect 3D landmarks + + log.info('loading 3D landmark generator files...') + landmark_detector_3d_68 = face_landmarks.FaceAlignment3D_68(gpu=opt_gpu) # -1 for CPU + log.info('generating 3D landmarks...') + st = time.time() + points_3d_68 = landmark_detector_3d_68.landmarks(im_resized, bbox_dim.to_xyxy()) + log.info(f'generated 3D landmarks in {(time.time() - st):.2f}s') + log.info('') + + + # ---------------------------------------------------------------------------- + # generate 3D GIF animation + + log.info('generating 3D animation...') + if not opt_fp_out: + fpp_im = Path(opt_fp_in) + fp_out = join(fpp_im.parent, f'{fpp_im.stem}_anim.gif') + else: + fp_out = opt_fp_out + st = time.time() + plot_utils.generate_3d_landmark_anim(np.array(points_3d_68), fp_out, + size=opt_gif_size, num_frames=opt_gif_frames) + log.info(f'Generated animation in {(time.time() - st):.2f}s') + log.info(f'Saved to: {fp_out}') + log.info('') + + + # ---------------------------------------------------------------------------- + # generate face vectors, only to test if feature extraction works + + log.info('initialize face recognition model...') + from app.processors import face_recognition + face_rec = face_recognition.RecognitionDLIB() + st = time.time() + log.info('generating face vector...') + vec = face_rec.vec(im_resized, bbox_dim) + log.info(f'generated face vector in {(time.time() - st):.2f}s') + log.info('') + + + # ---------------------------------------------------------------------------- + # generate 68 point landmarks using dlib + + log.info('initializing face landmarks 68 dlib...') + from app.processors import face_landmarks + landmark_detector_2d_68 = face_landmarks.Dlib2D_68() + log.info('generating 2D 68PT landmarks...') + st = time.time() + points_2d_68 = landmark_detector_2d_68.landmarks(im_resized, bbox_dim) + log.info(f'generated 2D 68PT face landmarks in {(time.time() - st):.2f}s') + log.info('') + + + # ---------------------------------------------------------------------------- + # generate pose from 68 point 2D landmarks + + if opt_run_pose: + log.info('initialize pose...') + from app.processors import face_pose + pose_detector = face_pose.FacePoseDLIB() + log.info('generating pose...') + st = time.time() + pose_data = pose_detector.pose(points_2d_68, dim) + log.info(f'generated pose {(time.time() - st):.2f}s') + log.info('') + + + # x + + + + # display + if opt_display: + + # draw bbox + + # draw 3d landmarks + im_landmarks_3d_68 = im_resized.copy() + draw_utils.draw_landmarks3D(im_landmarks_3d_68, points_3d_68) + draw_utils.draw_bbox(im_landmarks_3d_68, bbox_dim) + + # draw 2d landmarks + im_landmarks_2d_68 = im_resized.copy() + draw_utils.draw_landmarks2D(im_landmarks_2d_68, points_2d_68) + draw_utils.draw_bbox(im_landmarks_2d_68, bbox_dim) + + # draw pose + if opt_run_pose: + im_pose = im_resized.copy() + draw_utils.draw_pose(im_pose, pose_data['point_nose'], pose_data['points']) + draw_utils.draw_degrees(im_pose, pose_data) + + # draw animated GIF + im = Image.open(opt_fp_out) + im_frames = [] + duration = im.info['duration'] + try: + while True: + im.seek(len(im_frames)) + mypalette = im.getpalette() + im.putpalette(mypalette) + im_jpg = Image.new("RGB", im.size) + im_jpg.paste(im) + im_np = im_utils.pil2np(im_jpg.copy()) + im_frames.append(im_np) + except EOFError: + pass # end of GIF sequence + + n_frames = len(im_frames) + frame_number = 0 + + while True: + # show all images here + cv.imshow('Original', im_resized) + cv.imshow('2D 68PT Landmarks', im_landmarks_2d_68) + cv.imshow('3D 68PT Landmarks', im_landmarks_3d_68) + cv.imshow('Pose', im_pose) + cv.imshow('3D 68pt GIF', im_frames[frame_number]) + frame_number = (frame_number + 1) % n_frames + k = cv.waitKey(duration) & 0xFF + if k == 27 or k == ord('q'): # ESC + cv.destroyAllWindows() + sys.exit() + elif k != 255: + # any key to continue + break \ No newline at end of file diff --git a/megapixels/commands/demo/face_landmarks_3d.py b/megapixels/commands/demo/face_landmarks_3d.py new file mode 100644 index 00000000..22e09297 --- /dev/null +++ b/megapixels/commands/demo/face_landmarks_3d.py @@ -0,0 +1,219 @@ +""" +Crop images to prepare for training +""" + +import click +# from PIL import Image, ImageOps, ImageFilter, ImageDraw + +from app.settings import types +from app.utils import click_utils +from app.settings import app_cfg as cfg + + +@click.command() +@click.option('-i', '--input', 'opt_fp_in', default=None, required=True, + help='Image filepath') +@click.option('-o', '--output', 'opt_fp_out', default=None, + help='GIF output path') +@click.option('--size', 'opt_size', + type=(int, int), default=(300, 300), + help='Output image size') +@click.option('--gif-size', 'opt_gif_size', + type=(int, int), default=(480, 480), + help='GIF output size') +@click.option('--gif-frames', 'opt_gif_frames', default=15, + help='GIF frames') +@click.option('-g', '--gpu', 'opt_gpu', default=0, + help='GPU index') +@click.option('-f', '--force', 'opt_force', is_flag=True, + help='Force overwrite file') +@click.option('--display/--no-display', 'opt_display', is_flag=True, default=False, + help='Display detections to debug') +@click.pass_context +def cli(ctx, opt_fp_in, opt_fp_out, opt_gpu, opt_gif_frames, + opt_size, opt_gif_size, opt_force, opt_display): + """Generates 3D landmark animations from CSV files""" + + import sys + import os + from os.path import join + from pathlib import Path + import time + + from tqdm import tqdm + import numpy as np + import pandas as pd + import cv2 as cv + import dlib + from PIL import Image + import matplotlib.pyplot as plt + + from app.utils import logger_utils, file_utils, im_utils, display_utils, draw_utils + from app.utils import plot_utils + from app.processors import face_detector, face_landmarks + from app.models.data_store import DataStore + + # TOOD add selective testing + opt_run_pose = True + opt_run_2d_68 = True + opt_run_3d_68 = True + opt_run_3d_68 = True + + + # ------------------------------------------------- + # init here + + + log = logger_utils.Logger.getLogger() + + # load image + im = cv.imread(opt_fp_in) + im_resized = im_utils.resize(im, width=opt_size[0], height=opt_size[1]) + + + # ---------------------------------------------------------------------------- + # detect face + + face_detector = face_detector.DetectorDLIBCNN(gpu=opt_gpu) # -1 for CPU + log.info('detecting face...') + st = time.time() + bboxes = face_detector.detect(im_resized, largest=True) + bbox = bboxes[0] + dim = im_resized.shape[:2][::-1] + bbox_dim = bbox.to_dim(dim) + if not bbox: + log.error('no face detected') + return + else: + log.info(f'Detected face in {(time.time() - st):.2f}s') + log.info('') + + + # ---------------------------------------------------------------------------- + # detect 3D landmarks + + log.info('loading 3D landmark generator files...') + landmark_detector_3d_68 = face_landmarks.FaceAlignment3D_68(gpu=opt_gpu) # -1 for CPU + log.info('generating 3D landmarks...') + st = time.time() + points_3d_68 = landmark_detector_3d_68.landmarks(im_resized, bbox_dim.to_xyxy()) + log.info(f'generated 3D landmarks in {(time.time() - st):.2f}s') + log.info('') + + + # ---------------------------------------------------------------------------- + # generate 3D GIF animation + + log.info('generating 3D animation...') + if not opt_fp_out: + fpp_im = Path(opt_fp_in) + fp_out = join(fpp_im.parent, f'{fpp_im.stem}_anim.gif') + else: + fp_out = opt_fp_out + st = time.time() + plot_utils.generate_3d_landmark_anim(np.array(points_3d_68), fp_out, + size=opt_gif_size, num_frames=opt_gif_frames) + log.info(f'Generated animation in {(time.time() - st):.2f}s') + log.info(f'Saved to: {fp_out}') + log.info('') + + + # ---------------------------------------------------------------------------- + # generate face vectors, only to test if feature extraction works + + log.info('initialize face recognition model...') + from app.processors import face_recognition + face_rec = face_recognition.RecognitionDLIB() + st = time.time() + log.info('generating face vector...') + vec = face_rec.vec(im_resized, bbox_dim) + log.info(f'generated face vector in {(time.time() - st):.2f}s') + log.info('') + + + # ---------------------------------------------------------------------------- + # generate 68 point landmarks using dlib + + log.info('initializing face landmarks 68 dlib...') + from app.processors import face_landmarks + landmark_detector_2d_68 = face_landmarks.Dlib2D_68() + log.info('generating 2D 68PT landmarks...') + st = time.time() + points_2d_68 = landmark_detector_2d_68.landmarks(im_resized, bbox_dim) + log.info(f'generated 2D 68PT face landmarks in {(time.time() - st):.2f}s') + log.info('') + + + # ---------------------------------------------------------------------------- + # generate pose from 68 point 2D landmarks + + if opt_run_pose: + log.info('initialize pose...') + from app.processors import face_pose + pose_detector = face_pose.FacePoseDLIB() + log.info('generating pose...') + st = time.time() + pose_data = pose_detector.pose(points_2d_68, dim) + log.info(f'generated pose {(time.time() - st):.2f}s') + log.info('') + + + # x + + + + # display + if opt_display: + + # draw bbox + + # draw 3d landmarks + im_landmarks_3d_68 = im_resized.copy() + draw_utils.draw_landmarks3D(im_landmarks_3d_68, points_3d_68) + draw_utils.draw_bbox(im_landmarks_3d_68, bbox_dim) + + # draw 2d landmarks + im_landmarks_2d_68 = im_resized.copy() + draw_utils.draw_landmarks2D(im_landmarks_2d_68, points_2d_68) + draw_utils.draw_bbox(im_landmarks_2d_68, bbox_dim) + + # draw pose + if opt_run_pose: + im_pose = im_resized.copy() + draw_utils.draw_pose(im_pose, pose_data['point_nose'], pose_data['points']) + draw_utils.draw_degrees(im_pose, pose_data) + + # draw animated GIF + im = Image.open(opt_fp_out) + im_frames = [] + duration = im.info['duration'] + try: + while True: + im.seek(len(im_frames)) + mypalette = im.getpalette() + im.putpalette(mypalette) + im_jpg = Image.new("RGB", im.size) + im_jpg.paste(im) + im_np = im_utils.pil2np(im_jpg.copy()) + im_frames.append(im_np) + except EOFError: + pass # end of GIF sequence + + n_frames = len(im_frames) + frame_number = 0 + + while True: + # show all images here + cv.imshow('Original', im_resized) + cv.imshow('2D 68PT Landmarks', im_landmarks_2d_68) + cv.imshow('3D 68PT Landmarks', im_landmarks_3d_68) + cv.imshow('Pose', im_pose) + cv.imshow('3D 68pt GIF', im_frames[frame_number]) + frame_number = (frame_number + 1) % n_frames + k = cv.waitKey(duration) & 0xFF + if k == 27 or k == ord('q'): # ESC + cv.destroyAllWindows() + sys.exit() + elif k != 255: + # any key to continue + break \ No newline at end of file diff --git a/megapixels/commands/demo/face_pose.py b/megapixels/commands/demo/face_pose.py new file mode 100644 index 00000000..3918adac --- /dev/null +++ b/megapixels/commands/demo/face_pose.py @@ -0,0 +1,128 @@ +""" +Crop images to prepare for training +""" + +import click +# from PIL import Image, ImageOps, ImageFilter, ImageDraw + +from app.settings import types +from app.utils import click_utils +from app.settings import app_cfg as cfg + + +@click.command() +@click.option('-i', '--input', 'opt_fp_in', default=None, required=True, + help='Image filepath') +@click.option('-o', '--output', 'opt_fp_out', default=None, + help='GIF output path') +@click.option('--size', 'opt_size', + type=(int, int), default=(300, 300), + help='Output image size') +@click.option('-g', '--gpu', 'opt_gpu', default=0, + help='GPU index') +@click.option('-f', '--force', 'opt_force', is_flag=True, + help='Force overwrite file') +@click.option('--display/--no-display', 'opt_display', is_flag=True, default=False, + help='Display detections to debug') +@click.pass_context +def cli(ctx, opt_fp_in, opt_fp_out, opt_gpu, opt_size, opt_force, opt_display): + """Face pose demo""" + + import sys + import os + from os.path import join + from pathlib import Path + import time + + from tqdm import tqdm + import numpy as np + import pandas as pd + import cv2 as cv + import dlib + from PIL import Image + import matplotlib.pyplot as plt + + from app.utils import logger_utils, file_utils, im_utils, display_utils, draw_utils + from app.utils import plot_utils + from app.processors import face_detector, face_landmarks + from app.models.data_store import DataStore + + + log = logger_utils.Logger.getLogger() + + + # ------------------------------------------------- + # load image + + im = cv.imread(opt_fp_in) + im_resized = im_utils.resize(im, width=opt_size[0], height=opt_size[1]) + + + # ---------------------------------------------------------------------------- + # detect face + + face_detector = face_detector.DetectorDLIBCNN(gpu=opt_gpu) # -1 for CPU + bboxes = face_detector.detect(im_resized, largest=True) + bbox = bboxes[0] + dim = im_resized.shape[:2][::-1] + bbox_dim = bbox.to_dim(dim) + if not bbox: + log.error('no face detected') + return + + + # ---------------------------------------------------------------------------- + # generate 68 point landmarks using dlib + + from app.processors import face_landmarks + landmark_detector_2d_68 = face_landmarks.Dlib2D_68() + points_2d_68 = landmark_detector_2d_68.landmarks(im_resized, bbox_dim) + + + # ---------------------------------------------------------------------------- + # generate pose from 68 point 2D landmarks + + from app.processors import face_pose + pose_detector = face_pose.FacePoseDLIB() + pose_data = pose_detector.pose(points_2d_68, dim) + + # ---------------------------------------------------------------------------- + # output + + log.info(f'Face coords: {bbox_dim} face') + log.info(f'pitch: {pose_data["pitch"]}, roll: {pose_data["roll"]}, yaw: {pose_data["yaw"]}') + + + # ---------------------------------------------------------------------------- + # draw + + # draw 2d landmarks + im_landmarks_2d_68 = im_resized.copy() + draw_utils.draw_landmarks2D(im_landmarks_2d_68, points_2d_68) + draw_utils.draw_bbox(im_landmarks_2d_68, bbox_dim) + + # draw pose + im_pose = im_resized.copy() + draw_utils.draw_pose(im_pose, pose_data['point_nose'], pose_data['points']) + draw_utils.draw_degrees(im_pose, pose_data) + + + # ---------------------------------------------------------------------------- + # save + + if opt_fp_out: + # save pose only + cv.imwrite(opt_fp_out, im_pose) + + + # ---------------------------------------------------------------------------- + # display + + if opt_display: + + + # show all images here + cv.imshow('Original', im_resized) + cv.imshow('2D 68PT Landmarks', im_landmarks_2d_68) + cv.imshow('Pose', im_pose) + display_utils.handle_keyboard() \ No newline at end of file diff --git a/megapixels/commands/demo/face_vector.py b/megapixels/commands/demo/face_vector.py new file mode 100644 index 00000000..1104f923 --- /dev/null +++ b/megapixels/commands/demo/face_vector.py @@ -0,0 +1,79 @@ +""" +Crop images to prepare for training +""" + +import click +# from PIL import Image, ImageOps, ImageFilter, ImageDraw + +from app.settings import types +from app.utils import click_utils +from app.settings import app_cfg as cfg + + +@click.command() +@click.option('-i', '--input', 'opt_fp_in', default=None, required=True, + help='Image filepath') +@click.option('--size', 'opt_size', + type=(int, int), default=(300, 300), + help='Output image size') +@click.option('-g', '--gpu', 'opt_gpu', default=0, + help='GPU index') +@click.option('--display/--no-display', 'opt_display', is_flag=True, default=False, + help='Display detections to debug') +@click.pass_context +def cli(ctx, opt_fp_in, opt_gpu, opt_size, opt_display): + """Demo generating face vector""" + + import sys + import os + from os.path import join + from pathlib import Path + import time + + import numpy as np + import cv2 as cv + import dlib # NB: keep a reference in main file if using dlib detector processors + + from app.utils import logger_utils, file_utils, im_utils, display_utils, draw_utils + from app.utils import plot_utils + from app.processors import face_detector + from app.models.data_store import DataStore + + # ------------------------------------------------- + # init here + + log = logger_utils.Logger.getLogger() + + # ---------------------------------------------------------------------------- + # load image + + im = cv.imread(opt_fp_in) + im_resized = im_utils.resize(im, width=opt_size[0], height=opt_size[1]) + + + # ---------------------------------------------------------------------------- + # detect face + + face_detector = face_detector.DetectorDLIBCNN(gpu=opt_gpu) # -1 for CPU + bboxes = face_detector.detect(im_resized, largest=True) + bbox = bboxes[0] + dim = im_resized.shape[:2][::-1] + bbox_dim = bbox.to_dim(dim) + if not bbox: + log.error('no face detected') + return + + + # ---------------------------------------------------------------------------- + # generate face vectors, only to test if feature extraction works + + from app.processors import face_recognition + face_rec = face_recognition.RecognitionDLIB() + vec = face_rec.vec(im_resized, bbox_dim) + log.info(f'generated vector. showing vec[0:10]:') + log.info(f'\n{vec[0:10]}') + + if opt_display: + draw_utils.draw_bbox(im_resized, bbox_dim) + cv.imshow('Original', im_resized) + display_utils.handle_keyboard() \ No newline at end of file diff --git a/megapixels/commands/visualize/plot_3d_landmarks.py b/megapixels/commands/visualize/plot_3d_landmarks.py new file mode 100644 index 00000000..a0f9e555 --- /dev/null +++ b/megapixels/commands/visualize/plot_3d_landmarks.py @@ -0,0 +1,89 @@ +""" +Crop images to prepare for training +""" + +import click +# from PIL import Image, ImageOps, ImageFilter, ImageDraw + +from app.settings import types +from app.utils import click_utils +from app.settings import app_cfg as cfg + +color_filters = {'color': 1, 'gray': 2, 'all': 3} + +@click.command() +@click.option('-i', '--input', 'opt_fp_in', default=None, + help='Override enum input filename CSV') +@click.option('-o', '--output', 'opt_fp_out', default=None, + help='Override enum output filename CSV') +@click.option('-m', '--media', 'opt_dir_media', default=None, + help='Override enum media directory') +@click.option('--store', 'opt_data_store', + type=cfg.DataStoreVar, + default=click_utils.get_default(types.DataStore.HDD), + show_default=True, + help=click_utils.show_help(types.Dataset)) +@click.option('--dataset', 'opt_dataset', + type=cfg.DatasetVar, + required=True, + show_default=True, + help=click_utils.show_help(types.Dataset)) +@click.option('--size', 'opt_size', + type=(int, int), default=(480, 480), + help='Output image size') +@click.option('--slice', 'opt_slice', type=(int, int), default=(None, None), + help='Slice list of files') +@click.option('--display/--no-display', 'opt_display', is_flag=True, default=False, + help='Display detections to debug') +@click.option('-f', '--force', 'opt_force', is_flag=True, + help='Force overwrite file') +@click.pass_context +def cli(ctx, opt_fp_in, opt_dir_media, opt_fp_out, opt_data_store, opt_dataset, + opt_size, opt_slice, opt_display, opt_force): + """Generates 3D landmark animations from CSV files""" + + import sys + import os + from os.path import join + from pathlib import Path + + from tqdm import tqdm + import numpy as np + import pandas as pd + import matplotlib.pyplot as plt + + from app.utils import logger_utils, file_utils, im_utils, display_utils, draw_utils + from app.utils import plot_utils + from app.processors import face_detector + from app.models.data_store import DataStore + + # ------------------------------------------------- + # init here + + log = logger_utils.Logger.getLogger() + + # set storage location + data_store = DataStore(opt_data_store, opt_dataset) + # load file records + fp_record = data_store.metadata(types.Metadata.FILE_RECORD) # file_record.csv + df_records = pd.read_csv(fp_record).set_index('index') + # load ROI data + fp_roi = data_store.metadata(types.Metadata.FACE_ROI) # face_roi.csv + df_rois = pd.read_csv(fp_roi).set_index('index') + fp_landmark_3d = data_store.metadata(types.Metadata.FACE_LANDMARK_3D_68) # face_landmark_3d_68.csv + df_landmarks_3d = pd.read_csv(fp_landmark_3d).set_index('index') + if opt_slice: + df_landmarks_3d = df_landmarks_3d[opt_slice[0]:opt_slice[1]] # slice if you want + log.debug('processing {:,} groups'.format(len(df_landmarks_3d))) + + # get filepath out + #fp_out = data_store.metadata(types.Metadata.FACE_ROI) if opt_fp_out is None else opt_fp_out + fp_out = '/home/adam/Downloads/3d.gif' + + + for roi_index, df_3d in tqdm(df_landmarks_3d.iterrows(), total=len(df_landmarks_3d)): + log.debug(f'roi_index: {roi_index}') + # unflatten points + points_3d = np.array([(df_3d[f'x{i}'], df_3d[f'y{i}'], df_3d[f'z{i}']) for i in range(1, 68)]) + plot_utils.generate_3d_landmark_anim(points_3d, fp_out, size=(300,300)) + break \ No newline at end of file diff --git a/megapixels/notebooks/face_analysis/3d_face_plot.ipynb b/megapixels/notebooks/face_analysis/3d_face_plot.ipynb index f136015f..591b8706 100644 --- a/megapixels/notebooks/face_analysis/3d_face_plot.ipynb +++ b/megapixels/notebooks/face_analysis/3d_face_plot.ipynb @@ -11,18 +11,9 @@ }, { "cell_type": "code", - "execution_count": 88, + "execution_count": 1, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The autoreload extension is already loaded. To reload it, use:\n", - " %reload_ext autoreload\n" - ] - } - ], + "outputs": [], "source": [ "%load_ext autoreload\n", "%autoreload 2\n", @@ -53,12 +44,14 @@ "from skimage import io\n", "from tqdm import tqdm_notebook as tqdm\n", "from IPython.display import clear_output\n", - "from pathlib import Path" + "from pathlib import Path\n", + "\n", + "sys.path.append('/work/megapixels_dev/megapixels/')" ] }, { "cell_type": "code", - "execution_count": 89, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -70,7 +63,7 @@ }, { "cell_type": "code", - "execution_count": 90, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -81,36 +74,17 @@ }, { "cell_type": "code", - "execution_count": 276, + "execution_count": 4, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 276, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "fp_im = '/data_store_hdd/datasets/people/vgg_face2/media/original/test/n000009/0012_01.jpg'\n", - "im = cv.imread(fp_im)\n", - "plt.imshow(im)" + "im = cv.imread(fp_im)" ] }, { "cell_type": "code", - "execution_count": 287, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 288, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -164,809 +138,106 @@ }, { "cell_type": "code", - "execution_count": 367, + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "from app.utils import im_utils" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "im = cv.imread(fp_im)\n", + "im_resized = im_utils.resize(im, width=300, height=300)\n", + "im_rgb = cv.cvtColor(im, cv.COLOR_BGR2RGB)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "#import dlib\n", + "from app.processors import face_detector" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "#face_detector = face_detector.DetectorDLIBCNN(gpu=0) # -1 for CPU\n", + "face_detector = face_detector.DetectorCVDNN()" + ] + }, + { + "cell_type": "code", + "execution_count": 29, "metadata": {}, "outputs": [ { - "data": { - "application/javascript": [ - "/* Put everything inside the global mpl namespace */\n", - "window.mpl = {};\n", - "\n", - "\n", - "mpl.get_websocket_type = function() {\n", - " if (typeof(WebSocket) !== 'undefined') {\n", - " return WebSocket;\n", - " } else if (typeof(MozWebSocket) !== 'undefined') {\n", - " return MozWebSocket;\n", - " } else {\n", - " alert('Your browser does not have WebSocket support.' +\n", - " 'Please try Chrome, Safari or Firefox ≥ 6. ' +\n", - " 'Firefox 4 and 5 are also supported but you ' +\n", - " 'have to enable WebSockets in about:config.');\n", - " };\n", - "}\n", - "\n", - "mpl.figure = function(figure_id, websocket, ondownload, parent_element) {\n", - " this.id = figure_id;\n", - "\n", - " this.ws = websocket;\n", - "\n", - " this.supports_binary = (this.ws.binaryType != undefined);\n", - "\n", - " if (!this.supports_binary) {\n", - " var warnings = document.getElementById(\"mpl-warnings\");\n", - " if (warnings) {\n", - " warnings.style.display = 'block';\n", - " warnings.textContent = (\n", - " \"This browser does not support binary websocket messages. \" +\n", - " \"Performance may be slow.\");\n", - " }\n", - " }\n", - "\n", - " this.imageObj = new Image();\n", - "\n", - " this.context = undefined;\n", - " this.message = undefined;\n", - " this.canvas = undefined;\n", - " this.rubberband_canvas = undefined;\n", - " this.rubberband_context = undefined;\n", - " this.format_dropdown = undefined;\n", - "\n", - " this.image_mode = 'full';\n", - "\n", - " this.root = $('
');\n", - " this._root_extra_style(this.root)\n", - " this.root.attr('style', 'display: inline-block');\n", - "\n", - " $(parent_element).append(this.root);\n", - "\n", - " this._init_header(this);\n", - " this._init_canvas(this);\n", - " this._init_toolbar(this);\n", - "\n", - " var fig = this;\n", - "\n", - " this.waiting = false;\n", - "\n", - " this.ws.onopen = function () {\n", - " fig.send_message(\"supports_binary\", {value: fig.supports_binary});\n", - " fig.send_message(\"send_image_mode\", {});\n", - " if (mpl.ratio != 1) {\n", - " fig.send_message(\"set_dpi_ratio\", {'dpi_ratio': mpl.ratio});\n", - " }\n", - " fig.send_message(\"refresh\", {});\n", - " }\n", - "\n", - " this.imageObj.onload = function() {\n", - " if (fig.image_mode == 'full') {\n", - " // Full images could contain transparency (where diff images\n", - " // almost always do), so we need to clear the canvas so that\n", - " // there is no ghosting.\n", - " fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n", - " }\n", - " fig.context.drawImage(fig.imageObj, 0, 0);\n", - " };\n", - "\n", - " this.imageObj.onunload = function() {\n", - " fig.ws.close();\n", - " }\n", - "\n", - " this.ws.onmessage = this._make_on_message_function(this);\n", - "\n", - " this.ondownload = ondownload;\n", - "}\n", - "\n", - "mpl.figure.prototype._init_header = function() {\n", - " var titlebar = $(\n", - " '
');\n", - " var titletext = $(\n", - " '
');\n", - " titlebar.append(titletext)\n", - " this.root.append(titlebar);\n", - " this.header = titletext[0];\n", - "}\n", - "\n", - "\n", - "\n", - "mpl.figure.prototype._canvas_extra_style = function(canvas_div) {\n", - "\n", - "}\n", - "\n", - "\n", - "mpl.figure.prototype._root_extra_style = function(canvas_div) {\n", - "\n", - "}\n", - "\n", - "mpl.figure.prototype._init_canvas = function() {\n", - " var fig = this;\n", - "\n", - " var canvas_div = $('
');\n", - "\n", - " canvas_div.attr('style', 'position: relative; clear: both; outline: 0');\n", - "\n", - " function canvas_keyboard_event(event) {\n", - " return fig.key_event(event, event['data']);\n", - " }\n", - "\n", - " canvas_div.keydown('key_press', canvas_keyboard_event);\n", - " canvas_div.keyup('key_release', canvas_keyboard_event);\n", - " this.canvas_div = canvas_div\n", - " this._canvas_extra_style(canvas_div)\n", - " this.root.append(canvas_div);\n", - "\n", - " var canvas = $('');\n", - " canvas.addClass('mpl-canvas');\n", - " canvas.attr('style', \"left: 0; top: 0; z-index: 0; outline: 0\")\n", - "\n", - " this.canvas = canvas[0];\n", - " this.context = canvas[0].getContext(\"2d\");\n", - "\n", - " var backingStore = this.context.backingStorePixelRatio ||\n", - "\tthis.context.webkitBackingStorePixelRatio ||\n", - "\tthis.context.mozBackingStorePixelRatio ||\n", - "\tthis.context.msBackingStorePixelRatio ||\n", - "\tthis.context.oBackingStorePixelRatio ||\n", - "\tthis.context.backingStorePixelRatio || 1;\n", - "\n", - " mpl.ratio = (window.devicePixelRatio || 1) / backingStore;\n", - "\n", - " var rubberband = $('');\n", - " rubberband.attr('style', \"position: absolute; left: 0; top: 0; z-index: 1;\")\n", - "\n", - " var pass_mouse_events = true;\n", - "\n", - " canvas_div.resizable({\n", - " start: function(event, ui) {\n", - " pass_mouse_events = false;\n", - " },\n", - " resize: function(event, ui) {\n", - " fig.request_resize(ui.size.width, ui.size.height);\n", - " },\n", - " stop: function(event, ui) {\n", - " pass_mouse_events = true;\n", - " fig.request_resize(ui.size.width, ui.size.height);\n", - " },\n", - " });\n", - "\n", - " function mouse_event_fn(event) {\n", - " if (pass_mouse_events)\n", - " return fig.mouse_event(event, event['data']);\n", - " }\n", - "\n", - " rubberband.mousedown('button_press', mouse_event_fn);\n", - " rubberband.mouseup('button_release', mouse_event_fn);\n", - " // Throttle sequential mouse events to 1 every 20ms.\n", - " rubberband.mousemove('motion_notify', mouse_event_fn);\n", - "\n", - " rubberband.mouseenter('figure_enter', mouse_event_fn);\n", - " rubberband.mouseleave('figure_leave', mouse_event_fn);\n", - "\n", - " canvas_div.on(\"wheel\", function (event) {\n", - " event = event.originalEvent;\n", - " event['data'] = 'scroll'\n", - " if (event.deltaY < 0) {\n", - " event.step = 1;\n", - " } else {\n", - " event.step = -1;\n", - " }\n", - " mouse_event_fn(event);\n", - " });\n", - "\n", - " canvas_div.append(canvas);\n", - " canvas_div.append(rubberband);\n", - "\n", - " this.rubberband = rubberband;\n", - " this.rubberband_canvas = rubberband[0];\n", - " this.rubberband_context = rubberband[0].getContext(\"2d\");\n", - " this.rubberband_context.strokeStyle = \"#000000\";\n", - "\n", - " this._resize_canvas = function(width, height) {\n", - " // Keep the size of the canvas, canvas container, and rubber band\n", - " // canvas in synch.\n", - " canvas_div.css('width', width)\n", - " canvas_div.css('height', height)\n", - "\n", - " canvas.attr('width', width * mpl.ratio);\n", - " canvas.attr('height', height * mpl.ratio);\n", - " canvas.attr('style', 'width: ' + width + 'px; height: ' + height + 'px;');\n", - "\n", - " rubberband.attr('width', width);\n", - " rubberband.attr('height', height);\n", - " }\n", - "\n", - " // Set the figure to an initial 600x600px, this will subsequently be updated\n", - " // upon first draw.\n", - " this._resize_canvas(600, 600);\n", - "\n", - " // Disable right mouse context menu.\n", - " $(this.rubberband_canvas).bind(\"contextmenu\",function(e){\n", - " return false;\n", - " });\n", - "\n", - " function set_focus () {\n", - " canvas.focus();\n", - " canvas_div.focus();\n", - " }\n", - "\n", - " window.setTimeout(set_focus, 100);\n", - "}\n", - "\n", - "mpl.figure.prototype._init_toolbar = function() {\n", - " var fig = this;\n", - "\n", - " var nav_element = $('
')\n", - " nav_element.attr('style', 'width: 100%');\n", - " this.root.append(nav_element);\n", - "\n", - " // Define a callback function for later on.\n", - " function toolbar_event(event) {\n", - " return fig.toolbar_button_onclick(event['data']);\n", - " }\n", - " function toolbar_mouse_event(event) {\n", - " return fig.toolbar_button_onmouseover(event['data']);\n", - " }\n", - "\n", - " for(var toolbar_ind in mpl.toolbar_items) {\n", - " var name = mpl.toolbar_items[toolbar_ind][0];\n", - " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", - " var image = mpl.toolbar_items[toolbar_ind][2];\n", - " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", - "\n", - " if (!name) {\n", - " // put a spacer in here.\n", - " continue;\n", - " }\n", - " var button = $('