{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3D Face Plot\n",
    "\n",
    "Attenzione visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "import os\n",
    "from os.path import join\n",
    "import sys\n",
    "import time\n",
    "from random import randint\n",
    "import random\n",
    "\n",
    "import cv2 as cv\n",
    "import numpy as np\n",
    "import imutils\n",
    "import matplotlib.animation\n",
    "%matplotlib notebook\n",
    "from glob import glob\n",
    "from matplotlib import cbook\n",
    "from matplotlib import cm\n",
    "#from matplotlib.colors import LightSource\n",
    "import face_alignment\n",
    "import numpy as np\n",
    "\n",
    "from mpl_toolkits.mplot3d import Axes3D\n",
    "import matplotlib.pyplot as plt\n",
    "import mpl_toolkits.mplot3d.axes3d as p3\n",
    "from matplotlib import animation\n",
    "\n",
    "from skimage import io\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "from IPython.display import clear_output\n",
    "from pathlib import Path\n",
    "\n",
    "sys.path.append('/work/megapixels_dev/megapixels/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate random hex colors\n",
    "def rhex():\n",
    "  r = lambda: random.randint(0,255)\n",
    "  return '#%02X%02X%02X' % (r(), r(), r())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# init 3d face\n",
    "# Run the 3D face alignment on a test image, without CUDA.\n",
    "fa = face_alignment.FaceAlignment(face_alignment.LandmarksType._3D, device='cuda:0', flip_input=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "fp_im = '/data_store_hdd/datasets/people/vgg_face2/media/original/test/n000009/0012_01.jpg'\n",
    "im = cv.imread(fp_im)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_3d_face_plain(im, lm):\n",
    "  preds = lm\n",
    "  fig = plt.figure(figsize=plt.figaspect(.5))\n",
    "  ax = fig.add_subplot(1, 2, 1)\n",
    "  ax.imshow(im)\n",
    "  ax.plot(preds[0:17,0],preds[0:17,1],marker='o',markersize=6,linestyle='-',color='w',lw=2)\n",
    "  ax.plot(preds[17:22,0],preds[17:22,1],marker='o',markersize=6,linestyle='-',color='w',lw=2)\n",
    "  ax.plot(preds[22:27,0],preds[22:27,1],marker='o',markersize=6,linestyle='-',color='w',lw=2)\n",
    "  ax.plot(preds[27:31,0],preds[27:31,1],marker='o',markersize=6,linestyle='-',color='w',lw=2)\n",
    "  ax.plot(preds[31:36,0],preds[31:36,1],marker='o',markersize=6,linestyle='-',color='w',lw=2)\n",
    "  ax.plot(preds[36:42,0],preds[36:42,1],marker='o',markersize=6,linestyle='-',color='w',lw=2)\n",
    "  ax.plot(preds[42:48,0],preds[42:48,1],marker='o',markersize=6,linestyle='-',color='w',lw=2)\n",
    "  ax.plot(preds[48:60,0],preds[48:60,1],marker='o',markersize=6,linestyle='-',color='w',lw=2)\n",
    "  ax.plot(preds[60:68,0],preds[60:68,1],marker='o',markersize=6,linestyle='-',color='w',lw=2) \n",
    "  ax.axis('off')\n",
    "\n",
    "  ax = fig.add_subplot(1, 2, 2, projection='3d')\n",
    "  surf = ax.scatter(preds[:,0]*1.2,preds[:,1],preds[:,2],c=\"cyan\", alpha=1.0, edgecolor='b')\n",
    "  ax.plot3D(preds[:17,0]*1.2,preds[:17,1], preds[:17,2], color='blue' )\n",
    "  ax.plot3D(preds[17:22,0]*1.2,preds[17:22,1],preds[17:22,2], color='blue')\n",
    "  ax.plot3D(preds[22:27,0]*1.2,preds[22:27,1],preds[22:27,2], color='blue')\n",
    "  ax.plot3D(preds[27:31,0]*1.2,preds[27:31,1],preds[27:31,2], color='blue')\n",
    "  ax.plot3D(preds[31:36,0]*1.2,preds[31:36,1],preds[31:36,2], color='blue')\n",
    "  ax.plot3D(preds[36:42,0]*1.2,preds[36:42,1],preds[36:42,2], color='blue')\n",
    "  ax.plot3D(preds[42:48,0]*1.2,preds[42:48,1],preds[42:48,2], color='blue')\n",
    "  ax.plot3D(preds[48:,0]*1.2,preds[48:,1],preds[48:,2], color='blue' )\n",
    "  \n",
    "  # pad\n",
    "  xmm = (np.min(lm[:,0]),np.max(lm[:,0]))\n",
    "  ymm = (np.min(lm[:,1]),np.max(lm[:,1]))\n",
    "  zmm = (np.min(lm[:,2]),np.max(lm[:,2]))\n",
    "  \n",
    "  print(xmm, ymm, zmm)\n",
    "#   ax.set_xticks([])\n",
    "#   ax.set_yticks([])\n",
    "#   ax.set_zticks([])\n",
    "  plt.setp( ax.get_xticklabels(), visible=False)\n",
    "  plt.setp( ax.get_yticklabels(), visible=False)\n",
    "  #ax.set_xlim(xmm[0]-50, xmm[1]+50)\n",
    "  #ax.set_ylim(ymm[0]-50, ymm[1]+50)\n",
    "  #ax.set_ylim(zmm[0]- .1*zmm[0],zmm[1] + .1*zmm[1])\n",
    "  #ax.set_ylim(103, 275)\n",
    "  #ax.set_zlim((-100,100))\n",
    "  ax.view_init(elev=15., azim=135.)\n",
    "\n",
    "  plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from app.utils import im_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "im = cv.imread(fp_im)\n",
    "im_resized = im_utils.resize(im, width=300, height=300)\n",
    "im_rgb = cv.cvtColor(im, cv.COLOR_BGR2RGB)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import dlib\n",
    "from app.processors import face_detector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#face_detector = face_detector.DetectorDLIBCNN(gpu=0)  # -1 for CPU\n",
    "face_detector = face_detector.DetectorCVDNN()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "detecting face...\n"
     ]
    }
   ],
   "source": [
    "print('detecting face...')\n",
    "st = time.time()\n",
    "bboxes = face_detector.detect(im_resized, largest=True)\n",
    "bbox = bboxes[0].to_dim(im_resized.shape[:2][::-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(72, 73, 197, 225)\n"
     ]
    }
   ],
   "source": [
    "bbox = bbox.to_xyxy()\n",
    "print(bbox)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "points = fa.get_landmarks_from_image(im_resized, [bbox] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lm = fa.get_landmarks(im_rgb)[-1]\n",
    "generate_3d_face_plain(im_rgb, lm)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "im = cv.imread(fp_im)\n",
    "im_rgb = cv.cvtColor(im, cv.COLOR_BGR2RGB)\n",
    "lm = fa.get_landmarks(im_rgb)[-1]\n",
    "generate_3d_face_plain(im_rgb, lm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    " # line weight\n",
    "def generate_3d_face(lm, fp_out, num_frames=30, dpi=72, stroke_weight=2, size=(480,480),\n",
    "  mark_size=10, mark_type='.', mark_clr=(0,255,0), fps=10, transparent=False):\n",
    "  '''Generates 3D plot of face landmarks\n",
    "  '''\n",
    "  # convert opencv BGR numpy image to RGB\n",
    "  bg_color = '#%02x%02x%02x' % (0,0,0)\n",
    "  mark_clr = '#%02x%02x%02x' % (0,255,255)\n",
    "  \n",
    "  # scale to make larger\n",
    "  #lm = np.array([1.2*x,y,z] for x,y,z in list(lm))\n",
    "  \n",
    "  # center x,y,z\n",
    "  xmm = (np.min(lm[:,0]),np.max(lm[:,0]))\n",
    "  ymm = (np.min(lm[:,1]),np.max(lm[:,1]))\n",
    "  zmm = (np.min(lm[:,2]),np.max(lm[:,2]))\n",
    "  \n",
    "  # make copy of landmarks\n",
    "  lm_orig = lm.copy()\n",
    "  xmm = (np.min(lm_orig[:,0]),np.max(lm_orig[:,0]))\n",
    "  ymm = (np.min(lm_orig[:,1]),np.max(lm_orig[:,1]))\n",
    "  zmm = (np.min(lm_orig[:,2]),np.max(lm_orig[:,2]))\n",
    "  \n",
    "  # swap the y and z components to improve 3d rotation angles for matplotlib\n",
    "  lm = np.zeros_like(lm_orig).astype(np.uint8)\n",
    "  for i,p in enumerate(lm_orig):\n",
    "    x,y,z = p\n",
    "    lm[i] = np.array([x - xmm[0], z - zmm[0], y - ymm[0]])\n",
    "  \n",
    "  # Create plot\n",
    "  figsize = (size[0]/dpi, size[1]/dpi )\n",
    "  fig = plt.figure(figsize=figsize, dpi=dpi)  # frameon=False\n",
    "  #fig.set_size_inches(100/100, 1, forward=False)\n",
    "  fig.tight_layout()\n",
    "  fig.subplots_adjust(left=0, bottom=0, right=1, top=1, wspace=None, hspace=None)\n",
    "  ax = fig.add_subplot(111, projection='3d')\n",
    "  ax.set_facecolor(bg_color) # background color\n",
    "  \n",
    "  xscale, yscale, zscale = (1.2, 1.0, 1.0)\n",
    "  \n",
    "  # scatter plot the dots\n",
    "  # jaw line\n",
    "  mark_clr = '#%02x%02x%02x' % (0,255,0)  # green\n",
    "  ax.plot3D(lm[:17,0]*1.2,lm[:17,1], lm[:17,2],\n",
    "            marker=mark_type, markersize=mark_size, color=mark_clr,linewidth=stroke_weight)\n",
    "  # stage-right eyebrow\n",
    "  mark_clr = '#%02x%02x%02x' % (255,0,0)  # green\n",
    "  ax.plot3D(lm[17:22,0]*1.2,lm[17:22,1],lm[17:22,2],\n",
    "            marker=mark_type, markersize=mark_size, color=mark_clr,linewidth=stroke_weight)\n",
    "  # stage-left eyebrow\n",
    "  mark_clr = '#%02x%02x%02x' % (255,255,0)  # yellow\n",
    "  ax.plot3D(lm[22:27,0]*1.2,lm[22:27,1],lm[22:27,2], \n",
    "            marker=mark_type, markersize=mark_size, color=mark_clr,linewidth=stroke_weight)\n",
    "  # nose ridge\n",
    "  mark_clr = '#%02x%02x%02x' % (0,0,255)  # blue\n",
    "  ax.plot3D(lm[27:31,0]*1.2,lm[27:31,1],lm[27:31,2],\n",
    "            marker=mark_type, markersize=mark_size, color=mark_clr,linewidth=stroke_weight)\n",
    "  # nose-bottom\n",
    "  mark_clr = '#%02x%02x%02x' % (255,0,255)  # magenta\n",
    "  ax.plot3D(lm[31:36,0]*1.2,lm[31:36,1],lm[31:36,2],\n",
    "            marker=mark_type, markersize=mark_size, color=mark_clr,linewidth=stroke_weight)\n",
    "  # stage-left eye\n",
    "  mark_clr = '#%02x%02x%02x' % (0,255,255)  # cyan\n",
    "  px, py, pz = lm[36:42,0]*1.2,lm[36:42,1],lm[36:42,2]\n",
    "  px = np.append(px, lm[36,0]*1.2)\n",
    "  py = np.append(py, lm[36,1])\n",
    "  pz = np.append(pz, lm[36,2])\n",
    "  ax.plot3D(px, py, pz, marker=mark_type, markersize=mark_size, color=mark_clr,linewidth=stroke_weight)\n",
    "  \n",
    "  # stage-right eye\n",
    "  mark_clr = '#%02x%02x%02x' % (255,255,255)  # white\n",
    "  px, py, pz = lm[42:48,0]*1.2,lm[42:48,1],lm[42:48,2]\n",
    "  px = np.append(px, lm[42,0]*1.2)\n",
    "  py = np.append(py, lm[42,1])\n",
    "  pz = np.append(pz, lm[42,2])\n",
    "  ax.plot3D(px, py, pz, marker=mark_type, markersize=mark_size, color=mark_clr,linewidth=stroke_weight)\n",
    "  \n",
    "  # mouth\n",
    "  mark_clr = '#%02x%02x%02x' % (255,125,0)  # orange?\n",
    "  px, py, pz = lm[48:,0]*1.2,lm[48:,1],lm[48:,2]\n",
    "  px = np.append(px, lm[48,0]*1.2)\n",
    "  py = np.append(py, lm[48,1])\n",
    "  pz = np.append(pz, lm[48,2])\n",
    "  ax.plot3D(px, py, pz, marker=mark_type, markersize=mark_size, color=mark_clr, linewidth=stroke_weight)\n",
    "  \n",
    "  rh = '#00ff00'  # edge color\n",
    "  #ax.scatter(lm[:,0]*xscale,lm[:,1]*yscale,lm[:,2]*zscale, c=rh, alpha=1.0, s=35, edgecolor=rh)\n",
    "  #ax.scatter(lm[:,0]*xscale,lm[:,1]*yscale,lm[:,2]*zscale, c=rh, alpha=1.0, s=1)\n",
    "  \n",
    "  # center center x,y,z points\n",
    "  cx = ((xmm[0] - xmm[1]) // 2) + xmm[1]\n",
    "  cy = ((ymm[1] - ymm[0]) // 2) + ymm[0]\n",
    "  cz = ((zmm[1] - zmm[0]) // 2) + zmm[0]\n",
    "  \n",
    "  # set initial plot view\n",
    "  ax.view_init(elev=120., azim=70.)\n",
    "  \n",
    "  # remove ticks\n",
    "  ax.set_xticks([])\n",
    "  ax.set_yticks([])\n",
    "  ax.set_zticks([])\n",
    "    \n",
    "  # remove axis\n",
    "  ax.set_frame_on(False)\n",
    "  ax.set_axis_off()\n",
    "\n",
    "  # rotation increments: from 0 to 360 in num_frames\n",
    "  phi = np.linspace(0, 2*np.pi, num_frames)\n",
    "\n",
    "  def update(phi):\n",
    "    ax.view_init(180,phi*180./np.pi)\n",
    "  \n",
    "  ani = matplotlib.animation.FuncAnimation(fig, update, frames=phi)\n",
    "  savefig_kwargs = {'pad_inches': 0, 'transparent': transparent}\n",
    "  ani.save(fp_out, writer='imagemagick', fps=fps, savefig_kwargs=savefig_kwargs)\n",
    "  clear_output()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# filepaths\n",
    "dir_out = '/home/adam/Downloads/'\n",
    "fp_out = join(dir_out, '{}.gif'.format(Path(fp_im).stem))\n",
    "\n",
    "# generate 3D face\n",
    "st = time.time()\n",
    "generate_3d_face(lm,fp_out, num_frames=20, fps=10)\n",
    "print('time: {:.4f}'.format((time.time()-st)/1000))\n",
    "print(f'Saved file to {fp_out}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from PIL import Image\n",
    "fp_in = '/home/adam/Downloads/3d.gif'\n",
    "im = Image.open(fp_in)\n",
    "im_frames = []\n",
    "duration = im.info['duration']\n",
    "print(duration)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "im = Image.open(fp_in)\n",
    "#im_frame = im.seek(1)\n",
    "im.seek(im.tell() + 1)\n",
    "mypalette = im.getpalette()\n",
    "im.putpalette(mypalette)\n",
    "\n",
    "im_jpg = Image.new(\"RGB\", im.size)\n",
    "im_jpg.paste(im)\n",
    "\n",
    "im_jpg.save('/home/adam/Downloads/test.jpg')\n",
    "print(im_jpg)\n",
    "#plt.imshow(im_jpg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "im_jpg.getchannel(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.imshow(im_jpg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import dlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dlib.DLIB_USE_CUDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from app.settings import app_cfg as cfg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:megapixels]",
   "language": "python",
   "name": "conda-env-megapixels-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}