{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# 3D Face Plot\n", "\n", "Process faces for Georgetown Color of Surveillance" ] }, { "cell_type": "code", "execution_count": 128, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "The autoreload extension is already loaded. To reload it, use:\n", " %reload_ext autoreload\n" ] } ], "source": [ "%load_ext autoreload\n", "%autoreload 2\n", "import os\n", "from os.path import join\n", "import sys\n", "import time\n", "import cv2 as cv\n", "import numpy as np\n", "import imutils\n", "import matplotlib.animation\n", "%matplotlib notebook\n", "from glob import glob\n", "from matplotlib import cbook\n", "from matplotlib import cm\n", "from matplotlib.colors import LightSource\n", "from random import randint\n", "sys.path.append('/megapixels/3rdparty/face-alignment')\n", "import face_alignment\n", "import numpy as np\n", "from mpl_toolkits.mplot3d import Axes3D\n", "import matplotlib.pyplot as plt\n", "import mpl_toolkits.mplot3d.axes3d as p3\n", "from matplotlib import animation\n", "import random\n", "from skimage import io\n", "from tqdm import tqdm_notebook as tqdm\n", "from IPython.display import clear_output\n", "from pathlib import Path\n" ] }, { "cell_type": "code", "execution_count": 129, "metadata": {}, "outputs": [], "source": [ "# init 3d face\n", "# Run the 3D face alignment on a test image, without CUDA.\n", "fa = face_alignment.FaceAlignment(face_alignment.LandmarksType._3D, \n", " enable_cuda=True, flip_input=True)" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "100\n" ] } ], "source": [ "data_bodega = '/megapixels/data_bodega/'\n", "fp = join(data_bodega,'images/senators/*.jpg')\n", "face_files = glob(fp,recursive=True)\n", "print(len(face_files))" ] }, { "cell_type": "code", "execution_count": 97, "metadata": {}, "outputs": [], "source": [ "\n", "def ensure_dir(d):\n", " \"\"\"Create directory if not exist\n", " :param d: path to directory\n", " \"\"\"\n", " if not os.path.exists(d):\n", " os.makedirs(d, exist_ok=True)" ] }, { "cell_type": "code", "execution_count": 109, "metadata": {}, "outputs": [], "source": [ "def generate_3d_face(fpath,output_dir,anim=False,ext='png'):\n", " # load image\n", " im = io.imread(fpath)\n", " \n", " # generate 3d predictions\n", " preds = fa.get_landmarks(im)\n", " if preds is None:\n", " return\n", " preds = preds[-1]\n", " \n", " # plot style\n", " num_frames = 60\n", " lw = 2 # line weight\n", " bg_color = '#%02x%02x%02x' % (60,59,110)\n", " mark_clr = '#%02x%02x%02x' % (255,255,255)\n", " mark_clr = '#%02x%02x%02x' % (0,255,0)\n", " #mark_type='$\\star$'\n", " #mark_size = 20\n", " mark_type='o'\n", " mark_size = 10\n", " rh = '#ffffff'\n", " dpi = 72\n", " figsize = (16,16)\n", " \n", " # center x,y,z\n", " xmm = (np.min(preds[:,0]),np.max(preds[:,0]))\n", " ymm = (np.min(preds[:,1]),np.max(preds[:,1]))\n", " zmm = (np.min(preds[:,2]),np.max(preds[:,2]))\n", " \n", " preds_orig = preds.copy()\n", " xmm_sc = (1.2*np.min(preds[:,0]),1.2*np.max(preds_orig[:,0]))\n", " xmm = (np.min(preds_orig[:,0]),np.max(preds_orig[:,0]))\n", " ymm = (np.min(preds_orig[:,1]),np.max(preds_orig[:,1]))\n", " zmm = (np.min(preds_orig[:,2]),np.max(preds_orig[:,2]))\n", " \n", " # swap the y and z components to improve 3d rotation angles\n", " preds = np.zeros_like(preds_orig).astype(np.uint8)\n", " for i,p in enumerate(preds_orig):\n", " x,y,z = p\n", " #preds[i] = np.array([x - xmm[0], y - ymm[0], z - zmm[0]]) # ?\n", " preds[i] = np.array([x - xmm[0], z - zmm[0], y - ymm[0]])\n", " \n", " # Create plot\n", " fig = plt.figure(figsize=figsize,dpi=dpi)\n", " fig.tight_layout()\n", " ax = fig.add_subplot(111, projection='3d')\n", " ax.set_facecolor(bg_color) # background color\n", " \n", " preds_plot = np.zeros_like(preds)\n", " for i,p in enumerate(preds):\n", " x,y,z = p\n", " preds_plot[i] = np.array([x,y,z])\n", "\n", " \n", " # scatter plot the dots\n", " ax.plot3D(preds_plot[:17,0]*1.2,preds_plot[:17,1], preds_plot[:17,2],\n", " marker=mark_type,markersize=mark_size,color=mark_clr,linewidth=lw)\n", " ax.plot3D(preds_plot[17:22,0]*1.2,preds_plot[17:22,1],preds_plot[17:22,2],\n", " marker=mark_type,markersize=mark_size,color=mark_clr,linewidth=lw)\n", " ax.plot3D(preds_plot[22:27,0]*1.2,preds_plot[22:27,1],preds_plot[22:27,2], \n", " marker=mark_type,markersize=mark_size,color=mark_clr,linewidth=lw)\n", " ax.plot3D(preds_plot[27:31,0]*1.2,preds_plot[27:31,1],preds_plot[27:31,2],\n", " marker=mark_type,markersize=mark_size,color=mark_clr,linewidth=lw)\n", " ax.plot3D(preds_plot[31:36,0]*1.2,preds_plot[31:36,1],preds_plot[31:36,2],\n", " marker=mark_type,markersize=mark_size,color=mark_clr,linewidth=lw)\n", " ax.plot3D(preds_plot[36:42,0]*1.2,preds_plot[36:42,1],preds_plot[36:42,2],\n", " marker=mark_type,markersize=mark_size,color=mark_clr,linewidth=lw)\n", " ax.plot3D(preds_plot[42:48,0]*1.2,preds_plot[42:48,1],preds_plot[42:48,2],\n", " marker=mark_type,markersize=mark_size,color=mark_clr,linewidth=lw)\n", " ax.plot3D(preds_plot[48:,0]*1.2,preds_plot[48:,1],preds_plot[48:,2],\n", " marker=mark_type,markersize=mark_size,color=mark_clr, linewidth=lw)\n", "\n", " \n", " ax.scatter(preds_plot[:,0]*1.2,preds_plot[:,1],preds_plot[:,2],c=rh, alpha=1.0, s=35, edgecolor=rh)\n", " \n", " # center points\n", " cx = ((xmm[0] - xmm[1]) // 2) + xmm[1]\n", " cy = ((ymm[1] - ymm[0]) // 2) + ymm[0]\n", " cz = ((zmm[1] - zmm[0]) // 2) + zmm[0]\n", " \n", " # ?\n", " ax.view_init(elev=120., azim=70.)\n", " \n", " ax.set_xticks([])\n", " ax.set_yticks([])\n", " ax.set_zticks([])\n", " ax.set_axis_off()\n", " ax.set_xlabel('x')\n", " ax.set_ylabel('y')\n", " ax.set_zlabel('z')\n", "\n", " \n", " phis = np.linspace(0, 2*np.pi, num_frames)\n", " \n", " if anim:\n", " def update(phi):\n", " ax.view_init(180,phi*180./np.pi)\n", " fname_out = join(output_dir,'{}.gif'.format(Path(fpath).stem))\n", " ani = matplotlib.animation.FuncAnimation(fig, update, frames=phis)\n", " ani.save(fname_out, writer='imagemagick', fps=10)\n", " del ani\n", " else:\n", " for i,phi in enumerate(phis):\n", " # ext (jpg,png,pdf)\n", " person_name = Path(fpath).stem\n", " fname_out = join(output_dir,person_name,'{}.{}'.format(str(i).zfill(4), ext))\n", " ensure_dir(Path(fname_out).parent)\n", " ax.view_init(180,phi*180./np.pi)\n", " fig.savefig(fname_out, transparent=True)" ] }, { "cell_type": "code", "execution_count": 110, "metadata": {}, "outputs": [], "source": [ "output_dir = join(data_bodega,'output','senators_3d_points_frames')" ] }, { "cell_type": "code", "execution_count": 123, "metadata": { "scrolled": false }, "outputs": [ { "data": { "application/javascript": [ "/* Put everything inside the global mpl namespace */\n", "window.mpl = {};\n", "\n", "\n", "mpl.get_websocket_type = function() {\n", " if (typeof(WebSocket) !== 'undefined') {\n", " return WebSocket;\n", " } else if (typeof(MozWebSocket) !== 'undefined') {\n", " return MozWebSocket;\n", " } else {\n", " alert('Your browser does not have WebSocket support.' +\n", " 'Please try Chrome, Safari or Firefox ≥ 6. ' +\n", " 'Firefox 4 and 5 are also supported but you ' +\n", " 'have to enable WebSockets in about:config.');\n", " };\n", "}\n", "\n", "mpl.figure = function(figure_id, websocket, ondownload, parent_element) {\n", " this.id = figure_id;\n", "\n", " this.ws = websocket;\n", "\n", " this.supports_binary = (this.ws.binaryType != undefined);\n", "\n", " if (!this.supports_binary) {\n", " var warnings = document.getElementById(\"mpl-warnings\");\n", " if (warnings) {\n", " warnings.style.display = 'block';\n", " warnings.textContent = (\n", " \"This browser does not support binary websocket messages. \" +\n", " \"Performance may be slow.\");\n", " }\n", " }\n", "\n", " this.imageObj = new Image();\n", "\n", " this.context = undefined;\n", " this.message = undefined;\n", " this.canvas = undefined;\n", " this.rubberband_canvas = undefined;\n", " this.rubberband_context = undefined;\n", " this.format_dropdown = undefined;\n", "\n", " this.image_mode = 'full';\n", "\n", " this.root = $('
');\n", " this._root_extra_style(this.root)\n", " this.root.attr('style', 'display: inline-block');\n", "\n", " $(parent_element).append(this.root);\n", "\n", " this._init_header(this);\n", " this._init_canvas(this);\n", " this._init_toolbar(this);\n", "\n", " var fig = this;\n", "\n", " this.waiting = false;\n", "\n", " this.ws.onopen = function () {\n", " fig.send_message(\"supports_binary\", {value: fig.supports_binary});\n", " fig.send_message(\"send_image_mode\", {});\n", " if (mpl.ratio != 1) {\n", " fig.send_message(\"set_dpi_ratio\", {'dpi_ratio': mpl.ratio});\n", " }\n", " fig.send_message(\"refresh\", {});\n", " }\n", "\n", " this.imageObj.onload = function() {\n", " if (fig.image_mode == 'full') {\n", " // Full images could contain transparency (where diff images\n", " // almost always do), so we need to clear the canvas so that\n", " // there is no ghosting.\n", " fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n", " }\n", " fig.context.drawImage(fig.imageObj, 0, 0);\n", " };\n", "\n", " this.imageObj.onunload = function() {\n", " fig.ws.close();\n", " }\n", "\n", " this.ws.onmessage = this._make_on_message_function(this);\n", "\n", " this.ondownload = ondownload;\n", "}\n", "\n", "mpl.figure.prototype._init_header = function() {\n", " var titlebar = $(\n", " '');\n", " var titletext = $(\n", " '');\n", " titlebar.append(titletext)\n", " this.root.append(titlebar);\n", " this.header = titletext[0];\n", "}\n", "\n", "\n", "\n", "mpl.figure.prototype._canvas_extra_style = function(canvas_div) {\n", "\n", "}\n", "\n", "\n", "mpl.figure.prototype._root_extra_style = function(canvas_div) {\n", "\n", "}\n", "\n", "mpl.figure.prototype._init_canvas = function() {\n", " var fig = this;\n", "\n", " var canvas_div = $('');\n", "\n", " canvas_div.attr('style', 'position: relative; clear: both; outline: 0');\n", "\n", " function canvas_keyboard_event(event) {\n", " return fig.key_event(event, event['data']);\n", " }\n", "\n", " canvas_div.keydown('key_press', canvas_keyboard_event);\n", " canvas_div.keyup('key_release', canvas_keyboard_event);\n", " this.canvas_div = canvas_div\n", " this._canvas_extra_style(canvas_div)\n", " this.root.append(canvas_div);\n", "\n", " var canvas = $('');\n", " canvas.addClass('mpl-canvas');\n", " canvas.attr('style', \"left: 0; top: 0; z-index: 0; outline: 0\")\n", "\n", " this.canvas = canvas[0];\n", " this.context = canvas[0].getContext(\"2d\");\n", "\n", " var backingStore = this.context.backingStorePixelRatio ||\n", "\tthis.context.webkitBackingStorePixelRatio ||\n", "\tthis.context.mozBackingStorePixelRatio ||\n", "\tthis.context.msBackingStorePixelRatio ||\n", "\tthis.context.oBackingStorePixelRatio ||\n", "\tthis.context.backingStorePixelRatio || 1;\n", "\n", " mpl.ratio = (window.devicePixelRatio || 1) / backingStore;\n", "\n", " var rubberband = $('');\n", " rubberband.attr('style', \"position: absolute; left: 0; top: 0; z-index: 1;\")\n", "\n", " var pass_mouse_events = true;\n", "\n", " canvas_div.resizable({\n", " start: function(event, ui) {\n", " pass_mouse_events = false;\n", " },\n", " resize: function(event, ui) {\n", " fig.request_resize(ui.size.width, ui.size.height);\n", " },\n", " stop: function(event, ui) {\n", " pass_mouse_events = true;\n", " fig.request_resize(ui.size.width, ui.size.height);\n", " },\n", " });\n", "\n", " function mouse_event_fn(event) {\n", " if (pass_mouse_events)\n", " return fig.mouse_event(event, event['data']);\n", " }\n", "\n", " rubberband.mousedown('button_press', mouse_event_fn);\n", " rubberband.mouseup('button_release', mouse_event_fn);\n", " // Throttle sequential mouse events to 1 every 20ms.\n", " rubberband.mousemove('motion_notify', mouse_event_fn);\n", "\n", " rubberband.mouseenter('figure_enter', mouse_event_fn);\n", " rubberband.mouseleave('figure_leave', mouse_event_fn);\n", "\n", " canvas_div.on(\"wheel\", function (event) {\n", " event = event.originalEvent;\n", " event['data'] = 'scroll'\n", " if (event.deltaY < 0) {\n", " event.step = 1;\n", " } else {\n", " event.step = -1;\n", " }\n", " mouse_event_fn(event);\n", " });\n", "\n", " canvas_div.append(canvas);\n", " canvas_div.append(rubberband);\n", "\n", " this.rubberband = rubberband;\n", " this.rubberband_canvas = rubberband[0];\n", " this.rubberband_context = rubberband[0].getContext(\"2d\");\n", " this.rubberband_context.strokeStyle = \"#000000\";\n", "\n", " this._resize_canvas = function(width, height) {\n", " // Keep the size of the canvas, canvas container, and rubber band\n", " // canvas in synch.\n", " canvas_div.css('width', width)\n", " canvas_div.css('height', height)\n", "\n", " canvas.attr('width', width * mpl.ratio);\n", " canvas.attr('height', height * mpl.ratio);\n", " canvas.attr('style', 'width: ' + width + 'px; height: ' + height + 'px;');\n", "\n", " rubberband.attr('width', width);\n", " rubberband.attr('height', height);\n", " }\n", "\n", " // Set the figure to an initial 600x600px, this will subsequently be updated\n", " // upon first draw.\n", " this._resize_canvas(600, 600);\n", "\n", " // Disable right mouse context menu.\n", " $(this.rubberband_canvas).bind(\"contextmenu\",function(e){\n", " return false;\n", " });\n", "\n", " function set_focus () {\n", " canvas.focus();\n", " canvas_div.focus();\n", " }\n", "\n", " window.setTimeout(set_focus, 100);\n", "}\n", "\n", "mpl.figure.prototype._init_toolbar = function() {\n", " var fig = this;\n", "\n", " var nav_element = $('')\n", " nav_element.attr('style', 'width: 100%');\n", " this.root.append(nav_element);\n", "\n", " // Define a callback function for later on.\n", " function toolbar_event(event) {\n", " return fig.toolbar_button_onclick(event['data']);\n", " }\n", " function toolbar_mouse_event(event) {\n", " return fig.toolbar_button_onmouseover(event['data']);\n", " }\n", "\n", " for(var toolbar_ind in mpl.toolbar_items) {\n", " var name = mpl.toolbar_items[toolbar_ind][0];\n", " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", " var image = mpl.toolbar_items[toolbar_ind][2];\n", " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", "\n", " if (!name) {\n", " // put a spacer in here.\n", " continue;\n", " }\n", " var button = $('');\n", " button.addClass('ui-button ui-widget ui-state-default ui-corner-all ' +\n", " 'ui-button-icon-only');\n", " button.attr('role', 'button');\n", " button.attr('aria-disabled', 'false');\n", " button.click(method_name, toolbar_event);\n", " button.mouseover(tooltip, toolbar_mouse_event);\n", "\n", " var icon_img = $('');\n", " icon_img.addClass('ui-button-icon-primary ui-icon');\n", " icon_img.addClass(image);\n", " icon_img.addClass('ui-corner-all');\n", "\n", " var tooltip_span = $('');\n", " tooltip_span.addClass('ui-button-text');\n", " tooltip_span.html(tooltip);\n", "\n", " button.append(icon_img);\n", " button.append(tooltip_span);\n", "\n", " nav_element.append(button);\n", " }\n", "\n", " var fmt_picker_span = $('');\n", "\n", " var fmt_picker = $('');\n", " fmt_picker.addClass('mpl-toolbar-option ui-widget ui-widget-content');\n", " fmt_picker_span.append(fmt_picker);\n", " nav_element.append(fmt_picker_span);\n", " this.format_dropdown = fmt_picker[0];\n", "\n", " for (var ind in mpl.extensions) {\n", " var fmt = mpl.extensions[ind];\n", " var option = $(\n", " '', {selected: fmt === mpl.default_extension}).html(fmt);\n", " fmt_picker.append(option)\n", " }\n", "\n", " // Add hover states to the ui-buttons\n", " $( \".ui-button\" ).hover(\n", " function() { $(this).addClass(\"ui-state-hover\");},\n", " function() { $(this).removeClass(\"ui-state-hover\");}\n", " );\n", "\n", " var status_bar = $('