From 38746f284b17400d4e2555509ea60df5912b824a Mon Sep 17 00:00:00 2001
From: Jules Laplace <julescarbon@gmail.com>
Date: Fri, 14 Dec 2018 18:10:27 +0100
Subject: all the sql stuff communicating nicely

---
 megapixels/app/server/api.py | 72 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 72 insertions(+)
 create mode 100644 megapixels/app/server/api.py

(limited to 'megapixels/app/server/api.py')

diff --git a/megapixels/app/server/api.py b/megapixels/app/server/api.py
new file mode 100644
index 00000000..e7db11f1
--- /dev/null
+++ b/megapixels/app/server/api.py
@@ -0,0 +1,72 @@
+from flask import Blueprint, jsonify
+
+from app.models.sql_factory import list_datasets, get_dataset, get_table
+
+# from jinja2 import TemplateNotFound
+
+# import os
+# import sys
+# import json
+# import time
+# import argparse
+# import cv2 as cv
+# import numpy as np
+# from datetime import datetime
+# from flask import Flask, request, render_template, jsonify
+# from PIL import Image  # todo: try to remove PIL dependency
+# import re
+
+# sanitize_re = re.compile('[\W]+')
+# valid_exts = ['.gif', '.jpg', '.jpeg', '.png']
+
+# from dotenv import load_dotenv
+# load_dotenv()
+
+# from feature_extractor import FeatureExtractor
+
+# DEFAULT_LIMIT = 50
+
+api = Blueprint('api', __name__)
+
+@api.route('/')
+def index():
+  return jsonify({ 'datasets': list_datasets() })
+
+@api.route('/dataset/<dataset>/test', methods=['POST'])
+def test(dataset='test'):
+  dataset = get_dataset(dataset)
+  print('hiiiiii')
+  return jsonify({ 'test': 'OK', 'dataset': dataset })
+
+# @router.route('/<dataset>/face', methods=['POST'])
+# def upload(name):
+#   file = request.files['query_img']
+#   fn = file.filename
+#   if fn.endswith('blob'):
+#     fn = 'filename.jpg'
+
+#   basename, ext = os.path.splitext(fn)
+#   print("got {}, type {}".format(basename, ext))
+#   if ext.lower() not in valid_exts:
+#     return jsonify({ 'error': 'not an image' })
+
+#   uploaded_fn = datetime.now().isoformat() + "_" + basename
+#   uploaded_fn = sanitize_re.sub('', uploaded_fn)
+#   uploaded_img_path = "static/uploaded/" + uploaded_fn + ext
+#   uploaded_img_path = uploaded_img_path.lower()
+#   print('query: {}'.format(uploaded_img_path))
+
+#   img = Image.open(file.stream).convert('RGB')
+#   # img.save(uploaded_img_path)
+#   # vec = db.load_feature_vector_from_file(uploaded_img_path)
+#   vec = fe.extract(img)
+#   # print(vec.shape)
+
+#   results = db.search(vec, limit=limit)
+#   query = {
+#     'timing': time.time() - start,
+#   }
+#   print(results)
+#   return jsonify({
+#     'results': results,
+#   })
-- 
cgit v1.2.3-70-g09d2


From 2ee8cd6a77c3efed77e58d706f4ee76418770e54 Mon Sep 17 00:00:00 2001
From: Jules Laplace <julescarbon@gmail.com>
Date: Fri, 14 Dec 2018 18:17:15 +0100
Subject: sub apis workin

---
 megapixels/app/models/sql_factory.py | 11 +++++++----
 megapixels/app/server/api.py         | 15 +++++++++++----
 2 files changed, 18 insertions(+), 8 deletions(-)

(limited to 'megapixels/app/server/api.py')

diff --git a/megapixels/app/models/sql_factory.py b/megapixels/app/models/sql_factory.py
index 2a18d6af..e35c3e15 100644
--- a/megapixels/app/models/sql_factory.py
+++ b/megapixels/app/models/sql_factory.py
@@ -21,10 +21,7 @@ datasets = {}
 loaded = False
 
 def list_datasets():
-  return [{
-    'name': name,
-    'tables': list(datasets[name].tables.keys()),
-  } for name in datasets.keys()]
+  return [dataset.describe() for dataset in datasets.values()]
 
 def get_dataset(name):
   return datasets[name] if name in datasets else None
@@ -76,6 +73,12 @@ class SqlDataset:
       base_model = declarative_base(engine)
     self.base_model = base_model
 
+  def describe(self):
+    return {
+      'name': self.name,
+      'tables': list(self.tables.keys()),
+    }
+
   def get_table(self, type):
     if type in self.tables:
       return self.tables[type]
diff --git a/megapixels/app/server/api.py b/megapixels/app/server/api.py
index e7db11f1..428c53b1 100644
--- a/megapixels/app/server/api.py
+++ b/megapixels/app/server/api.py
@@ -32,11 +32,18 @@ api = Blueprint('api', __name__)
 def index():
   return jsonify({ 'datasets': list_datasets() })
 
-@api.route('/dataset/<dataset>/test', methods=['POST'])
-def test(dataset='test'):
-  dataset = get_dataset(dataset)
+@api.route('/dataset/<name>')
+def show(name):
+  dataset = get_dataset(name)
+  if dataset:
+    return jsonify(dataset.describe())
+  else:
+    return jsonify({ 'status': 404 })
+
+@api.route('/dataset/<name>/test', methods=['POST'])
+def test(name):
   print('hiiiiii')
-  return jsonify({ 'test': 'OK', 'dataset': dataset })
+  return jsonify({ 'test': 'OK', 'dataset': name })
 
 # @router.route('/<dataset>/face', methods=['POST'])
 # def upload(name):
-- 
cgit v1.2.3-70-g09d2


From 485cf0e4665c660d4e5e1fba00a95bc8036809c6 Mon Sep 17 00:00:00 2001
From: Jules Laplace <julescarbon@gmail.com>
Date: Sat, 15 Dec 2018 16:40:34 +0100
Subject: db stuff

---
 faiss/__init__.py                               |  0
 faiss/server.py                                 | 68 -----------------
 megapixels/app/server/api.py                    | 97 +++++++++----------------
 site/public/datasets/lfw/index.html             | 67 +++++++++++++----
 site/public/datasets/vgg_face2/index.html       | 84 +++++++++++++++++++++
 site/public/index.html                          |  1 +
 site/public/research/00_introduction/index.html | 86 ++++++++++++++++++++++
 site/public/research/index.html                 |  2 +-
 8 files changed, 259 insertions(+), 146 deletions(-)
 delete mode 100644 faiss/__init__.py
 delete mode 100644 faiss/server.py
 create mode 100644 site/public/datasets/vgg_face2/index.html
 create mode 100644 site/public/research/00_introduction/index.html

(limited to 'megapixels/app/server/api.py')

diff --git a/faiss/__init__.py b/faiss/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/faiss/server.py b/faiss/server.py
deleted file mode 100644
index a8c660fa..00000000
--- a/faiss/server.py
+++ /dev/null
@@ -1,68 +0,0 @@
-#!python
-
-import os
-import sys
-import json
-import time
-import argparse
-import cv2 as cv
-import numpy as np
-from datetime import datetime
-from flask import Flask, request, render_template, jsonify
-from PIL import Image  # todo: try to remove PIL dependency
-import re
-
-sanitize_re = re.compile('[\W]+')
-valid_exts = ['.gif', '.jpg', '.jpeg', '.png']
-
-from dotenv import load_dotenv
-load_dotenv()
-
-from feature_extractor import FeatureExtractor
-
-DEFAULT_LIMIT = 50
-
-app = Flask(__name__, static_url_path="/search/static", static_folder="static")
-
-# static api routes - this routing is actually handled in the JS
-@app.route('/', methods=['GET'])
-def index():
-  return app.send_static_file('metadata.html')
-
-# search using an uploaded file
-@app.route('/search/api/upload', methods=['POST'])
-def upload():
-  file = request.files['query_img']
-  fn = file.filename
-  if fn.endswith('blob'):
-    fn = 'filename.jpg'
-
-  basename, ext = os.path.splitext(fn)
-  print("got {}, type {}".format(basename, ext))
-  if ext.lower() not in valid_exts:
-    return jsonify({ 'error': 'not an image' })
-
-  uploaded_fn = datetime.now().isoformat() + "_" + basename
-  uploaded_fn = sanitize_re.sub('', uploaded_fn)
-  uploaded_img_path = "static/uploaded/" + uploaded_fn + ext
-  uploaded_img_path = uploaded_img_path.lower()
-  print('query: {}'.format(uploaded_img_path))
-
-  img = Image.open(file.stream).convert('RGB')
-  # img.save(uploaded_img_path)
-  # vec = db.load_feature_vector_from_file(uploaded_img_path)
-  vec = fe.extract(img)
-  # print(vec.shape)
-
-  results = db.search(vec, limit=limit)
-  query = {
-    'timing': time.time() - start,
-  }
-  print(results)
-  return jsonify({
-    'results': results,
-  })
-
-if __name__=="__main__":
-    app.run("0.0.0.0", debug=False)
-
diff --git a/megapixels/app/server/api.py b/megapixels/app/server/api.py
index 428c53b1..c5e27dd2 100644
--- a/megapixels/app/server/api.py
+++ b/megapixels/app/server/api.py
@@ -1,30 +1,13 @@
-from flask import Blueprint, jsonify
+import os
+import re
+import time
+from flask import Blueprint, request, jsonify
+from PIL import Image  # todo: try to remove PIL dependency
 
 from app.models.sql_factory import list_datasets, get_dataset, get_table
 
-# from jinja2 import TemplateNotFound
-
-# import os
-# import sys
-# import json
-# import time
-# import argparse
-# import cv2 as cv
-# import numpy as np
-# from datetime import datetime
-# from flask import Flask, request, render_template, jsonify
-# from PIL import Image  # todo: try to remove PIL dependency
-# import re
-
-# sanitize_re = re.compile('[\W]+')
-# valid_exts = ['.gif', '.jpg', '.jpeg', '.png']
-
-# from dotenv import load_dotenv
-# load_dotenv()
-
-# from feature_extractor import FeatureExtractor
-
-# DEFAULT_LIMIT = 50
+sanitize_re = re.compile('[\W]+')
+valid_exts = ['.gif', '.jpg', '.jpeg', '.png']
 
 api = Blueprint('api', __name__)
 
@@ -40,40 +23,32 @@ def show(name):
   else:
     return jsonify({ 'status': 404 })
 
-@api.route('/dataset/<name>/test', methods=['POST'])
-def test(name):
-  print('hiiiiii')
-  return jsonify({ 'test': 'OK', 'dataset': name })
-
-# @router.route('/<dataset>/face', methods=['POST'])
-# def upload(name):
-#   file = request.files['query_img']
-#   fn = file.filename
-#   if fn.endswith('blob'):
-#     fn = 'filename.jpg'
-
-#   basename, ext = os.path.splitext(fn)
-#   print("got {}, type {}".format(basename, ext))
-#   if ext.lower() not in valid_exts:
-#     return jsonify({ 'error': 'not an image' })
-
-#   uploaded_fn = datetime.now().isoformat() + "_" + basename
-#   uploaded_fn = sanitize_re.sub('', uploaded_fn)
-#   uploaded_img_path = "static/uploaded/" + uploaded_fn + ext
-#   uploaded_img_path = uploaded_img_path.lower()
-#   print('query: {}'.format(uploaded_img_path))
-
-#   img = Image.open(file.stream).convert('RGB')
-#   # img.save(uploaded_img_path)
-#   # vec = db.load_feature_vector_from_file(uploaded_img_path)
-#   vec = fe.extract(img)
-#   # print(vec.shape)
-
-#   results = db.search(vec, limit=limit)
-#   query = {
-#     'timing': time.time() - start,
-#   }
-#   print(results)
-#   return jsonify({
-#     'results': results,
-#   })
+@api.route('/dataset/<dataset>/face', methods=['POST'])
+def upload(name):
+  file = request.files['query_img']
+  fn = file.filename
+  if fn.endswith('blob'):
+    fn = 'filename.jpg'
+
+  basename, ext = os.path.splitext(fn)
+  print("got {}, type {}".format(basename, ext))
+  if ext.lower() not in valid_exts:
+    return jsonify({ 'error': 'not an image' })
+
+  img = Image.open(file.stream).convert('RGB')
+  
+  # vec = db.load_feature_vector_from_file(uploaded_img_path)
+  # vec = fe.extract(img)
+  # print(vec.shape)
+  # results = db.search(vec, limit=limit)
+
+  query = {
+    'timing': time.time() - start,
+  }
+  results = []
+
+  print(results)
+  return jsonify({
+    'query': query,
+    'results': results,
+  })
diff --git a/site/public/datasets/lfw/index.html b/site/public/datasets/lfw/index.html
index 39052b44..e080229f 100644
--- a/site/public/datasets/lfw/index.html
+++ b/site/public/datasets/lfw/index.html
@@ -26,23 +26,50 @@
   </header>
   <div class="content">
     
-  <section><h1>Labeled Faces in The Wild</h1>
-</section><section><div class='meta'><div><div class='gray'>Created</div><div>2007</div></div><div><div class='gray'>Images</div><div>13,233</div></div><div><div class='gray'>People</div><div>5,749</div></div><div><div class='gray'>Created From</div><div>Yahoo News images</div></div><div><div class='gray'>Search available</div><div>Searchable</div></div></div></section><section><p>Labeled Faces in The Wild (LFW) is amongst the most widely used facial recognition training datasets in the world and is the first of its kind to be created entirely from images that were posted online. The LFW dataset includes 13,233 images of 5,749 people that were collected between 2002-2004. Use the tools below to check if you were included in this dataset or scroll down to read the analysis.</p>
+  <section><h1>Labeled Faces in the Wild</h1>
+</section><section><div class='meta'><div><div class='gray'>Created</div><div>2007</div></div><div><div class='gray'>Images</div><div>13,233</div></div><div><div class='gray'>People</div><div>5,749</div></div><div><div class='gray'>Created From</div><div>Yahoo News images</div></div><div><div class='gray'>Search available</div><div>Searchable</div></div></div></section><section><p>Labeled Faces in The Wild (LFW) is amongst the most widely used facial recognition training datasets in the world and is the first of its kind to be created entirely from images posted online. The LFW dataset includes 13,233 images of 5,749 people that were collected between 2002-2004. Use the tools below to check if you were included in this dataset or scroll down to read the analysis.</p>
 <p>{INSERT IMAGE SEARCH MODULE}</p>
 <p>{INSERT TEXT SEARCH MODULE}</p>
-</section><section class='images'><div class='image'><img src='https://nyc3.digitaloceanspaces.com/megapixels/v1/datasets/lfw/assets/lfw_feature.jpg' alt='Eight out of 5,749 people in the Labeled Faces in the Wild dataset. The face recognition training dataset is created entirely from photos downloaded from the Internet.'><div class='caption'>Eight out of 5,749 people in the Labeled Faces in the Wild dataset. The face recognition training dataset is created entirely from photos downloaded from the Internet.</div></div></section><section><h2>INTRO</h2>
-<p>It began in 2002. Researchers at University of Massachusetts Amherst were developing algorithms for facial recognition and they needed more data. Between 2002-2004 they scraped Yahoo News for images of public figures. Two years later they cleaned up the dataset and repackaged it as Labeled Faces in the Wild (LFW).</p>
-<p>Since then the LFW dataset has become one of the most widely used datasets used for evaluating face recognition algorithms. The associated research paper “Labeled Faces in the Wild: A Database for Studying Face Recognition in Unconstrained Environments” has been cited 996 times reaching 45 different countries throughout the world.</p>
-<p>The faces come from news stories and are mostly celebrities from the entertainment industry, politicians, and villains. It’s a sampling of current affairs and breaking news that has come to pass. The images, detached from their original context now server a new purpose: to train, evaluate, and improve facial recognition.</p>
-<p>As the most widely used facial recognition dataset, it can be said that each individual in LFW has, in a small way, contributed to the current state of the art in facial recognition surveillance. John Cusack, Julianne Moore, Barry Bonds, Osama bin Laden, and even Moby are amongst these biometric pillars, exemplar faces provided the visual dimensions of a new computer vision future.</p>
-</section><section class='images'><div class='image'><img src='https://nyc3.digitaloceanspaces.com/megapixels/v1/datasets/lfw/assets/lfw_montage_1280.jpg' alt='The entire LFW dataset cropped to facial regions'><div class='caption'>The entire LFW dataset cropped to facial regions</div></div></section><section><p>In addition to commercial use as an evaluation tool, alll of the faces in LFW dataset are prepackaged into a popular machine learning code framework called scikit-learn.</p>
-<h2>Facts</h2>
-<p>The person with the most images is:
-The person with the least images is:</p>
-<h2>Commercial Use</h2>
+<pre><code>load file: lfw_names_gender_kg_min.csv
+Name, Images, Gender, Description
+</code></pre>
+</section><section class='images'><div class='image'><img src='https://nyc3.digitaloceanspaces.com/megapixels/v1/datasets/lfw/assets/lfw_feature.jpg' alt='Eighteen of the 5,749 people in the Labeled Faces in the Wild Dataset. The most widely used face dataset for benchmarking commercial face recognition algorithms.'><div class='caption'>Eighteen of the 5,749 people in the Labeled Faces in the Wild Dataset. The most widely used face dataset for benchmarking commercial face recognition algorithms.</div></div></section><section><h2>Intro</h2>
+<p>Three paragraphs describing the LFW dataset in a format that can be easily replicated for the other datasets. Nothing too custom. An analysis of the initial research papers with context relative to all the other dataset papers.</p>
+</section><section class='images'><div class='image'><img src='https://nyc3.digitaloceanspaces.com/megapixels/v1/datasets/lfw/assets/lfw_montage_everyone_1920.jpg' alt=' all 5,749 people in the LFW Dataset sorted from most to least images collected.'><div class='caption'> all 5,749 people in the LFW Dataset sorted from most to least images collected.</div></div></section><section><h2>LFW by the Numbers</h2>
+<ul>
+<li>Was first published in 2007</li>
+<li>Developed out of a prior dataset from Berkely called "Faces in the Wild" or "Names and Faces" [^lfw_original_paper]</li>
+<li>Includes 13,233 images and 5,749 different people [^lfw_website]</li>
+<li>There are about 3 men for every 1 woman (4,277 men and 1,472 women)[^lfw_website]</li>
+<li>The person with the most images is George W. Bush with 530</li>
+<li>Most people (70%) in the dataset have only 1 image</li>
+<li>Thre are 1,680 people in the dataset with 2 or more images [^lfw_website]</li>
+<li>Two out of 4 of the original authors received funding from the Office of Director of National Intelligence and IARPA for their 2016 LFW survey follow up report </li>
+<li>The LFW dataset includes over 500 actors, 30 models, 10 presidents, 24 football players, 124 basketball players, 11 kings, and 2 queens</li>
+<li>In all the LFW publications provided by the authors the words "ethics", "consent", and "privacy" appear 0 times [^lfw_original_paper], [^lfw_survey], [^lfw_tech_report] , [^lfw_website]</li>
+<li>The word "future" appears 71 times</li>
+</ul>
+<h1>Facts</h1>
+<ul>
+<li>Was created for the purpose of improving "unconstrained face recognition" [^lfw_original_paper]</li>
+<li>All images in LFW were obtained "in the wild" meaning without any consent from the subject or from the photographer</li>
+<li>The faces were detected using the Viola-Jones haarcascade face detector [^lfw_website] [^lfw_survey]</li>
+<li>Is considered the "most popular benchmark for face recognition" [^lfw_baidu]</li>
+<li>Is "the most widely used evaluation set in the field of facial recognition" [^lfw_pingan]</li>
+<li>Is used by several of the largest tech companies in the world including "Google, Facebook, Microsoft Research Asia, Baidu, Tencent, SenseTime, Face++ and Chinese University of Hong Kong." [^lfw_pingan]</li>
+</ul>
+<p>need citations</p>
+<ul>
+<li>All images were copied from Yahoo News between 2002 - 2004 [^lfw_original_paper]</li>
+<li>SenseTime, who has relied on LFW for benchmarking their facial recognition performance, is the leading provider of surveillance to the Chinese Government (need citation)</li>
+</ul>
+</section><section class='images'><div class='image'><img src='https://nyc3.digitaloceanspaces.com/megapixels/v1/datasets/lfw/assets/lfw_montage_top1_640.jpg' alt=' former President George W. Bush'><div class='caption'> former President George W. Bush</div></div>
+<div class='image'><img src='https://nyc3.digitaloceanspaces.com/megapixels/v1/datasets/lfw/assets/lfw_montage_top2_4_640.jpg' alt=' Colin Powel (236), Tony Blair (144), and Donald Rumsfeld (121)'><div class='caption'> Colin Powel (236), Tony Blair (144), and Donald Rumsfeld (121)</div></div></section><section><h2>People and Companies using the LFW Dataset</h2>
+<p>This section describes who is using the dataset and for what purposes. It should include specific examples of people or companies with citations and screenshots. This section is followed up by the graph, the map, and then the supplementary material.</p>
 <p>The LFW dataset is used by numerous companies for <a href="about/glossary#benchmarking">benchmarking</a> algorithms and in some cases <a href="about/glossary#training">training</a>. According to the benchmarking results page [^lfw_results] provided by the authors, over 2 dozen companies have contributed their benchmark results.</p>
 <p>According to BiometricUpdate.com [^lfw_pingan], LFW is "the most widely used evaluation set in the field of facial recognition, LFW attracts a few dozen teams from around the globe including Google, Facebook, Microsoft Research Asia, Baidu, Tencent, SenseTime, Face++ and Chinese University of Hong Kong."</p>
 <p>According to researchers at the Baidu Research – Institute of Deep Learning "LFW has been the most popular evaluation benchmark for face recognition, and played a very important role in facilitating the face recognition society to improve algorithm. [^lfw_baidu]."</p>
+<p>In addition to commercial use as an evaluation tool, alll of the faces in LFW dataset are prepackaged into a popular machine learning code framework called scikit-learn.</p>
 <pre><code>load file: lfw_commercial_use.csv
 name_display,company_url,example_url,country,description
 </code></pre>
@@ -83,13 +110,18 @@ name_display,company_url,example_url,country,description
 <p>The LFW face recognition training and evaluation dataset is a historically important face dataset as it was the first popular dataset to be created entirely from Internet images, paving the way for a global trend towards downloading anyone’s face from the Internet and adding it to a dataset. As will be evident with other datasets, LFW’s approach has now become the norm.</p>
 <p>For all the 5,000 people in this datasets, their face is forever a part of facial recognition history. It would be impossible to remove anyone from the dataset because it is so ubiquitous. For their rest of the lives and forever after, these 5,000 people will continue to be used for training facial recognition surveillance.</p>
 <h2>Right to Removal</h2>
-<p>If you are affected by disclosure of your identity in this dataset please do contact the authors, many state that they are willing to remove images upon request. The authors of the LFW can be reached from the emails posted in their paper:</p>
+<p>If you are affected by disclosure of your identity in this dataset please do contact the authors. Many have stated that they are willing to remove images upon request. The authors of the LFW dataset provide the following email for inquiries:</p>
 <p>You can use the following message to request removal from the dataset:</p>
+<p>To: Gary Huang <a href="mailto:mailto:gbhuang@cs.umass.edu">mailto:gbhuang@cs.umass.edu</a></p>
+<p>Subject: Request for Removal from LFW Face Dataset</p>
 <p>Dear [researcher name],</p>
-<p>I am writing to you about the "LFW Dataset". Recently I have discovered that your dataset includes my identity and no longer wish to be included in your dataset</p>
-<p>MegaPixels is an educational art project developed for academic purposes. In no way does this project aim to villify the researchers who produced the datasets. The aim of this project is to encourage discourse around ethics and consent in artificial intelligence by providing information about these datasets that is otherwise difficult to obtain or inaccessible to other researchers.</p>
+<p>I am writing to you about the "Labeled Faces in The Wild Dataset". Recently I discovered that your dataset includes my identity and I no longer wish to be included in your dataset.</p>
+<p>The dataset is being used thousands of companies around the world to improve facial recognition software including usage by governments for the purpose of law enforcement, national security, tracking consumers in retail environments, and tracking individuals through public spaces.</p>
+<p>My name as it appears in your dataset is [your name]. Please remove all images from your dataset and inform your newsletter subscribers to likewise update their copies.</p>
+<p>- [your name]</p>
+<hr>
 <h2>Supplementary Data</h2>
-<p>Sed ut perspiciatis, unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam eaque ipsa, quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt, explicabo. Nemo enim ipsam voluptatem, quia voluptas sit, aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos.</p>
+<p>Researchers, journ</p>
 <table>
 <thead><tr>
 <th style="text-align:left">Title</th>
@@ -221,6 +253,9 @@ imageio.imwrite(&#39;lfw_montage_full.png&#39;, montage)
 montage_960 = imutils.resize(montage, width=960)
 imageio.imwrite(&#39;lfw_montage_960.jpg&#39;, montage_960)
 </code></pre>
+<h2>Disclaimer</h2>
+<p>MegaPixels is an educational art project designed to encourage discourse about facial recognition datasets. Any ethical or legal issues should be directed to the researcher's parent organizations. Except where necessary for contact or clarity, the names of researchers have been subsituted by their parent organization. In no way does this project aim to villify researchers who produced the datasets.</p>
+<p>Read more about <a href="about/code-of-conduct">MegaPixels Code of Conduct</a></p>
 <div class="footnotes">
 <hr>
 <ol></ol>
diff --git a/site/public/datasets/vgg_face2/index.html b/site/public/datasets/vgg_face2/index.html
new file mode 100644
index 00000000..24a1059b
--- /dev/null
+++ b/site/public/datasets/vgg_face2/index.html
@@ -0,0 +1,84 @@
+<!doctype html>
+<html>
+<head>
+  <title>MegaPixels</title>
+  <meta charset="utf-8" />
+  <meta name="author" content="Adam Harvey" />
+  <meta name="description" content="A large scale image dataset for face recognition" />
+  <meta name="referrer" content="no-referrer" />
+  <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes" />
+  <link rel='stylesheet' href='/assets/css/fonts.css' />
+  <link rel='stylesheet' href='/assets/css/css.css' />
+</head>
+<body>
+  <header>
+    <a class='slogan' href="/">
+      <div class='logo'></div>
+      <div class='site_name'>MegaPixels</div>
+      <span class='sub'>The Darkside of Datasets</span>
+    </a>
+    <div class='links'>
+      <a href="/search/">Face Search</a>
+      <a href="/datasets/">Datasets</a>
+      <a href="/research/">Research</a>
+      <a href="/about/">About</a>
+    </div>
+  </header>
+  <div class="content">
+    
+  <section><h1>VGG Faces2</h1>
+</section><section><div class='meta'><div><div class='gray'>Created</div><div>2018</div></div><div><div class='gray'>Images</div><div>3.3M</div></div><div><div class='gray'>People</div><div>9,000</div></div><div><div class='gray'>Created From</div><div>Scraping search engines</div></div><div><div class='gray'>Search available</div><div>[Searchable](#)</div></div></div></section><section><p>VGG Face2 is the updated version of the VGG Face dataset and now includes over 3.3M face images from over 9K people. The identities were selected by taking the top 500K identities in Google's Knowledge Graph of celebrities and then selecting only the names that yielded enough training images. The dataset was created in the UK but funded by Office of Director of National Intelligence in the United States.</p>
+<p>{INSERT IMAGE SEARCH MODULE}</p>
+<p>{INSERT TEXT SEARCH MODULE}</p>
+<pre><code>load file: lfw_names_gender_kg_min.csv
+Name, Images, Gender, Description
+</code></pre>
+<h2>VGG Face2 by the Numbers</h2>
+<ul>
+<li>1,331 actresses, 139 presidents</li>
+<li>3 husbands and 16 wives</li>
+<li>2 snooker player</li>
+<li>1 guru</li>
+<li>1 pornographic actress</li>
+<li>3 computer programmer</li>
+</ul>
+<h1>Names and descriptions</h1>
+<ul>
+<li>The original VGGF2 name list has been updated with the results returned from Google Knowledge</li>
+<li>Names with a similarity score greater than 0.75 where automatically updated. Scores computed using <code>import difflib; seq = difflib.SequenceMatcher(a=a.lower(), b=b.lower()); score = seq.ratio()</code></li>
+<li>The 97 names with a score of 0.75 or lower were manually reviewed and includes name changes validating using Wikipedia.org results for names such as "Bruce Jenner" to "Caitlyn Jenner", spousal last-name changes, and discretionary changes to improve search results such as combining nicknames with full name when appropriate, for example changing "Aleksandar Petrović" to "Aleksandar 'Aco' Petrović" and minor changes such as "Mohammad Ali" to "Muhammad Ali"</li>
+<li>The 'Description` text was automatically added when the Knowledge Graph score was greater than 250</li>
+</ul>
+<h1>TODO</h1>
+<ul>
+<li>create name list, and populate with Knowledge graph information like LFW</li>
+<li>make list of interesting number stats, by the numbers</li>
+<li>make list of interesting important facts</li>
+<li>write intro abstract</li>
+<li>write analysis of usage</li>
+<li>find examples, citations, and screenshots of useage</li>
+<li>find list of companies using it for table</li>
+<li>create montages of the dataset, like LFW</li>
+<li>create right to removal information</li>
+</ul>
+</section>
+
+  </div>
+  <footer>
+    <div>
+      <a href="/">MegaPixels.cc</a>
+      <a href="/about/disclaimer/">Disclaimer</a>
+      <a href="/about/terms/">Terms of Use</a>
+      <a href="/about/privacy/">Privacy</a>
+      <a href="/about/">About</a>
+      <a href="/about/team/">Team</a>
+    </div>
+    <div>
+      MegaPixels &copy;2017-19 Adam R. Harvey /&nbsp;
+      <a href="https://ahprojects.com">ahprojects.com</a>
+    </div>
+  </footer>
+</body>
+
+<script src="/assets/js/app/site.js"></script>
+</html>
\ No newline at end of file
diff --git a/site/public/index.html b/site/public/index.html
index 51006b59..91ff467a 100644
--- a/site/public/index.html
+++ b/site/public/index.html
@@ -71,6 +71,7 @@
 
 <script src="/assets/js/vendor/three.min.js"></script>
 <script src="/assets/js/vendor/three.meshline.js"></script>
+<script src="/assets/js/vendor/oktween.js"></script>
 <script src="/assets/js/app/face.js"></script>
 
 <script src="/assets/js/app/site.js"></script>
diff --git a/site/public/research/00_introduction/index.html b/site/public/research/00_introduction/index.html
new file mode 100644
index 00000000..8f598f5b
--- /dev/null
+++ b/site/public/research/00_introduction/index.html
@@ -0,0 +1,86 @@
+<!doctype html>
+<html>
+<head>
+  <title>MegaPixels</title>
+  <meta charset="utf-8" />
+  <meta name="author" content="Adam Harvey" />
+  <meta name="description" content="" />
+  <meta name="referrer" content="no-referrer" />
+  <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes" />
+  <link rel='stylesheet' href='/assets/css/fonts.css' />
+  <link rel='stylesheet' href='/assets/css/css.css' />
+</head>
+<body>
+  <header>
+    <a class='slogan' href="/">
+      <div class='logo'></div>
+      <div class='site_name'>MegaPixels</div>
+      <span class='sub'>The Darkside of Datasets</span>
+    </a>
+    <div class='links'>
+      <a href="/search/">Face Search</a>
+      <a href="/datasets/">Datasets</a>
+      <a href="/research/">Research</a>
+      <a href="/about/">About</a>
+    </div>
+  </header>
+  <div class="content">
+    
+  <section>
+    <h1>Untitled Page</h1>
+    <div class='meta'>
+      <div>
+        <div class='gray'>Posted</div>
+        <div>2018-12-31</div>
+      </div>
+      <div>
+        <div class='gray'>By</div>
+        <div>Adam Harvey</div>
+      </div>
+      
+    </div>
+  </section>
+
+  <section><p>It was the early 2000s. Face recognition was new and no one seemed sure exactly how well it was going to perform in practice. In theory, face recognition was poised to be a game changer, a force multiplier,  a strategic military advantage, a way to make cities safer and to secure borders. This was the future John Ashcroft demanded with the Total Information Awareness act of the 2003 and that spooks had dreamed of for decades. It was a future that academics at Carnegie Mellon Universtiy and Colorado State University would help build. It was also a future that celebrities would play a significant role in building. And to the surprise of ordinary Internet users like myself and perhaps you, it was a future that millions of Internet users would unwittingly play role in creating.</p>
+<p>Now the future has arrived and it doesn't make sense. Facial recognition works yet it doesn't actually work. Facial recognition is cheap and accessible but also expensive and out of control. Facial recognition research has achieved headline grabbing superhuman accuracies over 99.9% yet facial recognition is also dangerously inaccurate. During a trial installation at Sudkreuz station in Berlin in 2018, 20% of the matches were wrong, a number so low that it should not have any connection to law enforcement or justice. And in London, the Metropolitan police had been using facial recognition software that mistakenly identified an alarming 98% of people as criminals <sup class="footnote-ref" id="fnref-met_police"><a href="#fn-met_police">1</a></sup>, which perhaps is a crime itself.</p>
+<p>MegaPixels is an online art project that explores the history of facial recognition from the perspective of datasets. To paraphrase the artist Trevor Paglen, whoever controls the dataset controls the meaning. MegaPixels aims to unravel the meanings behind the data and expose the darker corners of the biometric industry that have contributed to its growth. MegaPixels does not start with a conclusion, a moralistic slant, or a</p>
+<p>Whether or not to build facial recognition was a question that can no longer be asked. As an outspoken critic of face recognition I've developed, and hopefully furthered, my understanding during the last 10 years I've spent working with computer vision. Though I initially disagreed, I've come to see technocratic perspective as a non-negotiable reality. As Oren (nytimes article) wrote in NYT Op-Ed "the horse is out of the barn" and the only thing we can do collectively or individually is to steer towards the least worse outcome. Computational communication has entered a new era and it's both exciting and frightening to explore the potentials and opportunities. In 1997 getting access to 1 teraFLOPS of computational power would have cost you $55 million and required a strategic partnership with the Department of Defense. At the time of writing, anyone can rent 1 teraFLOPS on a cloud GPU marketplace for less than $1/day.  <sup class="footnote-ref" id="fnref-asci_option_red"><a href="#fn-asci_option_red">2</a></sup>.</p>
+<p>I hope that this project will illuminate the darker areas of strange world of facial recognition that have not yet received attention and encourage discourse in academic, industry, and . By no means do I believe discourse can save the day. Nor do I think creating artwork can. In fact, I'm not exactly sure what the outcome of this project will be. The project is not so much what I publish here but what happens after. This entire project is only a prologue.</p>
+<p>As McLuhan wrote, "You can't have a static, fixed position in the electric age". And in our hyper-connected age of mass surveillance, artificial intelligece, and unevenly distributed virtual futures the most irrational thing to be is rational. Increasingly the world is becoming a contradiction where people use surveillance to protest surveillance, use</p>
+<p>Like many projects, MegaPixels had spent years meandering between formats, unfeasible budgets, and was generally too niche of a subject. The basic idea for this project, as proposed to the original <a href="https://tacticaltech.org/projects/the-glass-room-nyc/">Glass Room</a> installation in 2016 in NYC, was to build an interactive mirror that showed people if they had been included in the <a href="/datasets/lfw">LFW</a> facial recognition dataset. The idea was based on my reaction to all the datasets I'd come across during research for the CV Dazzle project. I'd noticed strange datasets created for training and testing face detection algorithms. Most were created in labratory settings and their interpretation of face data was very strict.</p>
+<p>About the name</p>
+<p>About the funding</p>
+<p>About me</p>
+<p>About the team</p>
+<p>Conclusion</p>
+<h2>for other post</h2>
+<p>It was the early 2000s. Face recognition was new and no one seemed sure how well it was going to perform in practice. In theory, face recognition was poised to be a game changer, a force multiplier,  a strategic military advantage, a way to make cities safer and to secure the borders. It was the future that John Ashcroft demanded with the Total Information Awareness act of the 2003. It was a future that academics helped build. It was a future that celebrities helped build. And it was a future that</p>
+<p>A decade earlier the Department of Homeland Security and the Counterdrug Technology Development Program Office initated a feasibilty study called FERET (FacE REcognition Technology) to "develop automatic face recognition capabilities that could be employed to assist security, intelligence, and law enforcement personnel in the performance of their duties [^feret_website]."</p>
+<p>One problem with FERET dataset was that the photos were in controlled settings. For face recognition to work it would have to be used in uncontrolled settings. Even newer datasets such as the Multi-PIE (Pose, Illumination, and Expression) from Carnegie Mellon University included only indoor photos of  cooperative subjects. Not only were the photos completely unrealistic, CMU's Multi-Pie included only 18 individuals and cost $500 for academic use [^cmu_multipie_cost], took years to create, and required consent from every participant.</p>
+<div class="footnotes">
+<hr>
+<ol><li id="fn-met_police"><p>Sharman, Jon. "Metropolitan Police's facial recognition technology 98% inaccurate, figures show". 2018. <a href="https://www.independent.co.uk/news/uk/home-news/met-police-facial-recognition-success-south-wales-trial-home-office-false-positive-a8345036.html">https://www.independent.co.uk/news/uk/home-news/met-police-facial-recognition-success-south-wales-trial-home-office-false-positive-a8345036.html</a><a href="#fnref-met_police" class="footnote">&#8617;</a></p></li>
+<li id="fn-asci_option_red"><p>Calle, Dan. "Supercomptuers". 1997. <a href="http://ei.cs.vt.edu/~history/SUPERCOM.Calle.HTML">http://ei.cs.vt.edu/~history/SUPERCOM.Calle.HTML</a><a href="#fnref-asci_option_red" class="footnote">&#8617;</a></p></li>
+</ol>
+</div>
+</section>
+
+  </div>
+  <footer>
+    <div>
+      <a href="/">MegaPixels.cc</a>
+      <a href="/about/disclaimer/">Disclaimer</a>
+      <a href="/about/terms/">Terms of Use</a>
+      <a href="/about/privacy/">Privacy</a>
+      <a href="/about/">About</a>
+      <a href="/about/team/">Team</a>
+    </div>
+    <div>
+      MegaPixels &copy;2017-19 Adam R. Harvey /&nbsp;
+      <a href="https://ahprojects.com">ahprojects.com</a>
+    </div>
+  </footer>
+</body>
+
+<script src="/assets/js/app/site.js"></script>
+</html>
\ No newline at end of file
diff --git a/site/public/research/index.html b/site/public/research/index.html
index cf9546e1..59a5fee9 100644
--- a/site/public/research/index.html
+++ b/site/public/research/index.html
@@ -28,7 +28,7 @@
     
   <section><h1>Research Blog</h1>
 <h2>The darkside of datasets and the future of computer vision</h2>
-</section><div class='research_index'><a href='/research/01_from_1_to_100_pixels/'><section class='wide'><img src='https://nyc3.digitaloceanspaces.com/megapixels/v1/site/research/01_from_1_to_100_pixels/assets/intro.jpg' alt='Research post' /><section><h1>From 1 to 100 Pixels</h1><h2>Photographs are for romantics. For the rest of us, it's all about data.  And a photo contains a massive amount of information about who you are.</h2></section></section></a></div>
+</section><div class='research_index'><a href='/research/00_introduction/'><section class='wide'><img src='data:image/gif;base64,R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==' alt='Research post' /><section><h1>Untitled Page</h1><h2></h2></section></section></a><a href='/research/01_from_1_to_100_pixels/'><section class='wide'><img src='https://nyc3.digitaloceanspaces.com/megapixels/v1/site/research/01_from_1_to_100_pixels/assets/intro.jpg' alt='Research post' /><section><h1>From 1 to 100 Pixels</h1><h2>Photographs are for romantics. For the rest of us, it's all about data.  And a photo contains a massive amount of information about who you are.</h2></section></section></a></div>
 
   </div>
   <footer>
-- 
cgit v1.2.3-70-g09d2


From a53a598461a25e8bf1d0bd3e63c47642e3213aef Mon Sep 17 00:00:00 2001
From: Jules Laplace <julescarbon@gmail.com>
Date: Sat, 15 Dec 2018 22:47:28 +0100
Subject: client stuff

---
 client/actions.js            |  8 +++---
 client/index.js              | 60 ++++++++++++++++++++++++++++++++++++--------
 client/session.js            |  6 ++---
 client/store.js              | 26 ++++++++-----------
 client/util.js               |  2 ++
 megapixels/app/server/api.py | 26 ++++++++++++++++++-
 megapixels/cli_flask.py      |  1 +
 site/templates/research.html |  4 +++
 8 files changed, 98 insertions(+), 35 deletions(-)

(limited to 'megapixels/app/server/api.py')

diff --git a/client/actions.js b/client/actions.js
index ba899f06..3fd86e0d 100644
--- a/client/actions.js
+++ b/client/actions.js
@@ -1,9 +1,9 @@
 import * as search from './search/search.actions'
-import * as review from './review/review.actions'
-import * as metadata from './metadata/metadata.actions'
+// import * as review from './review/review.actions'
+// import * as metadata from './metadata/metadata.actions'
 
 export {
   search,
-  review,
-  metadata,
+  // review,
+  // metadata,
 }
diff --git a/client/index.js b/client/index.js
index eddc5fb2..03015988 100644
--- a/client/index.js
+++ b/client/index.js
@@ -3,17 +3,55 @@ import ReactDOM from 'react-dom'
 import { AppContainer } from 'react-hot-loader'
 import { Provider } from 'react-redux'
 
-import App from './app'
+// import App from './app'
 
-import { store, history } from './store'
+import { store } from './store'
 
-const container = document.createElement('div')
-document.body.appendChild(container)
+// const container = document.createElement('div')
+// document.body.appendChild(container)
 
-ReactDOM.render(
-  <AppContainer>
-    <Provider store={store}>
-      <App history={history} />
-    </Provider>
-  </AppContainer>, container
-)
+toArray(document.querySelectorAll('.applet')).forEach(el => {
+  try {
+    const payload = JSON.parse(el.dataSet.getItem('payload'))
+  } catch(e) {
+    return
+  }
+  switch (payload.command) {
+    case 'load file':
+      append_tabulator(el, payload)
+      break
+    default:
+      append_react_applet(el, payload)
+      break
+  }
+})
+
+function append_react_applet(el, payload) {
+  ReactDOM.render(
+    <AppContainer>
+      <Provider store={store}>
+        <App command={payload} />
+      </Provider>
+    </AppContainer>, el
+  )
+}
+function append_tabulator(el, payload) {
+  const table = new Tabulator(el, {
+    height:"311px",
+    layout:"fitColumns",
+    placeholder:"No Data Set",
+    columns:[
+      // {title:"Name", field:"name", sorter:"string", width:200},
+      // {title:"Progress", field:"progress", sorter:"number", formatter:"progress"},
+      // {title:"Gender", field:"gender", sorter:"string"},
+      // {title:"Rating", field:"rating", formatter:"star", align:"center", width:100},
+      // {title:"Favourite Color", field:"col", sorter:"string", sortable:false},
+      // {title:"Date Of Birth", field:"dob", sorter:"date", align:"center"},
+      // {title:"Driver", field:"car", align:"center", formatter:"tickCross", sorter:"boolean"},
+    ],
+  })
+  const columns = payload.fields.split(', ')
+  console.log(columns)
+  if ()
+  table.setData(path)
+}
\ No newline at end of file
diff --git a/client/session.js b/client/session.js
index 5bfae7eb..0fae31d2 100644
--- a/client/session.js
+++ b/client/session.js
@@ -1,5 +1,5 @@
-import Storage from 'store2'
+// import Storage from 'store2'
 
-const session = Storage.namespace('vcat.search')
+// const session = Storage.namespace('vcat.search')
 
-export default session
+// export default session
diff --git a/client/store.js b/client/store.js
index 043af351..9c0f78cd 100644
--- a/client/store.js
+++ b/client/store.js
@@ -1,30 +1,25 @@
 import { applyMiddleware, compose, combineReducers, createStore } from 'redux'
-import { connectRouter, routerMiddleware } from 'connected-react-router'
-import { createBrowserHistory } from 'history'
 import thunk from 'redux-thunk'
-import { login } from './util'
 
-import metadataReducer from './metadata/metadata.reducer'
-import searchReducer from './search/search.reducer'
-import reviewReducer from './review/review.reducer'
+// import metadataReducer from './metadata/metadata.reducer'
+// import searchReducer from './search/search.reducer'
+// import reviewReducer from './review/review.reducer'
 
 const rootReducer = combineReducers({
-  auth: (state = login()) => state,
-  metadata: metadataReducer,
-  search: searchReducer,
-  review: reviewReducer,
+  // auth: (state = login()) => state,
+  // metadata: metadataReducer,
+  // search: searchReducer,
+  // review: reviewReducer,
 })
 
-function configureStore(initialState = {}, history) {
+function configureStore(initialState = {}) {
   const composeEnhancers = window.__REDUX_DEVTOOLS_EXTENSION_COMPOSE__ || compose
 
   const store = createStore(
-    connectRouter(history)(rootReducer), // new root reducer with router state
     initialState,
     composeEnhancers(
       applyMiddleware(
         thunk,
-        routerMiddleware(history)
       ),
     ),
   )
@@ -32,7 +27,6 @@ function configureStore(initialState = {}, history) {
   return store
 }
 
-const history = createBrowserHistory()
-const store = configureStore({}, history)
+const store = configureStore({})
 
-export { store, history }
+export { store }
diff --git a/client/util.js b/client/util.js
index ad303c64..92b4addc 100644
--- a/client/util.js
+++ b/client/util.js
@@ -6,6 +6,8 @@ export const isAndroid = !!(navigator.userAgent.match(/Android/i))
 export const isMobile = isiPhone || isiPad || isAndroid
 export const isDesktop = !isMobile
 
+export const toArray = a => Array.prototype.slice.apply(a)
+
 const htmlClassList = document.body.parentNode.classList
 htmlClassList.add(isDesktop ? 'desktop' : 'mobile')
 
diff --git a/megapixels/app/server/api.py b/megapixels/app/server/api.py
index c5e27dd2..cd2b950b 100644
--- a/megapixels/app/server/api.py
+++ b/megapixels/app/server/api.py
@@ -23,8 +23,10 @@ def show(name):
   else:
     return jsonify({ 'status': 404 })
 
-@api.route('/dataset/<dataset>/face', methods=['POST'])
+@api.route('/dataset/<name>/face', methods=['POST'])
 def upload(name):
+  start = time.time()
+  dataset = get_dataset(name)
   file = request.files['query_img']
   fn = file.filename
   if fn.endswith('blob'):
@@ -42,6 +44,28 @@ def upload(name):
   # print(vec.shape)
   # results = db.search(vec, limit=limit)
 
+  # with the result we have an ID
+  # query the sql dataset for the UUID etc here
+
+  query = {
+    'timing': time.time() - start,
+  }
+  results = []
+
+  print(results)
+  return jsonify({
+    'query': query,
+    'results': results,
+  })
+
+@api.route('/dataset/<name>/name', methods=['GET'])
+def name_lookup(dataset):
+  start = time.time()
+  dataset = get_dataset(name)
+
+  # we have a query from the request query string...
+  # use this to do a like* query on the identities_meta table
+
   query = {
     'timing': time.time() - start,
   }
diff --git a/megapixels/cli_flask.py b/megapixels/cli_flask.py
index 369bec01..e80526c6 100644
--- a/megapixels/cli_flask.py
+++ b/megapixels/cli_flask.py
@@ -1,5 +1,6 @@
 # --------------------------------------------------------
 # wrapper for flask CLI API
+# NB: python cli_flask.py run
 # --------------------------------------------------------
 
 import click
diff --git a/site/templates/research.html b/site/templates/research.html
index 0bb9fa30..b2ea3a6b 100644
--- a/site/templates/research.html
+++ b/site/templates/research.html
@@ -21,3 +21,7 @@
 
   {{ content }}
 {% endblock %}
+
+{% block scripts %}
+<script src="/assets/js/applets.js"></script>
+{% endblock %}
\ No newline at end of file
-- 
cgit v1.2.3-70-g09d2


From 3b10acc73247ec703ed47f0423e7d255a91f074e Mon Sep 17 00:00:00 2001
From: Jules Laplace <julescarbon@gmail.com>
Date: Sun, 16 Dec 2018 00:11:20 +0100
Subject: tabulator

---
 client/index.js                                    |  29 +-
 megapixels/app/server/api.py                       |  17 +-
 megapixels/app/server/create.py                    |   7 +
 megapixels/app/site/builder.py                     |  10 +
 megapixels/app/site/parser.py                      |  36 +-
 megapixels/app/site/s3.py                          |   3 +
 site/assets/css/applets.css                        |   2 -
 site/assets/css/tabulator.css                      | 761 +++++++++++++++++++++
 site/public/about/credits/index.html               |   2 +-
 site/public/about/disclaimer/index.html            |   2 +-
 site/public/about/index.html                       |   2 +-
 site/public/about/press/index.html                 |   2 +-
 site/public/about/privacy/index.html               |   2 +-
 site/public/about/style/index.html                 |   2 +-
 site/public/about/terms/index.html                 |   2 +-
 site/public/datasets/lfw/index.html                |   6 +-
 site/public/datasets/vgg_face2/index.html          |   4 +-
 site/public/index.html                             |   2 +-
 site/public/research/00_introduction/index.html    |   2 +-
 .../research/01_from_1_to_100_pixels/index.html    |   2 +-
 site/public/research/index.html                    |   2 +-
 site/templates/layout.html                         |   1 +
 22 files changed, 862 insertions(+), 36 deletions(-)
 create mode 100755 site/assets/css/tabulator.css

(limited to 'megapixels/app/server/api.py')

diff --git a/client/index.js b/client/index.js
index bc57f548..2ee12c14 100644
--- a/client/index.js
+++ b/client/index.js
@@ -23,23 +23,24 @@ function appendTabulator(el, payload) {
     height: '311px',
     layout: 'fitColumns',
     placeholder: 'No Data Set',
-    columns: [
-      // {title:'Name', field:'name', sorter:'string', width:200},
-      // {title:'Progress', field:'progress', sorter:'number', formatter:'progress'},
-      // {title:'Gender', field:'gender', sorter:"string"},
-      // {title:"Rating", field:"rating", formatter:"star", align:"center", width:100},
-      // {title:"Favourite Color", field:"col", sorter:"string", sortable:false},
-      // {title:"Date Of Birth", field:"dob", sorter:"date", align:"center"},
-      // {title:"Driver", field:"car", align:"center", formatter:"tickCross", sorter:"boolean"},
-    ],
+    columns: payload.fields.split(', ').map(field => {
+      switch (field) {
+        default:
+          return { title: field, field: field.toLowerCase(), sorter: 'string' }
+      }
+    }),
+    // {title:'Name', field:'name', sorter:'string', width:200},
+    // {title:'Progress', field:'progress', sorter:'number', formatter:'progress'},
+    // {title:'Gender', field:'gender', sorter:"string"},
+    // {title:"Rating", field:"rating", formatter:"star", align:"center", width:100},
+    // {title:"Favourite Color", field:"col", sorter:"string", sortable:false},
+    // {title:"Date Of Birth", field:"dob", sorter:"date", align:"center"},
+    // {title:"Driver", field:"car", align:"center", formatter:"tickCross", sorter:"boolean"},
   })
   let path = payload.opt
   let columns = payload.fields.split(',').map(s => s.trim())
-  console.log(columns)
-  if (path[0] !== '/') {
-    console.log(path)
-  }
-  // table.setData(path)
+  console.log(path, columns)
+  table.setData(path)
 }
 
 function appendApplets() {
diff --git a/megapixels/app/server/api.py b/megapixels/app/server/api.py
index cd2b950b..cf8241bd 100644
--- a/megapixels/app/server/api.py
+++ b/megapixels/app/server/api.py
@@ -1,9 +1,12 @@
 import os
 import re
 import time
+import dlib
 from flask import Blueprint, request, jsonify
 from PIL import Image  # todo: try to remove PIL dependency
 
+from app.processors import face_recognition
+from app.processors import face_detector
 from app.models.sql_factory import list_datasets, get_dataset, get_table
 
 sanitize_re = re.compile('[\W]+')
@@ -39,8 +42,18 @@ def upload(name):
 
   img = Image.open(file.stream).convert('RGB')
   
-  # vec = db.load_feature_vector_from_file(uploaded_img_path)
-  # vec = fe.extract(img)
+  # Face detection
+  detector = face_detector.DetectorDLIBHOG()
+
+  # get detection as BBox object
+  bboxes = detector.detect(im, largest=True)
+  bbox = bboxes[0]
+  dim = im.shape[:2][::-1]
+  bbox = bbox.to_dim(dim)  # convert back to real dimensions
+
+  # face recognition/vector
+  recognition = face_recognition.RecognitionDLIB(gpu=-1)
+
   # print(vec.shape)
   # results = db.search(vec, limit=limit)
 
diff --git a/megapixels/app/server/create.py b/megapixels/app/server/create.py
index c1f41dc4..4b1333b9 100644
--- a/megapixels/app/server/create.py
+++ b/megapixels/app/server/create.py
@@ -7,6 +7,9 @@ from app.server.api import api
 db = SQLAlchemy()
 
 def create_app(script_info=None):
+  """
+  functional pattern for creating the flask app
+  """
   app = Flask(__name__, static_folder='static', static_url_path='')
   app.config['SQLALCHEMY_DATABASE_URI'] = connection_url
   app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
@@ -37,6 +40,10 @@ def create_app(script_info=None):
   return app
 
 def serve_page(file_relative_path_to_root):
+  """
+  trying to get this to serve /path/ with /path/index.html,
+  ...but it doesnt actually matter for production...
+  """
   if file_relative_path_to_root[-1] == '/':
     file_relative_path_to_root += 'index.html'
   return send_from_directory("static", file_relative_path_to_root)
diff --git a/megapixels/app/site/builder.py b/megapixels/app/site/builder.py
index 91df54c2..895f265b 100644
--- a/megapixels/app/site/builder.py
+++ b/megapixels/app/site/builder.py
@@ -15,6 +15,10 @@ env = Environment(
 )
 
 def build_page(fn, research_posts):
+  """
+  build a single page from markdown into the appropriate template
+  - writes it to site/public/
+  """
   metadata, sections = parser.read_metadata(fn)
 
   if metadata is None:
@@ -61,6 +65,9 @@ def build_page(fn, research_posts):
     file.write(html)
 
 def build_research_index(research_posts):
+  """
+  build the index of research (blog) posts
+  """
   metadata, sections = parser.read_metadata('../site/content/research/index.md')
   template = env.get_template("page.html")
   s3_path = s3.make_s3_path(cfg.S3_SITE_PATH, metadata['path'])
@@ -77,6 +84,9 @@ def build_research_index(research_posts):
     file.write(html)
 
 def build_site():
+  """
+  build the site! =^)
+  """
   research_posts = parser.read_research_post_index()
   for fn in glob.iglob(os.path.join(cfg.DIR_SITE_CONTENT, "**/*.md"), recursive=True):
     build_page(fn, research_posts)
diff --git a/megapixels/app/site/parser.py b/megapixels/app/site/parser.py
index d78cc402..40d9c7f6 100644
--- a/megapixels/app/site/parser.py
+++ b/megapixels/app/site/parser.py
@@ -11,6 +11,10 @@ renderer = mistune.Renderer(escape=False)
 markdown = mistune.Markdown(renderer=renderer)
 
 def fix_images(lines, s3_path):
+  """
+  do our own tranformation of the markdown around images to handle wide images etc
+  lines: markdown lines
+  """
   real_lines = []
   block = "\n\n".join(lines)
   for line in block.split("\n"):
@@ -29,6 +33,9 @@ def fix_images(lines, s3_path):
   return "\n".join(real_lines)
 
 def format_section(lines, s3_path, type=''):
+  """
+  format a normal markdown section
+  """
   if len(lines):
     lines = fix_images(lines, s3_path)
     if type:
@@ -38,13 +45,16 @@ def format_section(lines, s3_path, type=''):
   return ""
 
 def format_metadata(section):
+  """
+  format a metadata section (+ key: value pairs)
+  """
   meta = []
   for line in section.split('\n'):
     key, value = line[2:].split(': ', 1)
     meta.append("<div><div class='gray'>{}</div><div>{}</div></div>".format(key, value))
   return "<section><div class='meta'>{}</div></section>".format(''.join(meta)) 
 
-def format_applet(section):
+def format_applet(section, s3_path):
   payload = section.replace('```', '').strip().split('\n')
   applet = {}
   if ': ' in payload[0]:
@@ -56,10 +66,15 @@ def format_applet(section):
   if opt:
     applet['opt'] = opt
   if command == 'load file':
+    if opt[0] != '/':
+      applet['opt'] = s3_path + opt
     applet['fields'] = payload[1]
   return "<section><div class='applet' data-payload='{}'></div></section>".format(json.dumps(applet))
 
 def parse_markdown(sections, s3_path, skip_h1=False):
+  """
+  parse page into sections, preprocess the markdown to handle our modifications
+  """
   groups = []
   current_group = []
   for section in sections:
@@ -67,7 +82,7 @@ def parse_markdown(sections, s3_path, skip_h1=False):
       continue
     elif section.startswith('```'):
       groups.append(format_section(current_group, s3_path))
-      groups.append(format_applet(section))
+      groups.append(format_applet(section, s3_path))
       current_group = []
     elif section.startswith('+ '):
       groups.append(format_section(current_group, s3_path))
@@ -88,6 +103,9 @@ def parse_markdown(sections, s3_path, skip_h1=False):
   return content
 
 def parse_research_index(research_posts):
+  """
+  Generate an index file for the research pages
+  """
   content = "<div class='research_index'>"
   for post in research_posts:
     s3_path = s3.make_s3_path(cfg.S3_SITE_PATH, post['path'])
@@ -105,6 +123,9 @@ def parse_research_index(research_posts):
   return content
 
 def read_metadata(fn):
+  """
+  Read in read a markdown file and extract the metadata
+  """
   with open(fn, "r") as file:
     data = file.read()
     data = data.replace("\n ", "\n")
@@ -128,6 +149,9 @@ default_metadata = {
 }
 
 def parse_metadata_section(metadata, section):
+  """
+  parse a metadata key: value pair
+  """
   for line in section.split("\n"):
     if ': ' not in line:
       continue
@@ -135,6 +159,11 @@ def parse_metadata_section(metadata, section):
     metadata[key.lower()] = value
 
 def parse_metadata(fn, sections):
+  """
+  parse the metadata headers in a markdown file
+  (everything before the second ---------)
+  also generates appropriate urls for this page :)
+  """
   found_meta = False
   metadata = {}
   valid_sections = []
@@ -175,6 +204,9 @@ def parse_metadata(fn, sections):
   return metadata, valid_sections
 
 def read_research_post_index():
+  """
+  Generate an index of the research (blog) posts
+  """
   posts = []
   for fn in sorted(glob.glob('../site/content/research/*/index.md')):
     metadata, valid_sections = read_metadata(fn)
diff --git a/megapixels/app/site/s3.py b/megapixels/app/site/s3.py
index 99726a4d..5464d464 100644
--- a/megapixels/app/site/s3.py
+++ b/megapixels/app/site/s3.py
@@ -3,6 +3,9 @@ import glob
 import boto3
 
 def sync_directory(base_fn, s3_path, metadata):
+  """
+  Synchronize a local assets folder with S3
+  """
   fns = {}
   for fn in glob.glob(os.path.join(base_fn, 'assets/*')):
     fns[os.path.basename(fn)] = True
diff --git a/site/assets/css/applets.css b/site/assets/css/applets.css
index b437886b..e106b3dd 100644
--- a/site/assets/css/applets.css
+++ b/site/assets/css/applets.css
@@ -1,4 +1,2 @@
 .applet {
-  border: 1px solid #0f0;
-  color: #0f0;
 }
diff --git a/site/assets/css/tabulator.css b/site/assets/css/tabulator.css
new file mode 100755
index 00000000..8c123cf4
--- /dev/null
+++ b/site/assets/css/tabulator.css
@@ -0,0 +1,761 @@
+/* Tabulator v4.1.3 (c) Oliver Folkerd */
+.tabulator {
+  position: relative;
+  font-size: 14px;
+  text-align: left;
+  overflow: hidden;
+  -ms-transform: translatez(0);
+  transform: translatez(0);
+}
+
+.tabulator[tabulator-layout="fitDataFill"] .tabulator-tableHolder .tabulator-table {
+  min-width: 100%;
+}
+
+.tabulator.tabulator-block-select {
+  -webkit-user-select: none;
+     -moz-user-select: none;
+      -ms-user-select: none;
+          user-select: none;
+}
+
+.tabulator .tabulator-header {
+  position: relative;
+  box-sizing: border-box;
+  width: 100%;
+  border-bottom: 1px solid #999;
+  color: #ddd;
+  white-space: nowrap;
+  overflow: hidden;
+  -moz-user-select: none;
+  -khtml-user-select: none;
+  -webkit-user-select: none;
+  -o-user-select: none;
+}
+
+.tabulator .tabulator-header .tabulator-col {
+  display: inline-block;
+  position: relative;
+  box-sizing: border-box;
+  border-right: 1px solid #aaa;
+  text-align: left;
+  vertical-align: bottom;
+  overflow: hidden;
+}
+
+.tabulator .tabulator-header .tabulator-col.tabulator-moving {
+  position: absolute;
+  border: 1px solid #999;
+  background: rgba(80,20,10,0.2);
+  pointer-events: none;
+}
+
+.tabulator .tabulator-header .tabulator-col .tabulator-col-content {
+  box-sizing: border-box;
+  position: relative;
+  padding: 4px;
+}
+
+.tabulator .tabulator-header .tabulator-col .tabulator-col-content .tabulator-col-title {
+  box-sizing: border-box;
+  width: 100%;
+  white-space: nowrap;
+  overflow: hidden;
+  text-overflow: ellipsis;
+  vertical-align: bottom;
+}
+
+.tabulator .tabulator-header .tabulator-col .tabulator-col-content .tabulator-col-title .tabulator-title-editor {
+  box-sizing: border-box;
+  width: 100%;
+  border: 1px solid #999;
+  padding: 1px;
+  background: #fff;
+}
+
+.tabulator .tabulator-header .tabulator-col .tabulator-col-content .tabulator-arrow {
+  display: inline-block;
+  position: absolute;
+  top: 9px;
+  right: 8px;
+  width: 0;
+  height: 0;
+  border-left: 6px solid transparent;
+  border-right: 6px solid transparent;
+  border-bottom: 6px solid #bbb;
+}
+
+.tabulator .tabulator-header .tabulator-col.tabulator-col-group .tabulator-col-group-cols {
+  position: relative;
+  display: -ms-flexbox;
+  display: flex;
+  border-top: 1px solid #aaa;
+  overflow: hidden;
+}
+
+.tabulator .tabulator-header .tabulator-col.tabulator-col-group .tabulator-col-group-cols .tabulator-col:last-child {
+  margin-right: -1px;
+}
+
+.tabulator .tabulator-header .tabulator-col:first-child .tabulator-col-resize-handle.prev {
+  display: none;
+}
+
+.tabulator .tabulator-header .tabulator-col.ui-sortable-helper {
+  position: absolute;
+  border: 1px solid #aaa;
+}
+
+.tabulator .tabulator-header .tabulator-col .tabulator-header-filter {
+  position: relative;
+  box-sizing: border-box;
+  margin-top: 2px;
+  width: 100%;
+  text-align: center;
+}
+
+.tabulator .tabulator-header .tabulator-col .tabulator-header-filter textarea {
+  height: auto !important;
+}
+
+.tabulator .tabulator-header .tabulator-col .tabulator-header-filter svg {
+  margin-top: 3px;
+}
+
+.tabulator .tabulator-header .tabulator-col .tabulator-header-filter input::-ms-clear {
+  width: 0;
+  height: 0;
+}
+
+.tabulator .tabulator-header .tabulator-col.tabulator-sortable .tabulator-col-title {
+  padding-right: 25px;
+}
+
+.tabulator .tabulator-header .tabulator-col.tabulator-sortable:hover {
+  cursor: pointer;
+  background-color: #cdcdcd;
+}
+
+.tabulator .tabulator-header .tabulator-col.tabulator-sortable[aria-sort="none"] .tabulator-col-content .tabulator-arrow {
+  border-top: none;
+  border-bottom: 6px solid #bbb;
+}
+
+.tabulator .tabulator-header .tabulator-col.tabulator-sortable[aria-sort="asc"] .tabulator-col-content .tabulator-arrow {
+  border-top: none;
+  border-bottom: 6px solid #666;
+}
+
+.tabulator .tabulator-header .tabulator-col.tabulator-sortable[aria-sort="desc"] .tabulator-col-content .tabulator-arrow {
+  border-top: 6px solid #666;
+  border-bottom: none;
+}
+
+.tabulator .tabulator-header .tabulator-col.tabulator-col-vertical .tabulator-col-content .tabulator-col-title {
+  -webkit-writing-mode: vertical-rl;
+      -ms-writing-mode: tb-rl;
+          writing-mode: vertical-rl;
+  text-orientation: mixed;
+  display: -ms-flexbox;
+  display: flex;
+  -ms-flex-align: center;
+      align-items: center;
+  -ms-flex-pack: center;
+      justify-content: center;
+}
+
+.tabulator .tabulator-header .tabulator-col.tabulator-col-vertical.tabulator-col-vertical-flip .tabulator-col-title {
+  -ms-transform: rotate(180deg);
+      transform: rotate(180deg);
+}
+
+.tabulator .tabulator-header .tabulator-col.tabulator-col-vertical.tabulator-sortable .tabulator-col-title {
+  padding-right: 0;
+  padding-top: 20px;
+}
+
+.tabulator .tabulator-header .tabulator-col.tabulator-col-vertical.tabulator-sortable.tabulator-col-vertical-flip .tabulator-col-title {
+  padding-right: 0;
+  padding-bottom: 20px;
+}
+
+.tabulator .tabulator-header .tabulator-col.tabulator-col-vertical.tabulator-sortable .tabulator-arrow {
+  right: calc(50% - 6px);
+}
+
+.tabulator .tabulator-header .tabulator-frozen {
+  display: inline-block;
+  position: absolute;
+  z-index: 10;
+}
+
+.tabulator .tabulator-header .tabulator-frozen.tabulator-frozen-left {
+  border-right: 2px solid #aaa;
+}
+
+.tabulator .tabulator-header .tabulator-frozen.tabulator-frozen-right {
+  border-left: 2px solid #aaa;
+}
+
+.tabulator .tabulator-header .tabulator-calcs-holder {
+  box-sizing: border-box;
+  min-width: 400%;
+  background: rgba(80,20,10,0.2);
+  border-top: 1px solid #aaa;
+  border-bottom: 1px solid #aaa;
+  overflow: hidden;
+}
+
+.tabulator .tabulator-header .tabulator-calcs-holder .tabulator-row {
+  background: rgba(80,20,10,0.2);
+}
+
+.tabulator .tabulator-header .tabulator-calcs-holder .tabulator-row .tabulator-col-resize-handle {
+  display: none;
+}
+
+.tabulator .tabulator-header .tabulator-frozen-rows-holder {
+  min-width: 400%;
+}
+
+.tabulator .tabulator-header .tabulator-frozen-rows-holder:empty {
+  display: none;
+}
+
+.tabulator .tabulator-tableHolder {
+  position: relative;
+  width: 100%;
+  white-space: nowrap;
+  overflow: auto;
+  -webkit-overflow-scrolling: touch;
+}
+
+.tabulator .tabulator-tableHolder:focus {
+  outline: none;
+}
+
+.tabulator .tabulator-tableHolder .tabulator-placeholder {
+  box-sizing: border-box;
+  display: -ms-flexbox;
+  display: flex;
+  -ms-flex-align: center;
+      align-items: center;
+  width: 100%;
+}
+
+.tabulator .tabulator-tableHolder .tabulator-placeholder[tabulator-render-mode="virtual"] {
+  position: absolute;
+  top: 0;
+  left: 0;
+  height: 100%;
+}
+
+.tabulator .tabulator-tableHolder .tabulator-placeholder span {
+  display: inline-block;
+  margin: 0 auto;
+  padding: 10px;
+  color: #ccc;
+  font-weight: bold;
+  font-size: 20px;
+}
+
+.tabulator .tabulator-tableHolder .tabulator-table {
+  position: relative;
+  display: inline-block;
+  background-color: #fff;
+  white-space: nowrap;
+  overflow: visible;
+  color: #333;
+}
+
+.tabulator .tabulator-tableHolder .tabulator-table .tabulator-row.tabulator-calcs {
+  font-weight: bold;
+  background: rgba(80,20,10,0.2);
+}
+
+.tabulator .tabulator-tableHolder .tabulator-table .tabulator-row.tabulator-calcs.tabulator-calcs-top {
+  border-bottom: 2px solid #aaa;
+}
+
+.tabulator .tabulator-tableHolder .tabulator-table .tabulator-row.tabulator-calcs.tabulator-calcs-bottom {
+  border-top: 2px solid #aaa;
+}
+
+.tabulator .tabulator-footer {
+  padding: 5px 10px;
+  border-top: 1px solid #999;
+  text-align: right;
+  color: #555;
+  font-weight: bold;
+  white-space: nowrap;
+  -ms-user-select: none;
+      user-select: none;
+  -moz-user-select: none;
+  -khtml-user-select: none;
+  -webkit-user-select: none;
+  -o-user-select: none;
+}
+
+.tabulator .tabulator-footer .tabulator-calcs-holder {
+  box-sizing: border-box;
+  width: calc(100% + 20px);
+  margin: -5px -10px 5px -10px;
+  text-align: left;
+  background: rgba(80,20,10,0.2);
+  border-bottom: 1px solid #aaa;
+  border-top: 1px solid #aaa;
+  overflow: hidden;
+}
+
+.tabulator .tabulator-footer .tabulator-calcs-holder .tabulator-row {
+  background: rgba(80,20,10,0.2);
+}
+
+.tabulator .tabulator-footer .tabulator-calcs-holder .tabulator-row .tabulator-col-resize-handle {
+  display: none;
+}
+
+.tabulator .tabulator-footer .tabulator-calcs-holder:only-child {
+  margin-bottom: -5px;
+  border-bottom: none;
+}
+
+.tabulator .tabulator-footer .tabulator-pages {
+  margin: 0 7px;
+}
+
+.tabulator .tabulator-footer .tabulator-page {
+  display: inline-block;
+  margin: 0 2px;
+  padding: 2px 5px;
+  border: 1px solid #aaa;
+  border-radius: 3px;
+  background: rgba(255, 255, 255, 0.2);
+  color: #555;
+  font-family: inherit;
+  font-weight: inherit;
+  font-size: inherit;
+}
+
+.tabulator .tabulator-footer .tabulator-page.active {
+  color: #d00;
+}
+
+.tabulator .tabulator-footer .tabulator-page:disabled {
+  opacity: .5;
+}
+
+.tabulator .tabulator-footer .tabulator-page:not(.disabled):hover {
+  cursor: pointer;
+  background: rgba(0, 0, 0, 0.2);
+  color: #fff;
+}
+
+.tabulator .tabulator-col-resize-handle {
+  position: absolute;
+  right: 0;
+  top: 0;
+  bottom: 0;
+  width: 5px;
+}
+
+.tabulator .tabulator-col-resize-handle.prev {
+  left: 0;
+  right: auto;
+}
+
+.tabulator .tabulator-col-resize-handle:hover {
+  cursor: ew-resize;
+}
+
+.tabulator .tabulator-loader {
+  position: absolute;
+  display: -ms-flexbox;
+  display: flex;
+  -ms-flex-align: center;
+      align-items: center;
+  top: 0;
+  left: 0;
+  z-index: 100;
+  height: 100%;
+  width: 100%;
+  background: rgba(0, 0, 0, 0.4);
+  text-align: center;
+}
+
+.tabulator .tabulator-loader .tabulator-loader-msg {
+  display: inline-block;
+  margin: 0 auto;
+  padding: 10px 20px;
+  border-radius: 4px;
+  background: #fff;
+  font-weight: bold;
+  font-size: 16px;
+}
+
+.tabulator .tabulator-loader .tabulator-loader-msg.tabulator-loading {
+  border: 4px solid #333;
+  color: #000;
+}
+
+.tabulator .tabulator-loader .tabulator-loader-msg.tabulator-error {
+  color: #000;
+}
+
+.tabulator-row {
+  position: relative;
+  box-sizing: border-box;
+  min-height: 22px;
+  background-color: #fff;
+}
+
+.tabulator-row.tabulator-row-even {
+  background-color: #EFEFEF;
+}
+
+.tabulator-row.tabulator-selectable:hover {
+  background-color: #bbb;
+  cursor: pointer;
+}
+
+.tabulator-row.tabulator-selected {
+  background-color: #9ABCEA;
+}
+
+.tabulator-row.tabulator-selected:hover {
+  background-color: #769BCC;
+  cursor: pointer;
+}
+
+.tabulator-row.tabulator-row-moving {
+  border: 1px solid #000;
+  background: #fff;
+}
+
+.tabulator-row.tabulator-moving {
+  position: absolute;
+  border-top: 1px solid #aaa;
+  border-bottom: 1px solid #aaa;
+  pointer-events: none;
+  z-index: 15;
+}
+
+.tabulator-row .tabulator-row-resize-handle {
+  position: absolute;
+  right: 0;
+  bottom: 0;
+  left: 0;
+  height: 5px;
+}
+
+.tabulator-row .tabulator-row-resize-handle.prev {
+  top: 0;
+  bottom: auto;
+}
+
+.tabulator-row .tabulator-row-resize-handle:hover {
+  cursor: ns-resize;
+}
+
+.tabulator-row .tabulator-frozen {
+  display: inline-block;
+  position: absolute;
+  background-color: inherit;
+  z-index: 10;
+}
+
+.tabulator-row .tabulator-frozen.tabulator-frozen-left {
+  border-right: 2px solid #aaa;
+}
+
+.tabulator-row .tabulator-frozen.tabulator-frozen-right {
+  border-left: 2px solid #aaa;
+}
+
+.tabulator-row .tabulator-responsive-collapse {
+  box-sizing: border-box;
+  padding: 5px;
+  border-top: 1px solid #aaa;
+  border-bottom: 1px solid #aaa;
+}
+
+.tabulator-row .tabulator-responsive-collapse:empty {
+  display: none;
+}
+
+.tabulator-row .tabulator-responsive-collapse table {
+  font-size: 14px;
+}
+
+.tabulator-row .tabulator-responsive-collapse table tr td {
+  position: relative;
+}
+
+.tabulator-row .tabulator-responsive-collapse table tr td:first-of-type {
+  padding-right: 10px;
+}
+
+.tabulator-row .tabulator-cell {
+  display: inline-block;
+  position: relative;
+  box-sizing: border-box;
+  padding: 4px;
+  border-right: 1px solid #aaa;
+  vertical-align: middle;
+  white-space: nowrap;
+  overflow: hidden;
+  text-overflow: ellipsis;
+}
+
+.tabulator-row .tabulator-cell.tabulator-editing {
+  border: 1px solid #1D68CD;
+  padding: 0;
+}
+
+.tabulator-row .tabulator-cell.tabulator-editing input, .tabulator-row .tabulator-cell.tabulator-editing select {
+  border: 1px;
+  background: transparent;
+}
+
+.tabulator-row .tabulator-cell.tabulator-validation-fail {
+  border: 1px solid #dd0000;
+}
+
+.tabulator-row .tabulator-cell.tabulator-validation-fail input, .tabulator-row .tabulator-cell.tabulator-validation-fail select {
+  border: 1px;
+  background: transparent;
+  color: #dd0000;
+}
+
+.tabulator-row .tabulator-cell:first-child .tabulator-col-resize-handle.prev {
+  display: none;
+}
+
+.tabulator-row .tabulator-cell.tabulator-row-handle {
+  display: -ms-inline-flexbox;
+  display: inline-flex;
+  -ms-flex-align: center;
+      align-items: center;
+  -ms-flex-pack: center;
+      justify-content: center;
+  -moz-user-select: none;
+  -khtml-user-select: none;
+  -webkit-user-select: none;
+  -o-user-select: none;
+}
+
+.tabulator-row .tabulator-cell.tabulator-row-handle .tabulator-row-handle-box {
+  width: 80%;
+}
+
+.tabulator-row .tabulator-cell.tabulator-row-handle .tabulator-row-handle-box .tabulator-row-handle-bar {
+  width: 100%;
+  height: 3px;
+  margin-top: 2px;
+  background: #666;
+}
+
+.tabulator-row .tabulator-cell .tabulator-data-tree-branch {
+  display: inline-block;
+  vertical-align: middle;
+  height: 9px;
+  width: 7px;
+  margin-top: -9px;
+  margin-right: 5px;
+  border-bottom-left-radius: 1px;
+  border-left: 2px solid #aaa;
+  border-bottom: 2px solid #aaa;
+}
+
+.tabulator-row .tabulator-cell .tabulator-data-tree-control {
+  display: -ms-inline-flexbox;
+  display: inline-flex;
+  -ms-flex-pack: center;
+      justify-content: center;
+  -ms-flex-align: center;
+      align-items: center;
+  vertical-align: middle;
+  height: 11px;
+  width: 11px;
+  margin-right: 5px;
+  border: 1px solid #333;
+  border-radius: 2px;
+  background: rgba(0, 0, 0, 0.1);
+  overflow: hidden;
+}
+
+.tabulator-row .tabulator-cell .tabulator-data-tree-control:hover {
+  cursor: pointer;
+  background: rgba(0, 0, 0, 0.2);
+}
+
+.tabulator-row .tabulator-cell .tabulator-data-tree-control .tabulator-data-tree-control-collapse {
+  display: inline-block;
+  position: relative;
+  height: 7px;
+  width: 1px;
+  background: transparent;
+}
+
+.tabulator-row .tabulator-cell .tabulator-data-tree-control .tabulator-data-tree-control-collapse:after {
+  position: absolute;
+  content: "";
+  left: -3px;
+  top: 3px;
+  height: 1px;
+  width: 7px;
+  background: #333;
+}
+
+.tabulator-row .tabulator-cell .tabulator-data-tree-control .tabulator-data-tree-control-expand {
+  display: inline-block;
+  position: relative;
+  height: 7px;
+  width: 1px;
+  background: #333;
+}
+
+.tabulator-row .tabulator-cell .tabulator-data-tree-control .tabulator-data-tree-control-expand:after {
+  position: absolute;
+  content: "";
+  left: -3px;
+  top: 3px;
+  height: 1px;
+  width: 7px;
+  background: #333;
+}
+
+.tabulator-row .tabulator-cell .tabulator-responsive-collapse-toggle {
+  display: -ms-inline-flexbox;
+  display: inline-flex;
+  -ms-flex-align: center;
+      align-items: center;
+  -ms-flex-pack: center;
+      justify-content: center;
+  -moz-user-select: none;
+  -khtml-user-select: none;
+  -webkit-user-select: none;
+  -o-user-select: none;
+  height: 15px;
+  width: 15px;
+  border-radius: 4px;
+  background: #666;
+  color: #fff;
+  font-weight: bold;
+  font-size: 1.1em;
+}
+
+.tabulator-row .tabulator-cell .tabulator-responsive-collapse-toggle:hover {
+  opacity: .7;
+}
+
+.tabulator-row .tabulator-cell .tabulator-responsive-collapse-toggle.open .tabulator-responsive-collapse-toggle-close {
+  display: initial;
+}
+
+.tabulator-row .tabulator-cell .tabulator-responsive-collapse-toggle.open .tabulator-responsive-collapse-toggle-open {
+  display: none;
+}
+
+.tabulator-row .tabulator-cell .tabulator-responsive-collapse-toggle .tabulator-responsive-collapse-toggle-close {
+  display: none;
+}
+
+.tabulator-row.tabulator-group {
+  box-sizing: border-box;
+  border-bottom: 1px solid #999;
+  border-right: 1px solid #aaa;
+  border-top: 1px solid #999;
+  padding: 5px;
+  padding-left: 10px;
+  background: #ccc;
+  font-weight: bold;
+  min-width: 100%;
+}
+
+.tabulator-row.tabulator-group:hover {
+  cursor: pointer;
+  background-color: rgba(0, 0, 0, 0.1);
+}
+
+.tabulator-row.tabulator-group.tabulator-group-visible .tabulator-arrow {
+  margin-right: 10px;
+  border-left: 6px solid transparent;
+  border-right: 6px solid transparent;
+  border-top: 6px solid #666;
+  border-bottom: 0;
+}
+
+.tabulator-row.tabulator-group.tabulator-group-level-1 .tabulator-arrow {
+  margin-left: 20px;
+}
+
+.tabulator-row.tabulator-group.tabulator-group-level-2 .tabulator-arrow {
+  margin-left: 40px;
+}
+
+.tabulator-row.tabulator-group.tabulator-group-level-3 .tabulator-arrow {
+  margin-left: 60px;
+}
+
+.tabulator-row.tabulator-group.tabulator-group-level-4 .tabulator-arrow {
+  margin-left: 80px;
+}
+
+.tabulator-row.tabulator-group.tabulator-group-level-5 .tabulator-arrow {
+  margin-left: 100px;
+}
+
+.tabulator-row.tabulator-group .tabulator-arrow {
+  display: inline-block;
+  width: 0;
+  height: 0;
+  margin-right: 16px;
+  border-top: 6px solid transparent;
+  border-bottom: 6px solid transparent;
+  border-right: 0;
+  border-left: 6px solid #666;
+  vertical-align: middle;
+}
+
+.tabulator-row.tabulator-group span {
+  margin-left: 10px;
+  color: #d00;
+}
+
+.tabulator-edit-select-list {
+  position: absolute;
+  display: inline-block;
+  box-sizing: border-box;
+  max-height: 200px;
+  background: #fff;
+  border: 1px solid #aaa;
+  font-size: 14px;
+  overflow-y: auto;
+  -webkit-overflow-scrolling: touch;
+  z-index: 10000;
+}
+
+.tabulator-edit-select-list .tabulator-edit-select-list-item {
+  padding: 4px;
+  color: #333;
+}
+
+.tabulator-edit-select-list .tabulator-edit-select-list-item.active {
+  color: #fff;
+  background: #1D68CD;
+}
+
+.tabulator-edit-select-list .tabulator-edit-select-list-item:hover {
+  cursor: pointer;
+  color: #fff;
+  background: #1D68CD;
+}
+
+.tabulator-edit-select-list .tabulator-edit-select-list-group {
+  border-bottom: 1px solid #aaa;
+  padding: 4px;
+  padding-top: 6px;
+  color: #333;
+  font-weight: bold;
+}
diff --git a/site/public/about/credits/index.html b/site/public/about/credits/index.html
index 1b1f4d54..67e9dcb8 100644
--- a/site/public/about/credits/index.html
+++ b/site/public/about/credits/index.html
@@ -8,6 +8,7 @@
   <meta name="referrer" content="no-referrer" />
   <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes" />
   <link rel='stylesheet' href='/assets/css/fonts.css' />
+  <link rel='stylesheet' href='/assets/css/tabulator.css' />
   <link rel='stylesheet' href='/assets/css/css.css' />
   <link rel='stylesheet' href='/assets/css/applets.css' />
 </head>
@@ -19,7 +20,6 @@
       <span class='sub'>The Darkside of Datasets</span>
     </a>
     <div class='links'>
-      <a href="/search/">Face Search</a>
       <a href="/datasets/">Datasets</a>
       <a href="/research/">Research</a>
       <a href="/about/">About</a>
diff --git a/site/public/about/disclaimer/index.html b/site/public/about/disclaimer/index.html
index dfd136b6..d1252d94 100644
--- a/site/public/about/disclaimer/index.html
+++ b/site/public/about/disclaimer/index.html
@@ -8,6 +8,7 @@
   <meta name="referrer" content="no-referrer" />
   <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes" />
   <link rel='stylesheet' href='/assets/css/fonts.css' />
+  <link rel='stylesheet' href='/assets/css/tabulator.css' />
   <link rel='stylesheet' href='/assets/css/css.css' />
   <link rel='stylesheet' href='/assets/css/applets.css' />
 </head>
@@ -19,7 +20,6 @@
       <span class='sub'>The Darkside of Datasets</span>
     </a>
     <div class='links'>
-      <a href="/search/">Face Search</a>
       <a href="/datasets/">Datasets</a>
       <a href="/research/">Research</a>
       <a href="/about/">About</a>
diff --git a/site/public/about/index.html b/site/public/about/index.html
index 1b1f4d54..67e9dcb8 100644
--- a/site/public/about/index.html
+++ b/site/public/about/index.html
@@ -8,6 +8,7 @@
   <meta name="referrer" content="no-referrer" />
   <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes" />
   <link rel='stylesheet' href='/assets/css/fonts.css' />
+  <link rel='stylesheet' href='/assets/css/tabulator.css' />
   <link rel='stylesheet' href='/assets/css/css.css' />
   <link rel='stylesheet' href='/assets/css/applets.css' />
 </head>
@@ -19,7 +20,6 @@
       <span class='sub'>The Darkside of Datasets</span>
     </a>
     <div class='links'>
-      <a href="/search/">Face Search</a>
       <a href="/datasets/">Datasets</a>
       <a href="/research/">Research</a>
       <a href="/about/">About</a>
diff --git a/site/public/about/press/index.html b/site/public/about/press/index.html
index dca2aa3c..56fa792e 100644
--- a/site/public/about/press/index.html
+++ b/site/public/about/press/index.html
@@ -8,6 +8,7 @@
   <meta name="referrer" content="no-referrer" />
   <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes" />
   <link rel='stylesheet' href='/assets/css/fonts.css' />
+  <link rel='stylesheet' href='/assets/css/tabulator.css' />
   <link rel='stylesheet' href='/assets/css/css.css' />
   <link rel='stylesheet' href='/assets/css/applets.css' />
 </head>
@@ -19,7 +20,6 @@
       <span class='sub'>The Darkside of Datasets</span>
     </a>
     <div class='links'>
-      <a href="/search/">Face Search</a>
       <a href="/datasets/">Datasets</a>
       <a href="/research/">Research</a>
       <a href="/about/">About</a>
diff --git a/site/public/about/privacy/index.html b/site/public/about/privacy/index.html
index 86b645d5..969f2a87 100644
--- a/site/public/about/privacy/index.html
+++ b/site/public/about/privacy/index.html
@@ -8,6 +8,7 @@
   <meta name="referrer" content="no-referrer" />
   <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes" />
   <link rel='stylesheet' href='/assets/css/fonts.css' />
+  <link rel='stylesheet' href='/assets/css/tabulator.css' />
   <link rel='stylesheet' href='/assets/css/css.css' />
   <link rel='stylesheet' href='/assets/css/applets.css' />
 </head>
@@ -19,7 +20,6 @@
       <span class='sub'>The Darkside of Datasets</span>
     </a>
     <div class='links'>
-      <a href="/search/">Face Search</a>
       <a href="/datasets/">Datasets</a>
       <a href="/research/">Research</a>
       <a href="/about/">About</a>
diff --git a/site/public/about/style/index.html b/site/public/about/style/index.html
index ed61c270..d3784b31 100644
--- a/site/public/about/style/index.html
+++ b/site/public/about/style/index.html
@@ -8,6 +8,7 @@
   <meta name="referrer" content="no-referrer" />
   <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes" />
   <link rel='stylesheet' href='/assets/css/fonts.css' />
+  <link rel='stylesheet' href='/assets/css/tabulator.css' />
   <link rel='stylesheet' href='/assets/css/css.css' />
   <link rel='stylesheet' href='/assets/css/applets.css' />
 </head>
@@ -19,7 +20,6 @@
       <span class='sub'>The Darkside of Datasets</span>
     </a>
     <div class='links'>
-      <a href="/search/">Face Search</a>
       <a href="/datasets/">Datasets</a>
       <a href="/research/">Research</a>
       <a href="/about/">About</a>
diff --git a/site/public/about/terms/index.html b/site/public/about/terms/index.html
index dd095b64..8a1ef49d 100644
--- a/site/public/about/terms/index.html
+++ b/site/public/about/terms/index.html
@@ -8,6 +8,7 @@
   <meta name="referrer" content="no-referrer" />
   <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes" />
   <link rel='stylesheet' href='/assets/css/fonts.css' />
+  <link rel='stylesheet' href='/assets/css/tabulator.css' />
   <link rel='stylesheet' href='/assets/css/css.css' />
   <link rel='stylesheet' href='/assets/css/applets.css' />
 </head>
@@ -19,7 +20,6 @@
       <span class='sub'>The Darkside of Datasets</span>
     </a>
     <div class='links'>
-      <a href="/search/">Face Search</a>
       <a href="/datasets/">Datasets</a>
       <a href="/research/">Research</a>
       <a href="/about/">About</a>
diff --git a/site/public/datasets/lfw/index.html b/site/public/datasets/lfw/index.html
index e75e79cf..b375247b 100644
--- a/site/public/datasets/lfw/index.html
+++ b/site/public/datasets/lfw/index.html
@@ -8,6 +8,7 @@
   <meta name="referrer" content="no-referrer" />
   <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes" />
   <link rel='stylesheet' href='/assets/css/fonts.css' />
+  <link rel='stylesheet' href='/assets/css/tabulator.css' />
   <link rel='stylesheet' href='/assets/css/css.css' />
   <link rel='stylesheet' href='/assets/css/applets.css' />
 </head>
@@ -19,7 +20,6 @@
       <span class='sub'>The Darkside of Datasets</span>
     </a>
     <div class='links'>
-      <a href="/search/">Face Search</a>
       <a href="/datasets/">Datasets</a>
       <a href="/research/">Research</a>
       <a href="/about/">About</a>
@@ -29,7 +29,7 @@
     
   <section><h1>Labeled Faces in the Wild</h1>
 </section><section><div class='meta'><div><div class='gray'>Created</div><div>2007</div></div><div><div class='gray'>Images</div><div>13,233</div></div><div><div class='gray'>People</div><div>5,749</div></div><div><div class='gray'>Created From</div><div>Yahoo News images</div></div><div><div class='gray'>Search available</div><div>Searchable</div></div></div></section><section><p>Labeled Faces in The Wild (LFW) is amongst the most widely used facial recognition training datasets in the world and is the first of its kind to be created entirely from images posted online. The LFW dataset includes 13,233 images of 5,749 people that were collected between 2002-2004. Use the tools below to check if you were included in this dataset or scroll down to read the analysis.</p>
-</section><section><div class='applet' data-payload='{"command": "face_search"}'></div></section><section><div class='applet' data-payload='{"command": "name_search"}'></div></section><section><div class='applet' data-payload='{"command": "load file", "opt": "lfw_names_gender_kg_min.csv", "fields": "Name, Images, Gender, Description"}'></div></section><section class='images'><div class='image'><img src='https://nyc3.digitaloceanspaces.com/megapixels/v1/datasets/lfw/assets/lfw_feature.jpg' alt='Eighteen of the 5,749 people in the Labeled Faces in the Wild Dataset. The most widely used face dataset for benchmarking commercial face recognition algorithms.'><div class='caption'>Eighteen of the 5,749 people in the Labeled Faces in the Wild Dataset. The most widely used face dataset for benchmarking commercial face recognition algorithms.</div></div></section><section><h2>Intro</h2>
+</section><section><div class='applet' data-payload='{"command": "face_search"}'></div></section><section><div class='applet' data-payload='{"command": "name_search"}'></div></section><section><div class='applet' data-payload='{"command": "load file", "opt": "https://nyc3.digitaloceanspaces.com/megapixels/v1/datasets/lfw/assets/lfw_names_gender_kg_min.csv", "fields": "Name, Images, Gender, Description"}'></div></section><section class='images'><div class='image'><img src='https://nyc3.digitaloceanspaces.com/megapixels/v1/datasets/lfw/assets/lfw_feature.jpg' alt='Eighteen of the 5,749 people in the Labeled Faces in the Wild Dataset. The most widely used face dataset for benchmarking commercial face recognition algorithms.'><div class='caption'>Eighteen of the 5,749 people in the Labeled Faces in the Wild Dataset. The most widely used face dataset for benchmarking commercial face recognition algorithms.</div></div></section><section><h2>Intro</h2>
 <p>Three paragraphs describing the LFW dataset in a format that can be easily replicated for the other datasets. Nothing too custom. An analysis of the initial research papers with context relative to all the other dataset papers.</p>
 </section><section class='images'><div class='image'><img src='https://nyc3.digitaloceanspaces.com/megapixels/v1/datasets/lfw/assets/lfw_montage_everyone_1920.jpg' alt=' all 5,749 people in the LFW Dataset sorted from most to least images collected.'><div class='caption'> all 5,749 people in the LFW Dataset sorted from most to least images collected.</div></div></section><section><h2>LFW by the Numbers</h2>
 <ul>
@@ -66,7 +66,7 @@
 <p>According to BiometricUpdate.com [^lfw_pingan], LFW is "the most widely used evaluation set in the field of facial recognition, LFW attracts a few dozen teams from around the globe including Google, Facebook, Microsoft Research Asia, Baidu, Tencent, SenseTime, Face++ and Chinese University of Hong Kong."</p>
 <p>According to researchers at the Baidu Research – Institute of Deep Learning "LFW has been the most popular evaluation benchmark for face recognition, and played a very important role in facilitating the face recognition society to improve algorithm. [^lfw_baidu]."</p>
 <p>In addition to commercial use as an evaluation tool, alll of the faces in LFW dataset are prepackaged into a popular machine learning code framework called scikit-learn.</p>
-<pre><code>load file: lfw_commercial_use.csv
+<pre><code>load file: assets/lfw_commercial_use.csv
 name_display,company_url,example_url,country,description
 </code></pre>
 <table>
diff --git a/site/public/datasets/vgg_face2/index.html b/site/public/datasets/vgg_face2/index.html
index 63bb2cc0..46242505 100644
--- a/site/public/datasets/vgg_face2/index.html
+++ b/site/public/datasets/vgg_face2/index.html
@@ -8,6 +8,7 @@
   <meta name="referrer" content="no-referrer" />
   <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes" />
   <link rel='stylesheet' href='/assets/css/fonts.css' />
+  <link rel='stylesheet' href='/assets/css/tabulator.css' />
   <link rel='stylesheet' href='/assets/css/css.css' />
   <link rel='stylesheet' href='/assets/css/applets.css' />
 </head>
@@ -19,7 +20,6 @@
       <span class='sub'>The Darkside of Datasets</span>
     </a>
     <div class='links'>
-      <a href="/search/">Face Search</a>
       <a href="/datasets/">Datasets</a>
       <a href="/research/">Research</a>
       <a href="/about/">About</a>
@@ -29,7 +29,7 @@
     
   <section><h1>VGG Faces2</h1>
 </section><section><div class='meta'><div><div class='gray'>Created</div><div>2018</div></div><div><div class='gray'>Images</div><div>3.3M</div></div><div><div class='gray'>People</div><div>9,000</div></div><div><div class='gray'>Created From</div><div>Scraping search engines</div></div><div><div class='gray'>Search available</div><div>[Searchable](#)</div></div></div></section><section><p>VGG Face2 is the updated version of the VGG Face dataset and now includes over 3.3M face images from over 9K people. The identities were selected by taking the top 500K identities in Google's Knowledge Graph of celebrities and then selecting only the names that yielded enough training images. The dataset was created in the UK but funded by Office of Director of National Intelligence in the United States.</p>
-</section><section><div class='applet' data-payload='{"command": "face_search"}'></div></section><section><div class='applet' data-payload='{"command": "name_search"}'></div></section><section><div class='applet' data-payload='{"command": "load file", "opt": "lfw_names_gender_kg_min.csv", "fields": "Name, Images, Gender, Description"}'></div></section><section><h2>VGG Face2 by the Numbers</h2>
+</section><section><div class='applet' data-payload='{"command": "face_search"}'></div></section><section><div class='applet' data-payload='{"command": "name_search"}'></div></section><section><div class='applet' data-payload='{"command": "load file", "opt": "https://nyc3.digitaloceanspaces.com/megapixels/v1/datasets/vgg_face2/lfw_names_gender_kg_min.csv", "fields": "Name, Images, Gender, Description"}'></div></section><section><h2>VGG Face2 by the Numbers</h2>
 <ul>
 <li>1,331 actresses, 139 presidents</li>
 <li>3 husbands and 16 wives</li>
diff --git a/site/public/index.html b/site/public/index.html
index a745dd21..d60d50e9 100644
--- a/site/public/index.html
+++ b/site/public/index.html
@@ -8,6 +8,7 @@
   <meta name="referrer" content="no-referrer" />
   <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes" />
   <link rel='stylesheet' href='/assets/css/fonts.css' />
+  <link rel='stylesheet' href='/assets/css/tabulator.css' />
   <link rel='stylesheet' href='/assets/css/css.css' />
   <link rel='stylesheet' href='/assets/css/applets.css' />
 </head>
@@ -19,7 +20,6 @@
       <span class='sub'>The Darkside of Datasets</span>
     </a>
     <div class='links'>
-      <a href="/search/">Face Search</a>
       <a href="/datasets/">Datasets</a>
       <a href="/research/">Research</a>
       <a href="/about/">About</a>
diff --git a/site/public/research/00_introduction/index.html b/site/public/research/00_introduction/index.html
index 52a2416f..290208f8 100644
--- a/site/public/research/00_introduction/index.html
+++ b/site/public/research/00_introduction/index.html
@@ -8,6 +8,7 @@
   <meta name="referrer" content="no-referrer" />
   <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes" />
   <link rel='stylesheet' href='/assets/css/fonts.css' />
+  <link rel='stylesheet' href='/assets/css/tabulator.css' />
   <link rel='stylesheet' href='/assets/css/css.css' />
   <link rel='stylesheet' href='/assets/css/applets.css' />
 </head>
@@ -19,7 +20,6 @@
       <span class='sub'>The Darkside of Datasets</span>
     </a>
     <div class='links'>
-      <a href="/search/">Face Search</a>
       <a href="/datasets/">Datasets</a>
       <a href="/research/">Research</a>
       <a href="/about/">About</a>
diff --git a/site/public/research/01_from_1_to_100_pixels/index.html b/site/public/research/01_from_1_to_100_pixels/index.html
index 22869459..31df84bb 100644
--- a/site/public/research/01_from_1_to_100_pixels/index.html
+++ b/site/public/research/01_from_1_to_100_pixels/index.html
@@ -8,6 +8,7 @@
   <meta name="referrer" content="no-referrer" />
   <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes" />
   <link rel='stylesheet' href='/assets/css/fonts.css' />
+  <link rel='stylesheet' href='/assets/css/tabulator.css' />
   <link rel='stylesheet' href='/assets/css/css.css' />
   <link rel='stylesheet' href='/assets/css/applets.css' />
 </head>
@@ -19,7 +20,6 @@
       <span class='sub'>The Darkside of Datasets</span>
     </a>
     <div class='links'>
-      <a href="/search/">Face Search</a>
       <a href="/datasets/">Datasets</a>
       <a href="/research/">Research</a>
       <a href="/about/">About</a>
diff --git a/site/public/research/index.html b/site/public/research/index.html
index 62e3208a..17b4a2a0 100644
--- a/site/public/research/index.html
+++ b/site/public/research/index.html
@@ -8,6 +8,7 @@
   <meta name="referrer" content="no-referrer" />
   <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes" />
   <link rel='stylesheet' href='/assets/css/fonts.css' />
+  <link rel='stylesheet' href='/assets/css/tabulator.css' />
   <link rel='stylesheet' href='/assets/css/css.css' />
   <link rel='stylesheet' href='/assets/css/applets.css' />
 </head>
@@ -19,7 +20,6 @@
       <span class='sub'>The Darkside of Datasets</span>
     </a>
     <div class='links'>
-      <a href="/search/">Face Search</a>
       <a href="/datasets/">Datasets</a>
       <a href="/research/">Research</a>
       <a href="/about/">About</a>
diff --git a/site/templates/layout.html b/site/templates/layout.html
index a9fc89d6..75682849 100644
--- a/site/templates/layout.html
+++ b/site/templates/layout.html
@@ -8,6 +8,7 @@
   <meta name="referrer" content="no-referrer" />
   <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes" />
   <link rel='stylesheet' href='/assets/css/fonts.css' />
+  <link rel='stylesheet' href='/assets/css/tabulator.css' />
   <link rel='stylesheet' href='/assets/css/css.css' />
   <link rel='stylesheet' href='/assets/css/applets.css' />
 </head>
-- 
cgit v1.2.3-70-g09d2


From 4cf8581655c34698f8869bb364b6d436b881d17a Mon Sep 17 00:00:00 2001
From: Jules Laplace <julescarbon@gmail.com>
Date: Mon, 17 Dec 2018 00:35:19 +0100
Subject: returning results...!

---
 client/faceSearch/faceSearch.query.js    |  26 +++++++------
 megapixels/app/models/sql_factory.py     |  64 ++++++++++++++++++++++++++++---
 megapixels/app/processors/faiss.py       |  58 ++++++++++++++++++++++++++++
 megapixels/app/server/api.py             |  53 ++++++++++++++++++++-----
 megapixels/app/server/json_encoder.py    |  17 ++++++++
 megapixels/commands/faiss/build_faiss.py |  36 +----------------
 site/assets/img/ajax-loader.gif          | Bin 1849 -> 0 bytes
 site/assets/img/loader.gif               | Bin 0 -> 1849 bytes
 8 files changed, 193 insertions(+), 61 deletions(-)
 create mode 100644 megapixels/app/processors/faiss.py
 create mode 100644 megapixels/app/server/json_encoder.py
 delete mode 100644 site/assets/img/ajax-loader.gif
 create mode 100644 site/assets/img/loader.gif

(limited to 'megapixels/app/server/api.py')

diff --git a/client/faceSearch/faceSearch.query.js b/client/faceSearch/faceSearch.query.js
index 8302e437..425cb282 100644
--- a/client/faceSearch/faceSearch.query.js
+++ b/client/faceSearch/faceSearch.query.js
@@ -20,12 +20,12 @@ class FaceSearchQuery extends Component {
       if (file && file.type.match('image.*')) break
     }
     if (!file) return
-    var fr = new FileReader();
+    const fr = new FileReader()
     fr.onload = () => {
       fr.onload = null
       this.setState({ image: fr.result })
     }
-    fr.readAsDataURL(files[0]);
+    fr.readAsDataURL(files[0])
     this.props.actions.upload(this.props.payload, file)
   }
 
@@ -36,6 +36,7 @@ class FaceSearchQuery extends Component {
     if (image) {
       style.backgroundImage = 'url(' + image + ')'
       style.backgroundSize = 'cover'
+      style.opacity = 1
     }
     return (
       <div className='query row'>
@@ -44,9 +45,8 @@ class FaceSearchQuery extends Component {
           <div className='loading' style={style}>
             <Loader />
           </div>
-         :
-          <div style={style}>
-            <img src="/assets/img/icon_camera.svg" />
+         : <div style={style}>
+            {image ? null : <img src="/assets/img/icon_camera.svg" />}
             <input
               type="file"
               name="img"
@@ -57,18 +57,20 @@ class FaceSearchQuery extends Component {
           </div>
         }
         </div>
-        <div class='cta'>
+        <div className='cta'>
           <h2>Search This Dataset</h2>
           <h3>Searching {13456} images</h3>
           <p>
-            Use facial recognition to reverse search into the LFW dataset and see if it contains your photos.
+            {'Use facial recognition to reverse search into the LFW dataset '}
+            {'and see if it contains your photos.'}
           </p>
           <ol>
-          <li>Upload a photo of yourself</li>
-          <li>Use a photo similar to examples below</li>
-          <li>Only matches over 85% will be displayed</li>
-          <li>Read more tips to improve search results</li>
-          <li>Your search data is never stored and immediately cleared once you leave this page.</li>
+            <li>Upload a photo of yourself</li>
+            <li>Use a photo similar to examples below</li>
+            <li>Only matches over 85% will be displayed</li>
+            <li>Read more tips to improve search results</li>
+            <li>{'Your search data is never stored and immediately cleared '}
+            {'once you leave this page.'}</li>
           </ol>
           <p>
             Read more about <a href='/about/privacy/'>privacy</a>.
diff --git a/megapixels/app/models/sql_factory.py b/megapixels/app/models/sql_factory.py
index e35c3e15..0f7e73a0 100644
--- a/megapixels/app/models/sql_factory.py
+++ b/megapixels/app/models/sql_factory.py
@@ -19,6 +19,7 @@ connection_url = "mysql+mysqldb://{}:{}@{}/{}".format(
 
 datasets = {}
 loaded = False
+Session = None
 
 def list_datasets():
   return [dataset.describe() for dataset in datasets.values()]
@@ -31,10 +32,11 @@ def get_table(name, table_name):
   return dataset.get_table(table_name) if dataset else None
 
 def load_sql_datasets(replace=False, base_model=None):
-  global datasets, loaded
+  global datasets, loaded, Session
   if loaded:
     return datasets
-  engine = create_engine(connection_url) if replace else None
+  engine = create_engine(connection_url)
+  Session = sessionmaker(bind=engine)
   for path in glob.iglob(os.path.join(cfg.DIR_FAISS_METADATA, "*")):
     dataset = load_sql_dataset(path, replace, engine, base_model)
     datasets[dataset.name] = dataset
@@ -79,6 +81,27 @@ class SqlDataset:
       'tables': list(self.tables.keys()),
     }
 
+  def get_identity(self, id):
+    table = self.get_table('identity_meta')
+    identity = table.query.filter(table.image_id >= id).order_by(table.image_id.asc()).first().toJSON()
+    print(identity)
+    return {
+      'uuid': self.select('uuids', id),
+      'identity': identity,
+      'roi': self.select('roi', id),
+      'pose': self.select('pose', id),
+    }
+
+  def select(self, table, id):
+    table = self.get_table(table)
+    if not table:
+      return None
+    session = Session()
+    # for obj in session.query(table).filter_by(id=id):
+    print(table)
+    obj = session.query(table).filter(table.id == id).first()
+    return obj.toJSON()
+
   def get_table(self, type):
     if type in self.tables:
       return self.tables[type]
@@ -102,6 +125,11 @@ class SqlDataset:
       __tablename__ = self.name + "_uuid"
       id = Column(Integer, primary_key=True)
       uuid = Column(String(36), nullable=False)
+      def toJSON(self):
+        return {
+          'id': self.id,
+          'uuid': self.uuid,
+        }
     return UUID
 
   # ==> roi.csv <==
@@ -118,6 +146,17 @@ class SqlDataset:
       w = Column(Float, nullable=False)
       x = Column(Float, nullable=False)
       y = Column(Float, nullable=False)
+      def toJSON(self):
+        return {
+          'id': self.id,
+          'image_index': self.image_index,
+          'image_height': self.image_height,
+          'image_width': self.image_width,
+          'w': self.w,
+          'h': self.h,
+          'x': self.x,
+          'y': self.y,
+        }
     return ROI
 
   # ==> identity.csv <==
@@ -132,6 +171,15 @@ class SqlDataset:
       gender = Column(String(1), nullable=False)
       images = Column(Integer, nullable=False)
       image_id = Column(Integer, nullable=False)
+      def toJSON(self):
+        return {
+          'id': self.id,
+          'image_id': self.image_id,
+          'fullname': self.fullname,
+          'images': self.images,
+          'gender': self.gender,
+          'description': self.description,
+        }
     return Identity
 
   # ==> pose.csv <==
@@ -145,8 +193,12 @@ class SqlDataset:
       pitch = Column(Float, nullable=False)
       roll = Column(Float, nullable=False)
       yaw = Column(Float, nullable=False)
+      def toJSON(self):
+        return {
+          'id': self.id,
+          'image_id': self.image_id,
+          'pitch': self.pitch,
+          'roll': self.roll,
+          'yaw': self.yaw,
+        }
     return Pose
-
-
-# Session = sessionmaker(bind=engine)
-# session = Session()
diff --git a/megapixels/app/processors/faiss.py b/megapixels/app/processors/faiss.py
new file mode 100644
index 00000000..5156ad71
--- /dev/null
+++ b/megapixels/app/processors/faiss.py
@@ -0,0 +1,58 @@
+"""
+Index all of the FAISS datasets
+"""
+
+import os
+import glob
+import faiss
+import time
+import numpy as np
+
+from app.utils.file_utils import load_recipe, load_csv_safe
+from app.settings import app_cfg as cfg
+
+class DefaultRecipe:
+  def __init__(self):
+    self.dim = 128
+    self.factory_type = 'Flat'
+
+def build_all_faiss_databases():
+  datasets = []
+  for fn in glob.iglob(os.path.join(cfg.DIR_FAISS_METADATA, "*")):
+    name = os.path.basename(fn)
+    recipe_fn = os.path.join(cfg.DIR_FAISS_RECIPES, name + ".json")
+    if os.path.exists(recipe_fn):
+      build_faiss_database(name, load_recipe(recipe_fn))
+    else:
+      build_faiss_database(name, DefaultRecipe())
+
+def build_faiss_database(name, recipe):
+  vec_fn = os.path.join(cfg.DIR_FAISS_METADATA, name, "vecs.csv")
+  index_fn = os.path.join(cfg.DIR_FAISS_INDEXES, name + ".index")
+
+  index = faiss.index_factory(recipe.dim, recipe.factory_type)
+
+  keys, rows = load_csv_safe(vec_fn)
+  feats = np.array([ list(map(float, row[3].split(","))) for row in rows ]).astype('float32')
+  n, d = feats.shape
+
+  print("{}: training {} x {} dim vectors".format(name, n, d))
+  print(recipe.factory_type)
+
+  add_start = time.time()
+  index.add(feats)
+  add_end = time.time()
+  add_time = add_end - add_start
+  print("{}: add time: {:.1f}s".format(name, add_time))
+
+  faiss.write_index(index, index_fn)
+
+def load_faiss_databases():
+  faiss_datasets = {}
+  for fn in glob.iglob(os.path.join(cfg.DIR_FAISS_METADATA, "*")):
+    name = os.path.basename(fn)
+    index_fn = os.path.join(cfg.DIR_FAISS_INDEXES, name + ".index")
+    if os.path.exists(index_fn):
+      index = faiss.read_index(index_fn)
+      faiss_datasets[name] = index
+  return faiss_datasets
diff --git a/megapixels/app/server/api.py b/megapixels/app/server/api.py
index cf8241bd..36563910 100644
--- a/megapixels/app/server/api.py
+++ b/megapixels/app/server/api.py
@@ -2,18 +2,23 @@ import os
 import re
 import time
 import dlib
+import numpy as np
 from flask import Blueprint, request, jsonify
 from PIL import Image  # todo: try to remove PIL dependency
 
 from app.processors import face_recognition
 from app.processors import face_detector
-from app.models.sql_factory import list_datasets, get_dataset, get_table
+from app.processors.faiss import load_faiss_databases
+from app.models.sql_factory import load_sql_datasets, list_datasets, get_dataset, get_table
+from app.utils.im_utils import pil2np
 
 sanitize_re = re.compile('[\W]+')
 valid_exts = ['.gif', '.jpg', '.jpeg', '.png']
 
 api = Blueprint('api', __name__)
 
+faiss_datasets = load_faiss_databases()
+
 @api.route('/')
 def index():
   return jsonify({ 'datasets': list_datasets() })
@@ -26,10 +31,15 @@ def show(name):
   else:
     return jsonify({ 'status': 404 })
 
-@api.route('/dataset/<name>/face', methods=['POST'])
+@api.route('/dataset/<name>/face/', methods=['POST'])
 def upload(name):
   start = time.time()
   dataset = get_dataset(name)
+  if name not in faiss_datasets:
+    return jsonify({
+      'error': 'invalid dataset'  
+    })
+  faiss_dataset = faiss_datasets[name]
   file = request.files['query_img']
   fn = file.filename
   if fn.endswith('blob'):
@@ -40,22 +50,46 @@ def upload(name):
   if ext.lower() not in valid_exts:
     return jsonify({ 'error': 'not an image' })
 
-  img = Image.open(file.stream).convert('RGB')
+  im = Image.open(file.stream).convert('RGB')
+  im_np = pil2np(im)
   
   # Face detection
   detector = face_detector.DetectorDLIBHOG()
 
   # get detection as BBox object
-  bboxes = detector.detect(im, largest=True)
+  bboxes = detector.detect(im_np, largest=True)
   bbox = bboxes[0]
-  dim = im.shape[:2][::-1]
+  dim = im_np.shape[:2][::-1]
   bbox = bbox.to_dim(dim)  # convert back to real dimensions
 
   # face recognition/vector
   recognition = face_recognition.RecognitionDLIB(gpu=-1)
+  vec = recognition.vec(im_np, bbox)
+
+  # print(vec)
+  query = np.array([ vec ]).astype('float32')
+  
+  # query FAISS!
+  distances, indexes = faiss_dataset.search(query, 5)
+
+  if len(indexes) == 0:
+    print("weird, no results!")
+    return []
+
+  # get the results for this single query...
+  distances = distances[0]
+  indexes = indexes[0]
 
-  # print(vec.shape)
-  # results = db.search(vec, limit=limit)
+  if len(indexes) == 0:
+    print("no results!")
+    return []
+
+  lookup = {}
+  for _d, _i in zip(distances, indexes):
+    lookup[_i+1] = _d
+
+  print(distances)
+  print(indexes)
 
   # with the result we have an ID
   # query the sql dataset for the UUID etc here
@@ -63,12 +97,13 @@ def upload(name):
   query = {
     'timing': time.time() - start,
   }
-  results = []
+  results = [ dataset.get_identity(index) for index in indexes ]
 
   print(results)
   return jsonify({
-    'query': query,
     'results': results,
+    # 'distances': distances.tolist(),
+    # 'indexes': indexes.tolist(),
   })
 
 @api.route('/dataset/<name>/name', methods=['GET'])
diff --git a/megapixels/app/server/json_encoder.py b/megapixels/app/server/json_encoder.py
new file mode 100644
index 00000000..89af578a
--- /dev/null
+++ b/megapixels/app/server/json_encoder.py
@@ -0,0 +1,17 @@
+from sqlalchemy.ext.declarative import DeclarativeMeta
+from flask import json
+
+class AlchemyEncoder(json.JSONEncoder):
+  def default(self, o):
+    if isinstance(o.__class__, DeclarativeMeta):
+      data = {}
+      fields = o.__json__() if hasattr(o, '__json__') else dir(o)
+      for field in [f for f in fields if not f.startswith('_') and f not in ['metadata', 'query', 'query_class']]:
+        value = o.__getattribute__(field)
+        try:
+          json.dumps(value)
+          data[field] = value
+        except TypeError:
+          data[field] = None
+      return data
+    return json.JSONEncoder.default(self, o)
diff --git a/megapixels/commands/faiss/build_faiss.py b/megapixels/commands/faiss/build_faiss.py
index ec94c924..fc6b37ce 100644
--- a/megapixels/commands/faiss/build_faiss.py
+++ b/megapixels/commands/faiss/build_faiss.py
@@ -11,11 +11,7 @@ import numpy as np
 
 from app.utils.file_utils import load_recipe, load_csv_safe
 from app.settings import app_cfg as cfg
-
-class DefaultRecipe:
-  def __init__(self):
-    self.dim = 128
-    self.factory_type = 'Flat'
+from app.processors.faiss import build_all_faiss_databases
 
 @click.command()
 @click.pass_context
@@ -25,32 +21,4 @@ def cli(ctx):
     - uses the recipe above by default
     - however you can override this by adding a new recipe in faiss/recipes/{name}.json
   """
-  datasets = []
-  for fn in glob.iglob(os.path.join(cfg.DIR_FAISS_METADATA, "*")):
-    name = os.path.basename(fn)
-    recipe_fn = os.path.join(cfg.DIR_FAISS_RECIPES, name + ".json")
-    if os.path.exists(recipe_fn):
-      build_faiss(name, load_recipe(recipe_fn))
-    else:
-      build_faiss(name, DefaultRecipe())
-
-def build_faiss(name, recipe):
-  vec_fn = os.path.join(cfg.DIR_FAISS_METADATA, name, "vecs.csv")
-  index_fn = os.path.join(cfg.DIR_FAISS_INDEXES, name + ".index")
-
-  index = faiss.index_factory(recipe.dim, recipe.factory_type)
-
-  keys, rows = load_csv_safe(vec_fn)
-  feats = np.array([ list(map(float, row[3].split(","))) for row in rows ]).astype('float32')
-  n, d = feats.shape
-
-  print("{}: training {} x {} dim vectors".format(name, n, d))
-  print(recipe.factory_type)
-
-  add_start = time.time()
-  index.add(feats)
-  add_end = time.time()
-  add_time = add_end - add_start
-  print("{}: add time: {:.1f}s".format(name, add_time))
-
-  faiss.write_index(index, index_fn)
+  build_all_faiss_databases()
diff --git a/site/assets/img/ajax-loader.gif b/site/assets/img/ajax-loader.gif
deleted file mode 100644
index dc21df18..00000000
Binary files a/site/assets/img/ajax-loader.gif and /dev/null differ
diff --git a/site/assets/img/loader.gif b/site/assets/img/loader.gif
new file mode 100644
index 00000000..dc21df18
Binary files /dev/null and b/site/assets/img/loader.gif differ
-- 
cgit v1.2.3-70-g09d2


From d7df4ee5b9e24a9cdf2bf4d1bc2e73e97352afdc Mon Sep 17 00:00:00 2001
From: Jules Laplace <julescarbon@gmail.com>
Date: Mon, 17 Dec 2018 01:02:40 +0100
Subject: searches execute

---
 client/actions.js                      |  6 +-----
 client/faceSearch/faceSearch.result.js | 35 ++++++++++++++++++++++++++++++----
 megapixels/app/models/sql_factory.py   |  5 +++--
 megapixels/app/server/api.py           |  9 +++++----
 site/assets/css/applets.css            | 17 +++++++++++++++++
 5 files changed, 57 insertions(+), 15 deletions(-)

(limited to 'megapixels/app/server/api.py')

diff --git a/client/actions.js b/client/actions.js
index 37b4eb2e..bb011838 100644
--- a/client/actions.js
+++ b/client/actions.js
@@ -1,9 +1,5 @@
 import * as faceSearch from './faceSearch/faceSearch.actions'
-// import * as review from './review/review.actions'
-// import * as metadata from './metadata/metadata.actions'
 
 export {
-  // search,
-  // review,
-  // metadata,
+  faceSearch
 }
diff --git a/client/faceSearch/faceSearch.result.js b/client/faceSearch/faceSearch.result.js
index 844a5a70..2b223a46 100644
--- a/client/faceSearch/faceSearch.result.js
+++ b/client/faceSearch/faceSearch.result.js
@@ -1,17 +1,44 @@
 import React, { Component } from 'react'
 import { bindActionCreators } from 'redux'
 import { connect } from 'react-redux'
+import { courtesyS } from '../util'
 
 import * as actions from './faceSearch.actions'
 
 class FaceSearchResult extends Component {
-  componentDidMount() {
-  }
-
   render() {
+    const { dataset } = this.props.payload
+    const { distances, results } = this.props.result
+    if (!results) {
+      return (
+        <div className='result'></div>
+      )
+    }
+    if (!this.props.result.results.length) {
+      return (
+        <div className='result'>No results</div>
+      )
+    }
+    const els = results.map((result, i) => {
+      const distance = distances[i]
+      const { uuid } = result.uuid
+      const { fullname, gender, description, images } = result.identity
+      return (
+        <div>
+          <img src={'https://megapixels.nyc3.digitaloceanspaces.com/v1/media/' + dataset + '/' + uuid + '.jpg'} />
+          {fullname} {'('}{gender}{')'}<br/>
+          {description}<br/>
+          {courtesyS(images, 'image')}<br />
+          {distance}
+        </div>
+      )
+    })
+
     return (
       <div className='result'>
-        Result here
+        <div class='results'>
+          {els}
+        </div>
       </div>
     )
   }
diff --git a/megapixels/app/models/sql_factory.py b/megapixels/app/models/sql_factory.py
index 0f7e73a0..9a44941b 100644
--- a/megapixels/app/models/sql_factory.py
+++ b/megapixels/app/models/sql_factory.py
@@ -83,8 +83,8 @@ class SqlDataset:
 
   def get_identity(self, id):
     table = self.get_table('identity_meta')
-    identity = table.query.filter(table.image_id >= id).order_by(table.image_id.asc()).first().toJSON()
-    print(identity)
+    # id += 1
+    identity = table.query.filter(table.image_id <= id).order_by(table.image_id.desc()).first().toJSON()
     return {
       'uuid': self.select('uuids', id),
       'identity': identity,
@@ -100,6 +100,7 @@ class SqlDataset:
     # for obj in session.query(table).filter_by(id=id):
     print(table)
     obj = session.query(table).filter(table.id == id).first()
+    session.close()
     return obj.toJSON()
 
   def get_table(self, type):
diff --git a/megapixels/app/server/api.py b/megapixels/app/server/api.py
index 36563910..2f78ecd3 100644
--- a/megapixels/app/server/api.py
+++ b/megapixels/app/server/api.py
@@ -70,7 +70,7 @@ def upload(name):
   query = np.array([ vec ]).astype('float32')
   
   # query FAISS!
-  distances, indexes = faiss_dataset.search(query, 5)
+  distances, indexes = faiss_dataset.search(query, 10)
 
   if len(indexes) == 0:
     print("weird, no results!")
@@ -85,6 +85,7 @@ def upload(name):
     return []
 
   lookup = {}
+  ids = [i+1 for i in indexes]
   for _d, _i in zip(distances, indexes):
     lookup[_i+1] = _d
 
@@ -97,13 +98,13 @@ def upload(name):
   query = {
     'timing': time.time() - start,
   }
-  results = [ dataset.get_identity(index) for index in indexes ]
+  results = [ dataset.get_identity(id) for id in ids ]
 
   print(results)
   return jsonify({
     'results': results,
-    # 'distances': distances.tolist(),
-    # 'indexes': indexes.tolist(),
+    'distances': distances.tolist(),
+    'indexes': indexes.tolist(),
   })
 
 @api.route('/dataset/<name>/name', methods=['GET'])
diff --git a/site/assets/css/applets.css b/site/assets/css/applets.css
index 54508f44..a01703d5 100644
--- a/site/assets/css/applets.css
+++ b/site/assets/css/applets.css
@@ -17,6 +17,23 @@
   justify-content: flex-start;
 }
 
+.results {
+  display: flex;
+  flex-direction: row;
+  flex-wrap: wrap;
+}
+.results > div {
+  width: 200px;
+  margin-left: 20px;
+  margin-bottom: 40px;
+  font-size: 8pt;
+}
+.results > div img {
+  margin-bottom: 4px;
+}
+.results > div:nth-child(3n+1) {
+  margin-left: 0;
+}
 .query h2 {
   margin-top: 0; padding-top: 0;
 }
-- 
cgit v1.2.3-70-g09d2


From a2d4ad0499e4ee4f3f528b2ed7bc162d61026d09 Mon Sep 17 00:00:00 2001
From: Jules Laplace <julescarbon@gmail.com>
Date: Mon, 17 Dec 2018 01:13:32 +0100
Subject:  build

---
 client/faceSearch/faceSearch.result.js | 14 +++++++++++++-
 megapixels/app/server/api.py           | 14 ++++++++++----
 site/assets/css/applets.css            |  3 +++
 3 files changed, 26 insertions(+), 5 deletions(-)

(limited to 'megapixels/app/server/api.py')

diff --git a/client/faceSearch/faceSearch.result.js b/client/faceSearch/faceSearch.result.js
index 2b223a46..1882def0 100644
--- a/client/faceSearch/faceSearch.result.js
+++ b/client/faceSearch/faceSearch.result.js
@@ -5,10 +5,22 @@ import { courtesyS } from '../util'
 
 import * as actions from './faceSearch.actions'
 
+const errors = {
+  'bbox': "Sorry, we couldn't find a face in that image.  Please choose an image where the face is large and clear.",
+  'nomatch': "We didn't find a match.",
+  'default': "There was an error!",
+}
+
 class FaceSearchResult extends Component {
   render() {
     const { dataset } = this.props.payload
-    const { distances, results } = this.props.result
+    const { distances, results, error } = this.props.result
+    if (error) {
+      let errorMessage = errors[error.message] || errors.default
+      return (
+        <div className='result'>{errorMessage}</div>
+      )
+    }
     if (!results) {
       return (
         <div className='result'></div>
diff --git a/megapixels/app/server/api.py b/megapixels/app/server/api.py
index 2f78ecd3..bc60118c 100644
--- a/megapixels/app/server/api.py
+++ b/megapixels/app/server/api.py
@@ -58,6 +58,10 @@ def upload(name):
 
   # get detection as BBox object
   bboxes = detector.detect(im_np, largest=True)
+  if not len(bboxes):
+    return jsonify({
+      'error': 'bbox'
+    })
   bbox = bboxes[0]
   dim = im_np.shape[:2][::-1]
   bbox = bbox.to_dim(dim)  # convert back to real dimensions
@@ -73,16 +77,18 @@ def upload(name):
   distances, indexes = faiss_dataset.search(query, 10)
 
   if len(indexes) == 0:
-    print("weird, no results!")
-    return []
+    return jsonify({
+      'error': 'nomatch'
+    })
 
   # get the results for this single query...
   distances = distances[0]
   indexes = indexes[0]
 
   if len(indexes) == 0:
-    print("no results!")
-    return []
+    return jsonify({
+      'error': 'nomatch'
+    })
 
   lookup = {}
   ids = [i+1 for i in indexes]
diff --git a/site/assets/css/applets.css b/site/assets/css/applets.css
index a01703d5..ecba518c 100644
--- a/site/assets/css/applets.css
+++ b/site/assets/css/applets.css
@@ -30,6 +30,9 @@
 }
 .results > div img {
   margin-bottom: 4px;
+  width: 200px;
+  height: 200px;
+  background: rgba(255,255,255,0.05);
 }
 .results > div:nth-child(3n+1) {
   margin-left: 0;
-- 
cgit v1.2.3-70-g09d2


From 1c82f7ec6a603978322e16470547731e92e947c6 Mon Sep 17 00:00:00 2001
From: Jules Laplace <julescarbon@gmail.com>
Date: Mon, 17 Dec 2018 20:15:41 +0100
Subject: adding verbiage and timing

---
 client/faceSearch/faceSearch.result.js | 52 +++++++++++++++++++++++-----
 megapixels/app/models/sql_factory.py   |  2 +-
 megapixels/app/server/api.py           | 62 ++++++++++++++++++++--------------
 megapixels/app/site/parser.py          |  2 +-
 site/assets/css/applets.css            |  5 +++
 site/public/test/index.html            | 16 ++++-----
 site/public/test/style/index.html      | 10 ++++--
 7 files changed, 104 insertions(+), 45 deletions(-)

(limited to 'megapixels/app/server/api.py')

diff --git a/client/faceSearch/faceSearch.result.js b/client/faceSearch/faceSearch.result.js
index 1882def0..bc7831d9 100644
--- a/client/faceSearch/faceSearch.result.js
+++ b/client/faceSearch/faceSearch.result.js
@@ -4,31 +4,59 @@ import { connect } from 'react-redux'
 import { courtesyS } from '../util'
 
 import * as actions from './faceSearch.actions'
+import { Loader } from '../common'
 
 const errors = {
-  'bbox': "Sorry, we couldn't find a face in that image.  Please choose an image where the face is large and clear.",
-  'nomatch': "We didn't find a match.",
-  'default': "There was an error!",
+  bbox: (
+    <div>
+      <h2>No face found</h2>
+      {"Sorry, we didn't detect a face in that image. "}
+      {"Please choose an image where the face is large and clear."}
+    </div>
+  ),
+  nomatch: (
+    <div>
+      <h2>{"You're clear"}</h2>
+      {"No images in this dataset match your face.  We show only matches above 70% probability."}
+    </div>
+  ),
+  error: (
+    <div>
+      <h2>{"No matches found"}</h2>
+      {"Sorry, an error occured."}
+    </div>
+  ),
 }
 
 class FaceSearchResult extends Component {
   render() {
     const { dataset } = this.props.payload
-    const { distances, results, error } = this.props.result
+    const { query, distances, results, loading, error } = this.props.result
+    if (loading) {
+      return (
+        <div className='result'>
+          <div>
+            <Loader /><br />
+            <h2>Searching...</h2>
+          </div>
+        </div>
+      )
+    }
     if (error) {
-      let errorMessage = errors[error.message] || errors.default
+      console.log(error)
+      let errorMessage = errors[error] || errors.error
       return (
         <div className='result'>{errorMessage}</div>
       )
     }
     if (!results) {
       return (
-        <div className='result'></div>
+        <div className='result'>{errors.nomatch}</div>
       )
     }
     if (!this.props.result.results.length) {
       return (
-        <div className='result'>No results</div>
+        <div className='result'>{errors.nomatch}</div>
       )
     }
     const els = results.map((result, i) => {
@@ -48,7 +76,15 @@ class FaceSearchResult extends Component {
 
     return (
       <div className='result'>
-        <div class='results'>
+        <div className="about">
+          <h2>Did we find you?</h2>
+          {'These faces matched images in the '}
+          <b><tt>{dataset}</tt></b>
+          {' dataset with over 70% probability.'}
+          <br />
+          <small>Query took {query.timing.toFixed(2)} seconds</small>
+        </div>
+        <div className='results'>
           {els}
         </div>
       </div>
diff --git a/megapixels/app/models/sql_factory.py b/megapixels/app/models/sql_factory.py
index 9a44941b..02b722df 100644
--- a/megapixels/app/models/sql_factory.py
+++ b/megapixels/app/models/sql_factory.py
@@ -98,7 +98,7 @@ class SqlDataset:
       return None
     session = Session()
     # for obj in session.query(table).filter_by(id=id):
-    print(table)
+    # print(table)
     obj = session.query(table).filter(table.id == id).first()
     session.close()
     return obj.toJSON()
diff --git a/megapixels/app/server/api.py b/megapixels/app/server/api.py
index bc60118c..b3447eb1 100644
--- a/megapixels/app/server/api.py
+++ b/megapixels/app/server/api.py
@@ -15,24 +15,32 @@ from app.utils.im_utils import pil2np
 sanitize_re = re.compile('[\W]+')
 valid_exts = ['.gif', '.jpg', '.jpeg', '.png']
 
+LIMIT = 9
+THRESHOLD = 0.3
+
 api = Blueprint('api', __name__)
 
 faiss_datasets = load_faiss_databases()
 
 @api.route('/')
 def index():
+  """List the datasets and their fields"""
   return jsonify({ 'datasets': list_datasets() })
 
+
 @api.route('/dataset/<name>')
 def show(name):
+  """Show the data that a dataset will return"""
   dataset = get_dataset(name)
   if dataset:
     return jsonify(dataset.describe())
   else:
     return jsonify({ 'status': 404 })
 
+
 @api.route('/dataset/<name>/face/', methods=['POST'])
 def upload(name):
+  """Query an image against FAISS and return the matching identities"""
   start = time.time()
   dataset = get_dataset(name)
   if name not in faiss_datasets:
@@ -52,31 +60,39 @@ def upload(name):
 
   im = Image.open(file.stream).convert('RGB')
   im_np = pil2np(im)
-  
+
   # Face detection
   detector = face_detector.DetectorDLIBHOG()
 
   # get detection as BBox object
   bboxes = detector.detect(im_np, largest=True)
-  if not len(bboxes):
+  if not bboxes or not len(bboxes):
     return jsonify({
       'error': 'bbox'
     })
   bbox = bboxes[0]
+  if not bbox:
+    return jsonify({
+      'error': 'bbox'
+    })
+
   dim = im_np.shape[:2][::-1]
   bbox = bbox.to_dim(dim)  # convert back to real dimensions
+  print("got bbox")
+  if not bbox:
+    return jsonify({
+      'error': 'bbox'
+    })
 
-  # face recognition/vector
+  # extract 128-D vector
   recognition = face_recognition.RecognitionDLIB(gpu=-1)
   vec = recognition.vec(im_np, bbox)
-
-  # print(vec)
   query = np.array([ vec ]).astype('float32')
   
-  # query FAISS!
-  distances, indexes = faiss_dataset.search(query, 10)
+  # query FAISS
+  distances, indexes = faiss_dataset.search(query, LIMIT)
 
-  if len(indexes) == 0:
+  if len(indexes) == 0 or len(indexes[0]) == 0:
     return jsonify({
       'error': 'nomatch'
     })
@@ -85,36 +101,32 @@ def upload(name):
   distances = distances[0]
   indexes = indexes[0]
 
-  if len(indexes) == 0:
-    return jsonify({
-      'error': 'nomatch'
-    })
-
-  lookup = {}
-  ids = [i+1 for i in indexes]
+  dists = []
+  ids = []
   for _d, _i in zip(distances, indexes):
-    lookup[_i+1] = _d
+    if _d <= THRESHOLD:
+      dists.append(round(float(_d), 2))
+      ids.append(_i+1)
 
-  print(distances)
-  print(indexes)
+  results = [ dataset.get_identity(_i) for _i in ids ]
 
-  # with the result we have an ID
-  # query the sql dataset for the UUID etc here
+  print(distances)
+  print(ids)
 
   query = {
-    'timing': time.time() - start,
+    'timing': round(time.time() - start, 3),
   }
-  results = [ dataset.get_identity(id) for id in ids ]
-
   print(results)
   return jsonify({
+    'query': query,
     'results': results,
-    'distances': distances.tolist(),
-    'indexes': indexes.tolist(),
+    'distances': dists,
   })
 
+
 @api.route('/dataset/<name>/name', methods=['GET'])
 def name_lookup(dataset):
+  """Find a name in the dataset"""
   start = time.time()
   dataset = get_dataset(name)
 
diff --git a/megapixels/app/site/parser.py b/megapixels/app/site/parser.py
index ecfae0cb..b3d3a8c2 100644
--- a/megapixels/app/site/parser.py
+++ b/megapixels/app/site/parser.py
@@ -64,7 +64,7 @@ def format_applet(section, s3_path):
   else:
     command = payload[0]
     opt = None
-  if command == 'python':
+  if command == 'python' or command == 'javascript' or command == 'code':
     return format_section([ section ], s3_path)
 
   applet['command'] = command
diff --git a/site/assets/css/applets.css b/site/assets/css/applets.css
index ecba518c..edd5b709 100644
--- a/site/assets/css/applets.css
+++ b/site/assets/css/applets.css
@@ -18,6 +18,8 @@
 }
 
 .results {
+  margin-top: 10px;
+  padding-bottom: 10px;
   display: flex;
   flex-direction: row;
   flex-wrap: wrap;
@@ -27,6 +29,9 @@
   margin-left: 20px;
   margin-bottom: 40px;
   font-size: 8pt;
+  background: #000;
+  padding: 5px;
+  font-weight: 500;
 }
 .results > div img {
   margin-bottom: 4px;
diff --git a/site/public/test/index.html b/site/public/test/index.html
index b4d16036..41f8eda5 100644
--- a/site/public/test/index.html
+++ b/site/public/test/index.html
@@ -30,14 +30,14 @@
     
   <section><h1>Megapixels UI Tests</h1>
 <ul>
-<li><a href="/test/style">Style Guide</a></li>
-<li><a href="/test/csv">CSV</a></li>
-<li><a href="/test/datasets/">Dataset list</a></li>
-<li><a href="/test/citations/">Citation list</a></li>
-<li><a href="/test/map/">Citation map</a></li>
-<li><a href="/test/face_search/">Face search</a></li>
-<li><a href="/test/name_search/">Name search</a></li>
-<li><a href="/test/gallery/">Modal image gallery</a></li>
+<li><a href="/test/style/index.html">Style Guide</a></li>
+<li><a href="/test/csv/index.html">CSV</a></li>
+<li><a href="/test/datasets/index.html">Dataset list</a></li>
+<li><a href="/test/citations/index.html">Citation list</a></li>
+<li><a href="/test/map/index.html">Citation map</a></li>
+<li><a href="/test/face_search/index.html">Face search</a></li>
+<li><a href="/test/name_search/index.html">Name search</a></li>
+<li><a href="/test/gallery/index.html">Modal image gallery</a></li>
 </ul>
 </section>
 
diff --git a/site/public/test/style/index.html b/site/public/test/style/index.html
index 6d99a236..ab13a589 100644
--- a/site/public/test/style/index.html
+++ b/site/public/test/style/index.html
@@ -54,10 +54,16 @@
 <div class='image'><img src='https://nyc3.digitaloceanspaces.com/megapixels/v1/site/test/assets/man.jpg' alt='Person 3. Let me tell you about Person 3.  This person has a very long description with text which wraps like crazy'><div class='caption'>Person 3. Let me tell you about Person 3.  This person has a very long description with text which wraps like crazy</div></div></section><section><blockquote><p>est, qui dolorem ipsum, quia dolor sit amet consectetur adipisci[ng] velit, sed quia non-numquam [do] eius modi tempora inci[di]dunt, ut labore et dolore magnam aliquam quaerat voluptatem.</p>
 </blockquote>
 </section><section class='wide'><div class='image'><img src='https://nyc3.digitaloceanspaces.com/megapixels/v1/site/test/assets/wide-test.jpg' alt='This image is extremely wide and the text beneath it will wrap but thats fine because it can also contain <a href="https://example.com/">hyperlinks</a>! Yes, you read that right&mdash;hyperlinks! Lorem ipsum dolor sit amet ad volotesque sic hoc ad nauseam'><div class='caption'>This image is extremely wide and the text beneath it will wrap but that's fine because it can also contain <a href="https://example.com/">hyperlinks</a>! Yes, you read that right&mdash;hyperlinks! Lorem ipsum dolor sit amet ad volotesque sic hoc ad nauseam</div></div></section><section><p>Inline <code>code</code> has <code>back-ticks around</code> it.</p>
-</section><section class='applet_container'><div class='applet' data-payload='{"command": "javascript", "fields": ["var s = \"JavaScript syntax highlighting\";", "alert(s);"]}'></div></section><section><pre><code class="lang-python">s = &quot;Python syntax highlighting&quot;
+</section><section><pre><code class="lang-javascript">var s = &quot;JavaScript syntax highlighting&quot;;
+alert(s);
+</code></pre>
+</section><section><pre><code class="lang-python">s = &quot;Python syntax highlighting&quot;
 print(s)
 </code></pre>
-</section><section class='applet_container'><div class='applet' data-payload='{"command": "No language indicated, so no syntax highlighting. ", "fields": ["But let's throw in a <b>tag</b>."]}'></div></section><section><p>Horizontal rule</p>
+</section><section><pre><code class="lang-code">Generic code block. Note that code blocks that are not so marked will not appear.
+But let&#39;s throw in a &lt;b&gt;tag&lt;/b&gt;.
+</code></pre>
+</section><section><p>Horizontal rule</p>
 <hr>
 <p>Citations below here</p>
 <div class="footnotes">
-- 
cgit v1.2.3-70-g09d2


From b0120413faebeffb41db151d82b07775c754ad15 Mon Sep 17 00:00:00 2001
From: Jules Laplace <julescarbon@gmail.com>
Date: Mon, 17 Dec 2018 23:36:25 +0100
Subject: unicode

---
 README.md                    | 8 ++++++++
 megapixels/app/server/api.py | 2 +-
 2 files changed, 9 insertions(+), 1 deletion(-)

(limited to 'megapixels/app/server/api.py')

diff --git a/README.md b/README.md
index b3985f18..5c07fa75 100644
--- a/README.md
+++ b/README.md
@@ -23,6 +23,14 @@ mkdir -p /data_store_hdd/apps/megapixels/faiss/indexes
 mkdir -p /data_store_hdd/apps/megapixels/faiss/metadata
 ```
 
+### MySQL note
+
+You may need to set the database charset to `utf8mb4` in order to import the CSVs:
+
+```
+ALTER DATABASE megapixels CHARACTER SET = utf8mb4 COLLATE = utf8mb4_unicode_ci;
+```
+
 ## Building the site
 
 ```
diff --git a/megapixels/app/server/api.py b/megapixels/app/server/api.py
index b3447eb1..024c32cb 100644
--- a/megapixels/app/server/api.py
+++ b/megapixels/app/server/api.py
@@ -108,7 +108,7 @@ def upload(name):
       dists.append(round(float(_d), 2))
       ids.append(_i+1)
 
-  results = [ dataset.get_identity(_i) for _i in ids ]
+  results = [ dataset.get_identity(int(_i)) for _i in ids ]
 
   print(distances)
   print(ids)
-- 
cgit v1.2.3-70-g09d2


From dfce989731fc58268b40280af79eeaa0b80b333e Mon Sep 17 00:00:00 2001
From: Jules Laplace <julescarbon@gmail.com>
Date: Mon, 17 Dec 2018 23:49:57 +0100
Subject: unicode

---
 client/faceSearch/faceSearch.result.js |  6 ++----
 megapixels/app/server/api.py           | 14 ++++++++------
 2 files changed, 10 insertions(+), 10 deletions(-)

(limited to 'megapixels/app/server/api.py')

diff --git a/client/faceSearch/faceSearch.result.js b/client/faceSearch/faceSearch.result.js
index 2a85552d..d63f3265 100644
--- a/client/faceSearch/faceSearch.result.js
+++ b/client/faceSearch/faceSearch.result.js
@@ -50,11 +50,9 @@ class FaceSearchResult extends Component {
       )
     }
     if (!results) {
-      return (
-        <div className='result'>{errors.nomatch}</div>
-      )
+      return <div className='result'></div>
     }
-    if (!this.props.result.results.length) {
+    if (!results.length) {
       return (
         <div className='result'>{errors.nomatch}</div>
       )
diff --git a/megapixels/app/server/api.py b/megapixels/app/server/api.py
index 024c32cb..9c874d81 100644
--- a/megapixels/app/server/api.py
+++ b/megapixels/app/server/api.py
@@ -54,7 +54,7 @@ def upload(name):
     fn = 'filename.jpg'
 
   basename, ext = os.path.splitext(fn)
-  print("got {}, type {}".format(basename, ext))
+  # print("got {}, type {}".format(basename, ext))
   if ext.lower() not in valid_exts:
     return jsonify({ 'error': 'not an image' })
 
@@ -78,7 +78,7 @@ def upload(name):
 
   dim = im_np.shape[:2][::-1]
   bbox = bbox.to_dim(dim)  # convert back to real dimensions
-  print("got bbox")
+  # print("got bbox")
   if not bbox:
     return jsonify({
       'error': 'bbox'
@@ -110,13 +110,15 @@ def upload(name):
 
   results = [ dataset.get_identity(int(_i)) for _i in ids ]
 
-  print(distances)
-  print(ids)
+  # print(distances)
+  # print(ids)
 
   query = {
+    'bbox': bboxes[0],
+    'bbox_dim': bbox,
     'timing': round(time.time() - start, 3),
   }
-  print(results)
+  # print(results)
   return jsonify({
     'query': query,
     'results': results,
@@ -138,7 +140,7 @@ def name_lookup(dataset):
   }
   results = []
 
-  print(results)
+  # print(results)
   return jsonify({
     'query': query,
     'results': results,
-- 
cgit v1.2.3-70-g09d2


From d3be915bc5725a36dee867b07404725177783460 Mon Sep 17 00:00:00 2001
From: Jules Laplace <julescarbon@gmail.com>
Date: Mon, 17 Dec 2018 23:53:18 +0100
Subject: unicode

---
 README.md                    | 5 +++--
 megapixels/app/server/api.py | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

(limited to 'megapixels/app/server/api.py')

diff --git a/README.md b/README.md
index 5c07fa75..ed692e40 100644
--- a/README.md
+++ b/README.md
@@ -14,8 +14,9 @@ conda install faiss-cpu -c pytorch
 pip install numpy Pillow
 pip install dlib
 pip install requests simplejson click pdfminer.six
-pip install urllib3 flask flask_sqlalchemy
-pip install pymediainfo tqdm opencv-python imutils scikit-image python-dotenv imagehash scikit-learn colorlog
+pip install urllib3 flask flask_sqlalchemy mysql-connector
+pip install pymediainfo tqdm opencv-python imutils
+pip install scikit-image python-dotenv imagehash scikit-learn colorlog
 
 sudo apt-get install libmysqlclient-dev
 
diff --git a/megapixels/app/server/api.py b/megapixels/app/server/api.py
index 9c874d81..8ff06611 100644
--- a/megapixels/app/server/api.py
+++ b/megapixels/app/server/api.py
@@ -106,7 +106,7 @@ def upload(name):
   for _d, _i in zip(distances, indexes):
     if _d <= THRESHOLD:
       dists.append(round(float(_d), 2))
-      ids.append(_i+1)
+      ids.append(_i)
 
   results = [ dataset.get_identity(int(_i)) for _i in ids ]
 
-- 
cgit v1.2.3-70-g09d2


From 7b8e6f9a7d3eb36b72b53d5e754b9c7916b98ed7 Mon Sep 17 00:00:00 2001
From: Jules Laplace <julescarbon@gmail.com>
Date: Tue, 18 Dec 2018 01:03:10 +0100
Subject: namesearchg

---
 client/actions.js                         |  4 +-
 client/applet.js                          |  5 +-
 client/faceSearch/faceSearch.container.js |  2 +-
 client/faceSearch/faceSearch.result.js    | 16 +++++-
 client/index.js                           |  1 -
 client/nameSearch/index.js                |  5 ++
 client/nameSearch/nameSearch.actions.js   | 52 ++++++++++++++++++
 client/nameSearch/nameSearch.container.js | 24 +++++++++
 client/nameSearch/nameSearch.query.js     | 48 +++++++++++++++++
 client/nameSearch/nameSearch.reducer.js   | 32 +++++++++++
 client/nameSearch/nameSearch.result.js    | 88 +++++++++++++++++++++++++++++++
 client/store.js                           |  8 +--
 client/tables.js                          |  4 ++
 client/types.js                           |  3 ++
 megapixels/app/models/sql_factory.py      | 14 +++++
 megapixels/app/server/api.py              | 41 +++++++-------
 megapixels/app/settings/app_cfg.py        |  2 +-
 site/assets/css/applets.css               | 24 +++++++--
 18 files changed, 339 insertions(+), 34 deletions(-)
 create mode 100644 client/nameSearch/index.js
 create mode 100644 client/nameSearch/nameSearch.actions.js
 create mode 100644 client/nameSearch/nameSearch.container.js
 create mode 100644 client/nameSearch/nameSearch.query.js
 create mode 100644 client/nameSearch/nameSearch.reducer.js
 create mode 100644 client/nameSearch/nameSearch.result.js

(limited to 'megapixels/app/server/api.py')

diff --git a/client/actions.js b/client/actions.js
index bb011838..2be8229d 100644
--- a/client/actions.js
+++ b/client/actions.js
@@ -1,5 +1,7 @@
 import * as faceSearch from './faceSearch/faceSearch.actions'
+import * as nameSearch from './nameSearch/nameSearch.actions'
 
 export {
-  faceSearch
+  faceSearch,
+  nameSearch,
 }
diff --git a/client/applet.js b/client/applet.js
index 4d2a8e6c..80d40657 100644
--- a/client/applet.js
+++ b/client/applet.js
@@ -1,13 +1,16 @@
 import React, { Component } from 'react'
 
 import { Container as FaceSearchContainer } from './faceSearch'
+import { Container as NameSearchContainer } from './nameSearch'
 
 export default class Applet extends Component {
   render() {
-    console.log(this.props)
+    // console.log(this.props)
     switch (this.props.payload.cmd) {
       case 'face_search':
         return <FaceSearchContainer {...this.props} />
+      case 'name_search':
+        return <NameSearchContainer {...this.props} />
       default:
         return <pre style={{ color: '#0f0' }}>{'Megapixels'}</pre>
     }
diff --git a/client/faceSearch/faceSearch.container.js b/client/faceSearch/faceSearch.container.js
index f96961db..94c6eb9f 100644
--- a/client/faceSearch/faceSearch.container.js
+++ b/client/faceSearch/faceSearch.container.js
@@ -10,7 +10,7 @@ import FaceSearchResult from './faceSearch.result'
 class FaceSearchContainer extends Component {
   render() {
     const { payload } = this.props
-    console.log(payload)
+    // console.log(payload)
     return (
       <div className='searchContainer'>
         <FaceSearchQuery payload={payload} />
diff --git a/client/faceSearch/faceSearch.result.js b/client/faceSearch/faceSearch.result.js
index d63f3265..936bc8d2 100644
--- a/client/faceSearch/faceSearch.result.js
+++ b/client/faceSearch/faceSearch.result.js
@@ -32,6 +32,7 @@ class FaceSearchResult extends Component {
   render() {
     const { dataset } = this.props.payload
     const { query, distances, results, loading, error } = this.props.result
+    console.log(this.props.result)
     if (loading) {
       return (
         <div className='result'>
@@ -43,7 +44,7 @@ class FaceSearchResult extends Component {
       )
     }
     if (error) {
-      console.log(error)
+      // console.log(error)
       let errorMessage = errors[error] || errors.error
       return (
         <div className='result'>{errorMessage}</div>
@@ -60,10 +61,21 @@ class FaceSearchResult extends Component {
     const els = results.map((result, i) => {
       const distance = distances[i]
       const { uuid } = result.uuid
+      const { x, y, w, h } = result.roi
       const { fullname, gender, description, images } = result.identity
+      const bbox = {
+        left: (100 * x) + '%',
+        top: (100 * y) + '%',
+        width: (100 * w) + '%',
+        height: (100 * h) + '%',
+      }
+      // console.log(bbox)
       return (
         <div key={i}>
-          <img src={'https://megapixels.nyc3.digitaloceanspaces.com/v1/media/' + dataset + '/' + uuid + '.jpg'} />
+          <div className='img'>
+            <img src={'https://megapixels.nyc3.digitaloceanspaces.com/v1/media/' + dataset + '/' + uuid + '.jpg'} />
+            <div className='bbox' style={bbox} />
+          </div>
           {fullname} {'('}{gender}{')'}<br/>
           {description}<br/>
           {courtesyS(images, 'image')}{' in dataset'}<br />
diff --git a/client/index.js b/client/index.js
index 2beb5526..93341a77 100644
--- a/client/index.js
+++ b/client/index.js
@@ -38,7 +38,6 @@ function appendApplets(applets) {
         el.classList.add('loaded')
         break
       default:
-        console.log('react', el, payload)
         appendReactApplet(el, payload)
         el.classList.add('loaded')
         break
diff --git a/client/nameSearch/index.js b/client/nameSearch/index.js
new file mode 100644
index 00000000..8c6475e4
--- /dev/null
+++ b/client/nameSearch/index.js
@@ -0,0 +1,5 @@
+import Container from './nameSearch.container'
+
+export {
+  Container,
+}
diff --git a/client/nameSearch/nameSearch.actions.js b/client/nameSearch/nameSearch.actions.js
new file mode 100644
index 00000000..290ee38d
--- /dev/null
+++ b/client/nameSearch/nameSearch.actions.js
@@ -0,0 +1,52 @@
+// import fetchJsonp from 'fetch-jsonp'
+import * as types from '../types'
+// import { hashPath } from '../util'
+import { post } from '../util'
+// import querystring from 'query-string'
+
+// urls
+
+const url = {
+  search: (dataset, q) => process.env.API_HOST + '/api/dataset/' + dataset + '/name?q=' + encodeURIComponent(q),
+}
+export const publicUrl = {
+}
+
+// standard loading events
+
+const loading = (tag, offset) => ({
+  type: types.nameSearch.loading,
+  tag,
+  offset
+})
+const loaded = (tag, data, offset = 0) => ({
+  type: types.nameSearch.loaded,
+  tag,
+  data,
+  offset
+})
+const error = (tag, err) => ({
+  type: types.nameSearch.error,
+  tag,
+  err
+})
+
+// search UI functions
+
+export const updateOptions = opt => dispatch => {
+  dispatch({ type: types.nameSearch.update_options, opt })
+}
+
+// API functions
+
+export const search = (payload, q) => dispatch => {
+  const tag = 'result'
+  const fd = new FormData()
+  fd.append('q', q)
+  dispatch(loading(tag))
+  post(url.search(payload.dataset, q), fd)
+    .then(data => {
+      dispatch(loaded(tag, data))
+    })
+    .catch(err => dispatch(error(tag, err)))
+}
diff --git a/client/nameSearch/nameSearch.container.js b/client/nameSearch/nameSearch.container.js
new file mode 100644
index 00000000..b0de0c3a
--- /dev/null
+++ b/client/nameSearch/nameSearch.container.js
@@ -0,0 +1,24 @@
+import React, { Component } from 'react'
+import { bindActionCreators } from 'redux'
+import { connect } from 'react-redux'
+
+import * as actions from './nameSearch.actions'
+
+import NameSearchQuery from './nameSearch.query'
+import NameSearchResult from './nameSearch.result'
+
+class NameSearchContainer extends Component {
+  render() {
+    const { payload } = this.props
+    // console.log(payload)
+    return (
+      <div className='searchContainer'>
+        <NameSearchQuery payload={payload} />
+        <NameSearchResult payload={payload} />
+      </div>
+    )
+  }
+}
+
+
+export default NameSearchContainer
diff --git a/client/nameSearch/nameSearch.query.js b/client/nameSearch/nameSearch.query.js
new file mode 100644
index 00000000..b82e324b
--- /dev/null
+++ b/client/nameSearch/nameSearch.query.js
@@ -0,0 +1,48 @@
+import React, { Component } from 'react'
+import { bindActionCreators } from 'redux'
+import { connect } from 'react-redux'
+
+import * as actions from './nameSearch.actions'
+
+class NameSearchQuery extends Component {
+  state = {
+    value: null
+  }
+
+  handleInput(value) {
+    this.setState({ q: value })
+    if (value.length > 2) {
+      this.props.actions.search(this.props.payload, value)
+    }
+  }
+
+  render() {
+    return (
+      <div className='query'>
+        <h2>Find Your Name</h2>
+        <h3>Searching {13456} identities</h3>
+        <p>
+          {'Enter your name to see if you were included in this dataset..'}
+        </p>
+        <input
+          type="text"
+          class="q"
+          placeholder="Enter your name"
+          value={this.state.q}
+          onInput={e => this.handleInput(e.target.value)}
+        />
+      </div>
+    )
+  }
+}
+
+const mapStateToProps = state => ({
+  result: state.nameSearch.result,
+  options: state.nameSearch.options,
+})
+
+const mapDispatchToProps = dispatch => ({
+  actions: bindActionCreators({ ...actions }, dispatch),
+})
+
+export default connect(mapStateToProps, mapDispatchToProps)(NameSearchQuery)
diff --git a/client/nameSearch/nameSearch.reducer.js b/client/nameSearch/nameSearch.reducer.js
new file mode 100644
index 00000000..101c93ea
--- /dev/null
+++ b/client/nameSearch/nameSearch.reducer.js
@@ -0,0 +1,32 @@
+import * as types from '../types'
+
+const initialState = () => ({
+  query: {},
+  result: {},
+  loading: false,
+})
+
+export default function nameSearchReducer(state = initialState(), action) {
+  switch (action.type) {
+    case types.nameSearch.loading:
+      return {
+        ...state,
+        [action.tag]: { loading: true },
+      }
+
+    case types.nameSearch.loaded:
+      return {
+        ...state,
+        [action.tag]: action.data,
+      }
+
+    case types.nameSearch.error:
+      return {
+        ...state,
+        [action.tag]: { error: action.err },
+      }
+
+    default:
+      return state
+  }
+}
diff --git a/client/nameSearch/nameSearch.result.js b/client/nameSearch/nameSearch.result.js
new file mode 100644
index 00000000..9e20228c
--- /dev/null
+++ b/client/nameSearch/nameSearch.result.js
@@ -0,0 +1,88 @@
+import React, { Component } from 'react'
+import { bindActionCreators } from 'redux'
+import { connect } from 'react-redux'
+import { courtesyS } from '../util'
+
+import * as actions from './nameSearch.actions'
+import { Loader } from '../common'
+
+const errors = {
+  nomatch: (
+    <div>
+      <h3>Name not found</h3>
+      {"No names matched your query."}
+    </div>
+  ),
+  error: (
+    <div>
+      <h3>{"No matches found"}</h3>
+    </div>
+  ),
+}
+
+class NameSearchResult extends Component {
+  render() {
+    const { dataset } = this.props.payload
+    const { query, results, loading, error } = this.props.result
+    console.log(this.props.result)
+    if (loading) {
+      return (
+        <div className='result'>
+          <div>
+            <Loader />
+          </div>
+        </div>
+      )
+    }
+    if (error) {
+      console.log(error)
+      let errorMessage = errors[error] || errors.error
+      return (
+        <div className='result'>{errorMessage}</div>
+      )
+    }
+    if (!results) {
+      return <div className='result'></div>
+    }
+    if (!results.length) {
+      return (
+        <div className='result'>{errors.nomatch}</div>
+      )
+    }
+    const els = results.map((result, i) => {
+      const { uuid } = result.uuid
+      const { fullname, gender, description, images } = result.identity
+      return (
+        <div key={i}>
+          <img src={'https://megapixels.nyc3.digitaloceanspaces.com/v1/media/' + dataset + '/' + uuid + '.jpg'} />
+          {fullname} {'('}{gender}{')'}<br/>
+          {description}<br/>
+          {courtesyS(images, 'image')}{' in dataset'}<br />
+        </div>
+      )
+    })
+
+    return (
+      <div className='result'>
+        <div className="timing">
+          {'Search took '}{Math.round(query.timing * 1000) + ' ms'}
+        </div>
+        <div className='results'>
+          {els}
+        </div>
+      </div>
+    )
+  }
+}
+
+const mapStateToProps = state => ({
+  query: state.nameSearch.query,
+  result: state.nameSearch.result,
+  options: state.nameSearch.options,
+})
+
+const mapDispatchToProps = dispatch => ({
+  actions: bindActionCreators({ ...actions }, dispatch),
+})
+
+export default connect(mapStateToProps, mapDispatchToProps)(NameSearchResult)
diff --git a/client/store.js b/client/store.js
index 03f983a5..13612f2d 100644
--- a/client/store.js
+++ b/client/store.js
@@ -1,16 +1,12 @@
 import { applyMiddleware, compose, combineReducers, createStore } from 'redux'
 import thunk from 'redux-thunk'
 
-// import metadataReducer from './metadata/metadata.reducer'
 import faceSearchReducer from './faceSearch/faceSearch.reducer'
-// import reviewReducer from './review/review.reducer'
+import nameSearchReducer from './nameSearch/nameSearch.reducer'
 
 const rootReducer = combineReducers({
-  auth: (state = {}) => state,
-  // auth: (state = login()) => state,
-  // metadata: metadataReducer,
   faceSearch: faceSearchReducer,
-  // review: reviewReducer,
+  nameSearch: nameSearchReducer,
 })
 
 function configureStore(initialState = {}) {
diff --git a/client/tables.js b/client/tables.js
index 2a2699f9..b4c13887 100644
--- a/client/tables.js
+++ b/client/tables.js
@@ -74,4 +74,8 @@ export default function append(el, payload) {
         }
       })
   }
+
+  if (fields.length > 1 && fields[1].indexOf('filter')) {
+    const filter = fields[1].split(' ')
+  }
 }
diff --git a/client/types.js b/client/types.js
index d295d0d1..fb1fbe30 100644
--- a/client/types.js
+++ b/client/types.js
@@ -10,5 +10,8 @@ export const faceSearch = tagAsType('faceSearch', [
   'loading', 'loaded', 'error', 'update_options',
 ])
 
+export const nameSearch = tagAsType('nameSearch', [
+  'loading', 'loaded', 'error', 'update_options',
+])
 
 export const init = '@@INIT'
diff --git a/megapixels/app/models/sql_factory.py b/megapixels/app/models/sql_factory.py
index 414ef3a6..da95b539 100644
--- a/megapixels/app/models/sql_factory.py
+++ b/megapixels/app/models/sql_factory.py
@@ -97,6 +97,20 @@ class SqlDataset:
       'pose': self.select('pose', id),
     }
 
+  def search_name(self, q):
+    table = self.get_table('identity_meta')
+    uuid_table = self.get_table('uuids')
+
+    identity = table.query.filter(table.fullname.like(q)).order_by(table.fullname.desc()).limit(30)
+    identities = []
+    for row in identity:
+      uuid = uuid_table.query.filter(uuid_table.id == row.image_id).first()
+      identities.append({
+        'uuid': uuid.toJSON(),
+        'identity': row.toJSON(),
+      })
+    return identities
+
   def select(self, table, id):
     table = self.get_table(table)
     if not table:
diff --git a/megapixels/app/server/api.py b/megapixels/app/server/api.py
index 8ff06611..33cf45df 100644
--- a/megapixels/app/server/api.py
+++ b/megapixels/app/server/api.py
@@ -28,26 +28,26 @@ def index():
   return jsonify({ 'datasets': list_datasets() })
 
 
-@api.route('/dataset/<name>')
-def show(name):
+@api.route('/dataset/<dataset_name>')
+def show(dataset_name):
   """Show the data that a dataset will return"""
-  dataset = get_dataset(name)
+  dataset = get_dataset(dataset_name)
   if dataset:
     return jsonify(dataset.describe())
   else:
     return jsonify({ 'status': 404 })
 
 
-@api.route('/dataset/<name>/face/', methods=['POST'])
-def upload(name):
+@api.route('/dataset/<dataset_name>/face', methods=['POST'])
+def upload(dataset_name):
   """Query an image against FAISS and return the matching identities"""
   start = time.time()
-  dataset = get_dataset(name)
-  if name not in faiss_datasets:
+  dataset = get_dataset(dataset_name)
+  if dataset_name not in faiss_datasets:
     return jsonify({
       'error': 'invalid dataset'  
     })
-  faiss_dataset = faiss_datasets[name]
+  faiss_dataset = faiss_datasets[dataset_name]
   file = request.files['query_img']
   fn = file.filename
   if fn.endswith('blob'):
@@ -106,17 +106,21 @@ def upload(name):
   for _d, _i in zip(distances, indexes):
     if _d <= THRESHOLD:
       dists.append(round(float(_d), 2))
-      ids.append(_i)
+      ids.append(_i+1)
 
   results = [ dataset.get_identity(int(_i)) for _i in ids ]
 
   # print(distances)
   # print(ids)
 
+  # 'bbox': str(bboxes[0]),
+  # 'bbox_dim': str(bbox),
+  print(bboxes[0])
+  print(bbox)
+
   query = {
-    'bbox': bboxes[0],
-    'bbox_dim': bbox,
     'timing': round(time.time() - start, 3),
+    'bbox': str(bbox),
   }
   # print(results)
   return jsonify({
@@ -126,20 +130,21 @@ def upload(name):
   })
 
 
-@api.route('/dataset/<name>/name', methods=['GET'])
-def name_lookup(dataset):
+@api.route('/dataset/<dataset_name>/name', methods=['GET','POST'])
+def name_lookup(dataset_name):
   """Find a name in the dataset"""
   start = time.time()
-  dataset = get_dataset(name)
+  dataset = get_dataset(dataset_name)
 
-  # we have a query from the request query string...
-  # use this to do a like* query on the identities_meta table
+  q = request.args.get('q')
+  print(q)
 
   query = {
+    'q': q,
     'timing': time.time() - start,
   }
-  results = []
-
+  results = dataset.search_name(q + '%') if q else None
+  
   # print(results)
   return jsonify({
     'query': query,
diff --git a/megapixels/app/settings/app_cfg.py b/megapixels/app/settings/app_cfg.py
index d7752739..55fed166 100644
--- a/megapixels/app/settings/app_cfg.py
+++ b/megapixels/app/settings/app_cfg.py
@@ -89,7 +89,7 @@ CKPT_ZERO_PADDING = 9
 HASH_TREE_DEPTH = 3
 HASH_BRANCH_SIZE = 3
 
-DLIB_FACEREC_JITTERS = 25  # number of face recognition jitters
+DLIB_FACEREC_JITTERS = 5  # number of face recognition jitters
 DLIB_FACEREC_PADDING = 0.25  # default dlib
 
 POSE_MINMAX_YAW = (-25,25)
diff --git a/site/assets/css/applets.css b/site/assets/css/applets.css
index edd5b709..315d72e0 100644
--- a/site/assets/css/applets.css
+++ b/site/assets/css/applets.css
@@ -16,7 +16,15 @@
   flex-direction: row;
   justify-content: flex-start;
 }
-
+.q {
+  width: 100%;
+  padding: 5px;
+  font-size: 14pt;
+}
+.timing {
+  font-size: 9pt;
+  padding-top: 10px;
+}
 .results {
   margin-top: 10px;
   padding-bottom: 10px;
@@ -34,9 +42,10 @@
   font-weight: 500;
 }
 .results > div img {
+  display: block;
   margin-bottom: 4px;
-  width: 200px;
-  height: 200px;
+  width: 190px;
+  height: 190px;
   background: rgba(255,255,255,0.05);
 }
 .results > div:nth-child(3n+1) {
@@ -45,6 +54,15 @@
 .query h2 {
   margin-top: 0; padding-top: 0;
 }
+.img {
+  position: relative;
+}
+.img .bbox {
+  position: absolute;
+  color: rgba(255,0,0,1);
+  background: rgba(255,0,0,0.05);
+  border: 1px solid;
+}
 .cta {
   padding-left: 20px;
   font-size: 11pt;
-- 
cgit v1.2.3-70-g09d2


From 66d915e8c84c50572a3e575a62d40c3543598c6e Mon Sep 17 00:00:00 2001
From: Jules Laplace <julescarbon@gmail.com>
Date: Tue, 18 Dec 2018 01:05:43 +0100
Subject: bbox

---
 megapixels/app/server/api.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'megapixels/app/server/api.py')

diff --git a/megapixels/app/server/api.py b/megapixels/app/server/api.py
index 33cf45df..96fedeee 100644
--- a/megapixels/app/server/api.py
+++ b/megapixels/app/server/api.py
@@ -115,8 +115,8 @@ def upload(dataset_name):
 
   # 'bbox': str(bboxes[0]),
   # 'bbox_dim': str(bbox),
-  print(bboxes[0])
-  print(bbox)
+  # print(bboxes[0])
+  # print(bbox)
 
   query = {
     'timing': round(time.time() - start, 3),
-- 
cgit v1.2.3-70-g09d2


From f5090a87d6261216fd7d6842d0337753bbd3c918 Mon Sep 17 00:00:00 2001
From: Jules Laplace <julescarbon@gmail.com>
Date: Tue, 18 Dec 2018 01:08:05 +0100
Subject: bbox

---
 megapixels/app/server/api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'megapixels/app/server/api.py')

diff --git a/megapixels/app/server/api.py b/megapixels/app/server/api.py
index 96fedeee..805b24d4 100644
--- a/megapixels/app/server/api.py
+++ b/megapixels/app/server/api.py
@@ -120,7 +120,7 @@ def upload(dataset_name):
 
   query = {
     'timing': round(time.time() - start, 3),
-    'bbox': str(bbox),
+    # 'bbox': str(bbox),
   }
   # print(results)
   return jsonify({
-- 
cgit v1.2.3-70-g09d2


From ebafdcb75d7f3c440701be952f58b8b08c12d25c Mon Sep 17 00:00:00 2001
From: Jules Laplace <julescarbon@gmail.com>
Date: Tue, 18 Dec 2018 01:12:10 +0100
Subject: bbox

---
 megapixels/app/server/api.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

(limited to 'megapixels/app/server/api.py')

diff --git a/megapixels/app/server/api.py b/megapixels/app/server/api.py
index 805b24d4..9c0527c1 100644
--- a/megapixels/app/server/api.py
+++ b/megapixels/app/server/api.py
@@ -54,7 +54,7 @@ def upload(dataset_name):
     fn = 'filename.jpg'
 
   basename, ext = os.path.splitext(fn)
-  # print("got {}, type {}".format(basename, ext))
+  api.logger.info("got {}, type {}".format(basename, ext))
   if ext.lower() not in valid_exts:
     return jsonify({ 'error': 'not an image' })
 
@@ -78,7 +78,7 @@ def upload(dataset_name):
 
   dim = im_np.shape[:2][::-1]
   bbox = bbox.to_dim(dim)  # convert back to real dimensions
-  # print("got bbox")
+  api.logger.info("got bbox")
   if not bbox:
     return jsonify({
       'error': 'bbox'
@@ -122,7 +122,7 @@ def upload(dataset_name):
     'timing': round(time.time() - start, 3),
     # 'bbox': str(bbox),
   }
-  # print(results)
+  api.logger.info(results)
   return jsonify({
     'query': query,
     'results': results,
@@ -137,7 +137,7 @@ def name_lookup(dataset_name):
   dataset = get_dataset(dataset_name)
 
   q = request.args.get('q')
-  print(q)
+  # print(q)
 
   query = {
     'q': q,
-- 
cgit v1.2.3-70-g09d2


From bf3dd1399e4ef1db5fb8830004827fe603f73b2e Mon Sep 17 00:00:00 2001
From: Jules Laplace <julescarbon@gmail.com>
Date: Tue, 18 Dec 2018 01:15:38 +0100
Subject: bbox

---
 megapixels/app/server/api.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

(limited to 'megapixels/app/server/api.py')

diff --git a/megapixels/app/server/api.py b/megapixels/app/server/api.py
index 9c0527c1..35862837 100644
--- a/megapixels/app/server/api.py
+++ b/megapixels/app/server/api.py
@@ -54,7 +54,7 @@ def upload(dataset_name):
     fn = 'filename.jpg'
 
   basename, ext = os.path.splitext(fn)
-  api.logger.info("got {}, type {}".format(basename, ext))
+  # print("got {}, type {}".format(basename, ext))
   if ext.lower() not in valid_exts:
     return jsonify({ 'error': 'not an image' })
 
@@ -78,7 +78,7 @@ def upload(dataset_name):
 
   dim = im_np.shape[:2][::-1]
   bbox = bbox.to_dim(dim)  # convert back to real dimensions
-  api.logger.info("got bbox")
+  # print("got bbox")
   if not bbox:
     return jsonify({
       'error': 'bbox'
@@ -120,9 +120,9 @@ def upload(dataset_name):
 
   query = {
     'timing': round(time.time() - start, 3),
-    # 'bbox': str(bbox),
+    'bbox': str(bbox),
   }
-  api.logger.info(results)
+  # print(results)
   return jsonify({
     'query': query,
     'results': results,
-- 
cgit v1.2.3-70-g09d2