.

27 files changed, 1977 insertions, 74 deletions

diff --git a/megapixels/app/models/bbox.py b/megapixels/app/models/bbox.py
index 41b67416..55a92512 100644
--- a/megapixels/app/models/bbox.py
+++ b/megapixels/app/models/bbox.py
@@ -1,6 +1,9 @@
+import math
+
 from dlib import rectangle as dlib_rectangle
 import numpy as np
 
+
 class BBoxPoint:
 
   def __init__(self, x, y):
@@ -42,8 +45,12 @@ class BBox:
     self._tl = (x1, y1)
     self._br = (x2, y2)
     self._rect = (self._x1, self._y1, self._x2, self._y2)
+    self._area = self._width * self._height  # as percentage
 
-
+  @property
+  def area(self):
+    return self._area
+  
   @property
   def pt_tl(self):
     return self._tl
@@ -105,7 +112,12 @@ class BBox:
   # # Utils
 
   # def constrain(self, dim):
-
+  def distance(self, b):
+    a = self
+    dcx = self._cx - b.cx
+    dcy = self._cy - b.cy
+    d = int(math.sqrt(math.pow(dcx, 2) + math.pow(dcy, 2)))
+    return d
 
   # -----------------------------------------------------------------
   # Modify
@@ -117,26 +129,40 @@ class BBox:
     :returns (BBox) in pixel dimensions
     """
     # expand
-    rect_exp = list( (np.array(self._rect) + np.array([-amt, -amt, amt, amt])).astype('int'))
+    r = list( (np.array(self._rect) + np.array([-amt, -amt, amt, amt])).astype('int'))
     # outliers
     oob = list(range(4))
-    oob[0] = min(rect_exp[0], 0)
-    oob[1] = min(rect_exp[1], 0)
-    oob[2] = dim[0] - max(rect_exp[2], 2)
-    oob[3] = dim[1] - max(rect_exp[3], 3)
+    oob[0] = min(r[0], 0)
+    oob[1] = min(r[1], 0)
+    oob[2] = dim[0] - r[2]
+    oob[3] = dim[1] - r[3]
     oob = np.array(oob)
     oob[oob > 0] = 0
-    # amount
+    # absolute amount
     oob = np.absolute(oob)
-    # threshold
-    rect_exp[0] = max(rect_exp[0], 0)
-    rect_exp[1] = max(rect_exp[1], 0)
-    rect_exp[2] = min(rect_exp[2], dim[0])
-    rect_exp[3] = min(rect_exp[3], dim[1])
+    # threshold expanded rectangle
+    r[0] = max(r[0], 0)
+    r[1] = max(r[1], 0)
+    r[2] = min(r[2], dim[0])
+    r[3] = min(r[3], dim[1])
     # redistribute oob amounts
     oob = np.array([-oob[2], -oob[3], oob[0], oob[1]])
-    rect_exp = np.add(np.array(rect_exp), oob)
-    return BBox(*rect_exp)    
+    r = np.add(np.array(r), oob)
+    # find overage
+    oob[0] = min(r[0], 0)
+    oob[1] = min(r[1], 0)
+    oob[2] = dim[0] - r[2]
+    oob[3] = dim[1] - r[3]
+    oob = np.array(oob)
+    oob[oob > 0] = 0
+    oob = np.absolute(oob)
+    if np.array(oob).any():
+      m = np.max(oob)
+      adj = np.array([m, m, -m, -m])
+      # print(adj)
+      r = np.add(np.array(r), adj)
+
+    return BBox(*r)    
 
 
   # -----------------------------------------------------------------
@@ -156,23 +182,23 @@ class BBox:
   # -----------------------------------------------------------------
   # Format as
 
-  def as_xyxy(self):
+  def to_xyxy(self):
     """Converts BBox back to x1, y1, x2, y2 rect"""
     return (self._x1, self._y1, self._x2, self._y2)
 
-  def as_xywh(self):
+  def to_xywh(self):
     """Converts BBox back to haar type"""
     return (self._x1, self._y1, self._width, self._height)
 
-  def as_trbl(self):
+  def to_trbl(self):
     """Converts BBox to CSS (top, right, bottom, left)""" 
     return (self._y1, self._x2, self._y2, self._x1)
 
-  def as_dlib(self):
+  def to_dlib(self):
     """Converts BBox to dlib rect type"""
-    return dlib.rectangle(self._x1, self._y1, self._x2, self._y2)
+    return dlib_rectangle(self._x1, self._y1, self._x2, self._y2)
 
-  def as_yolo(self):
+  def to_yolo(self):
     """Converts BBox to normalized center x, center y, w, h"""
     return (self._cx, self._cy, self._width, self._height)
 
@@ -199,6 +225,13 @@ class BBox:
     return cls(*rect)
 
   @classmethod
+  def from_xyxy(cls, x1, y1, x2, y2):
+    """Converts x1, y1, x2, y2 to BBox
+    same as constructure but zprovided for conveniene
+    """
+    return cls(x1, y1, x2, y2)
+
+  @classmethod
   def from_xywh(cls, x, y, w, h):
     """Converts x1, y1, w, h to BBox
     :param rect: (list) x1, y1, w, h
@@ -227,8 +260,13 @@ class BBox:
     """ 
     rect = (rect.left(), rect.top(), rect.right(), rect.bottom())
     rect = cls.normalize(cls, rect, dim)
-    return cls(*rect)
+    return cls(*rect) 
+
+  def __str__(self):
+    return f'BBox: ({self._x1},{self._y1}), ({self._x2}, {self._y2}), width:{self._width}, height:{self._height}'
 
+  def __repr__(self):
+    return f'BBox: ({self._x1},{self._y1}), ({self._x2}, {self._y2}), width:{self._width}, height:{self._height}'
 
   def str(self):
     """Return BBox as a string "x1, y1, x2, y2" """
diff --git a/megapixels/app/models/data_store.py b/megapixels/app/models/data_store.py
new file mode 100644
index 00000000..7b6bef21
--- /dev/null
+++ b/megapixels/app/models/data_store.py
@@ -0,0 +1,67 @@
+import os
+from os.path import join
+import logging
+
+from app.settings import app_cfg as cfg
+from app.settings import types
+
+
+# -------------------------------------------------------------------------
+# Metadata and media files
+# -------------------------------------------------------------------------
+
+class DataStore:
+  # local data store
+  def __init__(self, opt_data_store, opt_dataset):
+    self.data_store = join(f'/data_store_{opt_data_store.name.lower()}')
+    self.dir_dataset = join(self.data_store, 'datasets', cfg.DIR_PEOPLE, opt_dataset.name.lower())
+    self.dir_media = join(self.dir_dataset, 'media')
+    self.dir_metadata = join(self.dir_dataset, 'metadata')
+
+  def metadata(self, enum_type):
+    return join(self.dir_metadata, f'{enum_type.name.lower()}.csv')
+
+  def metadata_dir(self):
+    return join(self.dir_metadata)
+
+  def media_images_original(self):
+    return join(self.dir_media, 'original')
+
+  def face(self, subdir, fn, ext):
+    return join(self.dir_media, 'original', subdir, f'{fn}.{ext}')
+
+  def face_crop(self, subdir, fn, ext):
+    return join(self.dir_media, 'cropped', subdir, f'{fn}.{ext}')
+
+  def face_uuid(self, uuid, ext):
+    return join(self.dir_media, 'uuid',f'{uuid}.{ext}')
+
+  def face_crop_uuid(self, uuid, ext):
+    return join(self.dir_media, 'uuid', f'{uuid}.{ext}')
+
+  def uuid_dir(self):
+    return join(self.dir_media, 'uuid')
+
+
+class DataStoreS3:
+  # S3 server
+  def __init__(self, opt_dataset):
+    self._dir_media = join(cfg.S3_HTTP_MEDIA_URL, opt_dataset.name.lower())
+    self._dir_metadata = join(cfg.S3_HTTP_METADATA_URL, opt_dataset.name.lower())
+
+  def metadata(self, opt_metadata_type, ext='csv'):
+    return join(self._dir_metadata, f'{opt_metadata_type.name.lower()}.{ext}')
+
+  def face(self, opt_uuid, ext='jpg'):
+    #return join(self._dir_media, 'original', f'{opt_uuid}.{ext}')
+    return join(self._dir_media, f'{opt_uuid}.{ext}')
+
+  def face_crop(self, opt_uuid, ext='jpg'):
+    # not currently using?
+    return join(self._dir_media, 'cropped', f'{opt_uuid}.{ext}')
+
+
+
+# -------------------------------------------------------------------------
+# Models
+# -------------------------------------------------------------------------
\ No newline at end of file
diff --git a/megapixels/app/models/dataset.py b/megapixels/app/models/dataset.py
new file mode 100644
index 00000000..eb0109a7
--- /dev/null
+++ b/megapixels/app/models/dataset.py
@@ -0,0 +1,229 @@
+"""
+Dataset model: container for all CSVs about a dataset
+"""
+import os
+import sys
+from os.path import join
+from pathlib import Path
+import logging
+
+import pandas as pd
+import numpy as np
+
+from app.settings import app_cfg as cfg
+from app.settings import types
+from app.models.bbox import BBox
+from app.utils import file_utils, im_utils
+from app.models.data_store import DataStore, DataStoreS3
+from app.utils.logger_utils import Logger
+
+# -------------------------------------------------------------------------
+# Dataset
+# -------------------------------------------------------------------------
+
+class Dataset:
+  
+  def __init__(self, opt_data_store, opt_dataset_type):
+    self._dataset_type = opt_dataset_type  # enum type
+    self.log = Logger.getLogger()
+    self._metadata = {}
+    self._face_vectors = []
+    self._nullframe = pd.DataFrame()  # empty placeholder
+    self.data_store = DataStore(opt_data_store, self._dataset_type)
+    self.data_store_s3 = DataStoreS3(self._dataset_type)
+
+  def load_face_vectors(self):
+    metadata_type = types.Metadata.FACE_VECTOR
+    fp_csv = self.data_store.metadata(metadata_type)
+    self.log.info(f'loading: {fp_csv}')
+    if Path(fp_csv).is_file():
+      self._metadata[metadata_type] = pd.read_csv(fp_csv).set_index('index')
+      # convert DataFrame to list of floats
+      self._face_vectors = self.df_vecs_to_dict(self._metadata[metadata_type])
+      self._face_vector_roi_idxs = self.df_vec_roi_idxs_to_dict(self._metadata[metadata_type])
+      self.log.info(f'build face vector dict: {len(self._face_vectors)}')
+      # remove the face vector column, it can be several GB of memory
+      self._metadata[metadata_type].drop('vec', axis=1, inplace=True)
+    else:
+      self.log.error(f'File not found: {fp_csv}. Exiting.')
+      sys.exit()
+
+  def load_records(self):
+    metadata_type = types.Metadata.FILE_RECORD
+    fp_csv = self.data_store.metadata(metadata_type)
+    self.log.info(f'loading: {fp_csv}')
+    if Path(fp_csv).is_file():
+      self._metadata[metadata_type] = pd.read_csv(fp_csv).set_index('index')
+    else:
+      self.log.error(f'File not found: {fp_csv}. Exiting.')
+      sys.exit()
+
+  def load_identities(self):
+    metadata_type = types.Metadata.IDENTITY
+    fp_csv = self.data_store.metadata(metadata_type)
+    self.log.info(f'loading: {fp_csv}')
+    if Path(fp_csv).is_file():
+      self._metadata[metadata_type] = pd.read_csv(fp_csv).set_index('index')
+    else:
+      self.log.error(f'File not found: {fp_csv}. Exiting.')
+      sys.exit()
+  
+  def metadata(self, opt_metadata_type):
+    return self._metadata.get(opt_metadata_type, None)
+
+  def index_to_record(self, index):
+    # get record meta
+    df_record = self._metadata[types.Metadata.FILE_RECORD]
+    ds_record = df_record.iloc[index]
+    identity_index = ds_record.identity_index
+    # get identity meta
+    df_identity = self._metadata[types.Metadata.IDENTITY]
+    # future datasets can have multiple identities per images
+    ds_identities = df_identity.iloc[identity_index]
+    # get filepath and S3 url
+    fp_im = self.data_store.face(ds_record.subdir, ds_record.fn, ds_record.ext)
+    s3_url = self.data_store_s3.face(ds_record.uuid) 
+    image_record = ImageRecord(ds_record, fp_im, s3_url, ds_identities=ds_identities)
+    return image_record
+
+  def vector_to_record(self, record_index):
+    '''Accumulates image and its metadata'''
+    df_face_vector = self._metadata[types.Metadata.FACE_VECTOR]
+    ds_face_vector = df_face_vector.iloc[vector_index]
+    # get the match's ROI index
+    image_index = ds_face_vector.image_index
+    # get the roi dataframe
+    df_face_roi = self._metadata[types.Metadata.FACE_ROI]
+    ds_roi = df_face_roi.iloc[image_index]
+    # create BBox
+    dim = (ds_roi.image_width, ds_roi.image_height)
+    bbox = BBox.from_xywh_dim(ds_roi.x, ds_roi.y, ds_roi.w, ds_roi.y, dim)
+    # use the ROI index to get identity index from the identity DataFrame
+    df_sha256 = self._metadata[types.Metadata.SHA256]
+    ds_sha256 = df_sha256.iloc[image_index]
+    sha256 = ds_sha256.sha256
+    # get the local filepath
+    df_filepath = self._metadata[types.Metadata.FILEPATH]
+    ds_file = df_filepath.iloc[image_index]
+    fp_im = self.data_store.face_image(ds_file.subdir, ds_file.fn, ds_file.ext)\
+    # get remote path
+    df_uuid = self._metadata[types.Metadata.UUID]
+    ds_uuid = df_uuid.iloc[image_index]
+    uuid = ds_uuid.uuid
+    fp_url = self.data_store_s3.face_image(uuid)
+    fp_url_crop = self.data_store_s3.face_image_crop(uuid)
+
+    image_record = ImageRecord(image_index, sha256, uuid, bbox, fp_im, fp_url)
+    # now get the identity index (if available)
+    identity_index = ds_sha256.identity_index
+    if identity_index > -1:
+      # then use the identity index to get the identity meta
+      df_identity = df_filepath = self._metadata[types.Metadata.IDENTITY]
+      ds_identity = df_identity.iloc[identity_index]
+      # get the name and description
+      name = ds_identity.fullname
+      desc = ds_identity.description
+      gender = ds_identity.gender
+      n_images = ds_identity.images
+      url = '(url)'  # TODO
+      age = '(age)' # TODO
+      nationality = '(nationality)'
+      identity = Identity(identity_index, name=name, desc=desc, gender=gender, n_images=n_images,
+        url=url, age=age, nationality=nationality)
+      image_record.identity = identity 
+    else:
+      self.log.info(f'no identity index: {ds_sha256}')
+
+    return image_record
+
+
+  def find_matches(self, query_vec, n_results=5, threshold=0.6):
+    image_records = []  # list of image matches w/identity if available
+    # find most similar feature vectors indexes
+    #match_idxs = self.similar(query_vec, n_results, threshold)
+    sim_scores = np.linalg.norm(np.array([query_vec]) - np.array(self._face_vectors), axis=1)
+    match_idxs = np.argpartition(sim_scores, n_results)[:n_results]
+
+    for match_idx in match_idxs:
+      # get the corresponding face vector row
+      roi_index = self._face_vector_roi_idxs[match_idx]
+      df_record = self._metadata[types.Metadata.FILE_RECORD]
+      ds_record = df_record.iloc[roi_index]
+      self.log.debug(f'find match index: {match_idx}, --> roi_index: {roi_index}')
+      fp_im = self.data_store.face(ds_record.subdir, ds_record.fn, ds_record.ext)
+      s3_url = self.data_store_s3.face(ds_record.uuid) 
+      image_record = ImageRecord(ds_record, fp_im, s3_url)
+      #roi_index = self._face_vector_roi_idxs[match_idx]
+      #image_record = self.roi_idx_to_record(roi_index)
+      image_records.append(image_record)
+    return image_records
+
+  # ----------------------------------------------------------------------
+  # utilities
+
+  def df_vecs_to_dict(self, df):
+    # convert the DataFrame CSV to float list of vecs
+    return [list(map(float,x.vec.split(','))) for x in df.itertuples()]
+
+  def df_vec_roi_idxs_to_dict(self, df):
+    # convert the DataFrame CSV to float list of vecs
+    #return [x.roi_index for x in df.itertuples()]
+    return [x.roi_index for x in df.itertuples()]
+
+  def similar(self, query_vec, n_results):
+    '''Finds most similar N indices of query face vector
+    :query_vec: (list) of 128 floating point numbers of face encoding
+    :n_results: (int) number of most similar indices to return
+    :returns (list) of (int) indices
+    '''
+    # uses np.linalg based on the ageitgey/face_recognition code
+    
+    return top_idxs
+
+
+
+class ImageRecord:
+
+  def __init__(self, ds_record, fp, url, ds_rois=None, ds_identities=None):
+    # maybe more other meta will go there
+    self.image_index = ds_record.index
+    self.sha256 = ds_record.sha256
+    self.uuid = ds_record.uuid
+    self.filepath = fp
+    self.url = url
+    self._identities = []
+    # image records contain ROIs
+    # ROIs are linked to identities
+
+    #self._identities = [Identity(x) for x in ds_identities]
+
+  @property
+  def identity(self, index):
+    return self._identity
+
+  def summarize(self):
+    '''Summarizes data for debugging'''
+    log = Logger.getLogger()
+    log.info(f'filepath: {self.filepath}')
+    log.info(f'sha256: {self.sha256}')
+    log.info(f'UUID: {self.uuid}')
+    log.info(f'S3 url: {self.url}')
+    for identity in self._identities:
+      log.info(f'fullname: {identity.fullname}')
+      log.info(f'description: {identity.description}')
+      log.info(f'gender: {identity.gender}')
+      log.info(f'images: {identity.n_images}')
+
+
+class Identity:
+
+  def __init__(self, idx, name='NA', desc='NA', gender='NA', n_images=1, 
+    url='NA', age='NA', nationality='NA'):
+    self.index = idx
+    self.name = name
+    self.description = desc
+    self.gender = gender
+    self.n_images = n_images
+    self.url = url
+    self.age = age
+    self.nationality = nationality
diff --git a/megapixels/app/models/sql_factory.py b/megapixels/app/models/sql_factory.py
new file mode 100644
index 00000000..da95b539
--- /dev/null
+++ b/megapixels/app/models/sql_factory.py
@@ -0,0 +1,224 @@
+import os
+import glob
+import time
+import pandas as pd
+
+from sqlalchemy import create_engine, Table, Column, String, Integer, DateTime, Float
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy.ext.declarative import declarative_base
+
+from app.utils.file_utils import load_recipe, load_csv_safe
+from app.settings import app_cfg as cfg
+
+connection_url = "mysql+mysqlconnector://{}:{}@{}/{}?charset=utf8mb4".format(
+  os.getenv("DB_USER"),
+  os.getenv("DB_PASS"),
+  os.getenv("DB_HOST"),
+  os.getenv("DB_NAME")
+)
+
+datasets = {}
+loaded = False
+Session = None
+
+def list_datasets():
+  return [dataset.describe() for dataset in datasets.values()]
+
+def get_dataset(name):
+  return datasets[name] if name in datasets else None
+
+def get_table(name, table_name):
+  dataset = get_dataset(name)
+  return dataset.get_table(table_name) if dataset else None
+
+def load_sql_datasets(replace=False, base_model=None):
+  global datasets, loaded, Session
+  if loaded:
+    return datasets
+  engine = create_engine(connection_url, encoding="utf-8")
+  # db.set_character_set('utf8')
+  # dbc = db.cursor()
+  # dbc.execute('SET NAMES utf8;')
+  # dbc.execute('SET CHARACTER SET utf8;')
+  # dbc.execute('SET character_set_connection=utf8;')
+  Session = sessionmaker(bind=engine)
+  for path in glob.iglob(os.path.join(cfg.DIR_FAISS_METADATA, "*")):
+    dataset = load_sql_dataset(path, replace, engine, base_model)
+    datasets[dataset.name] = dataset
+  loaded = True
+  return datasets
+
+def load_sql_dataset(path, replace=False, engine=None, base_model=None):
+  name = os.path.basename(path)
+  dataset = SqlDataset(name, base_model=base_model)
+
+  for fn in glob.iglob(os.path.join(path, "*.csv")):
+    key = os.path.basename(fn).replace(".csv", "")
+    table = dataset.get_table(key)
+    if table is None:
+      continue
+    if replace:
+      print('loading dataset {}'.format(fn))
+      df = pd.read_csv(fn)
+      # fix columns that are named "index", a sql reserved word
+      df.columns = table.__table__.columns.keys()
+      df.to_sql(name=table.__tablename__, con=engine, if_exists='replace', index=False)
+  return dataset
+
+class SqlDataset:
+  """
+  Bridge between the facial information CSVs connected to the datasets, and MySQL
+  - each dataset should have files that can be loaded into these database models
+  - names will be fixed to work in SQL (index -> id)
+  - we can then have more generic models for fetching this info after doing a FAISS query
+  """
+  def __init__(self, name, engine=None, base_model=None):
+    self.name = name
+    self.tables = {}
+    if base_model is None:
+      self.engine = create_engine(connection_url)
+      base_model = declarative_base(engine)
+    self.base_model = base_model
+
+  def describe(self):
+    return {
+      'name': self.name,
+      'tables': list(self.tables.keys()),
+    }
+
+  def get_identity(self, id):
+    table = self.get_table('identity_meta')
+    # id += 1
+    identity = table.query.filter(table.image_id <= id).order_by(table.image_id.desc()).first().toJSON()
+    return {
+      'uuid': self.select('uuids', id),
+      'identity': identity,
+      'roi': self.select('roi', id),
+      'pose': self.select('pose', id),
+    }
+
+  def search_name(self, q):
+    table = self.get_table('identity_meta')
+    uuid_table = self.get_table('uuids')
+
+    identity = table.query.filter(table.fullname.like(q)).order_by(table.fullname.desc()).limit(30)
+    identities = []
+    for row in identity:
+      uuid = uuid_table.query.filter(uuid_table.id == row.image_id).first()
+      identities.append({
+        'uuid': uuid.toJSON(),
+        'identity': row.toJSON(),
+      })
+    return identities
+
+  def select(self, table, id):
+    table = self.get_table(table)
+    if not table:
+      return None
+    session = Session()
+    # for obj in session.query(table).filter_by(id=id):
+    # print(table)
+    obj = session.query(table).filter(table.id == id).first()
+    session.close()
+    return obj.toJSON()
+
+  def get_table(self, type):
+    if type in self.tables:
+      return self.tables[type]
+    elif type == 'uuids':
+      self.tables[type] = self.uuid_table()
+    elif type == 'roi':
+      self.tables[type] = self.roi_table()
+    elif type == 'identity_meta':
+      self.tables[type] = self.identity_table()
+    elif type == 'pose':
+      self.tables[type] = self.pose_table()
+    else:
+      return None
+    return self.tables[type]
+
+  # ==> uuids.csv <==
+  # index,uuid
+  # 0,f03fd921-2d56-4e83-8115-f658d6a72287
+  def uuid_table(self):
+    class UUID(self.base_model):
+      __tablename__ = self.name + "_uuid"
+      id = Column(Integer, primary_key=True)
+      uuid = Column(String(36, convert_unicode=True), nullable=False)
+      def toJSON(self):
+        return {
+          'id': self.id,
+          'uuid': self.uuid,
+        }
+    return UUID
+
+  # ==> roi.csv <==
+  # index,h,image_height,image_index,image_width,w,x,y
+  # 0,0.33000000000000007,250,0,250,0.32999999999999996,0.33666666666666667,0.35
+  def roi_table(self):
+    class ROI(self.base_model):
+      __tablename__ = self.name + "_roi"
+      id = Column(Integer, primary_key=True)
+      h = Column(Float, nullable=False)
+      image_height = Column(Integer, nullable=False)
+      image_index = Column(Integer, nullable=False)
+      image_width = Column(Integer, nullable=False)
+      w = Column(Float, nullable=False)
+      x = Column(Float, nullable=False)
+      y = Column(Float, nullable=False)
+      def toJSON(self):
+        return {
+          'id': self.id,
+          'image_index': self.image_index,
+          'image_height': self.image_height,
+          'image_width': self.image_width,
+          'w': self.w,
+          'h': self.h,
+          'x': self.x,
+          'y': self.y,
+        }
+    return ROI
+
+  # ==> identity.csv <==
+  # index,fullname,description,gender,images,image_index
+  # 0,A. J. Cook,Canadian actress,f,1,0
+  def identity_table(self):
+    class Identity(self.base_model):
+      __tablename__ = self.name + "_identity"
+      id = Column(Integer, primary_key=True)
+      fullname = Column(String(36, convert_unicode=True), nullable=False)
+      description = Column(String(36, convert_unicode=True), nullable=False)
+      gender = Column(String(1, convert_unicode=True), nullable=False)
+      images = Column(Integer, nullable=False)
+      image_id = Column(Integer, nullable=False)
+      def toJSON(self):
+        return {
+          'id': self.id,
+          'image_id': self.image_id,
+          'fullname': self.fullname,
+          'images': self.images,
+          'gender': self.gender,
+          'description': self.description,
+        }
+    return Identity
+
+  # ==> pose.csv <==
+  # index,image_index,pitch,roll,yaw
+  # 0,0,11.16264458441435,10.415885631337728,22.99719032415318
+  def pose_table(self):
+    class Pose(self.base_model):
+      __tablename__ = self.name + "_pose"
+      id = Column(Integer, primary_key=True)
+      image_id = Column(Integer, primary_key=True)
+      pitch = Column(Float, nullable=False)
+      roll = Column(Float, nullable=False)
+      yaw = Column(Float, nullable=False)
+      def toJSON(self):
+        return {
+          'id': self.id,
+          'image_id': self.image_id,
+          'pitch': self.pitch,
+          'roll': self.roll,
+          'yaw': self.yaw,
+        }
+    return Pose
diff --git a/megapixels/app/processors/face_age.py b/megapixels/app/processors/face_age.py
new file mode 100644
index 00000000..222858a5
--- /dev/null
+++ b/megapixels/app/processors/face_age.py
@@ -0,0 +1,28 @@
+import os
+from os.path import join
+from pathlib import Path
+import math
+
+import cv2 as cv
+import numpy as np
+import imutils
+
+from app.utils import im_utils, logger_utils
+from app.models.bbox import BBox
+from app.settings import app_cfg as cfg
+from app.settings import types
+
+
+
+class FaceAge:
+
+  # Estimates face age
+
+  def __init__(self):
+    self.log = logger_utils.Logger.getLogger()
+    pass
+
+
+  def age(self):
+    # use enum typed emotions
+    return {'age': types.Age.ADULT, 'confidence': 0.5}
\ No newline at end of file
diff --git a/megapixels/app/processors/face_beauty.py b/megapixels/app/processors/face_beauty.py
new file mode 100644
index 00000000..a1ddd9f8
--- /dev/null
+++ b/megapixels/app/processors/face_beauty.py
@@ -0,0 +1,27 @@
+import os
+from os.path import join
+from pathlib import Path
+import math
+
+import cv2 as cv
+import numpy as np
+import imutils
+
+from app.utils import im_utils, logger_utils
+from app.models.bbox import BBox
+from app.settings import app_cfg as cfg
+from app.settings import types
+
+
+
+class FaceBeauty:
+
+  # Estimates beauty using CNN
+
+  def __init__(self):
+    self.log = logger_utils.Logger.getLogger()
+    pass
+
+
+  def beauty(self):
+    return 0.5
\ No newline at end of file
diff --git a/megapixels/app/processors/face_detector.py b/megapixels/app/processors/face_detector.py
index 02d068dc..3a90c557 100644
--- a/megapixels/app/processors/face_detector.py
+++ b/megapixels/app/processors/face_detector.py
@@ -4,71 +4,140 @@ from pathlib import Path
 
 import cv2 as cv
 import numpy as np
-import dlib
-# import imutils
+import imutils
+import operator
 
 from app.utils import im_utils, logger_utils
 from app.models.bbox import BBox
 from app.settings import app_cfg as cfg
+from app.settings import types
+
+
+class DetectorMTCNN:
+  
+  # https://github.com/ipazc/mtcnn
+  # pip install mtcnn
+
+  dnn_size = (300, 300)
+
+  def __init__(self, size=(400,400)):
+    from mtcnn.mtcnn import MTCNN
+    self.detector = MTCNN()
+
+  def detect(self, im, size=(400,400), conf_thresh=None, pyramids=None, largest=False):
+    '''Detects face using MTCNN and returns (list) of BBox
+    :param im: (numpy.ndarray) image
+    :returns list of BBox
+    '''
+    bboxes = []
+    #conf_thresh = self.conf_thresh if conf_thresh is None else conf_thresh
+    #pyramids = self.pyramids if pyramids is None else pyramids
+    dnn_size = self.dnn_size if size is None else size
+
+    im = im_utils.resize(im, width=dnn_size[0], height=dnn_size[1])
+    dim = im.shape[:2][::-1]
+    dets = self.detector.detect_faces(im)
+    for det in dets:
+      rect = det['box']
+      #keypoints = det['keypoints']  # not using here. see 'face_landmarks.py'
+      bbox = BBox.from_xywh_dim(*rect, dim)
+      bboxes.append(bbox)
+
+    if largest and len(bboxes) > 1:
+      # only keep largest
+      bboxes.sort(key=operator.attrgetter('area'), reverse=True)
+      bboxes = [bboxes[0]]
+
+    return bboxes
+
+
+class DetectorHaar:
+
+  im_size = (400, 400)
+  cascade_name = types.HaarCascade.FRONTAL
+
+  def __init__(self, cascade=types.HaarCascade.FRONTAL):
+    self.log = logger_utils.Logger.getLogger()
+
+  def detect(self, im, scale_factor=1.05, overlaps=5):
+    pass
+    
 
 class DetectorDLIBCNN:
+  
 
   dnn_size = (300, 300)
   pyramids = 0
   conf_thresh = 0.85
 
-  def __init__(self, opt_gpu):
+  def __init__(self, gpu=0):
+    import dlib
     self.log = logger_utils.Logger.getLogger()
     cuda_visible_devices = os.getenv('CUDA_VISIBLE_DEVICES', '')
-    os.environ['CUDA_VISIBLE_DEVICES'] = str(opt_gpu)
+    os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu)
     self.log.info('load model: {}'.format(cfg.DIR_MODELS_DLIB_CNN))
     self.detector = dlib.cnn_face_detection_model_v1(cfg.DIR_MODELS_DLIB_CNN)
     os.environ['CUDA_VISIBLE_DEVICES'] = cuda_visible_devices  # reset
 
-  def detect(self, im, opt_size=None, opt_conf_thresh=None, opt_pyramids=None):
-    rois = []
-    conf_thresh = self.conf_thresh if opt_conf_thresh is None else opt_conf_thresh
-    pyramids = self.pyramids if opt_pyramids is None else opt_pyramids
-    dnn_size = self.dnn_size if opt_size is None else opt_size
+  def detect(self, im, size=None, conf_thresh=None, pyramids=None, largest=False):
+    bboxes = []
+    conf_thresh = self.conf_thresh if conf_thresh is None else conf_thresh
+    pyramids = self.pyramids if pyramids is None else pyramids
+    dnn_size = self.dnn_size if size is None else size
     # resize image
     im = im_utils.resize(im, width=dnn_size[0], height=dnn_size[1])
     dim = im.shape[:2][::-1]
     im = im_utils.bgr2rgb(im)  # convert to RGB for dlib
     # run detector
-    mmod_rects = self.detector(im, 1)
+    mmod_rects = self.detector(im, pyramids)
     # sort results
     for mmod_rect in mmod_rects:
       if mmod_rect.confidence > conf_thresh:
         bbox = BBox.from_dlib_dim(mmod_rect.rect, dim)
-        rois.append(bbox)
-    return rois
+        bboxes.append(bbox)
+
+    if largest and len(bboxes) > 1:
+      # only keep largest
+      bboxes.sort(key=operator.attrgetter('area'), reverse=True)
+      bboxes = [bboxes[0]]
+
+    return bboxes
 
 
 class DetectorDLIBHOG:
 
   size = (320, 240)
   pyramids = 0
+  conf_thresh = 0.85
 
   def __init__(self):
+    import dlib
+    self.log = logger_utils.Logger.getLogger()
     self.detector = dlib.get_frontal_face_detector()
 
-  def detect(self, im, opt_size=None, opt_conf_thresh=None, opt_pyramids=0):
-    conf_thresh = self.conf_thresh if opt_conf_thresh is None else opt_conf_thresh
-    dnn_size = self.size if opt_size is None else opt_size
-    pyramids = self.pyramids if opt_pyramids is None else opt_pyramids
+  def detect(self, im, size=None, conf_thresh=None, pyramids=0, largest=False):
+    conf_thresh = self.conf_thresh if conf_thresh is None else conf_thresh
+    dnn_size = self.size if size is None else size
+    pyramids = self.pyramids if pyramids is None else pyramids
     
-    im = im_utils.resize(im, width=opt_size[0], height=opt_size[1])
+    im = im_utils.resize(im, width=dnn_size[0], height=dnn_size[1])
     dim = im.shape[:2][::-1]
     im = im_utils.bgr2rgb(im)  # ?
     hog_results = self.detector.run(im, pyramids)
     
-    rois = []
+    bboxes = []
     if len(hog_results[0]) > 0:
       for rect, score, direction in zip(*hog_results):
-        if score > opt_conf_thresh:
+        if score > conf_thresh:
           bbox = BBox.from_dlib_dim(rect, dim)
-          rois.append(bbox)
-    return rois
+          bboxes.append(bbox)
+    
+    if largest and len(bboxes) > 1:
+      # only keep largest
+      bboxes.sort(key=operator.attrgetter('area'), reverse=True)
+      bboxes = [bboxes[0]]
+
+    return bboxes
 
 class DetectorCVDNN:
 
@@ -79,25 +148,32 @@ class DetectorCVDNN:
   conf_thresh = 0.85
   
   def __init__(self):
+    import dlib
     fp_prototxt = join(cfg.DIR_MODELS_CAFFE, 'face_detect', 'opencv_face_detector.prototxt')
     fp_model = join(cfg.DIR_MODELS_CAFFE, 'face_detect', 'opencv_face_detector.caffemodel')
     self.net = cv.dnn.readNet(fp_prototxt, fp_model)
     self.net.setPreferableBackend(cv.dnn.DNN_BACKEND_OPENCV)
     self.net.setPreferableTarget(cv.dnn.DNN_TARGET_CPU)
 
-  def detect(self, im, opt_size=None, opt_conf_thresh=None):
+  def detect(self, im, size=None, conf_thresh=None, largest=False, pyramids=None):
     """Detects faces and returns (list) of (BBox)"""
-    conf_thresh = self.conf_thresh if opt_conf_thresh is None else opt_conf_thresh
-    dnn_size = self.size if opt_size is None else opt_size
+    conf_thresh = self.conf_thresh if conf_thresh is None else conf_thresh
+    dnn_size = self.size if size is None else size
     im = cv.resize(im, dnn_size)
     blob = cv.dnn.blobFromImage(im, self.dnn_scale, dnn_size, self.dnn_mean)
     self.net.setInput(blob)
     net_outputs = self.net.forward()
 
-    rois = []
+    bboxes = []
     for i in range(0, net_outputs.shape[2]):
       conf = net_outputs[0, 0, i, 2]
-      if conf > opt_conf_thresh:
+      if conf > conf_thresh:
         rect_norm = net_outputs[0, 0, i, 3:7]
-        rois.append(BBox(*rect_norm))
-    return rois
\ No newline at end of file
+        bboxes.append(BBox(*rect_norm))
+    
+    if largest and len(bboxes) > 1:
+      # only keep largest
+      bboxes.sort(key=operator.attrgetter('area'), reverse=True)
+      bboxes = [bboxes[0]]
+
+    return bboxes
\ No newline at end of file
diff --git a/megapixels/app/processors/face_emotion.py b/megapixels/app/processors/face_emotion.py
new file mode 100644
index 00000000..c45da9ba
--- /dev/null
+++ b/megapixels/app/processors/face_emotion.py
@@ -0,0 +1,28 @@
+import os
+from os.path import join
+from pathlib import Path
+import math
+
+import cv2 as cv
+import numpy as np
+import imutils
+
+from app.utils import im_utils, logger_utils
+from app.models.bbox import BBox
+from app.settings import app_cfg as cfg
+from app.settings import types
+
+
+
+class FaceEmotion:
+
+  # Estimates face emotion
+
+  def __init__(self):
+    self.log = logger_utils.Logger.getLogger()
+    pass
+
+
+  def emotion(self):
+    # use enum typed emotions
+    return {'emotion': types.Emotion.NEUTRAL, 'confidence': 0.5}
\ No newline at end of file
diff --git a/megapixels/app/processors/face_gender.py b/megapixels/app/processors/face_gender.py
new file mode 100644
index 00000000..ee152098
--- /dev/null
+++ b/megapixels/app/processors/face_gender.py
@@ -0,0 +1,27 @@
+import os
+from os.path import join
+from pathlib import Path
+import math
+
+import cv2 as cv
+import numpy as np
+import imutils
+
+from app.utils import im_utils, logger_utils
+from app.models.bbox import BBox
+from app.settings import app_cfg as cfg
+from app.settings import types
+
+
+
+class FaceGender:
+
+  # Estimates gender using CNN
+
+  def __init__(self):
+    self.log = logger_utils.Logger.getLogger()
+    pass
+
+
+  def gender(self):
+    return 'm'
\ No newline at end of file
diff --git a/megapixels/app/processors/face_landmarks.py b/megapixels/app/processors/face_landmarks.py
new file mode 100644
index 00000000..dfcb9ee8
--- /dev/null
+++ b/megapixels/app/processors/face_landmarks.py
@@ -0,0 +1,60 @@
+import os
+from os.path import join
+from pathlib import Path
+
+import cv2 as cv
+import numpy as np
+import imutils
+from app.utils import im_utils, logger_utils
+from app.models.bbox import BBox
+from app.settings import app_cfg as cfg
+from app.settings import types
+from app.models.bbox import BBox
+
+class LandmarksDLIB:
+
+  def __init__(self):
+    # init dlib
+    import dlib
+    self.log = logger_utils.Logger.getLogger()
+    self.predictor = dlib.shape_predictor(cfg.DIR_MODELS_DLIB_68PT)
+
+  def landmarks(self, im, bbox):
+    # Draw high-confidence faces
+    dim = im.shape[:2][::-1]
+    bbox = bbox.to_dlib()
+    im_gray = cv.cvtColor(im, cv.COLOR_BGR2GRAY)
+    landmarks = [[p.x, p.y] for p in self.predictor(im_gray, bbox).parts()]
+    return landmarks
+
+
+class LandmarksMTCNN:
+  
+  # https://github.com/ipazc/mtcnn
+  # pip install mtcnn
+
+  dnn_size = (400, 400)
+
+  def __init__(self, size=(400,400)):
+    from mtcnn.mtcnn import MTCNN
+    self.detector = MTCNN()
+
+  def detect(self, im, opt_size=None, opt_conf_thresh=None, opt_pyramids=None):
+    '''Detects face using MTCNN and returns (list) of BBox
+    :param im: (numpy.ndarray) image
+    :returns list of BBox
+    '''
+    rois = []
+    dnn_size = self.dnn_size if opt_size is None else opt_size
+    im = im_utils.resize(im, width=dnn_size[0], height=dnn_size[1])
+    dim = im.shape[:2][::-1]
+    
+    # run MTCNN
+    dets = self.detector.detect_faces(im)
+
+    for det in dets:
+      rect = det['box']
+      keypoints = det['keypoints']  # not using here. see 'face_landmarks.py'
+      bbox = BBox.from_xywh_dim(*rect, dim)
+      rois.append(bbox)
+    return rois
\ No newline at end of file
diff --git a/megapixels/app/processors/face_landmarks_3d.py b/megapixels/app/processors/face_landmarks_3d.py
new file mode 100644
index 00000000..84a423b0
--- /dev/null
+++ b/megapixels/app/processors/face_landmarks_3d.py
@@ -0,0 +1,27 @@
+import os
+from os.path import join
+from pathlib import Path
+import math
+
+import cv2 as cv
+import numpy as np
+import imutils
+
+from app.utils import im_utils, logger_utils
+from app.models.bbox import BBox
+from app.settings import app_cfg as cfg
+from app.settings import types
+
+
+
+class FaceLandmarks3D:
+
+  # Estimates 3D facial landmarks
+
+  def __init__(self):
+    self.log = logger_utils.Logger.getLogger()
+    pass
+
+
+  def landmarks(self):
+    return [1,2,3,4,100]
\ No newline at end of file
diff --git a/megapixels/app/processors/face_mesh.py b/megapixels/app/processors/face_mesh.py
new file mode 100644
index 00000000..2d3deb4f
--- /dev/null
+++ b/megapixels/app/processors/face_mesh.py
@@ -0,0 +1,27 @@
+import os
+from os.path import join
+from pathlib import Path
+import math
+
+import cv2 as cv
+import numpy as np
+import imutils
+
+from app.utils import im_utils, logger_utils
+from app.models.bbox import BBox
+from app.settings import app_cfg as cfg
+from app.settings import types
+
+
+
+class FaceMesh3D:
+
+  # Estimates 3D face mesh
+
+  def __init__(self):
+    self.log = logger_utils.Logger.getLogger()
+    pass
+
+
+  def mesh(self):
+    return [1,2,3,4,100]
\ No newline at end of file
diff --git a/megapixels/app/processors/face_pose.py b/megapixels/app/processors/face_pose.py
new file mode 100644
index 00000000..f2548b32
--- /dev/null
+++ b/megapixels/app/processors/face_pose.py
@@ -0,0 +1,104 @@
+import os
+from os.path import join
+from pathlib import Path
+import math
+
+import cv2 as cv
+import numpy as np
+import imutils
+
+from app.utils import im_utils, logger_utils
+from app.models.bbox import BBox
+from app.settings import app_cfg as cfg
+from app.settings import types
+
+
+
+class FacePoseDLIB:
+
+
+  dnn_size = (400, 400)
+
+  def __init__(self):
+    pass
+
+
+  def pose(self, landmarks, dim, project_points=False):
+    # computes pose using 6 / 68 points from dlib face landmarks
+    # based on learnopencv.com and 
+    # https://github.com/jerryhouuu/Face-Yaw-Roll-Pitch-from-Pose-Estimation-using-OpenCV/
+    # NB: not as accurate as MTCNN, see @jerryhouuu for ideas
+    
+    pose_points_idx = (30, 8, 36, 45, 48, 54)
+    axis = np.float32([[500,0,0], [0,500,0], [0,0,500]])
+    
+    # 3D model points.
+    model_points = np.array([
+        (0.0, 0.0, 0.0),             # Nose tip
+        (0.0, -330.0, -65.0),        # Chin
+        (-225.0, 170.0, -135.0),     # Left eye left corner
+        (225.0, 170.0, -135.0),      # Right eye right corne
+        (-150.0, -150.0, -125.0),    # Left Mouth corner
+        (150.0, -150.0, -125.0)      # Right mouth corner
+    ])
+    
+    # Assuming no lens distortion
+    dist_coeffs = np.zeros((4,1))
+
+    # find 6 pose points
+    pose_points = []
+    for j, idx in enumerate(pose_points_idx):
+      pt = landmarks[idx]
+      pose_points.append((pt[0], pt[1]))
+    pose_points = np.array(pose_points, dtype='double')  # convert to double
+    
+    # create camera matrix
+    focal_length = dim[0]
+    center = (dim[0]/2, dim[1]/2)
+    cam_mat = np.array(
+        [[focal_length, 0, center[0]],
+        [0, focal_length, center[1]],
+        [0, 1, 1]], dtype = "double")
+    
+    # solve PnP for rotation and translation
+    (success, rot_vec, tran_vec) = cv.solvePnP(model_points, pose_points, 
+                                               cam_mat, dist_coeffs, 
+                                               flags=cv.SOLVEPNP_ITERATIVE)
+
+    result = {}
+
+    # project points
+    if project_points:
+      pts_im, jac = cv.projectPoints(axis, rot_vec, tran_vec, cam_mat, dist_coeffs)
+      pts_model, jac2 = cv.projectPoints(model_points, rot_vec, tran_vec, cam_mat, dist_coeffs)
+      result['points_model'] = pts_model
+      result['points_image'] = pts_im
+      result['point_nose'] = tuple(landmarks[pose_points_idx[0]])
+
+    rvec_matrix = cv.Rodrigues(rot_vec)[0]
+    
+    # convert to degrees
+    proj_matrix = np.hstack((rvec_matrix, tran_vec))
+    eulerAngles = cv.decomposeProjectionMatrix(proj_matrix)[6] 
+    pitch, yaw, roll = [math.radians(x) for x in eulerAngles]
+    pitch = math.degrees(math.asin(math.sin(pitch)))
+    roll = -math.degrees(math.asin(math.sin(roll)))
+    yaw = math.degrees(math.asin(math.sin(yaw)))
+    degrees = {'pitch': pitch, 'roll': roll, 'yaw': yaw}
+    result['degrees'] = degrees
+
+    return result
+
+
+  def draw_pose(self, im, pts_im, pts_model, pt_nose):
+    cv.line(im, pt_nose, tuple(pts_im[1].ravel()), (0,255,0), 3) #GREEN
+    cv.line(im, pt_nose, tuple(pts_im[0].ravel()), (255,0,), 3) #BLUE
+    cv.line(im, pt_nose, tuple(pts_im[2].ravel()), (0,0,255), 3) #RED
+
+
+  def draw_degrees(self, im, degrees, color=(0,255,0)):
+    for i, item in enumerate(degrees.items()):
+      k, v = item
+      t = '{}: {:.2f}'.format(k, v)
+      origin = (10, 30 + (25 * i))
+      cv.putText(im, t, origin, cv.FONT_HERSHEY_SIMPLEX, 0.5, color, thickness=2, lineType=2)
\ No newline at end of file
diff --git a/megapixels/app/processors/face_recognition.py b/megapixels/app/processors/face_recognition.py
new file mode 100644
index 00000000..e0b9f752
--- /dev/null
+++ b/megapixels/app/processors/face_recognition.py
@@ -0,0 +1,56 @@
+import os
+from os.path import join
+from pathlib import Path
+
+import cv2 as cv
+import numpy as np
+import dlib
+import imutils
+
+from app.utils import im_utils, logger_utils
+from app.models.bbox import BBox
+from app.settings import app_cfg as cfg
+from app.settings import types
+    
+class RecognitionDLIB:
+
+  # https://github.com/davisking/dlib/blob/master/python_examples/face_recognition.py
+  # facerec.compute_face_descriptor(img, shape, 100, 0.25)
+
+  def __init__(self, gpu=0):
+    self.log = logger_utils.Logger.getLogger()
+    
+    if gpu > -1:
+      cuda_visible_devices = os.getenv('CUDA_VISIBLE_DEVICES', '')
+      os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu)
+    
+    self.predictor = dlib.shape_predictor(cfg.DIR_MODELS_DLIB_5PT)
+    self.facerec = dlib.face_recognition_model_v1(cfg.DIR_MODELS_DLIB_FACEREC_RESNET)
+    
+    if gpu > -1:
+      os.environ['CUDA_VISIBLE_DEVICES'] = cuda_visible_devices  # reset GPU env
+
+
+  def vec(self, im, bbox, width=100, 
+      jitters=cfg.DLIB_FACEREC_JITTERS, padding=cfg.DLIB_FACEREC_PADDING):
+    # Converts image and bbox into 128d vector
+    # scale the image so the face is always 100x100 pixels
+
+    #self.log.debug('compute scale')
+    scale = width / bbox.width
+    #im = cv.resize(im, (scale, scale), cv.INTER_LANCZOS4)
+    #self.log.debug('resize')
+    cv.resize(im, None, fx=scale, fy=scale, interpolation=cv.INTER_LANCZOS4) 
+    #self.log.debug('to dlib')
+    bbox_dlib = bbox.to_dlib()
+    #self.log.debug('precitor')
+    face_shape = self.predictor(im, bbox_dlib)
+    # vec = self.facerec.compute_face_descriptor(im, face_shape, jitters, padding)
+    #self.log.debug('vec')
+    vec = self.facerec.compute_face_descriptor(im, face_shape, jitters)
+    #vec = self.facerec.compute_face_descriptor(im, face_shape)
+    return vec
+
+
+  def similarity(self, query_enc, known_enc):
+    return np.linalg.norm(query_enc - known_enc, axis=1)
diff --git a/megapixels/app/processors/faiss.py b/megapixels/app/processors/faiss.py
new file mode 100644
index 00000000..5156ad71
--- /dev/null
+++ b/megapixels/app/processors/faiss.py
@@ -0,0 +1,58 @@
+"""
+Index all of the FAISS datasets
+"""
+
+import os
+import glob
+import faiss
+import time
+import numpy as np
+
+from app.utils.file_utils import load_recipe, load_csv_safe
+from app.settings import app_cfg as cfg
+
+class DefaultRecipe:
+  def __init__(self):
+    self.dim = 128
+    self.factory_type = 'Flat'
+
+def build_all_faiss_databases():
+  datasets = []
+  for fn in glob.iglob(os.path.join(cfg.DIR_FAISS_METADATA, "*")):
+    name = os.path.basename(fn)
+    recipe_fn = os.path.join(cfg.DIR_FAISS_RECIPES, name + ".json")
+    if os.path.exists(recipe_fn):
+      build_faiss_database(name, load_recipe(recipe_fn))
+    else:
+      build_faiss_database(name, DefaultRecipe())
+
+def build_faiss_database(name, recipe):
+  vec_fn = os.path.join(cfg.DIR_FAISS_METADATA, name, "vecs.csv")
+  index_fn = os.path.join(cfg.DIR_FAISS_INDEXES, name + ".index")
+
+  index = faiss.index_factory(recipe.dim, recipe.factory_type)
+
+  keys, rows = load_csv_safe(vec_fn)
+  feats = np.array([ list(map(float, row[3].split(","))) for row in rows ]).astype('float32')
+  n, d = feats.shape
+
+  print("{}: training {} x {} dim vectors".format(name, n, d))
+  print(recipe.factory_type)
+
+  add_start = time.time()
+  index.add(feats)
+  add_end = time.time()
+  add_time = add_end - add_start
+  print("{}: add time: {:.1f}s".format(name, add_time))
+
+  faiss.write_index(index, index_fn)
+
+def load_faiss_databases():
+  faiss_datasets = {}
+  for fn in glob.iglob(os.path.join(cfg.DIR_FAISS_METADATA, "*")):
+    name = os.path.basename(fn)
+    index_fn = os.path.join(cfg.DIR_FAISS_INDEXES, name + ".index")
+    if os.path.exists(index_fn):
+      index = faiss.read_index(index_fn)
+      faiss_datasets[name] = index
+  return faiss_datasets
diff --git a/megapixels/app/server/api.py b/megapixels/app/server/api.py
new file mode 100644
index 00000000..35862837
--- /dev/null
+++ b/megapixels/app/server/api.py
@@ -0,0 +1,152 @@
+import os
+import re
+import time
+import dlib
+import numpy as np
+from flask import Blueprint, request, jsonify
+from PIL import Image  # todo: try to remove PIL dependency
+
+from app.processors import face_recognition
+from app.processors import face_detector
+from app.processors.faiss import load_faiss_databases
+from app.models.sql_factory import load_sql_datasets, list_datasets, get_dataset, get_table
+from app.utils.im_utils import pil2np
+
+sanitize_re = re.compile('[\W]+')
+valid_exts = ['.gif', '.jpg', '.jpeg', '.png']
+
+LIMIT = 9
+THRESHOLD = 0.3
+
+api = Blueprint('api', __name__)
+
+faiss_datasets = load_faiss_databases()
+
+@api.route('/')
+def index():
+  """List the datasets and their fields"""
+  return jsonify({ 'datasets': list_datasets() })
+
+
+@api.route('/dataset/<dataset_name>')
+def show(dataset_name):
+  """Show the data that a dataset will return"""
+  dataset = get_dataset(dataset_name)
+  if dataset:
+    return jsonify(dataset.describe())
+  else:
+    return jsonify({ 'status': 404 })
+
+
+@api.route('/dataset/<dataset_name>/face', methods=['POST'])
+def upload(dataset_name):
+  """Query an image against FAISS and return the matching identities"""
+  start = time.time()
+  dataset = get_dataset(dataset_name)
+  if dataset_name not in faiss_datasets:
+    return jsonify({
+      'error': 'invalid dataset'  
+    })
+  faiss_dataset = faiss_datasets[dataset_name]
+  file = request.files['query_img']
+  fn = file.filename
+  if fn.endswith('blob'):
+    fn = 'filename.jpg'
+
+  basename, ext = os.path.splitext(fn)
+  # print("got {}, type {}".format(basename, ext))
+  if ext.lower() not in valid_exts:
+    return jsonify({ 'error': 'not an image' })
+
+  im = Image.open(file.stream).convert('RGB')
+  im_np = pil2np(im)
+
+  # Face detection
+  detector = face_detector.DetectorDLIBHOG()
+
+  # get detection as BBox object
+  bboxes = detector.detect(im_np, largest=True)
+  if not bboxes or not len(bboxes):
+    return jsonify({
+      'error': 'bbox'
+    })
+  bbox = bboxes[0]
+  if not bbox:
+    return jsonify({
+      'error': 'bbox'
+    })
+
+  dim = im_np.shape[:2][::-1]
+  bbox = bbox.to_dim(dim)  # convert back to real dimensions
+  # print("got bbox")
+  if not bbox:
+    return jsonify({
+      'error': 'bbox'
+    })
+
+  # extract 128-D vector
+  recognition = face_recognition.RecognitionDLIB(gpu=-1)
+  vec = recognition.vec(im_np, bbox)
+  query = np.array([ vec ]).astype('float32')
+  
+  # query FAISS
+  distances, indexes = faiss_dataset.search(query, LIMIT)
+
+  if len(indexes) == 0 or len(indexes[0]) == 0:
+    return jsonify({
+      'error': 'nomatch'
+    })
+
+  # get the results for this single query...
+  distances = distances[0]
+  indexes = indexes[0]
+
+  dists = []
+  ids = []
+  for _d, _i in zip(distances, indexes):
+    if _d <= THRESHOLD:
+      dists.append(round(float(_d), 2))
+      ids.append(_i+1)
+
+  results = [ dataset.get_identity(int(_i)) for _i in ids ]
+
+  # print(distances)
+  # print(ids)
+
+  # 'bbox': str(bboxes[0]),
+  # 'bbox_dim': str(bbox),
+  # print(bboxes[0])
+  # print(bbox)
+
+  query = {
+    'timing': round(time.time() - start, 3),
+    'bbox': str(bbox),
+  }
+  # print(results)
+  return jsonify({
+    'query': query,
+    'results': results,
+    'distances': dists,
+  })
+
+
+@api.route('/dataset/<dataset_name>/name', methods=['GET','POST'])
+def name_lookup(dataset_name):
+  """Find a name in the dataset"""
+  start = time.time()
+  dataset = get_dataset(dataset_name)
+
+  q = request.args.get('q')
+  # print(q)
+
+  query = {
+    'q': q,
+    'timing': time.time() - start,
+  }
+  results = dataset.search_name(q + '%') if q else None
+  
+  # print(results)
+  return jsonify({
+    'query': query,
+    'results': results,
+  })
diff --git a/megapixels/app/server/create.py b/megapixels/app/server/create.py
new file mode 100644
index 00000000..4b1333b9
--- /dev/null
+++ b/megapixels/app/server/create.py
@@ -0,0 +1,49 @@
+from flask import Flask, Blueprint, jsonify, send_from_directory
+from flask_sqlalchemy import SQLAlchemy
+from app.models.sql_factory import connection_url, load_sql_datasets
+
+from app.server.api import api
+
+db = SQLAlchemy()
+
+def create_app(script_info=None):
+  """
+  functional pattern for creating the flask app
+  """
+  app = Flask(__name__, static_folder='static', static_url_path='')
+  app.config['SQLALCHEMY_DATABASE_URI'] = connection_url
+  app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
+
+  db.init_app(app)
+  datasets = load_sql_datasets(replace=False, base_model=db.Model)
+
+  app.register_blueprint(api, url_prefix='/api')
+  app.add_url_rule('/<path:file_relative_path_to_root>', 'serve_page', serve_page, methods=['GET'])
+
+  @app.route('/', methods=['GET'])
+  def index():
+    return app.send_static_file('index.html')
+
+  @app.shell_context_processor
+  def shell_context():
+    return { 'app': app, 'db': db }
+
+  @app.route("/site-map")
+  def site_map():
+    links = []
+    for rule in app.url_map.iter_rules():
+      # url = url_for(rule.endpoint, **(rule.defaults or {}))
+      # print(url)
+      links.append((rule.endpoint))
+    return(jsonify(links))
+
+  return app
+
+def serve_page(file_relative_path_to_root):
+  """
+  trying to get this to serve /path/ with /path/index.html,
+  ...but it doesnt actually matter for production...
+  """
+  if file_relative_path_to_root[-1] == '/':
+    file_relative_path_to_root += 'index.html'
+  return send_from_directory("static", file_relative_path_to_root)
diff --git a/megapixels/app/server/json_encoder.py b/megapixels/app/server/json_encoder.py
new file mode 100644
index 00000000..89af578a
--- /dev/null
+++ b/megapixels/app/server/json_encoder.py
@@ -0,0 +1,17 @@
+from sqlalchemy.ext.declarative import DeclarativeMeta
+from flask import json
+
+class AlchemyEncoder(json.JSONEncoder):
+  def default(self, o):
+    if isinstance(o.__class__, DeclarativeMeta):
+      data = {}
+      fields = o.__json__() if hasattr(o, '__json__') else dir(o)
+      for field in [f for f in fields if not f.startswith('_') and f not in ['metadata', 'query', 'query_class']]:
+        value = o.__getattribute__(field)
+        try:
+          json.dumps(value)
+          data[field] = value
+        except TypeError:
+          data[field] = None
+      return data
+    return json.JSONEncoder.default(self, o)
diff --git a/megapixels/app/server/static b/megapixels/app/server/static
new file mode 120000
index 00000000..1dc7a639
--- /dev/null
+++ b/megapixels/app/server/static
@@ -0,0 +1 @@
+../../../site/public
\ No newline at end of file
diff --git a/megapixels/app/settings/app_cfg.py b/megapixels/app/settings/app_cfg.py
index 739ddce2..55fed166 100644
--- a/megapixels/app/settings/app_cfg.py
+++ b/megapixels/app/settings/app_cfg.py
@@ -2,27 +2,36 @@ import os
 from os.path import join
 import logging
 import collections
-
-import cv2 as cv
+from dotenv import load_dotenv
 
 from app.settings import types
 from app.utils import click_utils
 
+import codecs
+codecs.register(lambda name: codecs.lookup('utf8') if name == 'utf8mb4' else None)
 
 # -----------------------------------------------------------------------------
 # Enun lists used for custom Click Params
 # -----------------------------------------------------------------------------
 
 FaceDetectNetVar = click_utils.ParamVar(types.FaceDetectNet)
-
+HaarCascadeVar = click_utils.ParamVar(types.HaarCascade)
 LogLevelVar = click_utils.ParamVar(types.LogLevel)
+MetadataVar = click_utils.ParamVar(types.Metadata)
+DatasetVar = click_utils.ParamVar(types.Dataset)
+DataStoreVar = click_utils.ParamVar(types.DataStore)
 
 # # data_store
 DATA_STORE = '/data_store_hdd/'
+DATA_STORE_NAS = '/data_store_nas/'
+DATA_STORE_HDD = '/data_store_hdd/'
+DATA_STORE_SSD = '/data_store_ssd/'
 DIR_DATASETS = join(DATA_STORE,'datasets')
+DIR_DATSET_NAS = join(DIR_DATASETS, 'people')
 DIR_APPS = join(DATA_STORE,'apps')
 DIR_APP = join(DIR_APPS,'megapixels')
 DIR_MODELS = join(DIR_APP,'models')
+DIR_PEOPLE = 'people'
 
 # # Frameworks
 DIR_MODELS_CAFFE = join(DIR_MODELS,'caffe')
@@ -36,23 +45,39 @@ DIR_MODELS_DLIB = join(DIR_MODELS,'dlib')
 DIR_MODELS_DLIB_CNN = join(DIR_MODELS_DLIB, 'mmod_human_face_detector.dat')
 DIR_MODELS_DLIB_5PT = join(DIR_MODELS_DLIB, 'shape_predictor_5_face_landmarks.dat')
 DIR_MODELS_DLIB_68PT = join(DIR_MODELS_DLIB, 'shape_predictor_68_face_landmarks.dat')
+DIR_MODELS_DLIB_FACEREC_RESNET = join(DIR_MODELS_DLIB, 'dlib_face_recognition_resnet_model_v1.dat')
 
+DIR_FAISS = join(DIR_APP, 'faiss')
+DIR_FAISS_INDEXES = join(DIR_FAISS, 'indexes')
+DIR_FAISS_METADATA = join(DIR_FAISS, 'metadata')
+DIR_FAISS_RECIPES = join(DIR_FAISS, 'recipes')
 
 # Test images
 DIR_TEST_IMAGES = join(DIR_APP, 'test', 'images')
 
 # -----------------------------------------------------------------------------
+# .env config for keys
+# -----------------------------------------------------------------------------
+
+# DIR_DOTENV = join(DIR_APP, '.env')
+load_dotenv() # dotenv_path=DIR_DOTENV)
+
+# -----------------------------------------------------------------------------
 # Drawing, GUI settings
 # -----------------------------------------------------------------------------
 DIR_ASSETS = join(DIR_APP, 'assets')
 FP_FONT = join(DIR_ASSETS, 'font')
 
-
 # -----------------------------------------------------------------------------
 # click chair settings
 # -----------------------------------------------------------------------------
-DIR_COMMANDS_PROCESSOR_ADMIN = 'admin/commands'
-DIR_COMMANDS_PROCESSOR_DATASETS = 'datasets/commands'
+DIR_COMMANDS_CV = 'commands/cv'
+DIR_COMMANDS_ADMIN = 'commands/admin'
+DIR_COMMANDS_DATASETS = 'commands/datasets'
+DIR_COMMANDS_FAISS = 'commands/faiss'
+DIR_COMMANDS_MISC  = 'commands/misc'
+DIR_COMMANDS_SITE = 'commands/site'
+DIR_COMMANDS_DEMO = 'commands/demo'
 
 # -----------------------------------------------------------------------------
 # Filesystem settings
@@ -64,6 +89,16 @@ CKPT_ZERO_PADDING = 9
 HASH_TREE_DEPTH = 3
 HASH_BRANCH_SIZE = 3
 
+DLIB_FACEREC_JITTERS = 5  # number of face recognition jitters
+DLIB_FACEREC_PADDING = 0.25  # default dlib
+
+POSE_MINMAX_YAW = (-25,25)
+POSE_MINMAX_ROLL = (-15,15)
+POSE_MINMAX_PITCH = (-10,10)
+
+POSE_MINMAX_YAW = (-40,40)
+POSE_MINMAX_ROLL = (-35,35)
+POSE_MINMAX_PITCH = (-25,25)
 # -----------------------------------------------------------------------------
 # Logging options exposed for custom click Params
 # -----------------------------------------------------------------------------
@@ -87,4 +122,23 @@ black, red, green, yellow, blue, purple, cyan and white.
 bold, bold_{color}, fg_bold_{color}, bg_bold_{color}: Bold/bright colors.
 reset: Clear all formatting (both foreground and background colors).
 """
-LOGFILE_FORMAT = "%(log_color)s%(levelname)-8s%(reset)s %(cyan)s%(filename)s:%(lineno)s:%(bold_cyan)s%(funcName)s() %(reset)s%(message)s"
\ No newline at end of file
+LOGFILE_FORMAT = "%(log_color)s%(levelname)-8s%(reset)s %(cyan)s%(filename)s:%(lineno)s:%(bold_cyan)s%(funcName)s() %(reset)s%(message)s"
+
+# -----------------------------------------------------------------------------
+# S3 storage
+# -----------------------------------------------------------------------------
+S3_ROOT_URL = 's3://megapixels/v1/'
+S3_MEDIA_URL = join(S3_ROOT_URL, 'media')
+S3_METADATA_URL = join(S3_ROOT_URL, 'metadata')
+S3_HTTP_URL = 'https://megapixels.nyc3.digitaloceanspaces.com/v1/'
+S3_HTTP_MEDIA_URL = join(S3_HTTP_URL, 'media')
+S3_HTTP_METADATA_URL = join(S3_HTTP_URL, 'metadata')
+
+# -----------------------------------------------------------------------------
+# Static site generator
+# -----------------------------------------------------------------------------
+S3_SITE_PATH = "v1/site"
+S3_DATASETS_PATH = "v1" # datasets is already in the filename
+DIR_SITE_PUBLIC = "../site/public"
+DIR_SITE_CONTENT = "../site/content"
+DIR_SITE_TEMPLATES = "../site/templates"
diff --git a/megapixels/app/settings/types.py b/megapixels/app/settings/types.py
index 0c3d7942..0805c5bd 100644
--- a/megapixels/app/settings/types.py
+++ b/megapixels/app/settings/types.py
@@ -7,10 +7,9 @@ def find_type(name, enum_type):
   return None
 
 
-
 class FaceDetectNet(Enum):
   """Scene text detector networks"""
-  HAAR, DLIB_CNN, DLIB_HOG, CVDNN = range(4)
+  HAAR, DLIB_CNN, DLIB_HOG, CVDNN, MTCNN = range(5)
 
 class CVBackend(Enum):
   """OpenCV 3.4.2+ DNN target type"""
@@ -20,6 +19,18 @@ class CVTarget(Enum):
   """OpenCV 3.4.2+ DNN backend processor type"""
   CPU, OPENCL, OPENCL_FP16, MYRIAD = range(4)
 
+class HaarCascade(Enum):
+  FRONTAL, ALT, ALT2, PROFILE = range(4)
+
+
+# ---------------------------------------------------------------------
+# Storage
+# --------------------------------------------------------------------
+
+class DataStore(Enum):
+  """Storage devices. Paths are symlinked to root (eg /data_store_nas)"""
+  NAS, HDD, SSD, S3 = range(4)
+
 # ---------------------------------------------------------------------
 # Logger, monitoring
 # --------------------------------------------------------------------
@@ -27,3 +38,47 @@ class CVTarget(Enum):
 class LogLevel(Enum):
   """Loger vebosity"""
   DEBUG, INFO, WARN, ERROR, CRITICAL = range(5)
+
+
+# ---------------------------------------------------------------------
+# Metadata types
+# --------------------------------------------------------------------
+
+class Metadata(Enum):
+  IDENTITY, FILE_RECORD, FACE_VECTOR, FACE_POSE, FACE_ROI, FACE_LANDMARKS_68, \
+    FACE_LANDMARKS_3D = range(7)
+
+class Dataset(Enum):
+  LFW, VGG_FACE2, MSCELEB, UCCS, UMD_FACES = range(5)
+
+
+# ---------------------------------------------------------------------
+# Face analysis types
+# --------------------------------------------------------------------
+class FaceEmotion(Enum):
+  # Map these to text strings for web display
+  NEUTRAL, HAPPY, SAD, ANGRY, FRUSTURATED = range(5)
+
+class FaceBeauty(Enum):
+  # Map these to text strings for web display
+  AVERAGE, BELOW_AVERAGE, ABOVE_AVERAGE = range(3)
+
+class FaceYaw(Enum):
+  # Map these to text strings for web display
+  FAR_LEFT, LEFT, CENTER, RIGHT, FAR_RIGHT = range(5)
+
+class FacePitch(Enum):
+  # Map these to text strings for web display
+  FAR_DOWN, DOWN, CENTER, UP, FAR_UP = range(5)
+
+class FaceRoll(Enum):
+  # Map these to text strings for web display
+  FAR_DOWN, DOWN, CENTER, UP, FAR_UP = range(5)
+
+class FaceAge(Enum):
+  # Map these to text strings for web display
+  CHILD, TEENAGER, YOUNG_ADULT, ADULT, MATURE_ADULT, SENIOR = range(6)
+
+class Confidence(Enum):
+  # Map these to text strings for web display
+  VERY_LOW, LOW, MEDIUM, MEDIUM_HIGH, HIGH, VERY_HIGH = range(6)
\ No newline at end of file
diff --git a/megapixels/app/site/README.md b/megapixels/app/site/README.md
new file mode 100644
index 00000000..1a6d3a1e
--- /dev/null
+++ b/megapixels/app/site/README.md
@@ -0,0 +1,21 @@
+Megapixels Static Site Generator
+================================
+
+The index, blog, and about other pages are built using this static site generator.
+
+## Metadata
+
+```
+status: published|draft|private
+title: From 1 to 100 Pixels
+desc: High resolution insights from low resolution imagery
+slug: from-1-to-100-pixels
+published: 2018-12-04
+updated: 2018-12-04
+authors: Adam Harvey, Berit Gilma, Matthew Stender
+```
+
+## S3 Assets
+
+Static assets: `v1/site/about/assets/picture.jpg`
+Dataset assets: `v1/datasets/lfw/assets/picture.jpg`
diff --git a/megapixels/app/site/builder.py b/megapixels/app/site/builder.py
new file mode 100644
index 00000000..ff1a0c83
--- /dev/null
+++ b/megapixels/app/site/builder.py
@@ -0,0 +1,103 @@
+#!/usr/bin/python
+
+import os
+import glob
+from jinja2 import Environment, FileSystemLoader, select_autoescape
+
+import app.settings.app_cfg as cfg
+
+import app.site.s3 as s3
+import app.site.parser as parser
+
+env = Environment(
+  loader=FileSystemLoader(cfg.DIR_SITE_TEMPLATES),
+  autoescape=select_autoescape([])
+)
+
+def build_page(fn, research_posts):
+  """
+  build a single page from markdown into the appropriate template
+  - writes it to site/public/
+  - syncs any assets with s3
+  - handles certain index pages...
+  """
+  metadata, sections = parser.read_metadata(fn)
+
+  if metadata is None:
+    print("{} has no metadata".format(fn))
+    return
+
+  print(metadata['url'])
+
+  dirname = os.path.dirname(fn)
+  output_path = cfg.DIR_SITE_PUBLIC + metadata['url']
+  output_fn = os.path.join(output_path, "index.html")
+
+  skip_h1 = False
+
+  if metadata['url'] == '/':
+    template = env.get_template("home.html")
+  elif 'research/' in fn:
+    skip_h1 = True
+    template = env.get_template("research.html")
+  else:
+    template = env.get_template("page.html")
+
+  if 'datasets/' in fn:
+    s3_dir = cfg.S3_DATASETS_PATH
+  else:
+    s3_dir = cfg.S3_SITE_PATH
+
+  s3_path = s3.make_s3_path(s3_dir, metadata['path'])
+
+  if 'index.md' in fn:
+    s3.sync_directory(dirname, s3_dir, metadata)
+
+  content = parser.parse_markdown(sections, s3_path, skip_h1=skip_h1)
+
+  html = template.render(
+    metadata=metadata,
+    content=content,
+    research_posts=research_posts,
+    latest_research_post=research_posts[-1],
+  )
+
+  os.makedirs(output_path, exist_ok=True)
+  with open(output_fn, "w") as file:
+    file.write(html)
+
+def build_research_index(research_posts):
+  """
+  build the index of research (blog) posts
+  """
+  metadata, sections = parser.read_metadata('../site/content/research/index.md')
+  template = env.get_template("page.html")
+  s3_path = s3.make_s3_path(cfg.S3_SITE_PATH, metadata['path'])
+  content = parser.parse_markdown(sections, s3_path, skip_h1=False)
+  content += parser.parse_research_index(research_posts)
+  html = template.render(
+    metadata=metadata,
+    content=content,
+    research_posts=research_posts,
+    latest_research_post=research_posts[-1],
+  )
+  output_fn = cfg.DIR_SITE_PUBLIC + '/research/index.html'
+  with open(output_fn, "w") as file:
+    file.write(html)
+
+def build_site():
+  """
+  build the site! =^)
+  """
+  research_posts = parser.read_research_post_index()
+  for fn in glob.iglob(os.path.join(cfg.DIR_SITE_CONTENT, "**/*.md"), recursive=True):
+    build_page(fn, research_posts)
+  build_research_index(research_posts)
+
+def build_file(fn):
+  """
+  build just one page from a filename! =^)
+  """
+  research_posts = parser.read_research_post_index()
+  fn = os.path.join(cfg.DIR_SITE_CONTENT, fn)
+  build_page(fn, research_posts)
diff --git a/megapixels/app/site/parser.py b/megapixels/app/site/parser.py
new file mode 100644
index 00000000..b3d3a8c2
--- /dev/null
+++ b/megapixels/app/site/parser.py
@@ -0,0 +1,238 @@
+import os
+import re
+import glob
+import simplejson as json
+import mistune
+
+import app.settings.app_cfg as cfg
+import app.site.s3 as s3
+
+renderer = mistune.Renderer(escape=False)
+markdown = mistune.Markdown(renderer=renderer)
+
+def fix_images(lines, s3_path):
+  """
+  do our own tranformation of the markdown around images to handle wide images etc
+  lines: markdown lines
+  """
+  real_lines = []
+  block = "\n\n".join(lines)
+  for line in block.split("\n"):
+    if "![" in line:
+      line = line.replace('![', '')
+      alt_text, tail = line.split('](', 1)
+      url, tail = tail.split(')', 1)
+      if ':' in alt_text:
+        tail, alt_text = alt_text.split(':', 1)
+      img_tag = "<img src='{}' alt='{}'>".format(s3_path + url, alt_text.replace("'", ""))
+      if len(alt_text):
+        line = "<div class='image'>{}<div class='caption'>{}</div></div>".format(img_tag, alt_text)
+      else:
+        line = "<div class='image'>{}</div>".format(img_tag, alt_text)
+    real_lines.append(line)
+  return "\n".join(real_lines)
+
+def format_section(lines, s3_path, type=''):
+  """
+  format a normal markdown section
+  """
+  if len(lines):
+    lines = fix_images(lines, s3_path)
+    if type:
+      return "<section class='{}'>{}</section>".format(type, markdown(lines))
+    else:
+      return "<section>" + markdown(lines) + "</section>"
+  return ""
+
+def format_metadata(section):
+  """
+  format a metadata section (+ key: value pairs)
+  """
+  meta = []
+  for line in section.split('\n'):
+    key, value = line[2:].split(': ', 1)
+    meta.append("<div><div class='gray'>{}</div><div>{}</div></div>".format(key, value))
+  return "<section><div class='meta'>{}</div></section>".format(''.join(meta)) 
+
+def format_applet(section, s3_path):
+  # print(section)
+  payload = section.strip('```').strip().strip('```').strip().split('\n')
+  applet = {}
+  print(payload)
+  if ': ' in payload[0]:
+    command, opt = payload[0].split(': ')
+  else:
+    command = payload[0]
+    opt = None
+  if command == 'python' or command == 'javascript' or command == 'code':
+    return format_section([ section ], s3_path)
+
+  applet['command'] = command
+  if opt:
+    applet['opt'] = opt
+  if command == 'load_file':
+    if opt[0:4] != 'http':
+      applet['opt'] = s3_path + opt
+  if len(payload) > 1:
+    applet['fields'] = payload[1:]
+  return "<section class='applet_container'><div class='applet' data-payload='{}'></div></section>".format(json.dumps(applet))
+
+def parse_markdown(sections, s3_path, skip_h1=False):
+  """
+  parse page into sections, preprocess the markdown to handle our modifications
+  """
+  groups = []
+  current_group = []
+  for section in sections:
+    if skip_h1 and section.startswith('# '):
+      continue
+    elif section.strip().startswith('```'):
+      groups.append(format_section(current_group, s3_path))
+      current_group = []
+      current_group.append(section)
+      if section.strip().endswith('```'):
+        groups.append(format_applet("\n\n".join(current_group), s3_path))
+        current_group = []
+    elif section.strip().endswith('```'):
+      current_group.append(section)
+      groups.append(format_applet("\n\n".join(current_group), s3_path))
+      current_group = []
+    elif section.startswith('+ '):
+      groups.append(format_section(current_group, s3_path))
+      groups.append(format_metadata(section))
+      current_group = []
+    elif '![wide:' in section:
+      groups.append(format_section(current_group, s3_path))
+      groups.append(format_section([section], s3_path, type='wide'))
+      current_group = []
+    elif '![' in section:
+      groups.append(format_section(current_group, s3_path))
+      groups.append(format_section([section], s3_path, type='images'))
+      current_group = []
+    else:
+      current_group.append(section)
+  groups.append(format_section(current_group, s3_path))
+  content = "".join(groups)
+  return content
+
+def parse_research_index(research_posts):
+  """
+  Generate an index file for the research pages
+  """
+  content = "<div class='research_index'>"
+  for post in research_posts:
+    s3_path = s3.make_s3_path(cfg.S3_SITE_PATH, post['path'])
+    if 'image' in post:
+      post_image = s3_path + post['image']
+    else:
+      post_image = 'data:image/gif;base64,R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw=='
+    row = "<a href='{}'><section class='wide'><img src='{}' alt='Research post' /><section><h1>{}</h1><h2>{}</h2></section></section></a>".format(
+      post['path'],
+      post_image,
+      post['title'],
+      post['tagline'])
+    content += row
+  content += '</div>'
+  return content
+
+def read_metadata(fn):
+  """
+  Read in read a markdown file and extract the metadata
+  """
+  with open(fn, "r") as file:
+    data = file.read()
+    data = data.replace("\n ", "\n")
+    if "\n" in data:
+      data = data.replace("\r", "")
+    else:
+      data = data.replace("\r", "\n")
+    sections = data.split("\n\n")
+  return parse_metadata(fn, sections)
+
+default_metadata = {
+  'status': 'published',
+  'title': 'Untitled Page',
+  'desc': '',
+  'slug': '',
+  'published': '2018-12-31',
+  'updated': '2018-12-31',
+  'authors': 'Adam Harvey',
+  'sync': 'true',
+  'tagline': '',
+}
+
+def parse_metadata_section(metadata, section):
+  """
+  parse a metadata key: value pair
+  """
+  for line in section.split("\n"):
+    if ': ' not in line:
+      continue
+    key, value = line.split(': ', 1)
+    metadata[key.lower()] = value
+
+def parse_metadata(fn, sections):
+  """
+  parse the metadata headers in a markdown file
+  (everything before the second ---------)
+  also generates appropriate urls for this page :)
+  """
+  found_meta = False
+  metadata = {}
+  valid_sections = []
+  for section in sections:
+    if not found_meta and ': ' in section:
+      found_meta = True
+      parse_metadata_section(metadata, section)
+      continue
+    if '-----' in section:
+      continue
+    if found_meta:
+      valid_sections.append(section)
+
+  if 'title' not in metadata:
+    print('warning: {} has no title'.format(fn))
+  for key in default_metadata:
+    if key not in metadata:
+      metadata[key] = default_metadata[key]
+
+  basedir = os.path.dirname(fn.replace(cfg.DIR_SITE_CONTENT, ''))
+  basename = os.path.basename(fn)
+  if basedir == '/':
+    metadata['path'] = '/'
+    metadata['url'] = '/'
+  elif basename == 'index.md':
+    metadata['path'] = basedir + '/'
+    metadata['url'] = metadata['path']
+  else:
+    metadata['path'] = basedir + '/'
+    metadata['url'] = metadata['path'] + basename.replace('.md', '') + '/'
+
+  if metadata['status'] == 'published|draft|private':
+    metadata['status'] = 'published'
+
+  metadata['sync'] = metadata['sync'] != 'false'
+
+  metadata['author_html'] = '<br>'.join(metadata['authors'].split(','))
+
+  return metadata, valid_sections
+
+def read_research_post_index():
+  """
+  Generate an index of the research (blog) posts
+  """
+  posts = []
+  for fn in sorted(glob.glob('../site/content/research/*/index.md')):
+    metadata, valid_sections = read_metadata(fn)
+    if metadata is None or metadata['status'] == 'private' or metadata['status'] == 'draft':
+      continue
+    posts.append(metadata)
+  if not len(posts):
+    posts.append({
+      'title': 'Placeholder',
+      'slug': 'placeholder',
+      'date': 'Placeholder',
+      'url': '/',
+    })
+  return posts
+
diff --git a/megapixels/app/site/s3.py b/megapixels/app/site/s3.py
new file mode 100644
index 00000000..18133078
--- /dev/null
+++ b/megapixels/app/site/s3.py
@@ -0,0 +1,66 @@
+import os
+import glob
+import boto3
+
+def sync_directory(base_fn, s3_path, metadata):
+  """
+  Synchronize a local assets folder with S3
+  """
+  if not metadata['sync']:
+    return
+
+  fns = {}
+  for fn in glob.glob(os.path.join(base_fn, 'assets/*')):
+    # print(fn)
+    fns[os.path.basename(fn)] = True
+
+  remote_path = s3_path + metadata['url']
+
+  session = boto3.session.Session()
+
+  s3_client = session.client(
+    service_name='s3',
+    aws_access_key_id=os.getenv('S3_KEY'),
+    aws_secret_access_key=os.getenv('S3_SECRET'),
+    endpoint_url=os.getenv('S3_ENDPOINT'),
+    region_name=os.getenv('S3_REGION'),
+  )
+
+  directory = s3_client.list_objects(Bucket=os.getenv('S3_BUCKET'), Prefix=remote_path)
+  prefixes = []
+
+  if 'Contents' in directory:
+    for obj in directory['Contents']:
+      s3_fn = obj['Key']
+      # print(s3_fn)
+      fn = os.path.basename(s3_fn)
+      local_fn = os.path.join(base_fn, 'assets', fn)
+      if fn in fns:
+        del fns[fn]
+        if obj['LastModified'].timestamp() < os.path.getmtime(os.path.join(local_fn)):
+          print("s3 update {}".format(s3_fn))
+          s3_client.upload_file(
+            local_fn,
+            os.getenv('S3_BUCKET'),
+            s3_fn,
+            ExtraArgs={ 'ACL': 'public-read' })
+      else:
+        print("s3 delete {}".format(s3_fn))
+        response = s3_client.delete_object(
+          Bucket=os.getenv('S3_BUCKET'),
+          Key=s3_fn,
+        )
+
+  for fn in fns:
+    local_fn = os.path.join(base_fn, 'assets', fn)
+    s3_fn = os.path.join(remote_path, 'assets', fn)
+    print(s3_fn)
+    print("s3 create {}".format(s3_fn))
+    s3_client.upload_file(
+      local_fn,
+      os.getenv('S3_BUCKET'),
+      s3_fn,
+      ExtraArgs={ 'ACL': 'public-read' })
+
+def make_s3_path(s3_dir, metadata_path):
+  return "{}/{}/{}{}".format(os.getenv('S3_ENDPOINT'), os.getenv('S3_BUCKET'), s3_dir, metadata_path)
diff --git a/megapixels/app/utils/file_utils.py b/megapixels/app/utils/file_utils.py
index 773667b1..5c7b39d1 100644
--- a/megapixels/app/utils/file_utils.py
+++ b/megapixels/app/utils/file_utils.py
@@ -40,10 +40,16 @@ log = logging.getLogger(cfg.LOGGER_NAME)
 # File I/O read/write little helpers
 # ------------------------------------------
 
-def glob_multi(dir_in, exts):
+def glob_multi(dir_in, exts, recursive=False):
   files = []
-  for e in exts:
-    files.append(glob(join(dir_in, '*.{}'.format(e))))
+  for ext in exts:
+    if recursive:
+      fp_glob = join(dir_in, '**/*.{}'.format(ext))
+      log.info(f'glob {fp_glob}')
+      files +=  glob(fp_glob, recursive=True)
+    else:
+      fp_glob = join(dir_in, '*.{}'.format(ext))
+      files += glob(fp_glob)
   return files
 
 
@@ -77,7 +83,7 @@ def load_csv(fp_in, as_list=True):
   :returns: list of all CSV data
   """ 
   if not Path(fp_in).exists():
-    log.info('loading {}'.format(fp_in))
+    log.info('not found: {}'.format(fp_in))
   log.info('loading: {}'.format(fp_in))
   with open(fp_in, 'r') as fp:
     items = csv.DictReader(fp)
@@ -86,6 +92,50 @@ def load_csv(fp_in, as_list=True):
     log.info('returning {:,} items'.format(len(items)))
     return items
 
+def unfussy_csv_reader(reader):
+  """Loads a CSV while ignoring possible data errors
+  :param reader: Special reader for load_csv_safe which ignores CSV parse errors
+  """
+  while True:
+    try:
+      yield next(reader)
+    except StopIteration:
+      return
+    except csv.Error:
+      print(csv.Error)
+      # log the problem or whatever
+      continue
+
+def load_csv_safe(fp_in, keys=True, create=False):
+  """Loads a CSV while ignoring possible data errors
+  :param fp_in: string filepath to JSON file
+  :param keys: boolean set to false if the first line is not headers (for some reason)
+  :param create: boolean set to true to return an empty keys/values if the CSV does not exist
+  """ 
+  try:
+    with open(fp_in, 'r', newline='', encoding='utf-8') as f:
+      # reader = csv.reader( (line.replace('\0','') for line in f) )
+      reader = csv.reader(f)
+      lines = list(unfussy_csv_reader(reader))
+      if keys:
+        keys = lines[0]
+        lines = lines[1:]
+        return keys, lines
+      return lines
+  except:
+    if create:
+      if keys:
+        return {}, []
+      return []
+    raise
+
+def load_recipe(fp_in):
+  """Loads a JSON file as an object with properties accessible with dot syntax
+  :param fp_in: string filepath to JSON file
+  """ 
+  with open(path) as fh:
+    return json.load(fh, object_hook=lambda d: collections.namedtuple('X', d.keys())(*d.values()))
+
 
 def lazywrite(data, fp_out, sort_keys=True):
   """Writes JSON or Pickle data"""
@@ -175,7 +225,7 @@ def write_pickle(data, fp_out, ensure_path=True):
     pickle.dump(data, fp)
 
 
-def write_json(data, fp_out, minify=True, ensure_path=True, sort_keys=True):
+def write_json(data, fp_out, minify=True, ensure_path=True, sort_keys=True, verbose=False):
   """
   """
   if ensure_path:
@@ -185,6 +235,8 @@ def write_json(data, fp_out, minify=True, ensure_path=True, sort_keys=True):
       json.dump(data, fp, separators=(',',':'), sort_keys=sort_keys)
     else:
       json.dump(data, fp, indent=2, sort_keys=sort_keys)
+  if verbose:
+    log.info('Wrote JSON: {}'.format(fp_out))
 
 def write_csv(data, fp_out, header=None):
   """ """
@@ -277,7 +329,7 @@ def sha256(fp_in, block_size=65536):
   """
   sha256 = hashlib.sha256()
   with open(fp_in, 'rb') as fp:
-    for block in iter(lambda: f.read(block_size), b''):
+    for block in iter(lambda: fp.read(block_size), b''):
       sha256.update(block)
   return sha256.hexdigest()
 
diff --git a/megapixels/app/utils/im_utils.py b/megapixels/app/utils/im_utils.py
index a0f23cd2..d5e92aa3 100644
--- a/megapixels/app/utils/im_utils.py
+++ b/megapixels/app/utils/im_utils.py
@@ -22,6 +22,16 @@ import datetime
 
 
 
+def is_grayscale(im, threshold=5):
+  """Returns True if image is grayscale
+  :param im: (numpy.array) image
+  :return (bool) of if image is grayscale"""
+  b = im[:,:,0]
+  g = im[:,:,1]
+  mean = np.mean(np.abs(g - b))
+  return mean < threshold
+
+
 def compute_features(fe,frames,phashes,phash_thresh=1):
   """
   Get vector embedding using FeatureExtractor
@@ -40,7 +50,7 @@ def compute_features(fe,frames,phashes,phash_thresh=1):
   return vals
 
 
-def ensure_pil(im, bgr2rgb=False):
+def np2pil(im, swap=True):
   """Ensure image is Pillow format
     :param im: image in numpy or PIL.Image format
     :returns: image in Pillow RGB format
@@ -49,35 +59,44 @@ def ensure_pil(im, bgr2rgb=False):
       im.verify()
       return im
   except:
-    if bgr2rgb:
+    if swap:
       im = cv.cvtColor(im,cv.COLOR_BGR2RGB)
     return Image.fromarray(im.astype('uint8'), 'RGB')
 
-def ensure_np(im):
+def pil2np(im, swap=True):
   """Ensure image is Numpy.ndarry format
     :param im: image in numpy or PIL.Image format
     :returns: image in Numpy uint8 format
   """
   if type(im) == np.ndarray:
-      return im
-  return np.asarray(im, np.uint8)
+    return im
+  im = np.asarray(im, np.uint8)
+  if swap:
+    im = cv.cvtColor(im, cv.COLOR_RGB2BGR)
+  return im
 
 
-def resize(im,width=0,height=0):
+def resize(im, width=0, height=0):
   """resize image using imutils. Use w/h=[0 || None] to prioritize other edge size
     :param im: a Numpy.ndarray image
     :param wh: a tuple of (width, height)
   """
+  # TODO change to cv.resize and add algorithm choices
   w = width
   h = height
   if w is 0 and h is 0:
     return im
   elif w > 0 and h > 0:
-    return imutils.resize(im,width=w,height=h)
+    ws = im.shape[1] / w
+    hs = im.shape[0] / h
+    if ws > hs:
+      return imutils.resize(im, width=w)
+    else:
+      return imutils.resize(im, height=h)
   elif w > 0 and h is 0:
-    return imutils.resize(im,width=w)
+    return imutils.resize(im, width=w)
   elif w is 0 and h > 0:
-    return imutils.resize(im,height=h)
+    return imutils.resize(im, height=h)
   else:
     return im