Merge branch 'master' of github.com:adamhrv/megapixels_dev

92 files changed, 140 insertions, 160 deletions

diff --git a/builder/__init__.py b/builder/__init__.py
deleted file mode 100644
index e69de29b..00000000
--- a/builder/__init__.py
+++ /dev/null
diff --git a/builder/paths.py b/builder/paths.py
deleted file mode 100644
index 356f2f3d..00000000
--- a/builder/paths.py
+++ /dev/null
@@ -1,6 +0,0 @@
-
-s3_site_path = "v1/site"
-s3_datasets_path = "v1" # datasets is already in the filename
-public_path = "../site/public"
-content_path = "../site/content"
-template_path = "../site/templates"
diff --git a/scraper/client/actions.js b/client/actions.js
index ba899f06..ba899f06 100644
--- a/scraper/client/actions.js
+++ b/client/actions.js
diff --git a/scraper/client/app.js b/client/app.js
index 6c008ec6..6c008ec6 100644
--- a/scraper/client/app.js
+++ b/client/app.js
diff --git a/scraper/client/common/activeLink.component.js b/client/common/activeLink.component.js
index 59f63881..59f63881 100644
--- a/scraper/client/common/activeLink.component.js
+++ b/client/common/activeLink.component.js
diff --git a/scraper/client/common/classifier.component.js b/client/common/classifier.component.js
index af6a4934..af6a4934 100644
--- a/scraper/client/common/classifier.component.js
+++ b/client/common/classifier.component.js
diff --git a/scraper/client/common/common.css b/client/common/common.css
index 4b939df0..4b939df0 100644
--- a/scraper/client/common/common.css
+++ b/client/common/common.css
diff --git a/scraper/client/common/detectionBoxes.component.js b/client/common/detectionBoxes.component.js
index c4872ea8..c4872ea8 100644
--- a/scraper/client/common/detectionBoxes.component.js
+++ b/client/common/detectionBoxes.component.js
diff --git a/scraper/client/common/detectionList.component.js b/client/common/detectionList.component.js
index 416e66d8..416e66d8 100644
--- a/scraper/client/common/detectionList.component.js
+++ b/client/common/detectionList.component.js
diff --git a/scraper/client/common/footer.component.js b/client/common/footer.component.js
index 7c82b44b..7c82b44b 100644
--- a/scraper/client/common/footer.component.js
+++ b/client/common/footer.component.js
diff --git a/scraper/client/common/gate.component.js b/client/common/gate.component.js
index 9bf9287b..9bf9287b 100644
--- a/scraper/client/common/gate.component.js
+++ b/client/common/gate.component.js
diff --git a/scraper/client/common/header.component.js b/client/common/header.component.js
index 84fe306f..84fe306f 100644
--- a/scraper/client/common/header.component.js
+++ b/client/common/header.component.js
diff --git a/scraper/client/common/index.js b/client/common/index.js
index ad9fe5e1..ad9fe5e1 100644
--- a/scraper/client/common/index.js
+++ b/client/common/index.js
diff --git a/scraper/client/common/keyframe.component.js b/client/common/keyframe.component.js
index c77db3ac..c77db3ac 100644
--- a/scraper/client/common/keyframe.component.js
+++ b/client/common/keyframe.component.js
diff --git a/scraper/client/common/keyframes.component.js b/client/common/keyframes.component.js
index 62eda45e..62eda45e 100644
--- a/scraper/client/common/keyframes.component.js
+++ b/client/common/keyframes.component.js
diff --git a/scraper/client/common/loader.component.js b/client/common/loader.component.js
index 6795424b..6795424b 100644
--- a/scraper/client/common/loader.component.js
+++ b/client/common/loader.component.js
diff --git a/scraper/client/common/sidebar.component.js b/client/common/sidebar.component.js
index 487f3289..487f3289 100644
--- a/scraper/client/common/sidebar.component.js
+++ b/client/common/sidebar.component.js
diff --git a/scraper/client/common/table.component.js b/client/common/table.component.js
index 76a1d57c..76a1d57c 100644
--- a/scraper/client/common/table.component.js
+++ b/client/common/table.component.js
diff --git a/scraper/client/common/video.component.js b/client/common/video.component.js
index e5525bf6..e5525bf6 100644
--- a/scraper/client/common/video.component.js
+++ b/client/common/video.component.js
diff --git a/scraper/client/index.js b/client/index.js
index eddc5fb2..eddc5fb2 100644
--- a/scraper/client/index.js
+++ b/client/index.js
diff --git a/scraper/client/metadata/index.js b/client/metadata/index.js
index 0eef814e..0eef814e 100644
--- a/scraper/client/metadata/index.js
+++ b/client/metadata/index.js
diff --git a/scraper/client/session.js b/client/session.js
index 5bfae7eb..5bfae7eb 100644
--- a/scraper/client/session.js
+++ b/client/session.js
diff --git a/scraper/client/store.js b/client/store.js
index 043af351..043af351 100644
--- a/scraper/client/store.js
+++ b/client/store.js
diff --git a/scraper/client/types.js b/client/types.js
index e3c64691..e3c64691 100644
--- a/scraper/client/types.js
+++ b/client/types.js
diff --git a/scraper/client/util.js b/client/util.js
index ad303c64..ad303c64 100644
--- a/scraper/client/util.js
+++ b/client/util.js
diff --git a/megapixels/app/server/create.py b/megapixels/app/server/create.py
index 9efed669..c1f41dc4 100644
--- a/megapixels/app/server/create.py
+++ b/megapixels/app/server/create.py
@@ -1,4 +1,4 @@
-from flask import Flask, Blueprint, jsonify
+from flask import Flask, Blueprint, jsonify, send_from_directory
 from flask_sqlalchemy import SQLAlchemy
 from app.models.sql_factory import connection_url, load_sql_datasets
 
@@ -7,7 +7,7 @@ from app.server.api import api
 db = SQLAlchemy()
 
 def create_app(script_info=None):
-  app = Flask(__name__, static_url_path='')
+  app = Flask(__name__, static_folder='static', static_url_path='')
   app.config['SQLALCHEMY_DATABASE_URI'] = connection_url
   app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
 
@@ -15,6 +15,7 @@ def create_app(script_info=None):
   datasets = load_sql_datasets(replace=False, base_model=db.Model)
 
   app.register_blueprint(api, url_prefix='/api')
+  app.add_url_rule('/<path:file_relative_path_to_root>', 'serve_page', serve_page, methods=['GET'])
 
   @app.route('/', methods=['GET'])
   def index():
@@ -34,3 +35,8 @@ def create_app(script_info=None):
     return(jsonify(links))
 
   return app
+
+def serve_page(file_relative_path_to_root):
+  if file_relative_path_to_root[-1] == '/':
+    file_relative_path_to_root += 'index.html'
+  return send_from_directory("static", file_relative_path_to_root)
diff --git a/megapixels/app/settings/app_cfg.py b/megapixels/app/settings/app_cfg.py
index d60b83f8..50eaf576 100644
--- a/megapixels/app/settings/app_cfg.py
+++ b/megapixels/app/settings/app_cfg.py
@@ -54,12 +54,18 @@ DIR_FAISS_RECIPES = join(DIR_FAISS, 'recipes')
 DIR_TEST_IMAGES = join(DIR_APP, 'test', 'images')
 
 # -----------------------------------------------------------------------------
+# .env config for keys
+# -----------------------------------------------------------------------------
+
+# DIR_DOTENV = join(DIR_APP, '.env')
+load_dotenv() # dotenv_path=DIR_DOTENV)
+
+# -----------------------------------------------------------------------------
 # Drawing, GUI settings
 # -----------------------------------------------------------------------------
 DIR_ASSETS = join(DIR_APP, 'assets')
 FP_FONT = join(DIR_ASSETS, 'font')
 
-
 # -----------------------------------------------------------------------------
 # click chair settings
 # -----------------------------------------------------------------------------
@@ -68,6 +74,7 @@ DIR_COMMANDS_ADMIN = 'commands/admin'
 DIR_COMMANDS_DATASETS = 'commands/datasets'
 DIR_COMMANDS_FAISS = 'commands/faiss'
 DIR_COMMANDS_MISC  = 'commands/misc'
+DIR_COMMANDS_SITE = 'commands/site'
 
 # -----------------------------------------------------------------------------
 # Filesystem settings
@@ -79,7 +86,6 @@ CKPT_ZERO_PADDING = 9
 HASH_TREE_DEPTH = 3
 HASH_BRANCH_SIZE = 3
 
-
 DLIB_FACEREC_JITTERS = 5  # number of face recognition jitters
 DLIB_FACEREC_PADDING = 0.25  # default dlib
 
@@ -108,7 +114,6 @@ reset: Clear all formatting (both foreground and background colors).
 """
 LOGFILE_FORMAT = "%(log_color)s%(levelname)-8s%(reset)s %(cyan)s%(filename)s:%(lineno)s:%(bold_cyan)s%(funcName)s() %(reset)s%(message)s"
 
-
 # -----------------------------------------------------------------------------
 # S3 storage
 # -----------------------------------------------------------------------------
@@ -120,8 +125,10 @@ S3_HTTP_MEDIA_URL = join(S3_HTTP_URL, 'media')
 S3_HTTP_METADATA_URL = join(S3_HTTP_URL, 'metadata')
 
 # -----------------------------------------------------------------------------
-# .env config for keys
+# Static site generator
 # -----------------------------------------------------------------------------
-
-DIR_DOTENV = join(DIR_APP, '.env')
-load_dotenv(dotenv_path=DIR_DOTENV)
+S3_SITE_PATH = "v1/site"
+S3_DATASETS_PATH = "v1" # datasets is already in the filename
+DIR_SITE_PUBLIC = "../site/public"
+DIR_SITE_CONTENT = "../site/content"
+DIR_SITE_TEMPLATES = "../site/templates"
diff --git a/builder/README.md b/megapixels/app/site/README.md
index 1a6d3a1e..1a6d3a1e 100644
--- a/builder/README.md
+++ b/megapixels/app/site/README.md
diff --git a/builder/builder.py b/megapixels/app/site/builder.py
index 620fc710..42e25768 100644
--- a/builder/builder.py
+++ b/megapixels/app/site/builder.py
@@ -1,18 +1,16 @@
 #!/usr/bin/python
 
-from dotenv import load_dotenv
-load_dotenv()
-
 import os
 import glob
 from jinja2 import Environment, FileSystemLoader, select_autoescape
 
-import s3
-import parser
-from paths import *
+import app.settings.app_cfg as cfg
+
+import app.site.s3 as s3
+import app.site.parser as parser
 
 env = Environment(
-  loader=FileSystemLoader(template_path),
+  loader=FileSystemLoader(cfg.DIR_SITE_TEMPLATES),
   autoescape=select_autoescape([])
 )
 
@@ -26,7 +24,7 @@ def build_page(fn, research_posts):
   print(metadata['url'])
 
   dirname = os.path.dirname(fn)
-  output_path = public_path + metadata['url']
+  output_path = cfg.DIR_SITE_PUBLIC + metadata['url']
   output_fn = os.path.join(output_path, "index.html")
 
   skip_h1 = False
@@ -40,9 +38,9 @@ def build_page(fn, research_posts):
     template = env.get_template("page.html")
 
   if 'datasets/' in fn:
-    s3_dir = s3_datasets_path
+    s3_dir = cfg.S3_DATASETS_PATH
   else:
-    s3_dir = s3_site_path
+    s3_dir = cfg.S3_SITE_PATH
 
   s3_path = s3.make_s3_path(s3_dir, metadata['path'])
 
@@ -67,7 +65,7 @@ def build_page(fn, research_posts):
 def build_research_index(research_posts):
   metadata, sections = parser.read_metadata('../site/content/research/index.md')
   template = env.get_template("page.html")
-  s3_path = s3.make_s3_path(s3_site_path, metadata['path'])
+  s3_path = s3.make_s3_path(cfg.S3_SITE_PATH, metadata['path'])
   content = parser.parse_markdown(sections, s3_path, skip_h1=False)
   content += parser.parse_research_index(research_posts)
   html = template.render(
@@ -76,15 +74,12 @@ def build_research_index(research_posts):
     research_posts=research_posts,
     latest_research_post=research_posts[-1],
   )
-  output_fn = public_path + '/research/index.html'
+  output_fn = cfg.DIR_SITE_PUBLIC + '/research/index.html'
   with open(output_fn, "w") as file:
     file.write(html)
 
 def build_site():
   research_posts = parser.read_research_post_index()
-  for fn in glob.iglob(os.path.join(content_path, "**/*.md"), recursive=True):
+  for fn in glob.iglob(os.path.join(cfg.DIR_SITE_CONTENT, "**/*.md"), recursive=True):
     build_page(fn, research_posts)
   build_research_index(research_posts)
-
-if __name__ == '__main__':
-  build_site()
diff --git a/builder/parser.py b/megapixels/app/site/parser.py
index dd3643bf..d78cc402 100644
--- a/builder/parser.py
+++ b/megapixels/app/site/parser.py
@@ -1,10 +1,11 @@
 import os
 import re
 import glob
+import simplejson as json
 import mistune
 
-import s3
-from paths import *
+import app.settings.app_cfg as cfg
+import app.site.s3 as s3
 
 renderer = mistune.Renderer(escape=False)
 markdown = mistune.Markdown(renderer=renderer)
@@ -43,12 +44,31 @@ def format_metadata(section):
     meta.append("<div><div class='gray'>{}</div><div>{}</div></div>".format(key, value))
   return "<section><div class='meta'>{}</div></section>".format(''.join(meta)) 
 
+def format_applet(section):
+  payload = section.replace('```', '').strip().split('\n')
+  applet = {}
+  if ': ' in payload[0]:
+    command, opt = payload[0].split(': ')
+  else:
+    command = payload[0]
+    opt = None
+  applet['command'] = command
+  if opt:
+    applet['opt'] = opt
+  if command == 'load file':
+    applet['fields'] = payload[1]
+  return "<section><div class='applet' data-payload='{}'></div></section>".format(json.dumps(applet))
+
 def parse_markdown(sections, s3_path, skip_h1=False):
   groups = []
   current_group = []
   for section in sections:
     if skip_h1 and section.startswith('# '):
       continue
+    elif section.startswith('```'):
+      groups.append(format_section(current_group, s3_path))
+      groups.append(format_applet(section))
+      current_group = []
     elif section.startswith('+ '):
       groups.append(format_section(current_group, s3_path))
       groups.append(format_metadata(section))
@@ -70,7 +90,7 @@ def parse_markdown(sections, s3_path, skip_h1=False):
 def parse_research_index(research_posts):
   content = "<div class='research_index'>"
   for post in research_posts:
-    s3_path = s3.make_s3_path(s3_site_path, post['path'])
+    s3_path = s3.make_s3_path(cfg.S3_SITE_PATH, post['path'])
     if 'image' in post:
       post_image = s3_path + post['image']
     else:
@@ -134,7 +154,7 @@ def parse_metadata(fn, sections):
     if key not in metadata:
       metadata[key] = default_metadata[key]
 
-  basedir = os.path.dirname(fn.replace(content_path, ''))
+  basedir = os.path.dirname(fn.replace(cfg.DIR_SITE_CONTENT, ''))
   basename = os.path.basename(fn)
   if basedir == '/':
     metadata['path'] = '/'
diff --git a/builder/s3.py b/megapixels/app/site/s3.py
index 41ecdf61..99726a4d 100644
--- a/builder/s3.py
+++ b/megapixels/app/site/s3.py
@@ -1,17 +1,6 @@
 import os
 import glob
 import boto3
-from paths import *
-
-session = boto3.session.Session()
-
-s3_client = session.client(
-  service_name='s3',
-  aws_access_key_id=os.getenv('S3_KEY'),
-  aws_secret_access_key=os.getenv('S3_SECRET'),
-  endpoint_url=os.getenv('S3_ENDPOINT'),
-  region_name=os.getenv('S3_REGION'),
-)
 
 def sync_directory(base_fn, s3_path, metadata):
   fns = {}
@@ -23,6 +12,16 @@ def sync_directory(base_fn, s3_path, metadata):
 
   remote_path = s3_path + metadata['url']
 
+  session = boto3.session.Session()
+
+  s3_client = session.client(
+    service_name='s3',
+    aws_access_key_id=os.getenv('S3_KEY'),
+    aws_secret_access_key=os.getenv('S3_SECRET'),
+    endpoint_url=os.getenv('S3_ENDPOINT'),
+    region_name=os.getenv('S3_REGION'),
+  )
+
   directory = s3_client.list_objects(Bucket=os.getenv('S3_BUCKET'), Prefix=remote_path)
   prefixes = []
 
diff --git a/megapixels/cli_site.py b/megapixels/cli_site.py
new file mode 100644
index 00000000..5faf63cd
--- /dev/null
+++ b/megapixels/cli_site.py
@@ -0,0 +1,36 @@
+# --------------------------------------------------------
+# add/edit commands in commands/faiss directory
+# --------------------------------------------------------
+
+import click
+
+from app.settings import app_cfg as cfg
+from app.utils import logger_utils
+from app.models.click_factory import ClickSimple
+
+# click cli factory
+cc = ClickSimple.create(cfg.DIR_COMMANDS_SITE)
+
+# --------------------------------------------------------
+# CLI
+# --------------------------------------------------------
+@click.group(cls=cc, chain=False)
+@click.option('-v', '--verbose', 'verbosity', count=True, default=4, 
+  show_default=True,
+  help='Verbosity: -v DEBUG, -vv INFO, -vvv WARN, -vvvv ERROR, -vvvvv CRITICAL')
+@click.pass_context
+def cli(ctx, **kwargs):
+  """\033[1m\033[94mMegaPixels: FAISS Data Scripts\033[0m                                                
+  """
+  ctx.opts = {}
+  # init logger
+  logger_utils.Logger.create(verbosity=kwargs['verbosity'])
+
+
+
+# --------------------------------------------------------
+# Entrypoint
+# --------------------------------------------------------
+if __name__ == '__main__':
+    cli()
+
diff --git a/megapixels/commands/faiss/build_faiss.py b/megapixels/commands/faiss/build_faiss.py
index 96d3f99e..ec94c924 100644
--- a/megapixels/commands/faiss/build_faiss.py
+++ b/megapixels/commands/faiss/build_faiss.py
@@ -12,8 +12,6 @@ import numpy as np
 from app.utils.file_utils import load_recipe, load_csv_safe
 from app.settings import app_cfg as cfg
 
-engine = create_engine('sqlite:///:memory:')
-
 class DefaultRecipe:
   def __init__(self):
     self.dim = 128
diff --git a/megapixels/commands/site/build.py b/megapixels/commands/site/build.py
new file mode 100644
index 00000000..0a76a9ac
--- /dev/null
+++ b/megapixels/commands/site/build.py
@@ -0,0 +1,15 @@
+"""
+Build the static site
+"""
+
+import click
+
+from app.site.builder import build_site
+
+@click.command()
+@click.pass_context
+def cli(ctx):
+  """Build the static site
+  """
+  print('Building the site...')
+  build_site()
diff --git a/faiss/requirements.txt b/old/faiss/requirements.txt
index 1d60aabc..1d60aabc 100644
--- a/faiss/requirements.txt
+++ b/old/faiss/requirements.txt
diff --git a/faiss/run.sh b/old/faiss/run.sh
index 8f9e77e2..8f9e77e2 100644
--- a/faiss/run.sh
+++ b/old/faiss/run.sh
diff --git a/faiss/static/css/app.css b/old/faiss/static/css/app.css
index a3b24736..a3b24736 100644
--- a/faiss/static/css/app.css
+++ b/old/faiss/static/css/app.css
diff --git a/faiss/static/favicon.ico b/old/faiss/static/favicon.ico
index d97f2f59..d97f2f59 100644
--- a/faiss/static/favicon.ico
+++ b/old/faiss/static/favicon.ico
diff --git a/faiss/static/img/play.png b/old/faiss/static/img/play.png
index 40f76045..40f76045 100644
--- a/faiss/static/img/play.png
+++ b/old/faiss/static/img/play.png
diff --git a/faiss/static/index.html b/old/faiss/static/index.html
index cf59c628..cf59c628 100644
--- a/faiss/static/index.html
+++ b/old/faiss/static/index.html
diff --git a/faiss/static/js/app.js b/old/faiss/static/js/app.js
index 77164c76..77164c76 100644
--- a/faiss/static/js/app.js
+++ b/old/faiss/static/js/app.js
diff --git a/faiss/static/js/dataUriToBlob.js b/old/faiss/static/js/dataUriToBlob.js
index 80189b8d..80189b8d 100644
--- a/faiss/static/js/dataUriToBlob.js
+++ b/old/faiss/static/js/dataUriToBlob.js
diff --git a/faiss/static/js/metadata-app.js b/old/faiss/static/js/metadata-app.js
index fa2265fa..fa2265fa 100644
--- a/faiss/static/js/metadata-app.js
+++ b/old/faiss/static/js/metadata-app.js
diff --git a/faiss/static/js/store2.min.js b/old/faiss/static/js/store2.min.js
index 75e3ca37..75e3ca37 100644
--- a/faiss/static/js/store2.min.js
+++ b/old/faiss/static/js/store2.min.js
diff --git a/faiss/static/metadata.html b/old/faiss/static/metadata.html
index e74e1ee1..e74e1ee1 100644
--- a/faiss/static/metadata.html
+++ b/old/faiss/static/metadata.html
diff --git a/faiss/static/search.html b/old/faiss/static/search.html
index 056d06c1..056d06c1 100644
--- a/faiss/static/search.html
+++ b/old/faiss/static/search.html
diff --git a/faiss/util.py b/old/faiss/util.py
index 97afbc22..97afbc22 100644
--- a/faiss/util.py
+++ b/old/faiss/util.py
diff --git a/faiss/wsgi.py b/old/faiss/wsgi.py
index 371862fb..371862fb 100644
--- a/faiss/wsgi.py
+++ b/old/faiss/wsgi.py
diff --git a/server/app/README.md b/old/server/app/README.md
index 8bc70132..8bc70132 100644
--- a/server/app/README.md
+++ b/old/server/app/README.md
diff --git a/server/app/__init__.py b/old/server/app/__init__.py
index bce3f9ee..bce3f9ee 100644
--- a/server/app/__init__.py
+++ b/old/server/app/__init__.py
diff --git a/server/app/basemodels.py b/old/server/app/basemodels.py
index 475ab0c2..475ab0c2 100644
--- a/server/app/basemodels.py
+++ b/old/server/app/basemodels.py
diff --git a/server/app/favicon.ico b/old/server/app/favicon.ico
index 4d001b21..4d001b21 100644
--- a/server/app/favicon.ico
+++ b/old/server/app/favicon.ico
diff --git a/server/app/index.html b/old/server/app/index.html
index 3c1b0dfd..3c1b0dfd 100644
--- a/server/app/index.html
+++ b/old/server/app/index.html
diff --git a/server/app/main/__init__.py b/old/server/app/main/__init__.py
index a21e2754..a21e2754 100644
--- a/server/app/main/__init__.py
+++ b/old/server/app/main/__init__.py
diff --git a/server/app/main/errors.py b/old/server/app/main/errors.py
index 60b5f227..60b5f227 100644
--- a/server/app/main/errors.py
+++ b/old/server/app/main/errors.py
diff --git a/server/app/main/forms.py b/old/server/app/main/forms.py
index bc1399ad..bc1399ad 100644
--- a/server/app/main/forms.py
+++ b/old/server/app/main/forms.py
diff --git a/server/app/main/img_proc_config.py b/old/server/app/main/img_proc_config.py
index db124978..db124978 100644
--- a/server/app/main/img_proc_config.py
+++ b/old/server/app/main/img_proc_config.py
diff --git a/server/app/main/paths.py b/old/server/app/main/paths.py
index 69c21627..69c21627 100644
--- a/server/app/main/paths.py
+++ b/old/server/app/main/paths.py
diff --git a/server/app/main/tasks.py b/old/server/app/main/tasks.py
index 970e6988..970e6988 100644
--- a/server/app/main/tasks.py
+++ b/old/server/app/main/tasks.py
diff --git a/server/app/main/utils.py b/old/server/app/main/utils.py
index 510e5c23..510e5c23 100644
--- a/server/app/main/utils.py
+++ b/old/server/app/main/utils.py
diff --git a/server/app/main/views.py b/old/server/app/main/views.py
index 11a8ca53..11a8ca53 100644
--- a/server/app/main/views.py
+++ b/old/server/app/main/views.py
diff --git a/server/app/static/css/bootstrap.min.css b/old/server/app/static/css/bootstrap.min.css
index ed3905e0..ed3905e0 100644
--- a/server/app/static/css/bootstrap.min.css
+++ b/old/server/app/static/css/bootstrap.min.css
diff --git a/server/app/static/css/dullbrown-theme.css b/old/server/app/static/css/dullbrown-theme.css
index 98aff038..98aff038 100644
--- a/server/app/static/css/dullbrown-theme.css
+++ b/old/server/app/static/css/dullbrown-theme.css
diff --git a/server/app/static/css/projector.css b/old/server/app/static/css/projector.css
index 401f0dff..401f0dff 100644
--- a/server/app/static/css/projector.css
+++ b/old/server/app/static/css/projector.css
diff --git a/server/app/static/js/app.js b/old/server/app/static/js/app.js
index 454d5c37..454d5c37 100644
--- a/server/app/static/js/app.js
+++ b/old/server/app/static/js/app.js
diff --git a/server/app/static/js/upload.js b/old/server/app/static/js/upload.js
index 27437e43..27437e43 100644
--- a/server/app/static/js/upload.js
+++ b/old/server/app/static/js/upload.js
diff --git a/server/app/static/js/util.js b/old/server/app/static/js/util.js
index 851f634a..851f634a 100644
--- a/server/app/static/js/util.js
+++ b/old/server/app/static/js/util.js
diff --git a/server/app/static/js/vendor/ExifReader.js b/old/server/app/static/js/vendor/ExifReader.js
index a8343ede..a8343ede 100644
--- a/server/app/static/js/vendor/ExifReader.js
+++ b/old/server/app/static/js/vendor/ExifReader.js
diff --git a/server/app/static/js/vendor/canvas-to-blob.js b/old/server/app/static/js/vendor/canvas-to-blob.js
index 32913667..32913667 100644
--- a/server/app/static/js/vendor/canvas-to-blob.js
+++ b/old/server/app/static/js/vendor/canvas-to-blob.js
diff --git a/server/app/static/js/vendor/jquery-3.3.1.min.js b/old/server/app/static/js/vendor/jquery-3.3.1.min.js
index 4d9b3a25..4d9b3a25 100644
--- a/server/app/static/js/vendor/jquery-3.3.1.min.js
+++ b/old/server/app/static/js/vendor/jquery-3.3.1.min.js
diff --git a/server/app/static/js/vendor/nanobar.min.js b/old/server/app/static/js/vendor/nanobar.min.js
index 9329eadc..9329eadc 100644
--- a/server/app/static/js/vendor/nanobar.min.js
+++ b/old/server/app/static/js/vendor/nanobar.min.js
diff --git a/server/app/static/js/vendor/prefixfree.js b/old/server/app/static/js/vendor/prefixfree.js
index 40b6e693..40b6e693 100644
--- a/server/app/static/js/vendor/prefixfree.js
+++ b/old/server/app/static/js/vendor/prefixfree.js
diff --git a/server/app/templates/403.html b/old/server/app/templates/403.html
index f83c6dfc..f83c6dfc 100644
--- a/server/app/templates/403.html
+++ b/old/server/app/templates/403.html
diff --git a/server/app/templates/404.html b/old/server/app/templates/404.html
index a7f7d45a..a7f7d45a 100644
--- a/server/app/templates/404.html
+++ b/old/server/app/templates/404.html
diff --git a/server/app/templates/500.html b/old/server/app/templates/500.html
index b323c12c..b323c12c 100644
--- a/server/app/templates/500.html
+++ b/old/server/app/templates/500.html
diff --git a/server/app/templates/base.html b/old/server/app/templates/base.html
index 16402af8..16402af8 100644
--- a/server/app/templates/base.html
+++ b/old/server/app/templates/base.html
diff --git a/server/app/templates/celery.html b/old/server/app/templates/celery.html
index ddcd25cb..ddcd25cb 100644
--- a/server/app/templates/celery.html
+++ b/old/server/app/templates/celery.html
diff --git a/server/app/templates/display.html b/old/server/app/templates/display.html
index f73a6ca5..f73a6ca5 100644
--- a/server/app/templates/display.html
+++ b/old/server/app/templates/display.html
diff --git a/server/app/templates/index.html b/old/server/app/templates/index.html
index f740bb5b..f740bb5b 100644
--- a/server/app/templates/index.html
+++ b/old/server/app/templates/index.html
diff --git a/server/celery_worker.py b/old/server/celery_worker.py
index 1545a884..1545a884 100644
--- a/server/celery_worker.py
+++ b/old/server/celery_worker.py
diff --git a/server/config.py b/old/server/config.py
index 5042efb6..5042efb6 100644
--- a/server/config.py
+++ b/old/server/config.py
diff --git a/server/deploy.sh b/old/server/deploy.sh
index c2594cab..c2594cab 100755
--- a/server/deploy.sh
+++ b/old/server/deploy.sh
diff --git a/server/dulldream.wsgi.py b/old/server/dulldream.wsgi.py
index ed992528..ed992528 100644
--- a/server/dulldream.wsgi.py
+++ b/old/server/dulldream.wsgi.py
diff --git a/server/run-celery.sh b/old/server/run-celery.sh
index e38174fa..e38174fa 100755
--- a/server/run-celery.sh
+++ b/old/server/run-celery.sh
diff --git a/server/run-dev.sh b/old/server/run-dev.sh
index b4eb2a61..b4eb2a61 100755
--- a/server/run-dev.sh
+++ b/old/server/run-dev.sh
diff --git a/server/run-gunicorn.sh b/old/server/run-gunicorn.sh
index 64debabd..64debabd 100755
--- a/server/run-gunicorn.sh
+++ b/old/server/run-gunicorn.sh
diff --git a/server/run-redis.sh b/old/server/run-redis.sh
index e9ceb845..e9ceb845 100755
--- a/server/run-redis.sh
+++ b/old/server/run-redis.sh
diff --git a/server/run.py b/old/server/run.py
index c4c3e8d7..c4c3e8d7 100644
--- a/server/run.py
+++ b/old/server/run.py
diff --git a/site/public/about/style/index.html b/site/public/about/style/index.html
index f2c0d4b8..39a44380 100644
--- a/site/public/about/style/index.html
+++ b/site/public/about/style/index.html
@@ -51,16 +51,7 @@
 <div class='image'><img src='https://nyc3.digitaloceanspaces.com/megapixels/v1/site/about/assets/man.jpg' alt='Person 3. Let me tell you about Person 3.  This person has a very long description with text which wraps like crazy'><div class='caption'>Person 3. Let me tell you about Person 3.  This person has a very long description with text which wraps like crazy</div></div></section><section><blockquote><p>est, qui dolorem ipsum, quia dolor sit amet consectetur adipisci[ng] velit, sed quia non-numquam [do] eius modi tempora inci[di]dunt, ut labore et dolore magnam aliquam quaerat voluptatem.</p>
 </blockquote>
 </section><section class='wide'><div class='image'><img src='https://nyc3.digitaloceanspaces.com/megapixels/v1/site/about/assets/wide-test.jpg' alt='This image is extremely wide and the text beneath it will wrap but thats fine because it can also contain <a href="https://example.com/">hyperlinks</a>! Yes, you read that right&mdash;hyperlinks! Lorem ipsum dolor sit amet ad volotesque sic hoc ad nauseam'><div class='caption'>This image is extremely wide and the text beneath it will wrap but that's fine because it can also contain <a href="https://example.com/">hyperlinks</a>! Yes, you read that right&mdash;hyperlinks! Lorem ipsum dolor sit amet ad volotesque sic hoc ad nauseam</div></div></section><section><p>Inline <code>code</code> has <code>back-ticks around</code> it.</p>
-<pre><code class="lang-javascript">var s = &quot;JavaScript syntax highlighting&quot;;
-alert(s);
-</code></pre>
-<pre><code class="lang-python">s = &quot;Python syntax highlighting&quot;
-print(s)
-</code></pre>
-<pre><code>No language indicated, so no syntax highlighting. 
-But let's throw in a &lt;b&gt;tag&lt;/b&gt;.
-</code></pre>
-<p>Horizontal rule</p>
+</section><section><div class='applet' data-payload='{"command": "javascript"}'></div></section><section><div class='applet' data-payload='{"command": "python"}'></div></section><section><div class='applet' data-payload='{"command": "No language indicated, so no syntax highlighting. "}'></div></section><section><p>Horizontal rule</p>
 <hr>
 <p>Citations below here</p>
 <div class="footnotes">
diff --git a/site/public/datasets/lfw/index.html b/site/public/datasets/lfw/index.html
index e080229f..3c83acd3 100644
--- a/site/public/datasets/lfw/index.html
+++ b/site/public/datasets/lfw/index.html
@@ -28,12 +28,7 @@
     
   <section><h1>Labeled Faces in the Wild</h1>
 </section><section><div class='meta'><div><div class='gray'>Created</div><div>2007</div></div><div><div class='gray'>Images</div><div>13,233</div></div><div><div class='gray'>People</div><div>5,749</div></div><div><div class='gray'>Created From</div><div>Yahoo News images</div></div><div><div class='gray'>Search available</div><div>Searchable</div></div></div></section><section><p>Labeled Faces in The Wild (LFW) is amongst the most widely used facial recognition training datasets in the world and is the first of its kind to be created entirely from images posted online. The LFW dataset includes 13,233 images of 5,749 people that were collected between 2002-2004. Use the tools below to check if you were included in this dataset or scroll down to read the analysis.</p>
-<p>{INSERT IMAGE SEARCH MODULE}</p>
-<p>{INSERT TEXT SEARCH MODULE}</p>
-<pre><code>load file: lfw_names_gender_kg_min.csv
-Name, Images, Gender, Description
-</code></pre>
-</section><section class='images'><div class='image'><img src='https://nyc3.digitaloceanspaces.com/megapixels/v1/datasets/lfw/assets/lfw_feature.jpg' alt='Eighteen of the 5,749 people in the Labeled Faces in the Wild Dataset. The most widely used face dataset for benchmarking commercial face recognition algorithms.'><div class='caption'>Eighteen of the 5,749 people in the Labeled Faces in the Wild Dataset. The most widely used face dataset for benchmarking commercial face recognition algorithms.</div></div></section><section><h2>Intro</h2>
+</section><section><div class='applet' data-payload='{"command": "face_search"}'></div></section><section><div class='applet' data-payload='{"command": "name_search"}'></div></section><section><div class='applet' data-payload='{"command": "load file", "opt": "lfw_names_gender_kg_min.csv", "fields": "Name, Images, Gender, Description"}'></div></section><section class='images'><div class='image'><img src='https://nyc3.digitaloceanspaces.com/megapixels/v1/datasets/lfw/assets/lfw_feature.jpg' alt='Eighteen of the 5,749 people in the Labeled Faces in the Wild Dataset. The most widely used face dataset for benchmarking commercial face recognition algorithms.'><div class='caption'>Eighteen of the 5,749 people in the Labeled Faces in the Wild Dataset. The most widely used face dataset for benchmarking commercial face recognition algorithms.</div></div></section><section><h2>Intro</h2>
 <p>Three paragraphs describing the LFW dataset in a format that can be easily replicated for the other datasets. Nothing too custom. An analysis of the initial research papers with context relative to all the other dataset papers.</p>
 </section><section class='images'><div class='image'><img src='https://nyc3.digitaloceanspaces.com/megapixels/v1/datasets/lfw/assets/lfw_montage_everyone_1920.jpg' alt=' all 5,749 people in the LFW Dataset sorted from most to least images collected.'><div class='caption'> all 5,749 people in the LFW Dataset sorted from most to least images collected.</div></div></section><section><h2>LFW by the Numbers</h2>
 <ul>
@@ -224,36 +219,28 @@ name_display,company_url,example_url,country,description
 </tbody>
 </table>
 <h2>Code</h2>
-<pre><code class="lang-python">#!/usr/bin/python
-
-import numpy as np
+</section><section><div class='applet' data-payload='{"command": "python"}'></div></section><section><p>import numpy as np
 from sklearn.datasets import fetch_lfw_people
 import imageio
-import imutils
-
-# download LFW dataset (first run takes a while)
-lfw_people = fetch_lfw_people(min_faces_per_person=1, resize=1, color=True, funneled=False)
-
-# introspect dataset
-n_samples, h, w, c = lfw_people.images.shape
-print(&#39;{:,} images at {}x{}&#39;.format(n_samples, w, h))
+import imutils</p>
+<h1>download LFW dataset (first run takes a while)</h1>
+<p>lfw_people = fetch_lfw_people(min_faces_per_person=1, resize=1, color=True, funneled=False)</p>
+<h1>introspect dataset</h1>
+<p>n_samples, h, w, c = lfw_people.images.shape
+print('{:,} images at {}x{}'.format(n_samples, w, h))
 cols, rows = (176, 76)
-n_ims = cols * rows
-
-# build montages
-im_scale = 0.5
+n_ims = cols * rows</p>
+<h1>build montages</h1>
+<p>im_scale = 0.5
 ims = lfw_people.images[:n_ims
-montages = imutils.build_montages(ims, (int(w*im_scale, int(h*im_scale)), (cols, rows))
-montage = montages[0]
-
-# save full montage image
-imageio.imwrite(&#39;lfw_montage_full.png&#39;, montage)
-
-# make a smaller version
-montage_960 = imutils.resize(montage, width=960)
-imageio.imwrite(&#39;lfw_montage_960.jpg&#39;, montage_960)
-</code></pre>
-<h2>Disclaimer</h2>
+montages = imutils.build_montages(ims, (int(w<em>im_scale, int(h</em>im_scale)), (cols, rows))
+montage = montages[0]</p>
+<h1>save full montage image</h1>
+<p>imageio.imwrite('lfw_montage_full.png', montage)</p>
+<h1>make a smaller version</h1>
+<p>montage_960 = imutils.resize(montage, width=960)
+imageio.imwrite('lfw_montage_960.jpg', montage_960)</p>
+</section><section><div class='applet' data-payload='{"command": ""}'></div></section><section><h2>Disclaimer</h2>
 <p>MegaPixels is an educational art project designed to encourage discourse about facial recognition datasets. Any ethical or legal issues should be directed to the researcher's parent organizations. Except where necessary for contact or clarity, the names of researchers have been subsituted by their parent organization. In no way does this project aim to villify researchers who produced the datasets.</p>
 <p>Read more about <a href="about/code-of-conduct">MegaPixels Code of Conduct</a></p>
 <div class="footnotes">
diff --git a/site/public/datasets/vgg_face2/index.html b/site/public/datasets/vgg_face2/index.html
index 24a1059b..817fc9a0 100644
--- a/site/public/datasets/vgg_face2/index.html
+++ b/site/public/datasets/vgg_face2/index.html
@@ -28,12 +28,7 @@
     
   <section><h1>VGG Faces2</h1>
 </section><section><div class='meta'><div><div class='gray'>Created</div><div>2018</div></div><div><div class='gray'>Images</div><div>3.3M</div></div><div><div class='gray'>People</div><div>9,000</div></div><div><div class='gray'>Created From</div><div>Scraping search engines</div></div><div><div class='gray'>Search available</div><div>[Searchable](#)</div></div></div></section><section><p>VGG Face2 is the updated version of the VGG Face dataset and now includes over 3.3M face images from over 9K people. The identities were selected by taking the top 500K identities in Google's Knowledge Graph of celebrities and then selecting only the names that yielded enough training images. The dataset was created in the UK but funded by Office of Director of National Intelligence in the United States.</p>
-<p>{INSERT IMAGE SEARCH MODULE}</p>
-<p>{INSERT TEXT SEARCH MODULE}</p>
-<pre><code>load file: lfw_names_gender_kg_min.csv
-Name, Images, Gender, Description
-</code></pre>
-<h2>VGG Face2 by the Numbers</h2>
+</section><section><div class='applet' data-payload='{"command": "face_search"}'></div></section><section><div class='applet' data-payload='{"command": "name_search"}'></div></section><section><div class='applet' data-payload='{"command": "load file", "opt": "lfw_names_gender_kg_min.csv", "fields": "Name, Images, Gender, Description"}'></div></section><section><h2>VGG Face2 by the Numbers</h2>
 <ul>
 <li>1,331 actresses, 139 presidents</li>
 <li>3 husbands and 16 wives</li>
@@ -47,7 +42,7 @@ Name, Images, Gender, Description
 <li>The original VGGF2 name list has been updated with the results returned from Google Knowledge</li>
 <li>Names with a similarity score greater than 0.75 where automatically updated. Scores computed using <code>import difflib; seq = difflib.SequenceMatcher(a=a.lower(), b=b.lower()); score = seq.ratio()</code></li>
 <li>The 97 names with a score of 0.75 or lower were manually reviewed and includes name changes validating using Wikipedia.org results for names such as "Bruce Jenner" to "Caitlyn Jenner", spousal last-name changes, and discretionary changes to improve search results such as combining nicknames with full name when appropriate, for example changing "Aleksandar Petrović" to "Aleksandar 'Aco' Petrović" and minor changes such as "Mohammad Ali" to "Muhammad Ali"</li>
-<li>The 'Description` text was automatically added when the Knowledge Graph score was greater than 250</li>
+<li>The 'Description' text was automatically added when the Knowledge Graph score was greater than 250</li>
 </ul>
 <h1>TODO</h1>
 <ul>
diff --git a/site/public/datasets/vgg_faces2/index.html b/site/public/datasets/vgg_faces2/index.html
deleted file mode 100644
index 3f778f71..00000000
--- a/site/public/datasets/vgg_faces2/index.html
+++ /dev/null
@@ -1,63 +0,0 @@
-<!doctype html>
-<html>
-<head>
-  <title>MegaPixels</title>
-  <meta charset="utf-8" />
-  <meta name="author" content="Adam Harvey" />
-  <meta name="description" content="" />
-  <meta name="referrer" content="no-referrer" />
-  <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes" />
-  <link rel='stylesheet' href='/assets/css/fonts.css' />
-  <link rel='stylesheet' href='/assets/css/css.css' />
-</head>
-<body>
-  <header>
-    <a class='slogan' href="/">
-      <div class='logo'></div>
-      <div class='site_name'>MegaPixels</div>
-      <span class='sub'>The Darkside of Datasets</span>
-    </a>
-    <div class='links'>
-      <a href="/search/">Face Search</a>
-      <a href="/datasets/">Datasets</a>
-      <a href="/research/">Research</a>
-      <a href="/about/">About</a>
-    </div>
-  </header>
-  <div class="content">
-    
-  <section><h1>Labeled Faces in The Wild</h1>
-</section><section><div class='meta'><div><div class='gray'>Created</div><div>2007</div></div><div><div class='gray'>Images</div><div>13,233</div></div><div><div class='gray'>People</div><div>5,749</div></div><div><div class='gray'>Created From</div><div>Yahoo News images</div></div><div><div class='gray'>Search available</div><div>[Searchable](#)</div></div></div></section><section><p>Labeled Faces in The Wild is amongst the most widely used facial recognition training datasets in the world and is the first dataset of its kind to be created entirely from Internet photos. It includes 13,233 images of 5,749 people downloaded from the Internet, otherwise referred to by researchers as “The Wild”.</p>
-<h2>INTRO</h2>
-<p>It began in 2002. Researchers at University of Massachusetts Amherst were developing algorithms for facial recognition and they needed more data. Between 2002-2004 they scraped Yahoo News for images of public figures. Two years later they cleaned up the dataset and repackaged it as Labeled Faces in the Wild (LFW).</p>
-<p>Since then the LFW dataset has become one of the most widely used datasets used for evaluating face recognition algorithms. The associated research paper “Labeled Faces in the Wild: A Database for Studying Face Recognition in Unconstrained Environments” has been cited 996 times reaching 45 different countries throughout the world.</p>
-<p>The faces come from news stories and are mostly celebrities from the entertainment industry, politicians, and villains. It’s a sampling of current affairs and breaking news that has come to pass. The images, detached from their original context now server a new purpose: to train, evaluate, and improve facial recognition.</p>
-<p>As the most widely used facial recognition dataset, it can be said that each individual in LFW has, in a small way, contributed to the current state of the art in facial recognition surveillance. John Cusack, Julianne Moore, Barry Bonds, Osama bin Laden, and even Moby are amongst these biometric pillars, exemplar faces provided the visual dimensions of a new computer vision future.</p>
-<h2>Commercial Use</h2>
-<p>The dataset is used by numerous companies for benchmarking algorithms. According to the benchmarking results page <sup class="footnote-ref" id="fnref-lfw_results"><a href="#fn-lfw_results">1</a></sup> provided by the authors, there over 2 dozen commercial uses of the LFW face dataset.</p>
-<div class="footnotes">
-<hr>
-<ol><li id="fn-lfw_results"><p>"LFW Results". Accessed Dec 3, 2018. <a href="http://vis-www.cs.umass.edu/lfw/results.html">http://vis-www.cs.umass.edu/lfw/results.html</a><a href="#fnref-lfw_results" class="footnote">&#8617;</a></p></li>
-</ol>
-</div>
-</section>
-
-  </div>
-  <footer>
-    <div>
-      <a href="/">MegaPixels.cc</a>
-      <a href="/about/disclaimer/">Disclaimer</a>
-      <a href="/about/terms/">Terms of Use</a>
-      <a href="/about/privacy/">Privacy</a>
-      <a href="/about/">About</a>
-      <a href="/about/team/">Team</a>
-    </div>
-    <div>
-      MegaPixels &copy;2017-19 Adam R. Harvey /&nbsp;
-      <a href="https://ahprojects.com">ahprojects.com</a>
-    </div>
-  </footer>
-</body>
-
-<script src="/assets/js/app/site.js"></script>
-</html>
\ No newline at end of file