From e0b0b2f976c61225a178c7715caf2656a1f6741f Mon Sep 17 00:00:00 2001
From: Jules Laplace <julescarbon@gmail.com>
Date: Sat, 15 Dec 2018 21:32:51 +0100
Subject: moving stuff

---
 megapixels/app/settings/app_cfg.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

(limited to 'megapixels/app/settings/app_cfg.py')

diff --git a/megapixels/app/settings/app_cfg.py b/megapixels/app/settings/app_cfg.py
index 0507366f..1d3fbc4c 100644
--- a/megapixels/app/settings/app_cfg.py
+++ b/megapixels/app/settings/app_cfg.py
@@ -57,6 +57,13 @@ DIR_FAISS_RECIPES = join(DIR_FAISS, 'recipes')
 # Test images
 DIR_TEST_IMAGES = join(DIR_APP, 'test', 'images')
 
+# -----------------------------------------------------------------------------
+# .env config for keys
+# -----------------------------------------------------------------------------
+
+DIR_DOTENV = join(DIR_APP, '.env')
+load_dotenv(dotenv_path=DIR_DOTENV)
+
 # -----------------------------------------------------------------------------
 # Drawing, GUI settings
 # -----------------------------------------------------------------------------
@@ -69,6 +76,7 @@ FP_FONT = join(DIR_ASSETS, 'font')
 # -----------------------------------------------------------------------------
 DIR_COMMANDS_CV = 'commands/cv'
 DIR_COMMANDS_ADMIN = 'commands/admin'
+DIR_COMMANDS_BUILDER = 'commands/builder'
 DIR_COMMANDS_DATASETS = 'commands/datasets'
 DIR_COMMANDS_FAISS = 'commands/faiss'
 DIR_COMMANDS_MISC  = 'commands/misc'
@@ -118,10 +126,3 @@ LOGFILE_FORMAT = "%(log_color)s%(levelname)-8s%(reset)s %(cyan)s%(filename)s:%(l
 # -----------------------------------------------------------------------------
 S3_MEDIA_ROOT = 's3://megapixels/v1/media/'
 S3_METADATA_ROOT = 's3://megapixels/v1/metadata/'
-
-# -----------------------------------------------------------------------------
-# .env config for keys
-# -----------------------------------------------------------------------------
-
-DIR_DOTENV = join(DIR_APP, '.env')
-load_dotenv(dotenv_path=DIR_DOTENV)
-- 
cgit v1.2.3-70-g09d2


From 50a35da0c14664bd3e1392254afec21ef3880440 Mon Sep 17 00:00:00 2001
From: Jules Laplace <julescarbon@gmail.com>
Date: Sat, 15 Dec 2018 21:43:04 +0100
Subject: make builder a click script

---
 megapixels/app/builder/builder.py    | 25 ++++++++++---------------
 megapixels/app/builder/parser.py     |  6 +++---
 megapixels/app/builder/paths.py      |  6 ------
 megapixels/app/builder/s3.py         | 21 ++++++++++-----------
 megapixels/app/settings/app_cfg.py   | 13 ++++++++++---
 megapixels/cli_builder.py            | 36 ++++++++++++++++++++++++++++++++++++
 megapixels/commands/builder/build.py | 15 +++++++++++++++
 7 files changed, 84 insertions(+), 38 deletions(-)
 delete mode 100644 megapixels/app/builder/paths.py
 create mode 100644 megapixels/cli_builder.py
 create mode 100644 megapixels/commands/builder/build.py

(limited to 'megapixels/app/settings/app_cfg.py')

diff --git a/megapixels/app/builder/builder.py b/megapixels/app/builder/builder.py
index 620fc710..d1f882d6 100644
--- a/megapixels/app/builder/builder.py
+++ b/megapixels/app/builder/builder.py
@@ -1,18 +1,16 @@
 #!/usr/bin/python
 
-from dotenv import load_dotenv
-load_dotenv()
-
 import os
 import glob
 from jinja2 import Environment, FileSystemLoader, select_autoescape
 
-import s3
-import parser
-from paths import *
+import app.settings.app_cfg as cfg
+
+import app.builder.s3
+import app.builder.parser
 
 env = Environment(
-  loader=FileSystemLoader(template_path),
+  loader=FileSystemLoader(cfg.DIR_SITE_TEMPLATES),
   autoescape=select_autoescape([])
 )
 
@@ -26,7 +24,7 @@ def build_page(fn, research_posts):
   print(metadata['url'])
 
   dirname = os.path.dirname(fn)
-  output_path = public_path + metadata['url']
+  output_path = cfg.DIR_SITE_PUBLIC + metadata['url']
   output_fn = os.path.join(output_path, "index.html")
 
   skip_h1 = False
@@ -40,9 +38,9 @@ def build_page(fn, research_posts):
     template = env.get_template("page.html")
 
   if 'datasets/' in fn:
-    s3_dir = s3_datasets_path
+    s3_dir = cfg.S3_DATASETS_PATH
   else:
-    s3_dir = s3_site_path
+    s3_dir = cfg.S3_SITE_PATH
 
   s3_path = s3.make_s3_path(s3_dir, metadata['path'])
 
@@ -67,7 +65,7 @@ def build_page(fn, research_posts):
 def build_research_index(research_posts):
   metadata, sections = parser.read_metadata('../site/content/research/index.md')
   template = env.get_template("page.html")
-  s3_path = s3.make_s3_path(s3_site_path, metadata['path'])
+  s3_path = s3.make_s3_path(cfg.S3_SITE_PATH, metadata['path'])
   content = parser.parse_markdown(sections, s3_path, skip_h1=False)
   content += parser.parse_research_index(research_posts)
   html = template.render(
@@ -82,9 +80,6 @@ def build_research_index(research_posts):
 
 def build_site():
   research_posts = parser.read_research_post_index()
-  for fn in glob.iglob(os.path.join(content_path, "**/*.md"), recursive=True):
+  for fn in glob.iglob(os.path.join(cfg.DIR_SITE_CONTENT, "**/*.md"), recursive=True):
     build_page(fn, research_posts)
   build_research_index(research_posts)
-
-if __name__ == '__main__':
-  build_site()
diff --git a/megapixels/app/builder/parser.py b/megapixels/app/builder/parser.py
index dd3643bf..ea84f658 100644
--- a/megapixels/app/builder/parser.py
+++ b/megapixels/app/builder/parser.py
@@ -3,8 +3,8 @@ import re
 import glob
 import mistune
 
-import s3
-from paths import *
+import app.settings.app_cfg
+import app.builder.s3
 
 renderer = mistune.Renderer(escape=False)
 markdown = mistune.Markdown(renderer=renderer)
@@ -70,7 +70,7 @@ def parse_markdown(sections, s3_path, skip_h1=False):
 def parse_research_index(research_posts):
   content = "<div class='research_index'>"
   for post in research_posts:
-    s3_path = s3.make_s3_path(s3_site_path, post['path'])
+    s3_path = s3.make_s3_path(app_cfg.S3_SITE_PATH, post['path'])
     if 'image' in post:
       post_image = s3_path + post['image']
     else:
diff --git a/megapixels/app/builder/paths.py b/megapixels/app/builder/paths.py
deleted file mode 100644
index 356f2f3d..00000000
--- a/megapixels/app/builder/paths.py
+++ /dev/null
@@ -1,6 +0,0 @@
-
-s3_site_path = "v1/site"
-s3_datasets_path = "v1" # datasets is already in the filename
-public_path = "../site/public"
-content_path = "../site/content"
-template_path = "../site/templates"
diff --git a/megapixels/app/builder/s3.py b/megapixels/app/builder/s3.py
index 41ecdf61..99726a4d 100644
--- a/megapixels/app/builder/s3.py
+++ b/megapixels/app/builder/s3.py
@@ -1,17 +1,6 @@
 import os
 import glob
 import boto3
-from paths import *
-
-session = boto3.session.Session()
-
-s3_client = session.client(
-  service_name='s3',
-  aws_access_key_id=os.getenv('S3_KEY'),
-  aws_secret_access_key=os.getenv('S3_SECRET'),
-  endpoint_url=os.getenv('S3_ENDPOINT'),
-  region_name=os.getenv('S3_REGION'),
-)
 
 def sync_directory(base_fn, s3_path, metadata):
   fns = {}
@@ -23,6 +12,16 @@ def sync_directory(base_fn, s3_path, metadata):
 
   remote_path = s3_path + metadata['url']
 
+  session = boto3.session.Session()
+
+  s3_client = session.client(
+    service_name='s3',
+    aws_access_key_id=os.getenv('S3_KEY'),
+    aws_secret_access_key=os.getenv('S3_SECRET'),
+    endpoint_url=os.getenv('S3_ENDPOINT'),
+    region_name=os.getenv('S3_REGION'),
+  )
+
   directory = s3_client.list_objects(Bucket=os.getenv('S3_BUCKET'), Prefix=remote_path)
   prefixes = []
 
diff --git a/megapixels/app/settings/app_cfg.py b/megapixels/app/settings/app_cfg.py
index 1d3fbc4c..9e03357b 100644
--- a/megapixels/app/settings/app_cfg.py
+++ b/megapixels/app/settings/app_cfg.py
@@ -70,7 +70,6 @@ load_dotenv(dotenv_path=DIR_DOTENV)
 DIR_ASSETS = join(DIR_APP, 'assets')
 FP_FONT = join(DIR_ASSETS, 'font')
 
-
 # -----------------------------------------------------------------------------
 # click chair settings
 # -----------------------------------------------------------------------------
@@ -91,7 +90,6 @@ CKPT_ZERO_PADDING = 9
 HASH_TREE_DEPTH = 3
 HASH_BRANCH_SIZE = 3
 
-
 DLIB_FACEREC_JITTERS = 5  # number of face recognition jitters
 DLIB_FACEREC_PADDING = 0.25  # default dlib
 
@@ -120,9 +118,18 @@ reset: Clear all formatting (both foreground and background colors).
 """
 LOGFILE_FORMAT = "%(log_color)s%(levelname)-8s%(reset)s %(cyan)s%(filename)s:%(lineno)s:%(bold_cyan)s%(funcName)s() %(reset)s%(message)s"
 
-
 # -----------------------------------------------------------------------------
 # S3 storage
 # -----------------------------------------------------------------------------
 S3_MEDIA_ROOT = 's3://megapixels/v1/media/'
 S3_METADATA_ROOT = 's3://megapixels/v1/metadata/'
+
+# -----------------------------------------------------------------------------
+# Builder
+# -----------------------------------------------------------------------------
+
+S3_SITE_PATH = "v1/site"
+S3_DATASETS_PATH = "v1" # datasets is already in the filename
+DIR_SITE_PUBLIC = "../site/public"
+DIR_SITE_CONTENT = "../site/content"
+DIR_SITE_TEMPLATES = "../site/templates"
diff --git a/megapixels/cli_builder.py b/megapixels/cli_builder.py
new file mode 100644
index 00000000..8bc814f1
--- /dev/null
+++ b/megapixels/cli_builder.py
@@ -0,0 +1,36 @@
+# --------------------------------------------------------
+# add/edit commands in commands/faiss directory
+# --------------------------------------------------------
+
+import click
+
+from app.settings import app_cfg as cfg
+from app.utils import logger_utils
+from app.models.click_factory import ClickSimple
+
+# click cli factory
+cc = ClickSimple.create(cfg.DIR_COMMANDS_BUILDER)
+
+# --------------------------------------------------------
+# CLI
+# --------------------------------------------------------
+@click.group(cls=cc, chain=False)
+@click.option('-v', '--verbose', 'verbosity', count=True, default=4, 
+  show_default=True,
+  help='Verbosity: -v DEBUG, -vv INFO, -vvv WARN, -vvvv ERROR, -vvvvv CRITICAL')
+@click.pass_context
+def cli(ctx, **kwargs):
+  """\033[1m\033[94mMegaPixels: FAISS Data Scripts\033[0m                                                
+  """
+  ctx.opts = {}
+  # init logger
+  logger_utils.Logger.create(verbosity=kwargs['verbosity'])
+
+
+
+# --------------------------------------------------------
+# Entrypoint
+# --------------------------------------------------------
+if __name__ == '__main__':
+    cli()
+
diff --git a/megapixels/commands/builder/build.py b/megapixels/commands/builder/build.py
new file mode 100644
index 00000000..fc4fb302
--- /dev/null
+++ b/megapixels/commands/builder/build.py
@@ -0,0 +1,15 @@
+"""
+Build the static site
+"""
+
+import click
+
+from app.builder.builder import build_site
+
+@click.command()
+@click.pass_context
+def cli(ctx):
+  """Build the static site
+  """
+  print('Building the site...')
+  build_site()
-- 
cgit v1.2.3-70-g09d2


From 67cd70621b7dca679cd930de66d883e0610a1427 Mon Sep 17 00:00:00 2001
From: Jules Laplace <julescarbon@gmail.com>
Date: Sat, 15 Dec 2018 21:47:26 +0100
Subject: builds :)

---
 megapixels/app/builder/builder.py  |  2 +-
 megapixels/app/settings/app_cfg.py |  4 ++--
 site/public/research/index.html    | 20 +++-----------------
 3 files changed, 6 insertions(+), 20 deletions(-)

(limited to 'megapixels/app/settings/app_cfg.py')

diff --git a/megapixels/app/builder/builder.py b/megapixels/app/builder/builder.py
index cb5d8fe9..df609f60 100644
--- a/megapixels/app/builder/builder.py
+++ b/megapixels/app/builder/builder.py
@@ -74,7 +74,7 @@ def build_research_index(research_posts):
     research_posts=research_posts,
     latest_research_post=research_posts[-1],
   )
-  output_fn = public_path + '/research/index.html'
+  output_fn = cfg.DIR_SITE_PUBLIC + '/research/index.html'
   with open(output_fn, "w") as file:
     file.write(html)
 
diff --git a/megapixels/app/settings/app_cfg.py b/megapixels/app/settings/app_cfg.py
index 9e03357b..f861700c 100644
--- a/megapixels/app/settings/app_cfg.py
+++ b/megapixels/app/settings/app_cfg.py
@@ -61,8 +61,8 @@ DIR_TEST_IMAGES = join(DIR_APP, 'test', 'images')
 # .env config for keys
 # -----------------------------------------------------------------------------
 
-DIR_DOTENV = join(DIR_APP, '.env')
-load_dotenv(dotenv_path=DIR_DOTENV)
+# DIR_DOTENV = join(DIR_APP, '.env')
+load_dotenv() # dotenv_path=DIR_DOTENV)
 
 # -----------------------------------------------------------------------------
 # Drawing, GUI settings
diff --git a/site/public/research/index.html b/site/public/research/index.html
index 8b43ac17..59a5fee9 100644
--- a/site/public/research/index.html
+++ b/site/public/research/index.html
@@ -26,23 +26,9 @@
   </header>
   <div class="content">
     
-  <section>
-    <h1>Research</h1>
-    <div class='meta'>
-      <div>
-        <div class='gray'>Posted</div>
-        <div>2018-12-15</div>
-      </div>
-      <div>
-        <div class='gray'>By</div>
-        <div>Adam Harvey</div>
-      </div>
-      
-    </div>
-  </section>
-
-  <section><h2>The darkside of datasets and the future of computer vision</h2>
-</section>
+  <section><h1>Research Blog</h1>
+<h2>The darkside of datasets and the future of computer vision</h2>
+</section><div class='research_index'><a href='/research/00_introduction/'><section class='wide'><img src='data:image/gif;base64,R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==' alt='Research post' /><section><h1>Untitled Page</h1><h2></h2></section></section></a><a href='/research/01_from_1_to_100_pixels/'><section class='wide'><img src='https://nyc3.digitaloceanspaces.com/megapixels/v1/site/research/01_from_1_to_100_pixels/assets/intro.jpg' alt='Research post' /><section><h1>From 1 to 100 Pixels</h1><h2>Photographs are for romantics. For the rest of us, it's all about data.  And a photo contains a massive amount of information about who you are.</h2></section></section></a></div>
 
   </div>
   <footer>
-- 
cgit v1.2.3-70-g09d2


From b1e7dc570fe25749a2e1b02c9e859df6588b4660 Mon Sep 17 00:00:00 2001
From: Jules Laplace <julescarbon@gmail.com>
Date: Sat, 15 Dec 2018 22:04:41 +0100
Subject: move builder

---
 megapixels/app/builder/README.md     |  21 ----
 megapixels/app/builder/builder.py    |  85 ----------------
 megapixels/app/builder/parser.py     | 172 --------------------------------
 megapixels/app/builder/s3.py         |  60 -----------
 megapixels/app/settings/app_cfg.py   |   5 +-
 megapixels/app/site/README.md        |  21 ++++
 megapixels/app/site/builder.py       |  85 ++++++++++++++++
 megapixels/app/site/parser.py        | 187 +++++++++++++++++++++++++++++++++++
 megapixels/app/site/s3.py            |  60 +++++++++++
 megapixels/cli_builder.py            |  36 -------
 megapixels/cli_site.py               |  36 +++++++
 megapixels/commands/builder/build.py |  15 ---
 megapixels/commands/site/build.py    |  15 +++
 13 files changed, 406 insertions(+), 392 deletions(-)
 delete mode 100644 megapixels/app/builder/README.md
 delete mode 100644 megapixels/app/builder/builder.py
 delete mode 100644 megapixels/app/builder/parser.py
 delete mode 100644 megapixels/app/builder/s3.py
 create mode 100644 megapixels/app/site/README.md
 create mode 100644 megapixels/app/site/builder.py
 create mode 100644 megapixels/app/site/parser.py
 create mode 100644 megapixels/app/site/s3.py
 delete mode 100644 megapixels/cli_builder.py
 create mode 100644 megapixels/cli_site.py
 delete mode 100644 megapixels/commands/builder/build.py
 create mode 100644 megapixels/commands/site/build.py

(limited to 'megapixels/app/settings/app_cfg.py')

diff --git a/megapixels/app/builder/README.md b/megapixels/app/builder/README.md
deleted file mode 100644
index 1a6d3a1e..00000000
--- a/megapixels/app/builder/README.md
+++ /dev/null
@@ -1,21 +0,0 @@
-Megapixels Static Site Generator
-================================
-
-The index, blog, and about other pages are built using this static site generator.
-
-## Metadata
-
-```
-status: published|draft|private
-title: From 1 to 100 Pixels
-desc: High resolution insights from low resolution imagery
-slug: from-1-to-100-pixels
-published: 2018-12-04
-updated: 2018-12-04
-authors: Adam Harvey, Berit Gilma, Matthew Stender
-```
-
-## S3 Assets
-
-Static assets: `v1/site/about/assets/picture.jpg`
-Dataset assets: `v1/datasets/lfw/assets/picture.jpg`
diff --git a/megapixels/app/builder/builder.py b/megapixels/app/builder/builder.py
deleted file mode 100644
index df609f60..00000000
--- a/megapixels/app/builder/builder.py
+++ /dev/null
@@ -1,85 +0,0 @@
-#!/usr/bin/python
-
-import os
-import glob
-from jinja2 import Environment, FileSystemLoader, select_autoescape
-
-import app.settings.app_cfg as cfg
-
-import app.builder.s3 as s3
-import app.builder.parser as parser
-
-env = Environment(
-  loader=FileSystemLoader(cfg.DIR_SITE_TEMPLATES),
-  autoescape=select_autoescape([])
-)
-
-def build_page(fn, research_posts):
-  metadata, sections = parser.read_metadata(fn)
-
-  if metadata is None:
-    print("{} has no metadata".format(fn))
-    return
-
-  print(metadata['url'])
-
-  dirname = os.path.dirname(fn)
-  output_path = cfg.DIR_SITE_PUBLIC + metadata['url']
-  output_fn = os.path.join(output_path, "index.html")
-
-  skip_h1 = False
-
-  if metadata['url'] == '/':
-    template = env.get_template("home.html")
-  elif 'research/' in fn:
-    skip_h1 = True
-    template = env.get_template("research.html")
-  else:
-    template = env.get_template("page.html")
-
-  if 'datasets/' in fn:
-    s3_dir = cfg.S3_DATASETS_PATH
-  else:
-    s3_dir = cfg.S3_SITE_PATH
-
-  s3_path = s3.make_s3_path(s3_dir, metadata['path'])
-
-  if 'index.md' in fn:
-    s3.sync_directory(dirname, s3_dir, metadata)
-
-  content = parser.parse_markdown(sections, s3_path, skip_h1=skip_h1)
-
-  html = template.render(
-    metadata=metadata,
-    content=content,
-    research_posts=research_posts,
-    latest_research_post=research_posts[-1],
-  )
-
-  os.makedirs(output_path, exist_ok=True)
-  with open(output_fn, "w") as file:
-    file.write(html)
-
-  print("______")
-
-def build_research_index(research_posts):
-  metadata, sections = parser.read_metadata('../site/content/research/index.md')
-  template = env.get_template("page.html")
-  s3_path = s3.make_s3_path(cfg.S3_SITE_PATH, metadata['path'])
-  content = parser.parse_markdown(sections, s3_path, skip_h1=False)
-  content += parser.parse_research_index(research_posts)
-  html = template.render(
-    metadata=metadata,
-    content=content,
-    research_posts=research_posts,
-    latest_research_post=research_posts[-1],
-  )
-  output_fn = cfg.DIR_SITE_PUBLIC + '/research/index.html'
-  with open(output_fn, "w") as file:
-    file.write(html)
-
-def build_site():
-  research_posts = parser.read_research_post_index()
-  for fn in glob.iglob(os.path.join(cfg.DIR_SITE_CONTENT, "**/*.md"), recursive=True):
-    build_page(fn, research_posts)
-  build_research_index(research_posts)
diff --git a/megapixels/app/builder/parser.py b/megapixels/app/builder/parser.py
deleted file mode 100644
index 985111ac..00000000
--- a/megapixels/app/builder/parser.py
+++ /dev/null
@@ -1,172 +0,0 @@
-import os
-import re
-import glob
-import mistune
-
-import app.settings.app_cfg as cfg
-import app.builder.s3 as s3
-
-renderer = mistune.Renderer(escape=False)
-markdown = mistune.Markdown(renderer=renderer)
-
-def fix_images(lines, s3_path):
-  real_lines = []
-  block = "\n\n".join(lines)
-  for line in block.split("\n"):
-    if "![" in line:
-      line = line.replace('![', '')
-      alt_text, tail = line.split('](', 1)
-      url, tail = tail.split(')', 1)
-      if ':' in alt_text:
-        tail, alt_text = alt_text.split(':', 1)
-      img_tag = "<img src='{}' alt='{}'>".format(s3_path + url, alt_text.replace("'", ""))
-      if len(alt_text):
-        line = "<div class='image'>{}<div class='caption'>{}</div></div>".format(img_tag, alt_text)
-      else:
-        line = "<div class='image'>{}</div>".format(img_tag, alt_text)
-    real_lines.append(line)
-  return "\n".join(real_lines)
-
-def format_section(lines, s3_path, type=''):
-  if len(lines):
-    lines = fix_images(lines, s3_path)
-    if type:
-      return "<section class='{}'>{}</section>".format(type, markdown(lines))
-    else:
-      return "<section>" + markdown(lines) + "</section>"
-  return ""
-
-def format_metadata(section):
-  meta = []
-  for line in section.split('\n'):
-    key, value = line[2:].split(': ', 1)
-    meta.append("<div><div class='gray'>{}</div><div>{}</div></div>".format(key, value))
-  return "<section><div class='meta'>{}</div></section>".format(''.join(meta)) 
-
-def parse_markdown(sections, s3_path, skip_h1=False):
-  groups = []
-  current_group = []
-  for section in sections:
-    if skip_h1 and section.startswith('# '):
-      continue
-    elif section.startswith('+ '):
-      groups.append(format_section(current_group, s3_path))
-      groups.append(format_metadata(section))
-      current_group = []
-    elif '![wide:' in section:
-      groups.append(format_section(current_group, s3_path))
-      groups.append(format_section([section], s3_path, type='wide'))
-      current_group = []
-    elif '![' in section:
-      groups.append(format_section(current_group, s3_path))
-      groups.append(format_section([section], s3_path, type='images'))
-      current_group = []
-    else:
-      current_group.append(section)
-  groups.append(format_section(current_group, s3_path))
-  content = "".join(groups)
-  return content
-
-def parse_research_index(research_posts):
-  content = "<div class='research_index'>"
-  for post in research_posts:
-    s3_path = s3.make_s3_path(cfg.S3_SITE_PATH, post['path'])
-    if 'image' in post:
-      post_image = s3_path + post['image']
-    else:
-      post_image = 'data:image/gif;base64,R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw=='
-    row = "<a href='{}'><section class='wide'><img src='{}' alt='Research post' /><section><h1>{}</h1><h2>{}</h2></section></section></a>".format(
-      post['path'],
-      post_image,
-      post['title'],
-      post['tagline'])
-    content += row
-  content += '</div>'
-  return content
-
-def read_metadata(fn):
-  with open(fn, "r") as file:
-    data = file.read()
-    data = data.replace("\n ", "\n")
-    if "\n" in data:
-      data = data.replace("\r", "")
-    else:
-      data = data.replace("\r", "\n")
-    sections = data.split("\n\n")
-  return parse_metadata(fn, sections)
-
-default_metadata = {
-  'status': 'published',
-  'title': 'Untitled Page',
-  'desc': '',
-  'slug': '',
-  'published': '2018-12-31',
-  'updated': '2018-12-31',
-  'authors': 'Adam Harvey',
-  'sync': 'true',
-  'tagline': '',
-}
-
-def parse_metadata_section(metadata, section):
-  for line in section.split("\n"):
-    if ': ' not in line:
-      continue
-    key, value = line.split(': ', 1)
-    metadata[key.lower()] = value
-
-def parse_metadata(fn, sections):
-  found_meta = False
-  metadata = {}
-  valid_sections = []
-  for section in sections:
-    if not found_meta and ': ' in section:
-      found_meta = True
-      parse_metadata_section(metadata, section)
-      continue
-    if '-----' in section:
-      continue
-    if found_meta:
-      valid_sections.append(section)
-
-  if 'title' not in metadata:
-    print('warning: {} has no title'.format(fn))
-  for key in default_metadata:
-    if key not in metadata:
-      metadata[key] = default_metadata[key]
-
-  basedir = os.path.dirname(fn.replace(cfg.DIR_SITE_CONTENT, ''))
-  basename = os.path.basename(fn)
-  if basedir == '/':
-    metadata['path'] = '/'
-    metadata['url'] = '/'
-  elif basename == 'index.md':
-    metadata['path'] = basedir + '/'
-    metadata['url'] = metadata['path']
-  else:
-    metadata['path'] = basedir + '/'
-    metadata['url'] = metadata['path'] + basename.replace('.md', '') + '/'
-
-  if metadata['status'] == 'published|draft|private':
-    metadata['status'] = 'published'
-
-  metadata['sync'] = metadata['sync'] != 'false'
-
-  metadata['author_html'] = '<br>'.join(metadata['authors'].split(','))
-  return metadata, valid_sections
-
-def read_research_post_index():
-  posts = []
-  for fn in sorted(glob.glob('../site/content/research/*/index.md')):
-    metadata, valid_sections = read_metadata(fn)
-    if metadata is None or metadata['status'] == 'private' or metadata['status'] == 'draft':
-      continue
-    posts.append(metadata)
-  if not len(posts):
-    posts.append({
-      'title': 'Placeholder',
-      'slug': 'placeholder',
-      'date': 'Placeholder',
-      'url': '/',
-    })
-  return posts
-
diff --git a/megapixels/app/builder/s3.py b/megapixels/app/builder/s3.py
deleted file mode 100644
index 99726a4d..00000000
--- a/megapixels/app/builder/s3.py
+++ /dev/null
@@ -1,60 +0,0 @@
-import os
-import glob
-import boto3
-
-def sync_directory(base_fn, s3_path, metadata):
-  fns = {}
-  for fn in glob.glob(os.path.join(base_fn, 'assets/*')):
-    fns[os.path.basename(fn)] = True
-
-  if not metadata['sync']:
-    return
-
-  remote_path = s3_path + metadata['url']
-
-  session = boto3.session.Session()
-
-  s3_client = session.client(
-    service_name='s3',
-    aws_access_key_id=os.getenv('S3_KEY'),
-    aws_secret_access_key=os.getenv('S3_SECRET'),
-    endpoint_url=os.getenv('S3_ENDPOINT'),
-    region_name=os.getenv('S3_REGION'),
-  )
-
-  directory = s3_client.list_objects(Bucket=os.getenv('S3_BUCKET'), Prefix=remote_path)
-  prefixes = []
-
-  if 'Contents' in directory:
-    for obj in directory['Contents']:
-      s3_fn = obj['Key']
-      fn = os.path.basename(s3_fn)
-      local_fn = os.path.join(base_fn, 'assets', fn)
-      if fn in fns:
-        del fns[fn]
-        if obj['LastModified'].timestamp() < os.path.getmtime(os.path.join(local_fn)):
-          print("s3 update {}".format(s3_fn))
-          s3_client.upload_file(
-            local_fn,
-            os.getenv('S3_BUCKET'),
-            s3_fn,
-            ExtraArgs={ 'ACL': 'public-read' })
-      else:
-        print("s3 delete {}".format(s3_fn))
-        response = s3_client.delete_object(
-          Bucket=os.getenv('S3_BUCKET'),
-          Key=s3_fn,
-        )
-
-  for fn in fns:
-    local_fn = os.path.join(base_fn, 'assets', fn)
-    s3_fn = os.path.join(remote_path, 'assets', fn)
-    print("s3 create {}".format(s3_fn))
-    s3_client.upload_file(
-      local_fn,
-      os.getenv('S3_BUCKET'),
-      s3_fn,
-      ExtraArgs={ 'ACL': 'public-read' })
-
-def make_s3_path(s3_dir, metadata_path):
-  return "{}/{}/{}{}".format(os.getenv('S3_ENDPOINT'), os.getenv('S3_BUCKET'), s3_dir, metadata_path)
diff --git a/megapixels/app/settings/app_cfg.py b/megapixels/app/settings/app_cfg.py
index f861700c..a18e5875 100644
--- a/megapixels/app/settings/app_cfg.py
+++ b/megapixels/app/settings/app_cfg.py
@@ -75,7 +75,7 @@ FP_FONT = join(DIR_ASSETS, 'font')
 # -----------------------------------------------------------------------------
 DIR_COMMANDS_CV = 'commands/cv'
 DIR_COMMANDS_ADMIN = 'commands/admin'
-DIR_COMMANDS_BUILDER = 'commands/builder'
+DIR_COMMANDS_SITE = 'commands/site'
 DIR_COMMANDS_DATASETS = 'commands/datasets'
 DIR_COMMANDS_FAISS = 'commands/faiss'
 DIR_COMMANDS_MISC  = 'commands/misc'
@@ -125,9 +125,8 @@ S3_MEDIA_ROOT = 's3://megapixels/v1/media/'
 S3_METADATA_ROOT = 's3://megapixels/v1/metadata/'
 
 # -----------------------------------------------------------------------------
-# Builder
+# Static site generator
 # -----------------------------------------------------------------------------
-
 S3_SITE_PATH = "v1/site"
 S3_DATASETS_PATH = "v1" # datasets is already in the filename
 DIR_SITE_PUBLIC = "../site/public"
diff --git a/megapixels/app/site/README.md b/megapixels/app/site/README.md
new file mode 100644
index 00000000..1a6d3a1e
--- /dev/null
+++ b/megapixels/app/site/README.md
@@ -0,0 +1,21 @@
+Megapixels Static Site Generator
+================================
+
+The index, blog, and about other pages are built using this static site generator.
+
+## Metadata
+
+```
+status: published|draft|private
+title: From 1 to 100 Pixels
+desc: High resolution insights from low resolution imagery
+slug: from-1-to-100-pixels
+published: 2018-12-04
+updated: 2018-12-04
+authors: Adam Harvey, Berit Gilma, Matthew Stender
+```
+
+## S3 Assets
+
+Static assets: `v1/site/about/assets/picture.jpg`
+Dataset assets: `v1/datasets/lfw/assets/picture.jpg`
diff --git a/megapixels/app/site/builder.py b/megapixels/app/site/builder.py
new file mode 100644
index 00000000..df609f60
--- /dev/null
+++ b/megapixels/app/site/builder.py
@@ -0,0 +1,85 @@
+#!/usr/bin/python
+
+import os
+import glob
+from jinja2 import Environment, FileSystemLoader, select_autoescape
+
+import app.settings.app_cfg as cfg
+
+import app.builder.s3 as s3
+import app.builder.parser as parser
+
+env = Environment(
+  loader=FileSystemLoader(cfg.DIR_SITE_TEMPLATES),
+  autoescape=select_autoescape([])
+)
+
+def build_page(fn, research_posts):
+  metadata, sections = parser.read_metadata(fn)
+
+  if metadata is None:
+    print("{} has no metadata".format(fn))
+    return
+
+  print(metadata['url'])
+
+  dirname = os.path.dirname(fn)
+  output_path = cfg.DIR_SITE_PUBLIC + metadata['url']
+  output_fn = os.path.join(output_path, "index.html")
+
+  skip_h1 = False
+
+  if metadata['url'] == '/':
+    template = env.get_template("home.html")
+  elif 'research/' in fn:
+    skip_h1 = True
+    template = env.get_template("research.html")
+  else:
+    template = env.get_template("page.html")
+
+  if 'datasets/' in fn:
+    s3_dir = cfg.S3_DATASETS_PATH
+  else:
+    s3_dir = cfg.S3_SITE_PATH
+
+  s3_path = s3.make_s3_path(s3_dir, metadata['path'])
+
+  if 'index.md' in fn:
+    s3.sync_directory(dirname, s3_dir, metadata)
+
+  content = parser.parse_markdown(sections, s3_path, skip_h1=skip_h1)
+
+  html = template.render(
+    metadata=metadata,
+    content=content,
+    research_posts=research_posts,
+    latest_research_post=research_posts[-1],
+  )
+
+  os.makedirs(output_path, exist_ok=True)
+  with open(output_fn, "w") as file:
+    file.write(html)
+
+  print("______")
+
+def build_research_index(research_posts):
+  metadata, sections = parser.read_metadata('../site/content/research/index.md')
+  template = env.get_template("page.html")
+  s3_path = s3.make_s3_path(cfg.S3_SITE_PATH, metadata['path'])
+  content = parser.parse_markdown(sections, s3_path, skip_h1=False)
+  content += parser.parse_research_index(research_posts)
+  html = template.render(
+    metadata=metadata,
+    content=content,
+    research_posts=research_posts,
+    latest_research_post=research_posts[-1],
+  )
+  output_fn = cfg.DIR_SITE_PUBLIC + '/research/index.html'
+  with open(output_fn, "w") as file:
+    file.write(html)
+
+def build_site():
+  research_posts = parser.read_research_post_index()
+  for fn in glob.iglob(os.path.join(cfg.DIR_SITE_CONTENT, "**/*.md"), recursive=True):
+    build_page(fn, research_posts)
+  build_research_index(research_posts)
diff --git a/megapixels/app/site/parser.py b/megapixels/app/site/parser.py
new file mode 100644
index 00000000..add3f386
--- /dev/null
+++ b/megapixels/app/site/parser.py
@@ -0,0 +1,187 @@
+import os
+import re
+import glob
+import mistune
+
+import app.settings.app_cfg as cfg
+import app.builder.s3 as s3
+
+renderer = mistune.Renderer(escape=False)
+markdown = mistune.Markdown(renderer=renderer)
+
+def fix_images(lines, s3_path):
+  real_lines = []
+  block = "\n\n".join(lines)
+  for line in block.split("\n"):
+    if "![" in line:
+      line = line.replace('![', '')
+      alt_text, tail = line.split('](', 1)
+      url, tail = tail.split(')', 1)
+      if ':' in alt_text:
+        tail, alt_text = alt_text.split(':', 1)
+      img_tag = "<img src='{}' alt='{}'>".format(s3_path + url, alt_text.replace("'", ""))
+      if len(alt_text):
+        line = "<div class='image'>{}<div class='caption'>{}</div></div>".format(img_tag, alt_text)
+      else:
+        line = "<div class='image'>{}</div>".format(img_tag, alt_text)
+    real_lines.append(line)
+  return "\n".join(real_lines)
+
+def format_section(lines, s3_path, type=''):
+  if len(lines):
+    lines = fix_images(lines, s3_path)
+    if type:
+      return "<section class='{}'>{}</section>".format(type, markdown(lines))
+    else:
+      return "<section>" + markdown(lines) + "</section>"
+  return ""
+
+def format_metadata(section):
+  meta = []
+  for line in section.split('\n'):
+    key, value = line[2:].split(': ', 1)
+    meta.append("<div><div class='gray'>{}</div><div>{}</div></div>".format(key, value))
+  return "<section><div class='meta'>{}</div></section>".format(''.join(meta)) 
+
+def format_applet(section):
+  payload = section.replace('```', '').strip().split('\n')
+  if ': ' in payload[0]:
+    command, opt = payload[0].split(': ')
+  else:
+    command = payload[0]
+    opt = None
+  if command == 'load_file':
+    return "<section><div class='applet'>{}</div></section>"
+
+
+def parse_markdown(sections, s3_path, skip_h1=False):
+  groups = []
+  current_group = []
+  for section in sections:
+    if skip_h1 and section.startswith('# '):
+      continue
+    elif section.startsWith('```'):
+      groups.append(format_section(current_group, s3_path))
+      groups.append(format_applet(section))
+      current_group = []
+    elif section.startswith('+ '):
+      groups.append(format_section(current_group, s3_path))
+      groups.append(format_metadata(section))
+      current_group = []
+    elif '![wide:' in section:
+      groups.append(format_section(current_group, s3_path))
+      groups.append(format_section([section], s3_path, type='wide'))
+      current_group = []
+    elif '![' in section:
+      groups.append(format_section(current_group, s3_path))
+      groups.append(format_section([section], s3_path, type='images'))
+      current_group = []
+    else:
+      current_group.append(section)
+  groups.append(format_section(current_group, s3_path))
+  content = "".join(groups)
+  return content
+
+def parse_research_index(research_posts):
+  content = "<div class='research_index'>"
+  for post in research_posts:
+    s3_path = s3.make_s3_path(cfg.S3_SITE_PATH, post['path'])
+    if 'image' in post:
+      post_image = s3_path + post['image']
+    else:
+      post_image = 'data:image/gif;base64,R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw=='
+    row = "<a href='{}'><section class='wide'><img src='{}' alt='Research post' /><section><h1>{}</h1><h2>{}</h2></section></section></a>".format(
+      post['path'],
+      post_image,
+      post['title'],
+      post['tagline'])
+    content += row
+  content += '</div>'
+  return content
+
+def read_metadata(fn):
+  with open(fn, "r") as file:
+    data = file.read()
+    data = data.replace("\n ", "\n")
+    if "\n" in data:
+      data = data.replace("\r", "")
+    else:
+      data = data.replace("\r", "\n")
+    sections = data.split("\n\n")
+  return parse_metadata(fn, sections)
+
+default_metadata = {
+  'status': 'published',
+  'title': 'Untitled Page',
+  'desc': '',
+  'slug': '',
+  'published': '2018-12-31',
+  'updated': '2018-12-31',
+  'authors': 'Adam Harvey',
+  'sync': 'true',
+  'tagline': '',
+}
+
+def parse_metadata_section(metadata, section):
+  for line in section.split("\n"):
+    if ': ' not in line:
+      continue
+    key, value = line.split(': ', 1)
+    metadata[key.lower()] = value
+
+def parse_metadata(fn, sections):
+  found_meta = False
+  metadata = {}
+  valid_sections = []
+  for section in sections:
+    if not found_meta and ': ' in section:
+      found_meta = True
+      parse_metadata_section(metadata, section)
+      continue
+    if '-----' in section:
+      continue
+    if found_meta:
+      valid_sections.append(section)
+
+  if 'title' not in metadata:
+    print('warning: {} has no title'.format(fn))
+  for key in default_metadata:
+    if key not in metadata:
+      metadata[key] = default_metadata[key]
+
+  basedir = os.path.dirname(fn.replace(cfg.DIR_SITE_CONTENT, ''))
+  basename = os.path.basename(fn)
+  if basedir == '/':
+    metadata['path'] = '/'
+    metadata['url'] = '/'
+  elif basename == 'index.md':
+    metadata['path'] = basedir + '/'
+    metadata['url'] = metadata['path']
+  else:
+    metadata['path'] = basedir + '/'
+    metadata['url'] = metadata['path'] + basename.replace('.md', '') + '/'
+
+  if metadata['status'] == 'published|draft|private':
+    metadata['status'] = 'published'
+
+  metadata['sync'] = metadata['sync'] != 'false'
+
+  metadata['author_html'] = '<br>'.join(metadata['authors'].split(','))
+  return metadata, valid_sections
+
+def read_research_post_index():
+  posts = []
+  for fn in sorted(glob.glob('../site/content/research/*/index.md')):
+    metadata, valid_sections = read_metadata(fn)
+    if metadata is None or metadata['status'] == 'private' or metadata['status'] == 'draft':
+      continue
+    posts.append(metadata)
+  if not len(posts):
+    posts.append({
+      'title': 'Placeholder',
+      'slug': 'placeholder',
+      'date': 'Placeholder',
+      'url': '/',
+    })
+  return posts
+
diff --git a/megapixels/app/site/s3.py b/megapixels/app/site/s3.py
new file mode 100644
index 00000000..99726a4d
--- /dev/null
+++ b/megapixels/app/site/s3.py
@@ -0,0 +1,60 @@
+import os
+import glob
+import boto3
+
+def sync_directory(base_fn, s3_path, metadata):
+  fns = {}
+  for fn in glob.glob(os.path.join(base_fn, 'assets/*')):
+    fns[os.path.basename(fn)] = True
+
+  if not metadata['sync']:
+    return
+
+  remote_path = s3_path + metadata['url']
+
+  session = boto3.session.Session()
+
+  s3_client = session.client(
+    service_name='s3',
+    aws_access_key_id=os.getenv('S3_KEY'),
+    aws_secret_access_key=os.getenv('S3_SECRET'),
+    endpoint_url=os.getenv('S3_ENDPOINT'),
+    region_name=os.getenv('S3_REGION'),
+  )
+
+  directory = s3_client.list_objects(Bucket=os.getenv('S3_BUCKET'), Prefix=remote_path)
+  prefixes = []
+
+  if 'Contents' in directory:
+    for obj in directory['Contents']:
+      s3_fn = obj['Key']
+      fn = os.path.basename(s3_fn)
+      local_fn = os.path.join(base_fn, 'assets', fn)
+      if fn in fns:
+        del fns[fn]
+        if obj['LastModified'].timestamp() < os.path.getmtime(os.path.join(local_fn)):
+          print("s3 update {}".format(s3_fn))
+          s3_client.upload_file(
+            local_fn,
+            os.getenv('S3_BUCKET'),
+            s3_fn,
+            ExtraArgs={ 'ACL': 'public-read' })
+      else:
+        print("s3 delete {}".format(s3_fn))
+        response = s3_client.delete_object(
+          Bucket=os.getenv('S3_BUCKET'),
+          Key=s3_fn,
+        )
+
+  for fn in fns:
+    local_fn = os.path.join(base_fn, 'assets', fn)
+    s3_fn = os.path.join(remote_path, 'assets', fn)
+    print("s3 create {}".format(s3_fn))
+    s3_client.upload_file(
+      local_fn,
+      os.getenv('S3_BUCKET'),
+      s3_fn,
+      ExtraArgs={ 'ACL': 'public-read' })
+
+def make_s3_path(s3_dir, metadata_path):
+  return "{}/{}/{}{}".format(os.getenv('S3_ENDPOINT'), os.getenv('S3_BUCKET'), s3_dir, metadata_path)
diff --git a/megapixels/cli_builder.py b/megapixels/cli_builder.py
deleted file mode 100644
index 8bc814f1..00000000
--- a/megapixels/cli_builder.py
+++ /dev/null
@@ -1,36 +0,0 @@
-# --------------------------------------------------------
-# add/edit commands in commands/faiss directory
-# --------------------------------------------------------
-
-import click
-
-from app.settings import app_cfg as cfg
-from app.utils import logger_utils
-from app.models.click_factory import ClickSimple
-
-# click cli factory
-cc = ClickSimple.create(cfg.DIR_COMMANDS_BUILDER)
-
-# --------------------------------------------------------
-# CLI
-# --------------------------------------------------------
-@click.group(cls=cc, chain=False)
-@click.option('-v', '--verbose', 'verbosity', count=True, default=4, 
-  show_default=True,
-  help='Verbosity: -v DEBUG, -vv INFO, -vvv WARN, -vvvv ERROR, -vvvvv CRITICAL')
-@click.pass_context
-def cli(ctx, **kwargs):
-  """\033[1m\033[94mMegaPixels: FAISS Data Scripts\033[0m                                                
-  """
-  ctx.opts = {}
-  # init logger
-  logger_utils.Logger.create(verbosity=kwargs['verbosity'])
-
-
-
-# --------------------------------------------------------
-# Entrypoint
-# --------------------------------------------------------
-if __name__ == '__main__':
-    cli()
-
diff --git a/megapixels/cli_site.py b/megapixels/cli_site.py
new file mode 100644
index 00000000..5faf63cd
--- /dev/null
+++ b/megapixels/cli_site.py
@@ -0,0 +1,36 @@
+# --------------------------------------------------------
+# add/edit commands in commands/faiss directory
+# --------------------------------------------------------
+
+import click
+
+from app.settings import app_cfg as cfg
+from app.utils import logger_utils
+from app.models.click_factory import ClickSimple
+
+# click cli factory
+cc = ClickSimple.create(cfg.DIR_COMMANDS_SITE)
+
+# --------------------------------------------------------
+# CLI
+# --------------------------------------------------------
+@click.group(cls=cc, chain=False)
+@click.option('-v', '--verbose', 'verbosity', count=True, default=4, 
+  show_default=True,
+  help='Verbosity: -v DEBUG, -vv INFO, -vvv WARN, -vvvv ERROR, -vvvvv CRITICAL')
+@click.pass_context
+def cli(ctx, **kwargs):
+  """\033[1m\033[94mMegaPixels: FAISS Data Scripts\033[0m                                                
+  """
+  ctx.opts = {}
+  # init logger
+  logger_utils.Logger.create(verbosity=kwargs['verbosity'])
+
+
+
+# --------------------------------------------------------
+# Entrypoint
+# --------------------------------------------------------
+if __name__ == '__main__':
+    cli()
+
diff --git a/megapixels/commands/builder/build.py b/megapixels/commands/builder/build.py
deleted file mode 100644
index fc4fb302..00000000
--- a/megapixels/commands/builder/build.py
+++ /dev/null
@@ -1,15 +0,0 @@
-"""
-Build the static site
-"""
-
-import click
-
-from app.builder.builder import build_site
-
-@click.command()
-@click.pass_context
-def cli(ctx):
-  """Build the static site
-  """
-  print('Building the site...')
-  build_site()
diff --git a/megapixels/commands/site/build.py b/megapixels/commands/site/build.py
new file mode 100644
index 00000000..fc4fb302
--- /dev/null
+++ b/megapixels/commands/site/build.py
@@ -0,0 +1,15 @@
+"""
+Build the static site
+"""
+
+import click
+
+from app.builder.builder import build_site
+
+@click.command()
+@click.pass_context
+def cli(ctx):
+  """Build the static site
+  """
+  print('Building the site...')
+  build_site()
-- 
cgit v1.2.3-70-g09d2


From 898e6cdf8df0993f853b748d4e8a9c269fad0294 Mon Sep 17 00:00:00 2001
From: Jules Laplace <julescarbon@gmail.com>
Date: Sat, 15 Dec 2018 22:14:17 +0100
Subject: inject applet payload

---
 megapixels/app/settings/app_cfg.py        |  2 +-
 megapixels/app/site/builder.py            |  4 +--
 megapixels/app/site/parser.py             | 15 ++++++---
 megapixels/commands/site/build.py         |  2 +-
 site/public/about/style/index.html        | 11 +------
 site/public/datasets/lfw/index.html       | 51 ++++++++++++-------------------
 site/public/datasets/vgg_face2/index.html |  9 ++----
 7 files changed, 36 insertions(+), 58 deletions(-)

(limited to 'megapixels/app/settings/app_cfg.py')

diff --git a/megapixels/app/settings/app_cfg.py b/megapixels/app/settings/app_cfg.py
index a18e5875..d25936e6 100644
--- a/megapixels/app/settings/app_cfg.py
+++ b/megapixels/app/settings/app_cfg.py
@@ -75,10 +75,10 @@ FP_FONT = join(DIR_ASSETS, 'font')
 # -----------------------------------------------------------------------------
 DIR_COMMANDS_CV = 'commands/cv'
 DIR_COMMANDS_ADMIN = 'commands/admin'
-DIR_COMMANDS_SITE = 'commands/site'
 DIR_COMMANDS_DATASETS = 'commands/datasets'
 DIR_COMMANDS_FAISS = 'commands/faiss'
 DIR_COMMANDS_MISC  = 'commands/misc'
+DIR_COMMANDS_SITE = 'commands/site'
 
 # -----------------------------------------------------------------------------
 # Filesystem settings
diff --git a/megapixels/app/site/builder.py b/megapixels/app/site/builder.py
index df609f60..42e25768 100644
--- a/megapixels/app/site/builder.py
+++ b/megapixels/app/site/builder.py
@@ -6,8 +6,8 @@ from jinja2 import Environment, FileSystemLoader, select_autoescape
 
 import app.settings.app_cfg as cfg
 
-import app.builder.s3 as s3
-import app.builder.parser as parser
+import app.site.s3 as s3
+import app.site.parser as parser
 
 env = Environment(
   loader=FileSystemLoader(cfg.DIR_SITE_TEMPLATES),
diff --git a/megapixels/app/site/parser.py b/megapixels/app/site/parser.py
index add3f386..d78cc402 100644
--- a/megapixels/app/site/parser.py
+++ b/megapixels/app/site/parser.py
@@ -1,10 +1,11 @@
 import os
 import re
 import glob
+import simplejson as json
 import mistune
 
 import app.settings.app_cfg as cfg
-import app.builder.s3 as s3
+import app.site.s3 as s3
 
 renderer = mistune.Renderer(escape=False)
 markdown = mistune.Markdown(renderer=renderer)
@@ -45,14 +46,18 @@ def format_metadata(section):
 
 def format_applet(section):
   payload = section.replace('```', '').strip().split('\n')
+  applet = {}
   if ': ' in payload[0]:
     command, opt = payload[0].split(': ')
   else:
     command = payload[0]
     opt = None
-  if command == 'load_file':
-    return "<section><div class='applet'>{}</div></section>"
-
+  applet['command'] = command
+  if opt:
+    applet['opt'] = opt
+  if command == 'load file':
+    applet['fields'] = payload[1]
+  return "<section><div class='applet' data-payload='{}'></div></section>".format(json.dumps(applet))
 
 def parse_markdown(sections, s3_path, skip_h1=False):
   groups = []
@@ -60,7 +65,7 @@ def parse_markdown(sections, s3_path, skip_h1=False):
   for section in sections:
     if skip_h1 and section.startswith('# '):
       continue
-    elif section.startsWith('```'):
+    elif section.startswith('```'):
       groups.append(format_section(current_group, s3_path))
       groups.append(format_applet(section))
       current_group = []
diff --git a/megapixels/commands/site/build.py b/megapixels/commands/site/build.py
index fc4fb302..0a76a9ac 100644
--- a/megapixels/commands/site/build.py
+++ b/megapixels/commands/site/build.py
@@ -4,7 +4,7 @@ Build the static site
 
 import click
 
-from app.builder.builder import build_site
+from app.site.builder import build_site
 
 @click.command()
 @click.pass_context
diff --git a/site/public/about/style/index.html b/site/public/about/style/index.html
index f2c0d4b8..39a44380 100644
--- a/site/public/about/style/index.html
+++ b/site/public/about/style/index.html
@@ -51,16 +51,7 @@
 <div class='image'><img src='https://nyc3.digitaloceanspaces.com/megapixels/v1/site/about/assets/man.jpg' alt='Person 3. Let me tell you about Person 3.  This person has a very long description with text which wraps like crazy'><div class='caption'>Person 3. Let me tell you about Person 3.  This person has a very long description with text which wraps like crazy</div></div></section><section><blockquote><p>est, qui dolorem ipsum, quia dolor sit amet consectetur adipisci[ng] velit, sed quia non-numquam [do] eius modi tempora inci[di]dunt, ut labore et dolore magnam aliquam quaerat voluptatem.</p>
 </blockquote>
 </section><section class='wide'><div class='image'><img src='https://nyc3.digitaloceanspaces.com/megapixels/v1/site/about/assets/wide-test.jpg' alt='This image is extremely wide and the text beneath it will wrap but thats fine because it can also contain <a href="https://example.com/">hyperlinks</a>! Yes, you read that right&mdash;hyperlinks! Lorem ipsum dolor sit amet ad volotesque sic hoc ad nauseam'><div class='caption'>This image is extremely wide and the text beneath it will wrap but that's fine because it can also contain <a href="https://example.com/">hyperlinks</a>! Yes, you read that right&mdash;hyperlinks! Lorem ipsum dolor sit amet ad volotesque sic hoc ad nauseam</div></div></section><section><p>Inline <code>code</code> has <code>back-ticks around</code> it.</p>
-<pre><code class="lang-javascript">var s = &quot;JavaScript syntax highlighting&quot;;
-alert(s);
-</code></pre>
-<pre><code class="lang-python">s = &quot;Python syntax highlighting&quot;
-print(s)
-</code></pre>
-<pre><code>No language indicated, so no syntax highlighting. 
-But let's throw in a &lt;b&gt;tag&lt;/b&gt;.
-</code></pre>
-<p>Horizontal rule</p>
+</section><section><div class='applet' data-payload='{"command": "javascript"}'></div></section><section><div class='applet' data-payload='{"command": "python"}'></div></section><section><div class='applet' data-payload='{"command": "No language indicated, so no syntax highlighting. "}'></div></section><section><p>Horizontal rule</p>
 <hr>
 <p>Citations below here</p>
 <div class="footnotes">
diff --git a/site/public/datasets/lfw/index.html b/site/public/datasets/lfw/index.html
index e080229f..3c83acd3 100644
--- a/site/public/datasets/lfw/index.html
+++ b/site/public/datasets/lfw/index.html
@@ -28,12 +28,7 @@
     
   <section><h1>Labeled Faces in the Wild</h1>
 </section><section><div class='meta'><div><div class='gray'>Created</div><div>2007</div></div><div><div class='gray'>Images</div><div>13,233</div></div><div><div class='gray'>People</div><div>5,749</div></div><div><div class='gray'>Created From</div><div>Yahoo News images</div></div><div><div class='gray'>Search available</div><div>Searchable</div></div></div></section><section><p>Labeled Faces in The Wild (LFW) is amongst the most widely used facial recognition training datasets in the world and is the first of its kind to be created entirely from images posted online. The LFW dataset includes 13,233 images of 5,749 people that were collected between 2002-2004. Use the tools below to check if you were included in this dataset or scroll down to read the analysis.</p>
-<p>{INSERT IMAGE SEARCH MODULE}</p>
-<p>{INSERT TEXT SEARCH MODULE}</p>
-<pre><code>load file: lfw_names_gender_kg_min.csv
-Name, Images, Gender, Description
-</code></pre>
-</section><section class='images'><div class='image'><img src='https://nyc3.digitaloceanspaces.com/megapixels/v1/datasets/lfw/assets/lfw_feature.jpg' alt='Eighteen of the 5,749 people in the Labeled Faces in the Wild Dataset. The most widely used face dataset for benchmarking commercial face recognition algorithms.'><div class='caption'>Eighteen of the 5,749 people in the Labeled Faces in the Wild Dataset. The most widely used face dataset for benchmarking commercial face recognition algorithms.</div></div></section><section><h2>Intro</h2>
+</section><section><div class='applet' data-payload='{"command": "face_search"}'></div></section><section><div class='applet' data-payload='{"command": "name_search"}'></div></section><section><div class='applet' data-payload='{"command": "load file", "opt": "lfw_names_gender_kg_min.csv", "fields": "Name, Images, Gender, Description"}'></div></section><section class='images'><div class='image'><img src='https://nyc3.digitaloceanspaces.com/megapixels/v1/datasets/lfw/assets/lfw_feature.jpg' alt='Eighteen of the 5,749 people in the Labeled Faces in the Wild Dataset. The most widely used face dataset for benchmarking commercial face recognition algorithms.'><div class='caption'>Eighteen of the 5,749 people in the Labeled Faces in the Wild Dataset. The most widely used face dataset for benchmarking commercial face recognition algorithms.</div></div></section><section><h2>Intro</h2>
 <p>Three paragraphs describing the LFW dataset in a format that can be easily replicated for the other datasets. Nothing too custom. An analysis of the initial research papers with context relative to all the other dataset papers.</p>
 </section><section class='images'><div class='image'><img src='https://nyc3.digitaloceanspaces.com/megapixels/v1/datasets/lfw/assets/lfw_montage_everyone_1920.jpg' alt=' all 5,749 people in the LFW Dataset sorted from most to least images collected.'><div class='caption'> all 5,749 people in the LFW Dataset sorted from most to least images collected.</div></div></section><section><h2>LFW by the Numbers</h2>
 <ul>
@@ -224,36 +219,28 @@ name_display,company_url,example_url,country,description
 </tbody>
 </table>
 <h2>Code</h2>
-<pre><code class="lang-python">#!/usr/bin/python
-
-import numpy as np
+</section><section><div class='applet' data-payload='{"command": "python"}'></div></section><section><p>import numpy as np
 from sklearn.datasets import fetch_lfw_people
 import imageio
-import imutils
-
-# download LFW dataset (first run takes a while)
-lfw_people = fetch_lfw_people(min_faces_per_person=1, resize=1, color=True, funneled=False)
-
-# introspect dataset
-n_samples, h, w, c = lfw_people.images.shape
-print(&#39;{:,} images at {}x{}&#39;.format(n_samples, w, h))
+import imutils</p>
+<h1>download LFW dataset (first run takes a while)</h1>
+<p>lfw_people = fetch_lfw_people(min_faces_per_person=1, resize=1, color=True, funneled=False)</p>
+<h1>introspect dataset</h1>
+<p>n_samples, h, w, c = lfw_people.images.shape
+print('{:,} images at {}x{}'.format(n_samples, w, h))
 cols, rows = (176, 76)
-n_ims = cols * rows
-
-# build montages
-im_scale = 0.5
+n_ims = cols * rows</p>
+<h1>build montages</h1>
+<p>im_scale = 0.5
 ims = lfw_people.images[:n_ims
-montages = imutils.build_montages(ims, (int(w*im_scale, int(h*im_scale)), (cols, rows))
-montage = montages[0]
-
-# save full montage image
-imageio.imwrite(&#39;lfw_montage_full.png&#39;, montage)
-
-# make a smaller version
-montage_960 = imutils.resize(montage, width=960)
-imageio.imwrite(&#39;lfw_montage_960.jpg&#39;, montage_960)
-</code></pre>
-<h2>Disclaimer</h2>
+montages = imutils.build_montages(ims, (int(w<em>im_scale, int(h</em>im_scale)), (cols, rows))
+montage = montages[0]</p>
+<h1>save full montage image</h1>
+<p>imageio.imwrite('lfw_montage_full.png', montage)</p>
+<h1>make a smaller version</h1>
+<p>montage_960 = imutils.resize(montage, width=960)
+imageio.imwrite('lfw_montage_960.jpg', montage_960)</p>
+</section><section><div class='applet' data-payload='{"command": ""}'></div></section><section><h2>Disclaimer</h2>
 <p>MegaPixels is an educational art project designed to encourage discourse about facial recognition datasets. Any ethical or legal issues should be directed to the researcher's parent organizations. Except where necessary for contact or clarity, the names of researchers have been subsituted by their parent organization. In no way does this project aim to villify researchers who produced the datasets.</p>
 <p>Read more about <a href="about/code-of-conduct">MegaPixels Code of Conduct</a></p>
 <div class="footnotes">
diff --git a/site/public/datasets/vgg_face2/index.html b/site/public/datasets/vgg_face2/index.html
index 24a1059b..817fc9a0 100644
--- a/site/public/datasets/vgg_face2/index.html
+++ b/site/public/datasets/vgg_face2/index.html
@@ -28,12 +28,7 @@
     
   <section><h1>VGG Faces2</h1>
 </section><section><div class='meta'><div><div class='gray'>Created</div><div>2018</div></div><div><div class='gray'>Images</div><div>3.3M</div></div><div><div class='gray'>People</div><div>9,000</div></div><div><div class='gray'>Created From</div><div>Scraping search engines</div></div><div><div class='gray'>Search available</div><div>[Searchable](#)</div></div></div></section><section><p>VGG Face2 is the updated version of the VGG Face dataset and now includes over 3.3M face images from over 9K people. The identities were selected by taking the top 500K identities in Google's Knowledge Graph of celebrities and then selecting only the names that yielded enough training images. The dataset was created in the UK but funded by Office of Director of National Intelligence in the United States.</p>
-<p>{INSERT IMAGE SEARCH MODULE}</p>
-<p>{INSERT TEXT SEARCH MODULE}</p>
-<pre><code>load file: lfw_names_gender_kg_min.csv
-Name, Images, Gender, Description
-</code></pre>
-<h2>VGG Face2 by the Numbers</h2>
+</section><section><div class='applet' data-payload='{"command": "face_search"}'></div></section><section><div class='applet' data-payload='{"command": "name_search"}'></div></section><section><div class='applet' data-payload='{"command": "load file", "opt": "lfw_names_gender_kg_min.csv", "fields": "Name, Images, Gender, Description"}'></div></section><section><h2>VGG Face2 by the Numbers</h2>
 <ul>
 <li>1,331 actresses, 139 presidents</li>
 <li>3 husbands and 16 wives</li>
@@ -47,7 +42,7 @@ Name, Images, Gender, Description
 <li>The original VGGF2 name list has been updated with the results returned from Google Knowledge</li>
 <li>Names with a similarity score greater than 0.75 where automatically updated. Scores computed using <code>import difflib; seq = difflib.SequenceMatcher(a=a.lower(), b=b.lower()); score = seq.ratio()</code></li>
 <li>The 97 names with a score of 0.75 or lower were manually reviewed and includes name changes validating using Wikipedia.org results for names such as "Bruce Jenner" to "Caitlyn Jenner", spousal last-name changes, and discretionary changes to improve search results such as combining nicknames with full name when appropriate, for example changing "Aleksandar Petrović" to "Aleksandar 'Aco' Petrović" and minor changes such as "Mohammad Ali" to "Muhammad Ali"</li>
-<li>The 'Description` text was automatically added when the Knowledge Graph score was greater than 250</li>
+<li>The 'Description' text was automatically added when the Knowledge Graph score was greater than 250</li>
 </ul>
 <h1>TODO</h1>
 <ul>
-- 
cgit v1.2.3-70-g09d2