moving stuff

8 files changed, 366 insertions, 11 deletions

diff --git a/megapixels/app/builder/README.md b/megapixels/app/builder/README.md
new file mode 100644
index 00000000..1a6d3a1e
--- /dev/null
+++ b/megapixels/app/builder/README.md
@@ -0,0 +1,21 @@
+Megapixels Static Site Generator
+================================
+
+The index, blog, and about other pages are built using this static site generator.
+
+## Metadata
+
+```
+status: published|draft|private
+title: From 1 to 100 Pixels
+desc: High resolution insights from low resolution imagery
+slug: from-1-to-100-pixels
+published: 2018-12-04
+updated: 2018-12-04
+authors: Adam Harvey, Berit Gilma, Matthew Stender
+```
+
+## S3 Assets
+
+Static assets: `v1/site/about/assets/picture.jpg`
+Dataset assets: `v1/datasets/lfw/assets/picture.jpg`
diff --git a/megapixels/app/builder/builder.py b/megapixels/app/builder/builder.py
new file mode 100644
index 00000000..620fc710
--- /dev/null
+++ b/megapixels/app/builder/builder.py
@@ -0,0 +1,90 @@
+#!/usr/bin/python
+
+from dotenv import load_dotenv
+load_dotenv()
+
+import os
+import glob
+from jinja2 import Environment, FileSystemLoader, select_autoescape
+
+import s3
+import parser
+from paths import *
+
+env = Environment(
+  loader=FileSystemLoader(template_path),
+  autoescape=select_autoescape([])
+)
+
+def build_page(fn, research_posts):
+  metadata, sections = parser.read_metadata(fn)
+
+  if metadata is None:
+    print("{} has no metadata".format(fn))
+    return
+
+  print(metadata['url'])
+
+  dirname = os.path.dirname(fn)
+  output_path = public_path + metadata['url']
+  output_fn = os.path.join(output_path, "index.html")
+
+  skip_h1 = False
+
+  if metadata['url'] == '/':
+    template = env.get_template("home.html")
+  elif 'research/' in fn:
+    skip_h1 = True
+    template = env.get_template("research.html")
+  else:
+    template = env.get_template("page.html")
+
+  if 'datasets/' in fn:
+    s3_dir = s3_datasets_path
+  else:
+    s3_dir = s3_site_path
+
+  s3_path = s3.make_s3_path(s3_dir, metadata['path'])
+
+  if 'index.md' in fn:
+    s3.sync_directory(dirname, s3_dir, metadata)
+
+  content = parser.parse_markdown(sections, s3_path, skip_h1=skip_h1)
+
+  html = template.render(
+    metadata=metadata,
+    content=content,
+    research_posts=research_posts,
+    latest_research_post=research_posts[-1],
+  )
+
+  os.makedirs(output_path, exist_ok=True)
+  with open(output_fn, "w") as file:
+    file.write(html)
+
+  print("______")
+
+def build_research_index(research_posts):
+  metadata, sections = parser.read_metadata('../site/content/research/index.md')
+  template = env.get_template("page.html")
+  s3_path = s3.make_s3_path(s3_site_path, metadata['path'])
+  content = parser.parse_markdown(sections, s3_path, skip_h1=False)
+  content += parser.parse_research_index(research_posts)
+  html = template.render(
+    metadata=metadata,
+    content=content,
+    research_posts=research_posts,
+    latest_research_post=research_posts[-1],
+  )
+  output_fn = public_path + '/research/index.html'
+  with open(output_fn, "w") as file:
+    file.write(html)
+
+def build_site():
+  research_posts = parser.read_research_post_index()
+  for fn in glob.iglob(os.path.join(content_path, "**/*.md"), recursive=True):
+    build_page(fn, research_posts)
+  build_research_index(research_posts)
+
+if __name__ == '__main__':
+  build_site()
diff --git a/megapixels/app/builder/parser.py b/megapixels/app/builder/parser.py
new file mode 100644
index 00000000..dd3643bf
--- /dev/null
+++ b/megapixels/app/builder/parser.py
@@ -0,0 +1,172 @@
+import os
+import re
+import glob
+import mistune
+
+import s3
+from paths import *
+
+renderer = mistune.Renderer(escape=False)
+markdown = mistune.Markdown(renderer=renderer)
+
+def fix_images(lines, s3_path):
+  real_lines = []
+  block = "\n\n".join(lines)
+  for line in block.split("\n"):
+    if "![" in line:
+      line = line.replace('![', '')
+      alt_text, tail = line.split('](', 1)
+      url, tail = tail.split(')', 1)
+      if ':' in alt_text:
+        tail, alt_text = alt_text.split(':', 1)
+      img_tag = "<img src='{}' alt='{}'>".format(s3_path + url, alt_text.replace("'", ""))
+      if len(alt_text):
+        line = "<div class='image'>{}<div class='caption'>{}</div></div>".format(img_tag, alt_text)
+      else:
+        line = "<div class='image'>{}</div>".format(img_tag, alt_text)
+    real_lines.append(line)
+  return "\n".join(real_lines)
+
+def format_section(lines, s3_path, type=''):
+  if len(lines):
+    lines = fix_images(lines, s3_path)
+    if type:
+      return "<section class='{}'>{}</section>".format(type, markdown(lines))
+    else:
+      return "<section>" + markdown(lines) + "</section>"
+  return ""
+
+def format_metadata(section):
+  meta = []
+  for line in section.split('\n'):
+    key, value = line[2:].split(': ', 1)
+    meta.append("<div><div class='gray'>{}</div><div>{}</div></div>".format(key, value))
+  return "<section><div class='meta'>{}</div></section>".format(''.join(meta)) 
+
+def parse_markdown(sections, s3_path, skip_h1=False):
+  groups = []
+  current_group = []
+  for section in sections:
+    if skip_h1 and section.startswith('# '):
+      continue
+    elif section.startswith('+ '):
+      groups.append(format_section(current_group, s3_path))
+      groups.append(format_metadata(section))
+      current_group = []
+    elif '![wide:' in section:
+      groups.append(format_section(current_group, s3_path))
+      groups.append(format_section([section], s3_path, type='wide'))
+      current_group = []
+    elif '![' in section:
+      groups.append(format_section(current_group, s3_path))
+      groups.append(format_section([section], s3_path, type='images'))
+      current_group = []
+    else:
+      current_group.append(section)
+  groups.append(format_section(current_group, s3_path))
+  content = "".join(groups)
+  return content
+
+def parse_research_index(research_posts):
+  content = "<div class='research_index'>"
+  for post in research_posts:
+    s3_path = s3.make_s3_path(s3_site_path, post['path'])
+    if 'image' in post:
+      post_image = s3_path + post['image']
+    else:
+      post_image = 'data:image/gif;base64,R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw=='
+    row = "<a href='{}'><section class='wide'><img src='{}' alt='Research post' /><section><h1>{}</h1><h2>{}</h2></section></section></a>".format(
+      post['path'],
+      post_image,
+      post['title'],
+      post['tagline'])
+    content += row
+  content += '</div>'
+  return content
+
+def read_metadata(fn):
+  with open(fn, "r") as file:
+    data = file.read()
+    data = data.replace("\n ", "\n")
+    if "\n" in data:
+      data = data.replace("\r", "")
+    else:
+      data = data.replace("\r", "\n")
+    sections = data.split("\n\n")
+  return parse_metadata(fn, sections)
+
+default_metadata = {
+  'status': 'published',
+  'title': 'Untitled Page',
+  'desc': '',
+  'slug': '',
+  'published': '2018-12-31',
+  'updated': '2018-12-31',
+  'authors': 'Adam Harvey',
+  'sync': 'true',
+  'tagline': '',
+}
+
+def parse_metadata_section(metadata, section):
+  for line in section.split("\n"):
+    if ': ' not in line:
+      continue
+    key, value = line.split(': ', 1)
+    metadata[key.lower()] = value
+
+def parse_metadata(fn, sections):
+  found_meta = False
+  metadata = {}
+  valid_sections = []
+  for section in sections:
+    if not found_meta and ': ' in section:
+      found_meta = True
+      parse_metadata_section(metadata, section)
+      continue
+    if '-----' in section:
+      continue
+    if found_meta:
+      valid_sections.append(section)
+
+  if 'title' not in metadata:
+    print('warning: {} has no title'.format(fn))
+  for key in default_metadata:
+    if key not in metadata:
+      metadata[key] = default_metadata[key]
+
+  basedir = os.path.dirname(fn.replace(content_path, ''))
+  basename = os.path.basename(fn)
+  if basedir == '/':
+    metadata['path'] = '/'
+    metadata['url'] = '/'
+  elif basename == 'index.md':
+    metadata['path'] = basedir + '/'
+    metadata['url'] = metadata['path']
+  else:
+    metadata['path'] = basedir + '/'
+    metadata['url'] = metadata['path'] + basename.replace('.md', '') + '/'
+
+  if metadata['status'] == 'published|draft|private':
+    metadata['status'] = 'published'
+
+  metadata['sync'] = metadata['sync'] != 'false'
+
+  metadata['author_html'] = '<br>'.join(metadata['authors'].split(','))
+  return metadata, valid_sections
+
+def read_research_post_index():
+  posts = []
+  for fn in sorted(glob.glob('../site/content/research/*/index.md')):
+    metadata, valid_sections = read_metadata(fn)
+    if metadata is None or metadata['status'] == 'private' or metadata['status'] == 'draft':
+      continue
+    posts.append(metadata)
+  if not len(posts):
+    posts.append({
+      'title': 'Placeholder',
+      'slug': 'placeholder',
+      'date': 'Placeholder',
+      'url': '/',
+    })
+  return posts
+
diff --git a/megapixels/app/builder/paths.py b/megapixels/app/builder/paths.py
new file mode 100644
index 00000000..356f2f3d
--- /dev/null
+++ b/megapixels/app/builder/paths.py
@@ -0,0 +1,6 @@
+
+s3_site_path = "v1/site"
+s3_datasets_path = "v1" # datasets is already in the filename
+public_path = "../site/public"
+content_path = "../site/content"
+template_path = "../site/templates"
diff --git a/megapixels/app/builder/s3.py b/megapixels/app/builder/s3.py
new file mode 100644
index 00000000..41ecdf61
--- /dev/null
+++ b/megapixels/app/builder/s3.py
@@ -0,0 +1,61 @@
+import os
+import glob
+import boto3
+from paths import *
+
+session = boto3.session.Session()
+
+s3_client = session.client(
+  service_name='s3',
+  aws_access_key_id=os.getenv('S3_KEY'),
+  aws_secret_access_key=os.getenv('S3_SECRET'),
+  endpoint_url=os.getenv('S3_ENDPOINT'),
+  region_name=os.getenv('S3_REGION'),
+)
+
+def sync_directory(base_fn, s3_path, metadata):
+  fns = {}
+  for fn in glob.glob(os.path.join(base_fn, 'assets/*')):
+    fns[os.path.basename(fn)] = True
+
+  if not metadata['sync']:
+    return
+
+  remote_path = s3_path + metadata['url']
+
+  directory = s3_client.list_objects(Bucket=os.getenv('S3_BUCKET'), Prefix=remote_path)
+  prefixes = []
+
+  if 'Contents' in directory:
+    for obj in directory['Contents']:
+      s3_fn = obj['Key']
+      fn = os.path.basename(s3_fn)
+      local_fn = os.path.join(base_fn, 'assets', fn)
+      if fn in fns:
+        del fns[fn]
+        if obj['LastModified'].timestamp() < os.path.getmtime(os.path.join(local_fn)):
+          print("s3 update {}".format(s3_fn))
+          s3_client.upload_file(
+            local_fn,
+            os.getenv('S3_BUCKET'),
+            s3_fn,
+            ExtraArgs={ 'ACL': 'public-read' })
+      else:
+        print("s3 delete {}".format(s3_fn))
+        response = s3_client.delete_object(
+          Bucket=os.getenv('S3_BUCKET'),
+          Key=s3_fn,
+        )
+
+  for fn in fns:
+    local_fn = os.path.join(base_fn, 'assets', fn)
+    s3_fn = os.path.join(remote_path, 'assets', fn)
+    print("s3 create {}".format(s3_fn))
+    s3_client.upload_file(
+      local_fn,
+      os.getenv('S3_BUCKET'),
+      s3_fn,
+      ExtraArgs={ 'ACL': 'public-read' })
+
+def make_s3_path(s3_dir, metadata_path):
+  return "{}/{}/{}{}".format(os.getenv('S3_ENDPOINT'), os.getenv('S3_BUCKET'), s3_dir, metadata_path)
diff --git a/megapixels/app/server/create.py b/megapixels/app/server/create.py
index 9efed669..c1f41dc4 100644
--- a/megapixels/app/server/create.py
+++ b/megapixels/app/server/create.py
@@ -1,4 +1,4 @@
-from flask import Flask, Blueprint, jsonify
+from flask import Flask, Blueprint, jsonify, send_from_directory
 from flask_sqlalchemy import SQLAlchemy
 from app.models.sql_factory import connection_url, load_sql_datasets
 
@@ -7,7 +7,7 @@ from app.server.api import api
 db = SQLAlchemy()
 
 def create_app(script_info=None):
-  app = Flask(__name__, static_url_path='')
+  app = Flask(__name__, static_folder='static', static_url_path='')
   app.config['SQLALCHEMY_DATABASE_URI'] = connection_url
   app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
 
@@ -15,6 +15,7 @@ def create_app(script_info=None):
   datasets = load_sql_datasets(replace=False, base_model=db.Model)
 
   app.register_blueprint(api, url_prefix='/api')
+  app.add_url_rule('/<path:file_relative_path_to_root>', 'serve_page', serve_page, methods=['GET'])
 
   @app.route('/', methods=['GET'])
   def index():
@@ -34,3 +35,8 @@ def create_app(script_info=None):
     return(jsonify(links))
 
   return app
+
+def serve_page(file_relative_path_to_root):
+  if file_relative_path_to_root[-1] == '/':
+    file_relative_path_to_root += 'index.html'
+  return send_from_directory("static", file_relative_path_to_root)
diff --git a/megapixels/app/settings/app_cfg.py b/megapixels/app/settings/app_cfg.py
index 0507366f..1d3fbc4c 100644
--- a/megapixels/app/settings/app_cfg.py
+++ b/megapixels/app/settings/app_cfg.py
@@ -58,6 +58,13 @@ DIR_FAISS_RECIPES = join(DIR_FAISS, 'recipes')
 DIR_TEST_IMAGES = join(DIR_APP, 'test', 'images')
 
 # -----------------------------------------------------------------------------
+# .env config for keys
+# -----------------------------------------------------------------------------
+
+DIR_DOTENV = join(DIR_APP, '.env')
+load_dotenv(dotenv_path=DIR_DOTENV)
+
+# -----------------------------------------------------------------------------
 # Drawing, GUI settings
 # -----------------------------------------------------------------------------
 DIR_ASSETS = join(DIR_APP, 'assets')
@@ -69,6 +76,7 @@ FP_FONT = join(DIR_ASSETS, 'font')
 # -----------------------------------------------------------------------------
 DIR_COMMANDS_CV = 'commands/cv'
 DIR_COMMANDS_ADMIN = 'commands/admin'
+DIR_COMMANDS_BUILDER = 'commands/builder'
 DIR_COMMANDS_DATASETS = 'commands/datasets'
 DIR_COMMANDS_FAISS = 'commands/faiss'
 DIR_COMMANDS_MISC  = 'commands/misc'
@@ -118,10 +126,3 @@ LOGFILE_FORMAT = "%(log_color)s%(levelname)-8s%(reset)s %(cyan)s%(filename)s:%(l
 # -----------------------------------------------------------------------------
 S3_MEDIA_ROOT = 's3://megapixels/v1/media/'
 S3_METADATA_ROOT = 's3://megapixels/v1/metadata/'
-
-# -----------------------------------------------------------------------------
-# .env config for keys
-# -----------------------------------------------------------------------------
-
-DIR_DOTENV = join(DIR_APP, '.env')
-load_dotenv(dotenv_path=DIR_DOTENV)
diff --git a/megapixels/commands/faiss/build_faiss.py b/megapixels/commands/faiss/build_faiss.py
index 96d3f99e..ec94c924 100644
--- a/megapixels/commands/faiss/build_faiss.py
+++ b/megapixels/commands/faiss/build_faiss.py
@@ -12,8 +12,6 @@ import numpy as np
 from app.utils.file_utils import load_recipe, load_csv_safe
 from app.settings import app_cfg as cfg
 
-engine = create_engine('sqlite:///:memory:')
-
 class DefaultRecipe:
   def __init__(self):
     self.dim = 128