From b1e7dc570fe25749a2e1b02c9e859df6588b4660 Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Sat, 15 Dec 2018 22:04:41 +0100 Subject: move builder --- megapixels/app/site/parser.py | 187 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 megapixels/app/site/parser.py (limited to 'megapixels/app/site/parser.py') diff --git a/megapixels/app/site/parser.py b/megapixels/app/site/parser.py new file mode 100644 index 00000000..add3f386 --- /dev/null +++ b/megapixels/app/site/parser.py @@ -0,0 +1,187 @@ +import os +import re +import glob +import mistune + +import app.settings.app_cfg as cfg +import app.builder.s3 as s3 + +renderer = mistune.Renderer(escape=False) +markdown = mistune.Markdown(renderer=renderer) + +def fix_images(lines, s3_path): + real_lines = [] + block = "\n\n".join(lines) + for line in block.split("\n"): + if "![" in line: + line = line.replace('![', '') + alt_text, tail = line.split('](', 1) + url, tail = tail.split(')', 1) + if ':' in alt_text: + tail, alt_text = alt_text.split(':', 1) + img_tag = "{}".format(s3_path + url, alt_text.replace("'", "")) + if len(alt_text): + line = "
{}
{}
".format(img_tag, alt_text) + else: + line = "
{}
".format(img_tag, alt_text) + real_lines.append(line) + return "\n".join(real_lines) + +def format_section(lines, s3_path, type=''): + if len(lines): + lines = fix_images(lines, s3_path) + if type: + return "
{}
".format(type, markdown(lines)) + else: + return "
" + markdown(lines) + "
" + return "" + +def format_metadata(section): + meta = [] + for line in section.split('\n'): + key, value = line[2:].split(': ', 1) + meta.append("
{}
{}
".format(key, value)) + return "
{}
".format(''.join(meta)) + +def format_applet(section): + payload = section.replace('```', '').strip().split('\n') + if ': ' in payload[0]: + command, opt = payload[0].split(': ') + else: + command = payload[0] + opt = None + if command == 'load_file': + return "
{}
" + + +def parse_markdown(sections, s3_path, skip_h1=False): + groups = [] + current_group = [] + for section in sections: + if skip_h1 and section.startswith('# '): + continue + elif section.startsWith('```'): + groups.append(format_section(current_group, s3_path)) + groups.append(format_applet(section)) + current_group = [] + elif section.startswith('+ '): + groups.append(format_section(current_group, s3_path)) + groups.append(format_metadata(section)) + current_group = [] + elif '![wide:' in section: + groups.append(format_section(current_group, s3_path)) + groups.append(format_section([section], s3_path, type='wide')) + current_group = [] + elif '![' in section: + groups.append(format_section(current_group, s3_path)) + groups.append(format_section([section], s3_path, type='images')) + current_group = [] + else: + current_group.append(section) + groups.append(format_section(current_group, s3_path)) + content = "".join(groups) + return content + +def parse_research_index(research_posts): + content = "
" + for post in research_posts: + s3_path = s3.make_s3_path(cfg.S3_SITE_PATH, post['path']) + if 'image' in post: + post_image = s3_path + post['image'] + else: + post_image = 'data:image/gif;base64,R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==' + row = "
Research post

{}

{}

".format( + post['path'], + post_image, + post['title'], + post['tagline']) + content += row + content += '
' + return content + +def read_metadata(fn): + with open(fn, "r") as file: + data = file.read() + data = data.replace("\n ", "\n") + if "\n" in data: + data = data.replace("\r", "") + else: + data = data.replace("\r", "\n") + sections = data.split("\n\n") + return parse_metadata(fn, sections) + +default_metadata = { + 'status': 'published', + 'title': 'Untitled Page', + 'desc': '', + 'slug': '', + 'published': '2018-12-31', + 'updated': '2018-12-31', + 'authors': 'Adam Harvey', + 'sync': 'true', + 'tagline': '', +} + +def parse_metadata_section(metadata, section): + for line in section.split("\n"): + if ': ' not in line: + continue + key, value = line.split(': ', 1) + metadata[key.lower()] = value + +def parse_metadata(fn, sections): + found_meta = False + metadata = {} + valid_sections = [] + for section in sections: + if not found_meta and ': ' in section: + found_meta = True + parse_metadata_section(metadata, section) + continue + if '-----' in section: + continue + if found_meta: + valid_sections.append(section) + + if 'title' not in metadata: + print('warning: {} has no title'.format(fn)) + for key in default_metadata: + if key not in metadata: + metadata[key] = default_metadata[key] + + basedir = os.path.dirname(fn.replace(cfg.DIR_SITE_CONTENT, '')) + basename = os.path.basename(fn) + if basedir == '/': + metadata['path'] = '/' + metadata['url'] = '/' + elif basename == 'index.md': + metadata['path'] = basedir + '/' + metadata['url'] = metadata['path'] + else: + metadata['path'] = basedir + '/' + metadata['url'] = metadata['path'] + basename.replace('.md', '') + '/' + + if metadata['status'] == 'published|draft|private': + metadata['status'] = 'published' + + metadata['sync'] = metadata['sync'] != 'false' + + metadata['author_html'] = '
'.join(metadata['authors'].split(',')) + return metadata, valid_sections + +def read_research_post_index(): + posts = [] + for fn in sorted(glob.glob('../site/content/research/*/index.md')): + metadata, valid_sections = read_metadata(fn) + if metadata is None or metadata['status'] == 'private' or metadata['status'] == 'draft': + continue + posts.append(metadata) + if not len(posts): + posts.append({ + 'title': 'Placeholder', + 'slug': 'placeholder', + 'date': 'Placeholder', + 'url': '/', + }) + return posts + -- cgit v1.2.3-70-g09d2 From 898e6cdf8df0993f853b748d4e8a9c269fad0294 Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Sat, 15 Dec 2018 22:14:17 +0100 Subject: inject applet payload --- megapixels/app/settings/app_cfg.py | 2 +- megapixels/app/site/builder.py | 4 +-- megapixels/app/site/parser.py | 15 ++++++--- megapixels/commands/site/build.py | 2 +- site/public/about/style/index.html | 11 +------ site/public/datasets/lfw/index.html | 51 ++++++++++++------------------- site/public/datasets/vgg_face2/index.html | 9 ++---- 7 files changed, 36 insertions(+), 58 deletions(-) (limited to 'megapixels/app/site/parser.py') diff --git a/megapixels/app/settings/app_cfg.py b/megapixels/app/settings/app_cfg.py index a18e5875..d25936e6 100644 --- a/megapixels/app/settings/app_cfg.py +++ b/megapixels/app/settings/app_cfg.py @@ -75,10 +75,10 @@ FP_FONT = join(DIR_ASSETS, 'font') # ----------------------------------------------------------------------------- DIR_COMMANDS_CV = 'commands/cv' DIR_COMMANDS_ADMIN = 'commands/admin' -DIR_COMMANDS_SITE = 'commands/site' DIR_COMMANDS_DATASETS = 'commands/datasets' DIR_COMMANDS_FAISS = 'commands/faiss' DIR_COMMANDS_MISC = 'commands/misc' +DIR_COMMANDS_SITE = 'commands/site' # ----------------------------------------------------------------------------- # Filesystem settings diff --git a/megapixels/app/site/builder.py b/megapixels/app/site/builder.py index df609f60..42e25768 100644 --- a/megapixels/app/site/builder.py +++ b/megapixels/app/site/builder.py @@ -6,8 +6,8 @@ from jinja2 import Environment, FileSystemLoader, select_autoescape import app.settings.app_cfg as cfg -import app.builder.s3 as s3 -import app.builder.parser as parser +import app.site.s3 as s3 +import app.site.parser as parser env = Environment( loader=FileSystemLoader(cfg.DIR_SITE_TEMPLATES), diff --git a/megapixels/app/site/parser.py b/megapixels/app/site/parser.py index add3f386..d78cc402 100644 --- a/megapixels/app/site/parser.py +++ b/megapixels/app/site/parser.py @@ -1,10 +1,11 @@ import os import re import glob +import simplejson as json import mistune import app.settings.app_cfg as cfg -import app.builder.s3 as s3 +import app.site.s3 as s3 renderer = mistune.Renderer(escape=False) markdown = mistune.Markdown(renderer=renderer) @@ -45,14 +46,18 @@ def format_metadata(section): def format_applet(section): payload = section.replace('```', '').strip().split('\n') + applet = {} if ': ' in payload[0]: command, opt = payload[0].split(': ') else: command = payload[0] opt = None - if command == 'load_file': - return "
{}
" - + applet['command'] = command + if opt: + applet['opt'] = opt + if command == 'load file': + applet['fields'] = payload[1] + return "
".format(json.dumps(applet)) def parse_markdown(sections, s3_path, skip_h1=False): groups = [] @@ -60,7 +65,7 @@ def parse_markdown(sections, s3_path, skip_h1=False): for section in sections: if skip_h1 and section.startswith('# '): continue - elif section.startsWith('```'): + elif section.startswith('```'): groups.append(format_section(current_group, s3_path)) groups.append(format_applet(section)) current_group = [] diff --git a/megapixels/commands/site/build.py b/megapixels/commands/site/build.py index fc4fb302..0a76a9ac 100644 --- a/megapixels/commands/site/build.py +++ b/megapixels/commands/site/build.py @@ -4,7 +4,7 @@ Build the static site import click -from app.builder.builder import build_site +from app.site.builder import build_site @click.command() @click.pass_context diff --git a/site/public/about/style/index.html b/site/public/about/style/index.html index f2c0d4b8..39a44380 100644 --- a/site/public/about/style/index.html +++ b/site/public/about/style/index.html @@ -51,16 +51,7 @@
Person 3. Let me tell you about Person 3.  This person has a very long description with text which wraps like crazy
Person 3. Let me tell you about Person 3. This person has a very long description with text which wraps like crazy

est, qui dolorem ipsum, quia dolor sit amet consectetur adipisci[ng] velit, sed quia non-numquam [do] eius modi tempora inci[di]dunt, ut labore et dolore magnam aliquam quaerat voluptatem.

This image is extremely wide and the text beneath it will wrap but thats fine because it can also contain <a href="https://example.com/">hyperlinks</a>! Yes, you read that right—hyperlinks! Lorem ipsum dolor sit amet ad volotesque sic hoc ad nauseam
This image is extremely wide and the text beneath it will wrap but that's fine because it can also contain hyperlinks! Yes, you read that right—hyperlinks! Lorem ipsum dolor sit amet ad volotesque sic hoc ad nauseam

Inline code has back-ticks around it.

-
var s = "JavaScript syntax highlighting";
-alert(s);
-
-
s = "Python syntax highlighting"
-print(s)
-
-
No language indicated, so no syntax highlighting. 
-But let's throw in a <b>tag</b>.
-
-

Horizontal rule

+

Horizontal rule


Citations below here

diff --git a/site/public/datasets/lfw/index.html b/site/public/datasets/lfw/index.html index e080229f..3c83acd3 100644 --- a/site/public/datasets/lfw/index.html +++ b/site/public/datasets/lfw/index.html @@ -28,12 +28,7 @@

Labeled Faces in the Wild

Created
2007
Images
13,233
People
5,749
Created From
Yahoo News images
Search available
Searchable

Labeled Faces in The Wild (LFW) is amongst the most widely used facial recognition training datasets in the world and is the first of its kind to be created entirely from images posted online. The LFW dataset includes 13,233 images of 5,749 people that were collected between 2002-2004. Use the tools below to check if you were included in this dataset or scroll down to read the analysis.

-

{INSERT IMAGE SEARCH MODULE}

-

{INSERT TEXT SEARCH MODULE}

-
load file: lfw_names_gender_kg_min.csv
-Name, Images, Gender, Description
-
-
Eighteen of the 5,749 people in the Labeled Faces in the Wild Dataset. The most widely used face dataset for benchmarking commercial face recognition algorithms.
Eighteen of the 5,749 people in the Labeled Faces in the Wild Dataset. The most widely used face dataset for benchmarking commercial face recognition algorithms.

Intro

+
Eighteen of the 5,749 people in the Labeled Faces in the Wild Dataset. The most widely used face dataset for benchmarking commercial face recognition algorithms.
Eighteen of the 5,749 people in the Labeled Faces in the Wild Dataset. The most widely used face dataset for benchmarking commercial face recognition algorithms.

Intro

Three paragraphs describing the LFW dataset in a format that can be easily replicated for the other datasets. Nothing too custom. An analysis of the initial research papers with context relative to all the other dataset papers.

 all 5,749 people in the LFW Dataset sorted from most to least images collected.
all 5,749 people in the LFW Dataset sorted from most to least images collected.

LFW by the Numbers

    @@ -224,36 +219,28 @@ name_display,company_url,example_url,country,description

    Code

    -
    #!/usr/bin/python
    -
    -import numpy as np
    +

import numpy as np from sklearn.datasets import fetch_lfw_people import imageio -import imutils - -# download LFW dataset (first run takes a while) -lfw_people = fetch_lfw_people(min_faces_per_person=1, resize=1, color=True, funneled=False) - -# introspect dataset -n_samples, h, w, c = lfw_people.images.shape -print('{:,} images at {}x{}'.format(n_samples, w, h)) +import imutils

+

download LFW dataset (first run takes a while)

+

lfw_people = fetch_lfw_people(min_faces_per_person=1, resize=1, color=True, funneled=False)

+

introspect dataset

+

n_samples, h, w, c = lfw_people.images.shape +print('{:,} images at {}x{}'.format(n_samples, w, h)) cols, rows = (176, 76) -n_ims = cols * rows - -# build montages -im_scale = 0.5 +n_ims = cols * rows

+

build montages

+

im_scale = 0.5 ims = lfw_people.images[:n_ims -montages = imutils.build_montages(ims, (int(w*im_scale, int(h*im_scale)), (cols, rows)) -montage = montages[0] - -# save full montage image -imageio.imwrite('lfw_montage_full.png', montage) - -# make a smaller version -montage_960 = imutils.resize(montage, width=960) -imageio.imwrite('lfw_montage_960.jpg', montage_960) - -

Disclaimer

+montages = imutils.build_montages(ims, (int(wim_scale, int(him_scale)), (cols, rows)) +montage = montages[0]

+

save full montage image

+

imageio.imwrite('lfw_montage_full.png', montage)

+

make a smaller version

+

montage_960 = imutils.resize(montage, width=960) +imageio.imwrite('lfw_montage_960.jpg', montage_960)

+

Disclaimer

MegaPixels is an educational art project designed to encourage discourse about facial recognition datasets. Any ethical or legal issues should be directed to the researcher's parent organizations. Except where necessary for contact or clarity, the names of researchers have been subsituted by their parent organization. In no way does this project aim to villify researchers who produced the datasets.

Read more about MegaPixels Code of Conduct

diff --git a/site/public/datasets/vgg_face2/index.html b/site/public/datasets/vgg_face2/index.html index 24a1059b..817fc9a0 100644 --- a/site/public/datasets/vgg_face2/index.html +++ b/site/public/datasets/vgg_face2/index.html @@ -28,12 +28,7 @@

VGG Faces2

Created
2018
Images
3.3M
People
9,000
Created From
Scraping search engines
Search available
[Searchable](#)

VGG Face2 is the updated version of the VGG Face dataset and now includes over 3.3M face images from over 9K people. The identities were selected by taking the top 500K identities in Google's Knowledge Graph of celebrities and then selecting only the names that yielded enough training images. The dataset was created in the UK but funded by Office of Director of National Intelligence in the United States.

-

{INSERT IMAGE SEARCH MODULE}

-

{INSERT TEXT SEARCH MODULE}

-
load file: lfw_names_gender_kg_min.csv
-Name, Images, Gender, Description
-
-

VGG Face2 by the Numbers

+

VGG Face2 by the Numbers

  • 1,331 actresses, 139 presidents
  • 3 husbands and 16 wives
  • @@ -47,7 +42,7 @@ Name, Images, Gender, Description
  • The original VGGF2 name list has been updated with the results returned from Google Knowledge
  • Names with a similarity score greater than 0.75 where automatically updated. Scores computed using import difflib; seq = difflib.SequenceMatcher(a=a.lower(), b=b.lower()); score = seq.ratio()
  • The 97 names with a score of 0.75 or lower were manually reviewed and includes name changes validating using Wikipedia.org results for names such as "Bruce Jenner" to "Caitlyn Jenner", spousal last-name changes, and discretionary changes to improve search results such as combining nicknames with full name when appropriate, for example changing "Aleksandar Petrović" to "Aleksandar 'Aco' Petrović" and minor changes such as "Mohammad Ali" to "Muhammad Ali"
  • -
  • The 'Description` text was automatically added when the Knowledge Graph score was greater than 250
  • +
  • The 'Description' text was automatically added when the Knowledge Graph score was greater than 250

TODO

    -- cgit v1.2.3-70-g09d2 From 3b10acc73247ec703ed47f0423e7d255a91f074e Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Sun, 16 Dec 2018 00:11:20 +0100 Subject: tabulator --- client/index.js | 29 +- megapixels/app/server/api.py | 17 +- megapixels/app/server/create.py | 7 + megapixels/app/site/builder.py | 10 + megapixels/app/site/parser.py | 36 +- megapixels/app/site/s3.py | 3 + site/assets/css/applets.css | 2 - site/assets/css/tabulator.css | 761 +++++++++++++++++++++ site/public/about/credits/index.html | 2 +- site/public/about/disclaimer/index.html | 2 +- site/public/about/index.html | 2 +- site/public/about/press/index.html | 2 +- site/public/about/privacy/index.html | 2 +- site/public/about/style/index.html | 2 +- site/public/about/terms/index.html | 2 +- site/public/datasets/lfw/index.html | 6 +- site/public/datasets/vgg_face2/index.html | 4 +- site/public/index.html | 2 +- site/public/research/00_introduction/index.html | 2 +- .../research/01_from_1_to_100_pixels/index.html | 2 +- site/public/research/index.html | 2 +- site/templates/layout.html | 1 + 22 files changed, 862 insertions(+), 36 deletions(-) create mode 100755 site/assets/css/tabulator.css (limited to 'megapixels/app/site/parser.py') diff --git a/client/index.js b/client/index.js index bc57f548..2ee12c14 100644 --- a/client/index.js +++ b/client/index.js @@ -23,23 +23,24 @@ function appendTabulator(el, payload) { height: '311px', layout: 'fitColumns', placeholder: 'No Data Set', - columns: [ - // {title:'Name', field:'name', sorter:'string', width:200}, - // {title:'Progress', field:'progress', sorter:'number', formatter:'progress'}, - // {title:'Gender', field:'gender', sorter:"string"}, - // {title:"Rating", field:"rating", formatter:"star", align:"center", width:100}, - // {title:"Favourite Color", field:"col", sorter:"string", sortable:false}, - // {title:"Date Of Birth", field:"dob", sorter:"date", align:"center"}, - // {title:"Driver", field:"car", align:"center", formatter:"tickCross", sorter:"boolean"}, - ], + columns: payload.fields.split(', ').map(field => { + switch (field) { + default: + return { title: field, field: field.toLowerCase(), sorter: 'string' } + } + }), + // {title:'Name', field:'name', sorter:'string', width:200}, + // {title:'Progress', field:'progress', sorter:'number', formatter:'progress'}, + // {title:'Gender', field:'gender', sorter:"string"}, + // {title:"Rating", field:"rating", formatter:"star", align:"center", width:100}, + // {title:"Favourite Color", field:"col", sorter:"string", sortable:false}, + // {title:"Date Of Birth", field:"dob", sorter:"date", align:"center"}, + // {title:"Driver", field:"car", align:"center", formatter:"tickCross", sorter:"boolean"}, }) let path = payload.opt let columns = payload.fields.split(',').map(s => s.trim()) - console.log(columns) - if (path[0] !== '/') { - console.log(path) - } - // table.setData(path) + console.log(path, columns) + table.setData(path) } function appendApplets() { diff --git a/megapixels/app/server/api.py b/megapixels/app/server/api.py index cd2b950b..cf8241bd 100644 --- a/megapixels/app/server/api.py +++ b/megapixels/app/server/api.py @@ -1,9 +1,12 @@ import os import re import time +import dlib from flask import Blueprint, request, jsonify from PIL import Image # todo: try to remove PIL dependency +from app.processors import face_recognition +from app.processors import face_detector from app.models.sql_factory import list_datasets, get_dataset, get_table sanitize_re = re.compile('[\W]+') @@ -39,8 +42,18 @@ def upload(name): img = Image.open(file.stream).convert('RGB') - # vec = db.load_feature_vector_from_file(uploaded_img_path) - # vec = fe.extract(img) + # Face detection + detector = face_detector.DetectorDLIBHOG() + + # get detection as BBox object + bboxes = detector.detect(im, largest=True) + bbox = bboxes[0] + dim = im.shape[:2][::-1] + bbox = bbox.to_dim(dim) # convert back to real dimensions + + # face recognition/vector + recognition = face_recognition.RecognitionDLIB(gpu=-1) + # print(vec.shape) # results = db.search(vec, limit=limit) diff --git a/megapixels/app/server/create.py b/megapixels/app/server/create.py index c1f41dc4..4b1333b9 100644 --- a/megapixels/app/server/create.py +++ b/megapixels/app/server/create.py @@ -7,6 +7,9 @@ from app.server.api import api db = SQLAlchemy() def create_app(script_info=None): + """ + functional pattern for creating the flask app + """ app = Flask(__name__, static_folder='static', static_url_path='') app.config['SQLALCHEMY_DATABASE_URI'] = connection_url app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False @@ -37,6 +40,10 @@ def create_app(script_info=None): return app def serve_page(file_relative_path_to_root): + """ + trying to get this to serve /path/ with /path/index.html, + ...but it doesnt actually matter for production... + """ if file_relative_path_to_root[-1] == '/': file_relative_path_to_root += 'index.html' return send_from_directory("static", file_relative_path_to_root) diff --git a/megapixels/app/site/builder.py b/megapixels/app/site/builder.py index 91df54c2..895f265b 100644 --- a/megapixels/app/site/builder.py +++ b/megapixels/app/site/builder.py @@ -15,6 +15,10 @@ env = Environment( ) def build_page(fn, research_posts): + """ + build a single page from markdown into the appropriate template + - writes it to site/public/ + """ metadata, sections = parser.read_metadata(fn) if metadata is None: @@ -61,6 +65,9 @@ def build_page(fn, research_posts): file.write(html) def build_research_index(research_posts): + """ + build the index of research (blog) posts + """ metadata, sections = parser.read_metadata('../site/content/research/index.md') template = env.get_template("page.html") s3_path = s3.make_s3_path(cfg.S3_SITE_PATH, metadata['path']) @@ -77,6 +84,9 @@ def build_research_index(research_posts): file.write(html) def build_site(): + """ + build the site! =^) + """ research_posts = parser.read_research_post_index() for fn in glob.iglob(os.path.join(cfg.DIR_SITE_CONTENT, "**/*.md"), recursive=True): build_page(fn, research_posts) diff --git a/megapixels/app/site/parser.py b/megapixels/app/site/parser.py index d78cc402..40d9c7f6 100644 --- a/megapixels/app/site/parser.py +++ b/megapixels/app/site/parser.py @@ -11,6 +11,10 @@ renderer = mistune.Renderer(escape=False) markdown = mistune.Markdown(renderer=renderer) def fix_images(lines, s3_path): + """ + do our own tranformation of the markdown around images to handle wide images etc + lines: markdown lines + """ real_lines = [] block = "\n\n".join(lines) for line in block.split("\n"): @@ -29,6 +33,9 @@ def fix_images(lines, s3_path): return "\n".join(real_lines) def format_section(lines, s3_path, type=''): + """ + format a normal markdown section + """ if len(lines): lines = fix_images(lines, s3_path) if type: @@ -38,13 +45,16 @@ def format_section(lines, s3_path, type=''): return "" def format_metadata(section): + """ + format a metadata section (+ key: value pairs) + """ meta = [] for line in section.split('\n'): key, value = line[2:].split(': ', 1) meta.append("
    {}
    {}
    ".format(key, value)) return "
    {}
    ".format(''.join(meta)) -def format_applet(section): +def format_applet(section, s3_path): payload = section.replace('```', '').strip().split('\n') applet = {} if ': ' in payload[0]: @@ -56,10 +66,15 @@ def format_applet(section): if opt: applet['opt'] = opt if command == 'load file': + if opt[0] != '/': + applet['opt'] = s3_path + opt applet['fields'] = payload[1] return "
    ".format(json.dumps(applet)) def parse_markdown(sections, s3_path, skip_h1=False): + """ + parse page into sections, preprocess the markdown to handle our modifications + """ groups = [] current_group = [] for section in sections: @@ -67,7 +82,7 @@ def parse_markdown(sections, s3_path, skip_h1=False): continue elif section.startswith('```'): groups.append(format_section(current_group, s3_path)) - groups.append(format_applet(section)) + groups.append(format_applet(section, s3_path)) current_group = [] elif section.startswith('+ '): groups.append(format_section(current_group, s3_path)) @@ -88,6 +103,9 @@ def parse_markdown(sections, s3_path, skip_h1=False): return content def parse_research_index(research_posts): + """ + Generate an index file for the research pages + """ content = "
    " for post in research_posts: s3_path = s3.make_s3_path(cfg.S3_SITE_PATH, post['path']) @@ -105,6 +123,9 @@ def parse_research_index(research_posts): return content def read_metadata(fn): + """ + Read in read a markdown file and extract the metadata + """ with open(fn, "r") as file: data = file.read() data = data.replace("\n ", "\n") @@ -128,6 +149,9 @@ default_metadata = { } def parse_metadata_section(metadata, section): + """ + parse a metadata key: value pair + """ for line in section.split("\n"): if ': ' not in line: continue @@ -135,6 +159,11 @@ def parse_metadata_section(metadata, section): metadata[key.lower()] = value def parse_metadata(fn, sections): + """ + parse the metadata headers in a markdown file + (everything before the second ---------) + also generates appropriate urls for this page :) + """ found_meta = False metadata = {} valid_sections = [] @@ -175,6 +204,9 @@ def parse_metadata(fn, sections): return metadata, valid_sections def read_research_post_index(): + """ + Generate an index of the research (blog) posts + """ posts = [] for fn in sorted(glob.glob('../site/content/research/*/index.md')): metadata, valid_sections = read_metadata(fn) diff --git a/megapixels/app/site/s3.py b/megapixels/app/site/s3.py index 99726a4d..5464d464 100644 --- a/megapixels/app/site/s3.py +++ b/megapixels/app/site/s3.py @@ -3,6 +3,9 @@ import glob import boto3 def sync_directory(base_fn, s3_path, metadata): + """ + Synchronize a local assets folder with S3 + """ fns = {} for fn in glob.glob(os.path.join(base_fn, 'assets/*')): fns[os.path.basename(fn)] = True diff --git a/site/assets/css/applets.css b/site/assets/css/applets.css index b437886b..e106b3dd 100644 --- a/site/assets/css/applets.css +++ b/site/assets/css/applets.css @@ -1,4 +1,2 @@ .applet { - border: 1px solid #0f0; - color: #0f0; } diff --git a/site/assets/css/tabulator.css b/site/assets/css/tabulator.css new file mode 100755 index 00000000..8c123cf4 --- /dev/null +++ b/site/assets/css/tabulator.css @@ -0,0 +1,761 @@ +/* Tabulator v4.1.3 (c) Oliver Folkerd */ +.tabulator { + position: relative; + font-size: 14px; + text-align: left; + overflow: hidden; + -ms-transform: translatez(0); + transform: translatez(0); +} + +.tabulator[tabulator-layout="fitDataFill"] .tabulator-tableHolder .tabulator-table { + min-width: 100%; +} + +.tabulator.tabulator-block-select { + -webkit-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + user-select: none; +} + +.tabulator .tabulator-header { + position: relative; + box-sizing: border-box; + width: 100%; + border-bottom: 1px solid #999; + color: #ddd; + white-space: nowrap; + overflow: hidden; + -moz-user-select: none; + -khtml-user-select: none; + -webkit-user-select: none; + -o-user-select: none; +} + +.tabulator .tabulator-header .tabulator-col { + display: inline-block; + position: relative; + box-sizing: border-box; + border-right: 1px solid #aaa; + text-align: left; + vertical-align: bottom; + overflow: hidden; +} + +.tabulator .tabulator-header .tabulator-col.tabulator-moving { + position: absolute; + border: 1px solid #999; + background: rgba(80,20,10,0.2); + pointer-events: none; +} + +.tabulator .tabulator-header .tabulator-col .tabulator-col-content { + box-sizing: border-box; + position: relative; + padding: 4px; +} + +.tabulator .tabulator-header .tabulator-col .tabulator-col-content .tabulator-col-title { + box-sizing: border-box; + width: 100%; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + vertical-align: bottom; +} + +.tabulator .tabulator-header .tabulator-col .tabulator-col-content .tabulator-col-title .tabulator-title-editor { + box-sizing: border-box; + width: 100%; + border: 1px solid #999; + padding: 1px; + background: #fff; +} + +.tabulator .tabulator-header .tabulator-col .tabulator-col-content .tabulator-arrow { + display: inline-block; + position: absolute; + top: 9px; + right: 8px; + width: 0; + height: 0; + border-left: 6px solid transparent; + border-right: 6px solid transparent; + border-bottom: 6px solid #bbb; +} + +.tabulator .tabulator-header .tabulator-col.tabulator-col-group .tabulator-col-group-cols { + position: relative; + display: -ms-flexbox; + display: flex; + border-top: 1px solid #aaa; + overflow: hidden; +} + +.tabulator .tabulator-header .tabulator-col.tabulator-col-group .tabulator-col-group-cols .tabulator-col:last-child { + margin-right: -1px; +} + +.tabulator .tabulator-header .tabulator-col:first-child .tabulator-col-resize-handle.prev { + display: none; +} + +.tabulator .tabulator-header .tabulator-col.ui-sortable-helper { + position: absolute; + border: 1px solid #aaa; +} + +.tabulator .tabulator-header .tabulator-col .tabulator-header-filter { + position: relative; + box-sizing: border-box; + margin-top: 2px; + width: 100%; + text-align: center; +} + +.tabulator .tabulator-header .tabulator-col .tabulator-header-filter textarea { + height: auto !important; +} + +.tabulator .tabulator-header .tabulator-col .tabulator-header-filter svg { + margin-top: 3px; +} + +.tabulator .tabulator-header .tabulator-col .tabulator-header-filter input::-ms-clear { + width: 0; + height: 0; +} + +.tabulator .tabulator-header .tabulator-col.tabulator-sortable .tabulator-col-title { + padding-right: 25px; +} + +.tabulator .tabulator-header .tabulator-col.tabulator-sortable:hover { + cursor: pointer; + background-color: #cdcdcd; +} + +.tabulator .tabulator-header .tabulator-col.tabulator-sortable[aria-sort="none"] .tabulator-col-content .tabulator-arrow { + border-top: none; + border-bottom: 6px solid #bbb; +} + +.tabulator .tabulator-header .tabulator-col.tabulator-sortable[aria-sort="asc"] .tabulator-col-content .tabulator-arrow { + border-top: none; + border-bottom: 6px solid #666; +} + +.tabulator .tabulator-header .tabulator-col.tabulator-sortable[aria-sort="desc"] .tabulator-col-content .tabulator-arrow { + border-top: 6px solid #666; + border-bottom: none; +} + +.tabulator .tabulator-header .tabulator-col.tabulator-col-vertical .tabulator-col-content .tabulator-col-title { + -webkit-writing-mode: vertical-rl; + -ms-writing-mode: tb-rl; + writing-mode: vertical-rl; + text-orientation: mixed; + display: -ms-flexbox; + display: flex; + -ms-flex-align: center; + align-items: center; + -ms-flex-pack: center; + justify-content: center; +} + +.tabulator .tabulator-header .tabulator-col.tabulator-col-vertical.tabulator-col-vertical-flip .tabulator-col-title { + -ms-transform: rotate(180deg); + transform: rotate(180deg); +} + +.tabulator .tabulator-header .tabulator-col.tabulator-col-vertical.tabulator-sortable .tabulator-col-title { + padding-right: 0; + padding-top: 20px; +} + +.tabulator .tabulator-header .tabulator-col.tabulator-col-vertical.tabulator-sortable.tabulator-col-vertical-flip .tabulator-col-title { + padding-right: 0; + padding-bottom: 20px; +} + +.tabulator .tabulator-header .tabulator-col.tabulator-col-vertical.tabulator-sortable .tabulator-arrow { + right: calc(50% - 6px); +} + +.tabulator .tabulator-header .tabulator-frozen { + display: inline-block; + position: absolute; + z-index: 10; +} + +.tabulator .tabulator-header .tabulator-frozen.tabulator-frozen-left { + border-right: 2px solid #aaa; +} + +.tabulator .tabulator-header .tabulator-frozen.tabulator-frozen-right { + border-left: 2px solid #aaa; +} + +.tabulator .tabulator-header .tabulator-calcs-holder { + box-sizing: border-box; + min-width: 400%; + background: rgba(80,20,10,0.2); + border-top: 1px solid #aaa; + border-bottom: 1px solid #aaa; + overflow: hidden; +} + +.tabulator .tabulator-header .tabulator-calcs-holder .tabulator-row { + background: rgba(80,20,10,0.2); +} + +.tabulator .tabulator-header .tabulator-calcs-holder .tabulator-row .tabulator-col-resize-handle { + display: none; +} + +.tabulator .tabulator-header .tabulator-frozen-rows-holder { + min-width: 400%; +} + +.tabulator .tabulator-header .tabulator-frozen-rows-holder:empty { + display: none; +} + +.tabulator .tabulator-tableHolder { + position: relative; + width: 100%; + white-space: nowrap; + overflow: auto; + -webkit-overflow-scrolling: touch; +} + +.tabulator .tabulator-tableHolder:focus { + outline: none; +} + +.tabulator .tabulator-tableHolder .tabulator-placeholder { + box-sizing: border-box; + display: -ms-flexbox; + display: flex; + -ms-flex-align: center; + align-items: center; + width: 100%; +} + +.tabulator .tabulator-tableHolder .tabulator-placeholder[tabulator-render-mode="virtual"] { + position: absolute; + top: 0; + left: 0; + height: 100%; +} + +.tabulator .tabulator-tableHolder .tabulator-placeholder span { + display: inline-block; + margin: 0 auto; + padding: 10px; + color: #ccc; + font-weight: bold; + font-size: 20px; +} + +.tabulator .tabulator-tableHolder .tabulator-table { + position: relative; + display: inline-block; + background-color: #fff; + white-space: nowrap; + overflow: visible; + color: #333; +} + +.tabulator .tabulator-tableHolder .tabulator-table .tabulator-row.tabulator-calcs { + font-weight: bold; + background: rgba(80,20,10,0.2); +} + +.tabulator .tabulator-tableHolder .tabulator-table .tabulator-row.tabulator-calcs.tabulator-calcs-top { + border-bottom: 2px solid #aaa; +} + +.tabulator .tabulator-tableHolder .tabulator-table .tabulator-row.tabulator-calcs.tabulator-calcs-bottom { + border-top: 2px solid #aaa; +} + +.tabulator .tabulator-footer { + padding: 5px 10px; + border-top: 1px solid #999; + text-align: right; + color: #555; + font-weight: bold; + white-space: nowrap; + -ms-user-select: none; + user-select: none; + -moz-user-select: none; + -khtml-user-select: none; + -webkit-user-select: none; + -o-user-select: none; +} + +.tabulator .tabulator-footer .tabulator-calcs-holder { + box-sizing: border-box; + width: calc(100% + 20px); + margin: -5px -10px 5px -10px; + text-align: left; + background: rgba(80,20,10,0.2); + border-bottom: 1px solid #aaa; + border-top: 1px solid #aaa; + overflow: hidden; +} + +.tabulator .tabulator-footer .tabulator-calcs-holder .tabulator-row { + background: rgba(80,20,10,0.2); +} + +.tabulator .tabulator-footer .tabulator-calcs-holder .tabulator-row .tabulator-col-resize-handle { + display: none; +} + +.tabulator .tabulator-footer .tabulator-calcs-holder:only-child { + margin-bottom: -5px; + border-bottom: none; +} + +.tabulator .tabulator-footer .tabulator-pages { + margin: 0 7px; +} + +.tabulator .tabulator-footer .tabulator-page { + display: inline-block; + margin: 0 2px; + padding: 2px 5px; + border: 1px solid #aaa; + border-radius: 3px; + background: rgba(255, 255, 255, 0.2); + color: #555; + font-family: inherit; + font-weight: inherit; + font-size: inherit; +} + +.tabulator .tabulator-footer .tabulator-page.active { + color: #d00; +} + +.tabulator .tabulator-footer .tabulator-page:disabled { + opacity: .5; +} + +.tabulator .tabulator-footer .tabulator-page:not(.disabled):hover { + cursor: pointer; + background: rgba(0, 0, 0, 0.2); + color: #fff; +} + +.tabulator .tabulator-col-resize-handle { + position: absolute; + right: 0; + top: 0; + bottom: 0; + width: 5px; +} + +.tabulator .tabulator-col-resize-handle.prev { + left: 0; + right: auto; +} + +.tabulator .tabulator-col-resize-handle:hover { + cursor: ew-resize; +} + +.tabulator .tabulator-loader { + position: absolute; + display: -ms-flexbox; + display: flex; + -ms-flex-align: center; + align-items: center; + top: 0; + left: 0; + z-index: 100; + height: 100%; + width: 100%; + background: rgba(0, 0, 0, 0.4); + text-align: center; +} + +.tabulator .tabulator-loader .tabulator-loader-msg { + display: inline-block; + margin: 0 auto; + padding: 10px 20px; + border-radius: 4px; + background: #fff; + font-weight: bold; + font-size: 16px; +} + +.tabulator .tabulator-loader .tabulator-loader-msg.tabulator-loading { + border: 4px solid #333; + color: #000; +} + +.tabulator .tabulator-loader .tabulator-loader-msg.tabulator-error { + color: #000; +} + +.tabulator-row { + position: relative; + box-sizing: border-box; + min-height: 22px; + background-color: #fff; +} + +.tabulator-row.tabulator-row-even { + background-color: #EFEFEF; +} + +.tabulator-row.tabulator-selectable:hover { + background-color: #bbb; + cursor: pointer; +} + +.tabulator-row.tabulator-selected { + background-color: #9ABCEA; +} + +.tabulator-row.tabulator-selected:hover { + background-color: #769BCC; + cursor: pointer; +} + +.tabulator-row.tabulator-row-moving { + border: 1px solid #000; + background: #fff; +} + +.tabulator-row.tabulator-moving { + position: absolute; + border-top: 1px solid #aaa; + border-bottom: 1px solid #aaa; + pointer-events: none; + z-index: 15; +} + +.tabulator-row .tabulator-row-resize-handle { + position: absolute; + right: 0; + bottom: 0; + left: 0; + height: 5px; +} + +.tabulator-row .tabulator-row-resize-handle.prev { + top: 0; + bottom: auto; +} + +.tabulator-row .tabulator-row-resize-handle:hover { + cursor: ns-resize; +} + +.tabulator-row .tabulator-frozen { + display: inline-block; + position: absolute; + background-color: inherit; + z-index: 10; +} + +.tabulator-row .tabulator-frozen.tabulator-frozen-left { + border-right: 2px solid #aaa; +} + +.tabulator-row .tabulator-frozen.tabulator-frozen-right { + border-left: 2px solid #aaa; +} + +.tabulator-row .tabulator-responsive-collapse { + box-sizing: border-box; + padding: 5px; + border-top: 1px solid #aaa; + border-bottom: 1px solid #aaa; +} + +.tabulator-row .tabulator-responsive-collapse:empty { + display: none; +} + +.tabulator-row .tabulator-responsive-collapse table { + font-size: 14px; +} + +.tabulator-row .tabulator-responsive-collapse table tr td { + position: relative; +} + +.tabulator-row .tabulator-responsive-collapse table tr td:first-of-type { + padding-right: 10px; +} + +.tabulator-row .tabulator-cell { + display: inline-block; + position: relative; + box-sizing: border-box; + padding: 4px; + border-right: 1px solid #aaa; + vertical-align: middle; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +.tabulator-row .tabulator-cell.tabulator-editing { + border: 1px solid #1D68CD; + padding: 0; +} + +.tabulator-row .tabulator-cell.tabulator-editing input, .tabulator-row .tabulator-cell.tabulator-editing select { + border: 1px; + background: transparent; +} + +.tabulator-row .tabulator-cell.tabulator-validation-fail { + border: 1px solid #dd0000; +} + +.tabulator-row .tabulator-cell.tabulator-validation-fail input, .tabulator-row .tabulator-cell.tabulator-validation-fail select { + border: 1px; + background: transparent; + color: #dd0000; +} + +.tabulator-row .tabulator-cell:first-child .tabulator-col-resize-handle.prev { + display: none; +} + +.tabulator-row .tabulator-cell.tabulator-row-handle { + display: -ms-inline-flexbox; + display: inline-flex; + -ms-flex-align: center; + align-items: center; + -ms-flex-pack: center; + justify-content: center; + -moz-user-select: none; + -khtml-user-select: none; + -webkit-user-select: none; + -o-user-select: none; +} + +.tabulator-row .tabulator-cell.tabulator-row-handle .tabulator-row-handle-box { + width: 80%; +} + +.tabulator-row .tabulator-cell.tabulator-row-handle .tabulator-row-handle-box .tabulator-row-handle-bar { + width: 100%; + height: 3px; + margin-top: 2px; + background: #666; +} + +.tabulator-row .tabulator-cell .tabulator-data-tree-branch { + display: inline-block; + vertical-align: middle; + height: 9px; + width: 7px; + margin-top: -9px; + margin-right: 5px; + border-bottom-left-radius: 1px; + border-left: 2px solid #aaa; + border-bottom: 2px solid #aaa; +} + +.tabulator-row .tabulator-cell .tabulator-data-tree-control { + display: -ms-inline-flexbox; + display: inline-flex; + -ms-flex-pack: center; + justify-content: center; + -ms-flex-align: center; + align-items: center; + vertical-align: middle; + height: 11px; + width: 11px; + margin-right: 5px; + border: 1px solid #333; + border-radius: 2px; + background: rgba(0, 0, 0, 0.1); + overflow: hidden; +} + +.tabulator-row .tabulator-cell .tabulator-data-tree-control:hover { + cursor: pointer; + background: rgba(0, 0, 0, 0.2); +} + +.tabulator-row .tabulator-cell .tabulator-data-tree-control .tabulator-data-tree-control-collapse { + display: inline-block; + position: relative; + height: 7px; + width: 1px; + background: transparent; +} + +.tabulator-row .tabulator-cell .tabulator-data-tree-control .tabulator-data-tree-control-collapse:after { + position: absolute; + content: ""; + left: -3px; + top: 3px; + height: 1px; + width: 7px; + background: #333; +} + +.tabulator-row .tabulator-cell .tabulator-data-tree-control .tabulator-data-tree-control-expand { + display: inline-block; + position: relative; + height: 7px; + width: 1px; + background: #333; +} + +.tabulator-row .tabulator-cell .tabulator-data-tree-control .tabulator-data-tree-control-expand:after { + position: absolute; + content: ""; + left: -3px; + top: 3px; + height: 1px; + width: 7px; + background: #333; +} + +.tabulator-row .tabulator-cell .tabulator-responsive-collapse-toggle { + display: -ms-inline-flexbox; + display: inline-flex; + -ms-flex-align: center; + align-items: center; + -ms-flex-pack: center; + justify-content: center; + -moz-user-select: none; + -khtml-user-select: none; + -webkit-user-select: none; + -o-user-select: none; + height: 15px; + width: 15px; + border-radius: 4px; + background: #666; + color: #fff; + font-weight: bold; + font-size: 1.1em; +} + +.tabulator-row .tabulator-cell .tabulator-responsive-collapse-toggle:hover { + opacity: .7; +} + +.tabulator-row .tabulator-cell .tabulator-responsive-collapse-toggle.open .tabulator-responsive-collapse-toggle-close { + display: initial; +} + +.tabulator-row .tabulator-cell .tabulator-responsive-collapse-toggle.open .tabulator-responsive-collapse-toggle-open { + display: none; +} + +.tabulator-row .tabulator-cell .tabulator-responsive-collapse-toggle .tabulator-responsive-collapse-toggle-close { + display: none; +} + +.tabulator-row.tabulator-group { + box-sizing: border-box; + border-bottom: 1px solid #999; + border-right: 1px solid #aaa; + border-top: 1px solid #999; + padding: 5px; + padding-left: 10px; + background: #ccc; + font-weight: bold; + min-width: 100%; +} + +.tabulator-row.tabulator-group:hover { + cursor: pointer; + background-color: rgba(0, 0, 0, 0.1); +} + +.tabulator-row.tabulator-group.tabulator-group-visible .tabulator-arrow { + margin-right: 10px; + border-left: 6px solid transparent; + border-right: 6px solid transparent; + border-top: 6px solid #666; + border-bottom: 0; +} + +.tabulator-row.tabulator-group.tabulator-group-level-1 .tabulator-arrow { + margin-left: 20px; +} + +.tabulator-row.tabulator-group.tabulator-group-level-2 .tabulator-arrow { + margin-left: 40px; +} + +.tabulator-row.tabulator-group.tabulator-group-level-3 .tabulator-arrow { + margin-left: 60px; +} + +.tabulator-row.tabulator-group.tabulator-group-level-4 .tabulator-arrow { + margin-left: 80px; +} + +.tabulator-row.tabulator-group.tabulator-group-level-5 .tabulator-arrow { + margin-left: 100px; +} + +.tabulator-row.tabulator-group .tabulator-arrow { + display: inline-block; + width: 0; + height: 0; + margin-right: 16px; + border-top: 6px solid transparent; + border-bottom: 6px solid transparent; + border-right: 0; + border-left: 6px solid #666; + vertical-align: middle; +} + +.tabulator-row.tabulator-group span { + margin-left: 10px; + color: #d00; +} + +.tabulator-edit-select-list { + position: absolute; + display: inline-block; + box-sizing: border-box; + max-height: 200px; + background: #fff; + border: 1px solid #aaa; + font-size: 14px; + overflow-y: auto; + -webkit-overflow-scrolling: touch; + z-index: 10000; +} + +.tabulator-edit-select-list .tabulator-edit-select-list-item { + padding: 4px; + color: #333; +} + +.tabulator-edit-select-list .tabulator-edit-select-list-item.active { + color: #fff; + background: #1D68CD; +} + +.tabulator-edit-select-list .tabulator-edit-select-list-item:hover { + cursor: pointer; + color: #fff; + background: #1D68CD; +} + +.tabulator-edit-select-list .tabulator-edit-select-list-group { + border-bottom: 1px solid #aaa; + padding: 4px; + padding-top: 6px; + color: #333; + font-weight: bold; +} diff --git a/site/public/about/credits/index.html b/site/public/about/credits/index.html index 1b1f4d54..67e9dcb8 100644 --- a/site/public/about/credits/index.html +++ b/site/public/about/credits/index.html @@ -8,6 +8,7 @@ + @@ -19,7 +20,6 @@ The Darkside of Datasets