From 3b10acc73247ec703ed47f0423e7d255a91f074e Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Sun, 16 Dec 2018 00:11:20 +0100 Subject: tabulator --- megapixels/app/server/api.py | 17 +++++++++++++++-- megapixels/app/server/create.py | 7 +++++++ megapixels/app/site/builder.py | 10 ++++++++++ megapixels/app/site/parser.py | 36 ++++++++++++++++++++++++++++++++++-- megapixels/app/site/s3.py | 3 +++ 5 files changed, 69 insertions(+), 4 deletions(-) (limited to 'megapixels') diff --git a/megapixels/app/server/api.py b/megapixels/app/server/api.py index cd2b950b..cf8241bd 100644 --- a/megapixels/app/server/api.py +++ b/megapixels/app/server/api.py @@ -1,9 +1,12 @@ import os import re import time +import dlib from flask import Blueprint, request, jsonify from PIL import Image # todo: try to remove PIL dependency +from app.processors import face_recognition +from app.processors import face_detector from app.models.sql_factory import list_datasets, get_dataset, get_table sanitize_re = re.compile('[\W]+') @@ -39,8 +42,18 @@ def upload(name): img = Image.open(file.stream).convert('RGB') - # vec = db.load_feature_vector_from_file(uploaded_img_path) - # vec = fe.extract(img) + # Face detection + detector = face_detector.DetectorDLIBHOG() + + # get detection as BBox object + bboxes = detector.detect(im, largest=True) + bbox = bboxes[0] + dim = im.shape[:2][::-1] + bbox = bbox.to_dim(dim) # convert back to real dimensions + + # face recognition/vector + recognition = face_recognition.RecognitionDLIB(gpu=-1) + # print(vec.shape) # results = db.search(vec, limit=limit) diff --git a/megapixels/app/server/create.py b/megapixels/app/server/create.py index c1f41dc4..4b1333b9 100644 --- a/megapixels/app/server/create.py +++ b/megapixels/app/server/create.py @@ -7,6 +7,9 @@ from app.server.api import api db = SQLAlchemy() def create_app(script_info=None): + """ + functional pattern for creating the flask app + """ app = Flask(__name__, static_folder='static', static_url_path='') app.config['SQLALCHEMY_DATABASE_URI'] = connection_url app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False @@ -37,6 +40,10 @@ def create_app(script_info=None): return app def serve_page(file_relative_path_to_root): + """ + trying to get this to serve /path/ with /path/index.html, + ...but it doesnt actually matter for production... + """ if file_relative_path_to_root[-1] == '/': file_relative_path_to_root += 'index.html' return send_from_directory("static", file_relative_path_to_root) diff --git a/megapixels/app/site/builder.py b/megapixels/app/site/builder.py index 91df54c2..895f265b 100644 --- a/megapixels/app/site/builder.py +++ b/megapixels/app/site/builder.py @@ -15,6 +15,10 @@ env = Environment( ) def build_page(fn, research_posts): + """ + build a single page from markdown into the appropriate template + - writes it to site/public/ + """ metadata, sections = parser.read_metadata(fn) if metadata is None: @@ -61,6 +65,9 @@ def build_page(fn, research_posts): file.write(html) def build_research_index(research_posts): + """ + build the index of research (blog) posts + """ metadata, sections = parser.read_metadata('../site/content/research/index.md') template = env.get_template("page.html") s3_path = s3.make_s3_path(cfg.S3_SITE_PATH, metadata['path']) @@ -77,6 +84,9 @@ def build_research_index(research_posts): file.write(html) def build_site(): + """ + build the site! =^) + """ research_posts = parser.read_research_post_index() for fn in glob.iglob(os.path.join(cfg.DIR_SITE_CONTENT, "**/*.md"), recursive=True): build_page(fn, research_posts) diff --git a/megapixels/app/site/parser.py b/megapixels/app/site/parser.py index d78cc402..40d9c7f6 100644 --- a/megapixels/app/site/parser.py +++ b/megapixels/app/site/parser.py @@ -11,6 +11,10 @@ renderer = mistune.Renderer(escape=False) markdown = mistune.Markdown(renderer=renderer) def fix_images(lines, s3_path): + """ + do our own tranformation of the markdown around images to handle wide images etc + lines: markdown lines + """ real_lines = [] block = "\n\n".join(lines) for line in block.split("\n"): @@ -29,6 +33,9 @@ def fix_images(lines, s3_path): return "\n".join(real_lines) def format_section(lines, s3_path, type=''): + """ + format a normal markdown section + """ if len(lines): lines = fix_images(lines, s3_path) if type: @@ -38,13 +45,16 @@ def format_section(lines, s3_path, type=''): return "" def format_metadata(section): + """ + format a metadata section (+ key: value pairs) + """ meta = [] for line in section.split('\n'): key, value = line[2:].split(': ', 1) meta.append("
{}
{}
".format(key, value)) return "
{}
".format(''.join(meta)) -def format_applet(section): +def format_applet(section, s3_path): payload = section.replace('```', '').strip().split('\n') applet = {} if ': ' in payload[0]: @@ -56,10 +66,15 @@ def format_applet(section): if opt: applet['opt'] = opt if command == 'load file': + if opt[0] != '/': + applet['opt'] = s3_path + opt applet['fields'] = payload[1] return "
".format(json.dumps(applet)) def parse_markdown(sections, s3_path, skip_h1=False): + """ + parse page into sections, preprocess the markdown to handle our modifications + """ groups = [] current_group = [] for section in sections: @@ -67,7 +82,7 @@ def parse_markdown(sections, s3_path, skip_h1=False): continue elif section.startswith('```'): groups.append(format_section(current_group, s3_path)) - groups.append(format_applet(section)) + groups.append(format_applet(section, s3_path)) current_group = [] elif section.startswith('+ '): groups.append(format_section(current_group, s3_path)) @@ -88,6 +103,9 @@ def parse_markdown(sections, s3_path, skip_h1=False): return content def parse_research_index(research_posts): + """ + Generate an index file for the research pages + """ content = "
" for post in research_posts: s3_path = s3.make_s3_path(cfg.S3_SITE_PATH, post['path']) @@ -105,6 +123,9 @@ def parse_research_index(research_posts): return content def read_metadata(fn): + """ + Read in read a markdown file and extract the metadata + """ with open(fn, "r") as file: data = file.read() data = data.replace("\n ", "\n") @@ -128,6 +149,9 @@ default_metadata = { } def parse_metadata_section(metadata, section): + """ + parse a metadata key: value pair + """ for line in section.split("\n"): if ': ' not in line: continue @@ -135,6 +159,11 @@ def parse_metadata_section(metadata, section): metadata[key.lower()] = value def parse_metadata(fn, sections): + """ + parse the metadata headers in a markdown file + (everything before the second ---------) + also generates appropriate urls for this page :) + """ found_meta = False metadata = {} valid_sections = [] @@ -175,6 +204,9 @@ def parse_metadata(fn, sections): return metadata, valid_sections def read_research_post_index(): + """ + Generate an index of the research (blog) posts + """ posts = [] for fn in sorted(glob.glob('../site/content/research/*/index.md')): metadata, valid_sections = read_metadata(fn) diff --git a/megapixels/app/site/s3.py b/megapixels/app/site/s3.py index 99726a4d..5464d464 100644 --- a/megapixels/app/site/s3.py +++ b/megapixels/app/site/s3.py @@ -3,6 +3,9 @@ import glob import boto3 def sync_directory(base_fn, s3_path, metadata): + """ + Synchronize a local assets folder with S3 + """ fns = {} for fn in glob.glob(os.path.join(base_fn, 'assets/*')): fns[os.path.basename(fn)] = True -- cgit v1.2.3-70-g09d2