diff options
Diffstat (limited to 'megapixels')
| -rw-r--r-- | megapixels/app/server/api.py | 43 | ||||
| -rw-r--r-- | megapixels/app/server/create.py | 7 | ||||
| -rw-r--r-- | megapixels/app/site/builder.py | 12 | ||||
| -rw-r--r-- | megapixels/app/site/parser.py | 50 | ||||
| -rw-r--r-- | megapixels/app/site/s3.py | 3 | ||||
| -rw-r--r-- | megapixels/cli_flask.py | 1 |
6 files changed, 108 insertions, 8 deletions
diff --git a/megapixels/app/server/api.py b/megapixels/app/server/api.py index c5e27dd2..cf8241bd 100644 --- a/megapixels/app/server/api.py +++ b/megapixels/app/server/api.py @@ -1,9 +1,12 @@ import os import re import time +import dlib from flask import Blueprint, request, jsonify from PIL import Image # todo: try to remove PIL dependency +from app.processors import face_recognition +from app.processors import face_detector from app.models.sql_factory import list_datasets, get_dataset, get_table sanitize_re = re.compile('[\W]+') @@ -23,8 +26,10 @@ def show(name): else: return jsonify({ 'status': 404 }) -@api.route('/dataset/<dataset>/face', methods=['POST']) +@api.route('/dataset/<name>/face', methods=['POST']) def upload(name): + start = time.time() + dataset = get_dataset(name) file = request.files['query_img'] fn = file.filename if fn.endswith('blob'): @@ -37,11 +42,43 @@ def upload(name): img = Image.open(file.stream).convert('RGB') - # vec = db.load_feature_vector_from_file(uploaded_img_path) - # vec = fe.extract(img) + # Face detection + detector = face_detector.DetectorDLIBHOG() + + # get detection as BBox object + bboxes = detector.detect(im, largest=True) + bbox = bboxes[0] + dim = im.shape[:2][::-1] + bbox = bbox.to_dim(dim) # convert back to real dimensions + + # face recognition/vector + recognition = face_recognition.RecognitionDLIB(gpu=-1) + # print(vec.shape) # results = db.search(vec, limit=limit) + # with the result we have an ID + # query the sql dataset for the UUID etc here + + query = { + 'timing': time.time() - start, + } + results = [] + + print(results) + return jsonify({ + 'query': query, + 'results': results, + }) + +@api.route('/dataset/<name>/name', methods=['GET']) +def name_lookup(dataset): + start = time.time() + dataset = get_dataset(name) + + # we have a query from the request query string... + # use this to do a like* query on the identities_meta table + query = { 'timing': time.time() - start, } diff --git a/megapixels/app/server/create.py b/megapixels/app/server/create.py index c1f41dc4..4b1333b9 100644 --- a/megapixels/app/server/create.py +++ b/megapixels/app/server/create.py @@ -7,6 +7,9 @@ from app.server.api import api db = SQLAlchemy() def create_app(script_info=None): + """ + functional pattern for creating the flask app + """ app = Flask(__name__, static_folder='static', static_url_path='') app.config['SQLALCHEMY_DATABASE_URI'] = connection_url app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False @@ -37,6 +40,10 @@ def create_app(script_info=None): return app def serve_page(file_relative_path_to_root): + """ + trying to get this to serve /path/ with /path/index.html, + ...but it doesnt actually matter for production... + """ if file_relative_path_to_root[-1] == '/': file_relative_path_to_root += 'index.html' return send_from_directory("static", file_relative_path_to_root) diff --git a/megapixels/app/site/builder.py b/megapixels/app/site/builder.py index 42e25768..895f265b 100644 --- a/megapixels/app/site/builder.py +++ b/megapixels/app/site/builder.py @@ -15,6 +15,10 @@ env = Environment( ) def build_page(fn, research_posts): + """ + build a single page from markdown into the appropriate template + - writes it to site/public/ + """ metadata, sections = parser.read_metadata(fn) if metadata is None: @@ -60,9 +64,10 @@ def build_page(fn, research_posts): with open(output_fn, "w") as file: file.write(html) - print("______") - def build_research_index(research_posts): + """ + build the index of research (blog) posts + """ metadata, sections = parser.read_metadata('../site/content/research/index.md') template = env.get_template("page.html") s3_path = s3.make_s3_path(cfg.S3_SITE_PATH, metadata['path']) @@ -79,6 +84,9 @@ def build_research_index(research_posts): file.write(html) def build_site(): + """ + build the site! =^) + """ research_posts = parser.read_research_post_index() for fn in glob.iglob(os.path.join(cfg.DIR_SITE_CONTENT, "**/*.md"), recursive=True): build_page(fn, research_posts) diff --git a/megapixels/app/site/parser.py b/megapixels/app/site/parser.py index d78cc402..6c6ad688 100644 --- a/megapixels/app/site/parser.py +++ b/megapixels/app/site/parser.py @@ -11,6 +11,10 @@ renderer = mistune.Renderer(escape=False) markdown = mistune.Markdown(renderer=renderer) def fix_images(lines, s3_path): + """ + do our own tranformation of the markdown around images to handle wide images etc + lines: markdown lines + """ real_lines = [] block = "\n\n".join(lines) for line in block.split("\n"): @@ -29,6 +33,9 @@ def fix_images(lines, s3_path): return "\n".join(real_lines) def format_section(lines, s3_path, type=''): + """ + format a normal markdown section + """ if len(lines): lines = fix_images(lines, s3_path) if type: @@ -38,28 +45,41 @@ def format_section(lines, s3_path, type=''): return "" def format_metadata(section): + """ + format a metadata section (+ key: value pairs) + """ meta = [] for line in section.split('\n'): key, value = line[2:].split(': ', 1) meta.append("<div><div class='gray'>{}</div><div>{}</div></div>".format(key, value)) return "<section><div class='meta'>{}</div></section>".format(''.join(meta)) -def format_applet(section): - payload = section.replace('```', '').strip().split('\n') +def format_applet(section, s3_path): + print(section) + payload = section.strip('```').strip().split('\n') applet = {} + print(payload) if ': ' in payload[0]: command, opt = payload[0].split(': ') else: command = payload[0] opt = None + if command == 'python': + return format_section([ section ], s3_path) + applet['command'] = command if opt: applet['opt'] = opt if command == 'load file': + if opt[0] != '/': + applet['opt'] = s3_path + opt applet['fields'] = payload[1] return "<section><div class='applet' data-payload='{}'></div></section>".format(json.dumps(applet)) def parse_markdown(sections, s3_path, skip_h1=False): + """ + parse page into sections, preprocess the markdown to handle our modifications + """ groups = [] current_group = [] for section in sections: @@ -67,7 +87,14 @@ def parse_markdown(sections, s3_path, skip_h1=False): continue elif section.startswith('```'): groups.append(format_section(current_group, s3_path)) - groups.append(format_applet(section)) + current_group = [] + current_group.append(section) + if section.endswith('```'): + groups.append(format_applet("\n\n".join(current_group), s3_path)) + current_group = [] + elif section.endswith('```'): + current_group.append(section) + groups.append(format_applet("\n\n".join(current_group), s3_path)) current_group = [] elif section.startswith('+ '): groups.append(format_section(current_group, s3_path)) @@ -88,6 +115,9 @@ def parse_markdown(sections, s3_path, skip_h1=False): return content def parse_research_index(research_posts): + """ + Generate an index file for the research pages + """ content = "<div class='research_index'>" for post in research_posts: s3_path = s3.make_s3_path(cfg.S3_SITE_PATH, post['path']) @@ -105,6 +135,9 @@ def parse_research_index(research_posts): return content def read_metadata(fn): + """ + Read in read a markdown file and extract the metadata + """ with open(fn, "r") as file: data = file.read() data = data.replace("\n ", "\n") @@ -128,6 +161,9 @@ default_metadata = { } def parse_metadata_section(metadata, section): + """ + parse a metadata key: value pair + """ for line in section.split("\n"): if ': ' not in line: continue @@ -135,6 +171,11 @@ def parse_metadata_section(metadata, section): metadata[key.lower()] = value def parse_metadata(fn, sections): + """ + parse the metadata headers in a markdown file + (everything before the second ---------) + also generates appropriate urls for this page :) + """ found_meta = False metadata = {} valid_sections = [] @@ -175,6 +216,9 @@ def parse_metadata(fn, sections): return metadata, valid_sections def read_research_post_index(): + """ + Generate an index of the research (blog) posts + """ posts = [] for fn in sorted(glob.glob('../site/content/research/*/index.md')): metadata, valid_sections = read_metadata(fn) diff --git a/megapixels/app/site/s3.py b/megapixels/app/site/s3.py index 99726a4d..5464d464 100644 --- a/megapixels/app/site/s3.py +++ b/megapixels/app/site/s3.py @@ -3,6 +3,9 @@ import glob import boto3 def sync_directory(base_fn, s3_path, metadata): + """ + Synchronize a local assets folder with S3 + """ fns = {} for fn in glob.glob(os.path.join(base_fn, 'assets/*')): fns[os.path.basename(fn)] = True diff --git a/megapixels/cli_flask.py b/megapixels/cli_flask.py index 369bec01..e80526c6 100644 --- a/megapixels/cli_flask.py +++ b/megapixels/cli_flask.py @@ -1,5 +1,6 @@ # -------------------------------------------------------- # wrapper for flask CLI API +# NB: python cli_flask.py run # -------------------------------------------------------- import click |