import os import re import glob import simplejson as json import mistune import app.settings.app_cfg as cfg import app.site.s3 as s3 renderer = mistune.Renderer(escape=False) markdown = mistune.Markdown(renderer=renderer) def parse_markdown(metadata, sections, s3_path, skip_h1=False): """ parse page into sections, preprocess the markdown to handle our modifications """ groups = [] current_group = [] for section in sections: if skip_h1 and section.startswith('# '): continue elif section.strip().startswith('```'): groups.append(format_section(current_group, s3_path)) current_group = [] current_group.append(section) if section.strip().endswith('```'): groups.append(format_applet("\n\n".join(current_group), s3_path)) current_group = [] elif section.strip().endswith('```'): current_group.append(section) groups.append(format_applet("\n\n".join(current_group), s3_path)) current_group = [] elif section.startswith('+ '): groups.append(format_section(current_group, s3_path)) groups.append(format_metadata(section)) current_group = [] elif '![fullwidth:' in section: groups.append(format_section(current_group, s3_path)) groups.append(format_section([section], s3_path, type='fullwidth')) current_group = [] elif '![wide:' in section: groups.append(format_section(current_group, s3_path)) groups.append(format_section([section], s3_path, type='wide')) current_group = [] elif '![' in section: groups.append(format_section(current_group, s3_path)) groups.append(format_section([section], s3_path, type='images')) current_group = [] else: current_group.append(section) groups.append(format_section(current_group, s3_path)) content = "".join(groups) return content def fix_images(lines, s3_path): """ do our own tranformation of the markdown around images to handle wide images etc lines: markdown lines """ real_lines = [] block = "\n\n".join(lines) for line in block.split("\n"): if "![" in line: line = line.replace('![', '') alt_text, tail = line.split('](', 1) url, tail = tail.split(')', 1) if ':' in alt_text: tail, alt_text = alt_text.split(':', 1) img_tag = "{}".format(s3_path + url, alt_text.replace("'", "")) if len(alt_text): line = "
{}
{}
".format(img_tag, alt_text) else: line = "
{}
".format(img_tag, alt_text) real_lines.append(line) return "\n".join(real_lines) def format_section(lines, s3_path, type=''): """ format a normal markdown section """ if len(lines): lines = fix_images(lines, s3_path) if type: return "
{}
".format(type, markdown(lines)) else: return "
" + markdown(lines) + "
" return "" def format_metadata(section): """ format a metadata section (+ key: value pairs) """ meta = [] for line in section.split('\n'): key, value = line[2:].split(': ', 1) meta.append("
{}
{}
".format(key, value)) return "
{}
".format(''.join(meta)) def format_applet(section, s3_path): """ Format the applets, which load javascript modules like the map and CSVs """ # print(section) payload = section.strip('```').strip().strip('```').strip().split('\n') applet = {} print(payload) if ': ' in payload[0]: command, opt = payload[0].split(': ') else: command = payload[0] opt = None if command == 'python' or command == 'javascript' or command == 'code': return format_section([ section ], s3_path) if command == '': return '' applet['command'] = command if opt: applet['opt'] = opt if command == 'load_file': if opt[0:4] != 'http': applet['opt'] = s3_path + opt if len(payload) > 1: applet['fields'] = payload[1:] return "
".format(json.dumps(applet)) def parse_research_index(research_posts): """ Generate an index file for the research pages """ content = "
" for post in research_posts: print(post) s3_path = s3.make_s3_path(cfg.S3_SITE_PATH, post['path']) if 'image' in post: post_image = s3_path + post['image'] else: post_image = 'data:image/gif;base64,R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==' row = "
Research post

{}

{}

".format( post['path'], post_image, post['title'], post['tagline']) content += row content += '
' return content