import click from app.settings import app_cfg from app.utils.file_utils import load_text, write_json, write_text from os.path import join from functools import reduce from shutil import copyfile import os @click.command('info') # @click.option('-g', '--graph', 'opt_graph_path', required=True, # help='Graph name') @click.option('-o', '--output', 'opt_output_dir', required=False, default="animism", help='Output directory') @click.pass_context def cli(ctx, opt_output_dir): """Export a graph""" # ------------------------------------------------ # imports import datetime import json from distutils.dir_util import copy_tree # ------------------------------------------------ # export settings page_title = "Animism: Episode 1" page_name = "episode1" page_desc = "A Report on Migrating Souls in Museums and Moving Pictures. Animism on e-flux.com is the ninth iteration of the exhibition, which will be released in four episodes starting November 2020." page_image = "https://animism.e-flux.com/episode1/media/8ca4adc754093cc8578a3033de8f96af96180fbce838c480636ffd5d128d1937.jpg" site_url = "https://animism.e-flux.com/" + page_name page_url = "/" + page_name media_url = "/" + page_name + "/media" site_title = f"{page_title}: {page_desc}" site_path = opt_output_dir or datetime.datetime.now().strftime("animism_%Y%m%d%H%M") site_fp_root = join(app_cfg.DIR_EXPORTS, site_path) site_fp_static = join(site_fp_root, 'static') site_fp_out = join(site_fp_root, page_name) site_fp_media = join(site_fp_root, page_name, 'media') # ------------------------------------------------ # load the db db = export_db() prune_db(db) media_to_copy = rewrite_db_media(db, site_fp_media, media_url) db['urls'] = { 'audio': '/' + page_name + '/media/animism_episode_01.mp3' } media_to_copy['audio'] = { 'src': join(app_cfg.DIR_DATA_STORE, 'peaks/animism_episode_01_2810.mp3'), 'dst': join(site_fp_media, 'animism_episode_01.mp3'), } # ------------------------------------------------ # build the json for the e-flux search search_json = [{ "text": transcript_to_text(db), "url": site_url, "type": "text", "previewtitle": 'Animism, episode 1', "previewtext": page_desc, "previewimage": page_image, }] # ------------------------------------------------ # build the index.html index_html = load_text(join(app_cfg.DIR_STATIC, 'site.html'), split=False) index_html = index_html.replace('SITE_PATH', page_url) index_html = index_html.replace('SITE_JSON', json.dumps(db, separators=(',', ':'))) index_html = index_html.replace('PAGE_TITLE', page_title) index_html = index_html.replace('PAGE_DESCRIPTION', page_desc) index_html = index_html.replace('PAGE_SHARE_IMAGE', page_image) index_html = index_html.replace('PLAIN_CONTENT', plain_content(db, site_title)) index_html = index_html.replace('BUNDLE_PATH', join(page_url, 'bundle.js')) write_text(index_html, join(site_fp_out, 'index.html')) # ------------------------------------------------ # write all the json write_json(db, join(site_fp_out, 'index.json'), default=str, minify=False) write_json(search_json, join(site_fp_static, 'search.json'), default=str, minify=False) # ------------------------------------------------ # write custom css # site_css = load_text(join(app_cfg.DIR_STATIC, 'site.css'), split=False) # index_html = index_html.replace('SITE_PATH', page_url) # write_text(site_css, join(site_fp_out, 'site.css')) # ------------------------------------------------ # copy media from the exhibition copy_media(site_fp_media, media_to_copy) # ------------------------------------------------ # copy any static assets copy_tree(join(app_cfg.DIR_STATIC, 'fonts'), join(site_fp_static, 'fonts')) copy_tree(join(app_cfg.DIR_STATIC, 'img'), join(site_fp_static, 'img')) copyfile(join(app_cfg.DIR_STATIC, 'favicon.ico'), join(site_fp_root, 'favicon.ico')) # ------------------------------------------------ # build javascript print("Building javascript...") print(f'NODE_ENV=production node ./node_modules/webpack-cli/bin/cli.js --config ./webpack.config.site.js -o {site_fp_out}/bundle.js') os.chdir(app_cfg.DIR_PROJECT_ROOT) os.system(f'NODE_ENV=production node ./node_modules/webpack-cli/bin/cli.js --config ./webpack.config.site.js -o {site_fp_out}/bundle.js') print("Deploying site...") os.system(""" lftp -c "set ssl:verify-certificate false; set ftp:list-options -a; open ftp://animism:Agkp8#48@93.114.86.205; lcd ./data_store/exports/animism; cd /; mirror --reverse --use-cache --verbose --no-umask --ignore-time --parallel=2" """) print("Site export complete!") print(f"Site exported to: {site_fp_out}") ###################################################################### # Database Functions ###################################################################### def copy_media(fp_media, to_copy): os.makedirs(fp_media, exist_ok=True) print(f"Copying {len(to_copy.keys())} files") total_size = 0 for fp in to_copy.values(): copyfile(fp['src'], fp['dst']) total_size += os.path.getsize(fp['dst']) print(f"wrote {round(total_size / 1000000, 2)} MB") def rewrite_db_media(db, fp_out, url_out): """ Go over all the media and find any Upload objects. Figure out which to copy, and rewrite DB to use the export URL schema. """ to_copy = {} for item in IterateTable(db['media']): settings = item['settings'] # images - various sizes... we don't use fullsize anywhere if item['type'] == 'image': if 'fullsize' in settings: del settings['fullsize'] for field in app_cfg.IMAGE_UPLOAD_FIELDS: if field in settings: settings[field] = rewrite_upload(to_copy, settings[field], fp_out, url_out) # videos - poster images elif item['type'] == 'video': if 'poster' in settings: settings['poster'] = rewrite_upload(to_copy, settings['poster'], fp_out, url_out) # galleries - a bunch of lookups... we PROBABLY don't need image_lookup (fullsize) elif item['type'] == 'gallery': new_image_lookup = {} if 'display' in settings: settings['display'] = rewrite_upload(to_copy, settings['display'], fp_out, url_out) if 'thumbnail' in settings: settings['thumbnail'] = rewrite_upload(to_copy, settings['thumbnail'], fp_out, url_out) for id in settings['image_order']: id = str(id) if id in settings['image_lookup']: new_image_lookup[id] = rewrite_upload(to_copy, settings['image_lookup'][id], fp_out, url_out, png_only=True) settings['image_lookup'] = new_image_lookup for field in app_cfg.IMAGE_UPLOAD_GALLERY_LOOKUPS: for id in settings['image_order']: id = str(id) if id in settings[field]: settings[field][id] = rewrite_upload(to_copy, settings[field][id], fp_out, url_out) # files - singleton file uploads elif item['type'] == 'file': if 'file' in settings: settings['file'] = rewrite_upload(to_copy, settings['file'], fp_out, url_out) return to_copy def rewrite_upload(to_copy, item, fp_out, url_out, png_only=False): """ # rewriting uploads. they look like this: "fn": "koester.gif", "sha256": "c7c25e8d9be8b3e5db89df0f4a35f8a599dfdcf8bf9bc1f6c4137c7b6522d710", "tag": "file", "url": "/static/data_store/uploads/file/koester.gif", "username": "animism" """ if 'sha256' not in item: return item if png_only and item['ext'] != '.png': return sha = item['sha256'] out_fn = sha + item['ext'] out_obj = { "url": join(url_out, out_fn), } if sha not in to_copy: # print(f"SHA: {sha}") in_fn = item['fn'] in_path = join(app_cfg.DIR_UPLOADS, item['tag'], in_fn) if os.path.exists(in_path): to_copy[sha] = { "src": in_path, "dst": join(fp_out, out_fn) } else: print(f"Missing path: {in_path}") return out_obj def prune_db(db): """Remove random stuff from the JSON that doesn't need to be there - extraneous paragraphs - extraneous media """ seen_paras = {} seen_media = {} for a in IterateTable(db['annotation']): seen_paras[a['paragraph_id']] = True if 'media_id' in a['settings']: seen_media[a['settings']['media_id']] = True db['paragraph'] = filter_db(db, 'paragraph', seen_paras) db['media'] = filter_db(db, 'media', seen_media) def filter_db(db, table, seen): order = list(filter(lambda i: i in seen, db[table]['order'])) lookup = { id: db[table]['lookup'][id] for id in order } return { 'order': order, 'lookup': lookup } def export_db(): """Load the entire database and convert it to JSON""" from app.sql.common import db, Session, Episode, Venue, Annotation, Paragraph, Media, Upload session = Session() classes = [ Episode, Venue, Annotation, Paragraph, Media ] data = {} for c in classes: e_q = session.query(c) if c == Annotation or c == Paragraph: e_q = e_q.order_by(c.start_ts) e_list = e_q.all() order = list(map(get_id, e_list)) lookup = reduce(get_json_tup, e_list, {}) table_name = str(c.__table__) data[table_name] = { 'order': order, 'lookup': lookup } print(f"""exported {table_name} ({len(order)} rows)""") return data def sanitize_obj(data): if 'created_at' in data: del data['created_at'] if 'updated_at' in data: del data['updated_at'] return data def get_id(e): return e.id def get_json_tup(a,e): a[e.id] = sanitize_obj(e.toJSON()) return a def db_get(db, table, idx): """Get an indexed object out of our db table""" id = db[table]['order'][idx] return db[table]['lookup'][id] ###################################################################### # Plaintext helper functions ###################################################################### def transcript_to_text(db): s = "" para = "" last_pid = 0 section_count = 0 notes = [] # check each annotation for a in IterateTable(db['annotation']): # skip media annotations (for now..) if a['type'] not in app_cfg.TEXT_ANNOTATION_TYPES: continue # if it's a section heading or the paragraph id changed, append # print(f"{a['type']} {a['paragraph_id']}") if a['type'] == 'section_heading' or a['paragraph_id'] != last_pid: if len(para): s += para + "\n\n" para = "" last_pid = a['paragraph_id'] # if it's a new section, add a heading if a['type'] == 'section_heading': s += f"{app_cfg.ROMAN_NUMERALS[section_count]}: {a['text']}\n" section_count += 1 last_pid = a['paragraph_id'] # elif a['type'] == 'footnote': # para += f"{len(notes)+1} " # notes.append(a['text']) else: para += a['text'] + " " if len(para): s += para # if len(notes): # s += h(3, "Footnotes") # for i, note in enumerate(notes): # s += p(f"{i+1} " + note) return s ###################################################################### # HTML Helper Functions ###################################################################### def plain_content(db, title): # Episode, Venue, Annotation s = h(1, title) s += transcript_to_html(db) s += credits_to_html(db, 1) s += table_to_html(db, 'episode', 'Episodes', episode_to_html) s += table_to_html(db, 'venue', 'Venues', venue_to_html) return s def transcript_to_html(db): s = h(2, "Transcript") para = "" last_pid = 0 section_count = 0 notes = [] # check each annotation for a in IterateTable(db['annotation']): # skip media annotations (for now..) if a['type'] not in app_cfg.TEXT_ANNOTATION_TYPES: continue # if it's a section heading or the paragraph id changed, append # print(f"{a['type']} {a['paragraph_id']}") if a['type'] == 'section_heading' or a['paragraph_id'] != last_pid: if len(para): s += p(para) para = "" last_pid = a['paragraph_id'] # if it's a new section, add a heading if a['type'] == 'section_heading': s += h(3, f"{app_cfg.ROMAN_NUMERALS[section_count]}: {a['text']}") section_count += 1 last_pid = a['paragraph_id'] elif a['type'] == 'footnote': para += f"{len(notes)+1} " notes.append(a['text']) else: para += a['text'] + " " if len(para): s += p(para) if len(notes): s += h(3, "Footnotes") for i, note in enumerate(notes): s += p(f"{i+1} " + note) return s def credits_to_html(db, ep_num): e = db_get(db, 'episode', ep_num - 1) s = h(2, "Credits") s += pbr_to_paras(e['settings']['credits']) return s def episode_to_html(e): """Render an upcoming episode as plain HTML""" if len(e['title']): s = h(3, f"Episode {e['episode_number']}: {e['title']}") else: s = h(3, f"Episode {e['episode_number']}") s += p(e['release_date']) s += h(4, "Artists") s += pbr(e['settings']['artists']) return s def venue_to_html(e): """Render a venue as plain HTML""" s = h(3, e['title']) s += p(e['date']) s += h(4, "Artists") s += pbr(e['settings']['artists']) s += pbr_to_paras(e['settings']['credits']) return s ###################################################################### # HTML Helper Functions ###################################################################### def table_to_html(db, table, title, fn): """Convert a simple table list to HTML""" s = h(2, title) for e in IterateTable(db[table]): s += d(fn(e)) return d(s) # Helper functions that wrap stuff in HTML def pbr_to_paras(s): return "".join(list(map(pbr, to_paras(s)))) def to_paras(s): return s.replace("# ", "").split("\n\n") def d(s): return f"
{s}
" def h(n, s): return f"{s}" def br(s): return s.replace("\n","
") def p(s): return f"

{s}

" def pbr(s): return p(br(s)) def write_refresh(url, site_fp_out): write_text(f'', join(site_fp_out, 'index.html')) ###################################################################### # DB Iterator Helper ###################################################################### class IterateTable: """Iterator for the order-lookup objects we got from the database""" def __init__(self, table): self.table = table self.len = len(table['order']) self.index = -1 def __iter__(self): return self def __next__(self): self.index += 1 if self.index >= self.len: raise StopIteration id = self.table['order'][self.index] return self.table['lookup'][id]