path: root/animism-align/cli/commands/site/export.py

import click

from app.settings import app_cfg
from app.utils.file_utils import load_text, write_json, write_text
from os.path import join
from functools import reduce
from shutil import copyfile
import os

MEDIA_ANNOTATION_TYPES = [
  'image', 'carousel', 'grid', 'gallery',
  'video',
  'vitrine',
]

@click.command('info')
# @click.option('-g', '--graph', 'opt_graph_path', required=True,
#   help='Graph name')
@click.option('-o', '--output', 'opt_output_dir', required=False, default="animism",
  help='Output directory')
@click.option('-s', '--sync/--no-sync', 'opt_sync', required=False, default=True,
  help='Whether to sync over FTP')
@click.option('-j', '--js/--no-js', 'opt_js', required=False, default=False,
  help='Whether to rebuild the Javascript bundle')
@click.pass_context
def cli(ctx, opt_output_dir, opt_sync, opt_js):
  """Export a graph"""

  # ------------------------------------------------
  # imports

  import datetime
  import json
  from distutils.dir_util import copy_tree

  # ------------------------------------------------
  # export settings

  page_title = "Animism: Episode 1"
  page_name = "episode1"
  page_desc = "A Report on Migrating Souls in Museums and Moving Pictures. Animism on e-flux.com is the ninth iteration of the exhibition, presented digitally here."
  base_href = "https://animism.e-flux.com"
  page_image = base_href + "/episode1/media/8ca4adc754093cc8578a3033de8f96af96180fbce838c480636ffd5d128d1937.jpg"
  site_url = base_href + "/" + page_name

  page_url = "/" + page_name
  media_url = "/" + page_name + "/media"

  site_title = f"{page_title}: {page_desc}"
  site_path = opt_output_dir or datetime.datetime.now().strftime("animism_%Y%m%d%H%M")
  site_fp_root = join(app_cfg.DIR_EXPORTS, site_path)
  site_fp_static = join(site_fp_root, 'static')
  site_fp_out = join(site_fp_root, page_name)
  site_fp_media = join(site_fp_root, page_name, 'media')

  # ------------------------------------------------
  # load the db

  db = export_db()
  prune_db(db)
  media_to_copy = rewrite_db_media(db, site_fp_media, media_url)

  db['urls'] = { 'audio': '/' + page_name + '/media/animism_episode_01.mp3' }
  media_to_copy['audio'] = {
    'src': join(app_cfg.DIR_DATA_STORE, 'peaks/animism_episode_01_2810.mp3'),
    'dst': join(site_fp_media, 'animism_episode_01.mp3'),
  }

  # ------------------------------------------------
  # build the json for the e-flux search

  search_json = build_search_json(db, base_href, site_url, page_desc, page_image)

  # ------------------------------------------------
  # build the index.html

  index_html = load_text(join(app_cfg.DIR_STATIC, 'site.html'), split=False)
  index_html = index_html.replace('SITE_PATH', page_url)
  index_html = index_html.replace('SITE_JSON', json.dumps(db, separators=(',', ':')))
  index_html = index_html.replace('PAGE_TITLE', page_title)
  index_html = index_html.replace('PAGE_DESCRIPTION', page_desc)
  index_html = index_html.replace('PAGE_SHARE_IMAGE', page_image)
  index_html = index_html.replace('PLAIN_CONTENT', plain_content(db, site_title))
  index_html = index_html.replace('BUNDLE_PATH', join(page_url, 'bundle.js'))
  write_text(index_html, join(site_fp_out, 'index.html'))

  # ------------------------------------------------
  # write all the json

  write_json(db, join(site_fp_out, 'index.json'), default=str, minify=False)
  write_json(search_json, join(site_fp_static, 'search.json'), default=str, minify=False)

  # ------------------------------------------------
  # write custom css

  # site_css = load_text(join(app_cfg.DIR_STATIC, 'site.css'), split=False)
  # index_html = index_html.replace('SITE_PATH', page_url)
  # write_text(site_css, join(site_fp_out, 'site.css'))

  # ------------------------------------------------
  # copy media from the exhibition

  copy_media(site_fp_media, media_to_copy)

  # ------------------------------------------------
  # copy any static assets

  copy_tree(join(app_cfg.DIR_STATIC, 'fonts'), join(site_fp_static, 'fonts'))
  copy_tree(join(app_cfg.DIR_STATIC, 'img'), join(site_fp_static, 'img'))
  copyfile(join(app_cfg.DIR_STATIC, 'favicon.ico'), join(site_fp_root, 'favicon.ico'))

    print("Site export complete!")
    print(f"Site exported to: {site_fp_out}")
    return

  # ------------------------------------------------
  # build javascript

  if opt_js:
    print("Building javascript...")
    print(f'NODE_ENV=production node ./node_modules/webpack-cli/bin/cli.js --config ./webpack.config.site.js -o {site_fp_out}/bundle.js')
    os.chdir(app_cfg.DIR_PROJECT_ROOT)
    os.system(f'NODE_ENV=production node ./node_modules/webpack-cli/bin/cli.js --config ./webpack.config.site.js -o {site_fp_out}/bundle.js')
  else:
    print("Skipping JS build.")

  if opt_sync:
    print("Deploying site...")
    os.system("""
        lftp -c "set ssl:verify-certificate false; set ftp:list-options -a;
        open ftp://animism:Agkp8#48@93.114.86.205;
        lcd ./data_store/exports/animism;
        cd /;
        mirror --reverse --use-cache --verbose --no-umask --ignore-time --parallel=2"
      """)
  else:
    print("Skipping sync.")

  print("Site export complete!")
  print(f"Site exported to: {site_fp_out}")

######################################################################
# Search JSON
######################################################################

def build_search_json(db, base_href, site_url, page_desc, page_image):
  search_items = []
  search_items.append({
    "text": transcript_to_text(db),
    "url": site_url,
    "type": "text",
    "previewtitle": '<i>Animism</i>, episode 1',
    "previewtext": page_desc,
    "previewimage": page_image,
  })
  seen_media = {}
  for annotation in IterateTable(db['annotation']):
    if annotation['type'] not in MEDIA_ANNOTATION_TYPES:
      continue
    if 'media_id' not in annotation['settings']:
      continue
    media_id = annotation['settings']['media_id']
    if media_id in seen_media:
      continue
    seen_media[media_id] = True
    if media_id not in db['media']['lookup']:
      continue
    media = db['media']['lookup'][media_id]
    if 'hide_in_bibliography' in media['settings'] and media['settings']['hide_in_bibliography']:
      continue
    if 'thumbnail' not in media['settings']:
      continue
    start_ts = annotation['start_ts']
    search_items.append({
      "text": media['settings']['bibliography'],
      "url": site_url + "/#ts=" + str(start_ts),
      "type": annotation['type'],
      "previewtitle": media['title'],
      "previewtext": page_desc,
      "previewimage": base_href + media['settings']['thumbnail']['url'],
      })

  return search_items

######################################################################
# Database Functions
######################################################################

def copy_media(fp_media, to_copy):
  os.makedirs(fp_media, exist_ok=True)
  print(f"Copying {len(to_copy.keys())} files")
  total_size = 0
  for fp in to_copy.values():
    copyfile(fp['src'], fp['dst'])
    total_size += os.path.getsize(fp['dst'])
  print(f"wrote {round(total_size / 1000000, 2)} MB")

def rewrite_db_media(db, fp_out, url_out):
  """
  Go over all the media and find any Upload objects.
  Figure out which to copy, and rewrite DB to use the export URL schema.
  """
  to_copy = {}
  for item in IterateTable(db['media']):
    settings = item['settings']

    # images - various sizes... we don't use fullsize anywhere
    if item['type'] == 'image':
      if 'fullsize' in settings:
        del settings['fullsize']
      for field in app_cfg.IMAGE_UPLOAD_FIELDS:
        if field in settings:
          settings[field] = rewrite_upload(to_copy, settings[field], fp_out, url_out)
    # videos - poster images
    elif item['type'] == 'video':
      if 'poster' in settings:
        settings['poster'] = rewrite_upload(to_copy, settings['poster'], fp_out, url_out)
    # galleries - a bunch of lookups... we PROBABLY don't need image_lookup (fullsize)
    elif item['type'] == 'gallery':
      new_image_lookup = {}
      if 'display' in settings:
        settings['display'] = rewrite_upload(to_copy, settings['display'], fp_out, url_out)
      if 'thumbnail' in settings:
        settings['thumbnail'] = rewrite_upload(to_copy, settings['thumbnail'], fp_out, url_out)
      for id in settings['image_order']:
        id = str(id)
        if id in settings['image_lookup']:
          new_image_lookup[id] = rewrite_upload(to_copy, settings['image_lookup'][id], fp_out, url_out, png_only=True)
      settings['image_lookup'] = new_image_lookup

      for field in app_cfg.IMAGE_UPLOAD_GALLERY_LOOKUPS:
        for id in settings['image_order']:
          id = str(id)
          if id in settings[field]:
            settings[field][id] = rewrite_upload(to_copy, settings[field][id], fp_out, url_out)
    # files - singleton file uploads
    elif item['type'] == 'file':
      if 'file' in settings:
        settings['file'] = rewrite_upload(to_copy, settings['file'], fp_out, url_out)
  return to_copy

def rewrite_upload(to_copy, item, fp_out, url_out, png_only=False):
  """
  # rewriting uploads. they look like this:
  "fn": "koester.gif",
  "sha256": "c7c25e8d9be8b3e5db89df0f4a35f8a599dfdcf8bf9bc1f6c4137c7b6522d710",
  "tag": "file",
  "url": "/static/data_store/uploads/file/koester.gif",
  "username": "animism"
  """
  if 'sha256' not in item:
    return item
  if png_only and item['ext'] != '.png':
    return
  sha = item['sha256']
  out_fn = sha + item['ext']
  out_obj = {
    "url": join(url_out, out_fn),
  }

  if sha not in to_copy:
    # print(f"SHA: {sha}")
    in_fn = item['fn']
    in_path = join(app_cfg.DIR_UPLOADS, item['tag'], in_fn)
    if os.path.exists(in_path):
      to_copy[sha] = {
        "src": in_path,
        "dst": join(fp_out, out_fn)
      }
    else:
      print(f"Missing path: {in_path}")

  return out_obj

def prune_db(db):
  """Remove random stuff from the JSON that doesn't need to be there
  - extraneous paragraphs
  - extraneous media
  """
  seen_paras = {}
  seen_media = {}
  for a in IterateTable(db['annotation']):
    seen_paras[a['paragraph_id']] = True
    if 'media_id' in a['settings']:
      seen_media[a['settings']['media_id']] = True

  db['paragraph'] = filter_db(db, 'paragraph', seen_paras)
  db['media'] = filter_db(db, 'media', seen_media)

def filter_db(db, table, seen):
  order = list(filter(lambda i: i in seen, db[table]['order']))
  lookup = { id: db[table]['lookup'][id] for id in order }
  return { 'order': order, 'lookup': lookup }

def export_db():
  """Load the entire database and convert it to JSON"""
  from app.sql.common import db, Session, Episode, Venue, Annotation, Paragraph, Media, Upload

  session = Session()

  classes = [ Episode, Venue, Annotation, Paragraph, Media ]
  data = {}

  for c in classes:
    e_q = session.query(c)
    if c == Annotation or c == Paragraph:
      e_q = e_q.order_by(c.start_ts)
    e_list = e_q.all()
    order = list(map(get_id, e_list))
    lookup = reduce(get_json_tup, e_list, {})
    table_name = str(c.__table__)
    data[table_name] = { 'order': order, 'lookup': lookup }
    print(f"""exported {table_name} ({len(order)} rows)""")
  return data

def sanitize_obj(data):
  if 'created_at' in data:
    del data['created_at']
  if 'updated_at' in data:
    del data['updated_at']
  return data

def get_id(e):
  return e.id
def get_json_tup(a,e):
  a[e.id] = sanitize_obj(e.toJSON())
  return a

def db_get(db, table, idx):
  """Get an indexed object out of our db table"""
  id = db[table]['order'][idx]
  return db[table]['lookup'][id]

######################################################################
# Plaintext helper functions
######################################################################

def transcript_to_text(db):
  s = ""
  para = ""
  last_pid = 0
  section_count = 0
  notes = []
  # check each annotation
  for a in IterateTable(db['annotation']):
    # skip media annotations (for now..)
    if a['type'] not in app_cfg.TEXT_ANNOTATION_TYPES:
      continue
    # if it's a section heading or the paragraph id changed, append
    # print(f"{a['type']} {a['paragraph_id']}")
    if a['type'] == 'section_heading' or a['paragraph_id'] != last_pid:
      if len(para):
        s += para + "\n\n"
      para = ""
      last_pid = a['paragraph_id']
    # if it's a new section, add a heading
    if a['type'] == 'section_heading':
      s += f"{app_cfg.ROMAN_NUMERALS[section_count]}: {a['text']}\n"
      section_count += 1
      last_pid = a['paragraph_id']
    # elif a['type'] == 'footnote':
    #   para += f"<sup>{len(notes)+1}</sup> "
    #   notes.append(a['text'])
    else:
      para += a['text'] + " "
  if len(para):
    s += para
  # if len(notes):
  #   s += h(3, "Footnotes")
  #   for i, note in enumerate(notes):
  #     s += p(f"<sup>{i+1}</sup> " + note)
  return s

######################################################################
# HTML Helper Functions
######################################################################

def plain_content(db, title):
  # Episode, Venue, Annotation
  s = h(1, title)
  s += transcript_to_html(db)
  s += credits_to_html(db, 1)
  s += table_to_html(db, 'episode', 'Episodes', episode_to_html)
  s += table_to_html(db, 'venue', 'Venues', venue_to_html)
  return s

def transcript_to_html(db):
  s = h(2, "Transcript")
  para = ""
  last_pid = 0
  section_count = 0
  notes = []
  # check each annotation
  for a in IterateTable(db['annotation']):
    # skip media annotations (for now..)
    if a['type'] not in app_cfg.TEXT_ANNOTATION_TYPES:
      continue
    # if it's a section heading or the paragraph id changed, append
    # print(f"{a['type']} {a['paragraph_id']}")
    if a['type'] == 'section_heading' or a['paragraph_id'] != last_pid:
      if len(para):
        s += p(para)
      para = ""
      last_pid = a['paragraph_id']
    # if it's a new section, add a heading
    if a['type'] == 'section_heading':
      s += h(3, f"{app_cfg.ROMAN_NUMERALS[section_count]}: {a['text']}")
      section_count += 1
      last_pid = a['paragraph_id']
    elif a['type'] == 'footnote':
      para += f"<sup>{len(notes)+1}</sup> "
      notes.append(a['text'])
    else:
      para += a['text'] + " "
  if len(para):
    s += p(para)
  if len(notes):
    s += h(3, "Footnotes")
    for i, note in enumerate(notes):
      s += p(f"<sup>{i+1}</sup> " + note)
  return s

def credits_to_html(db, ep_num):
  e = db_get(db, 'episode', ep_num - 1)
  s = h(2, "Credits")
  s += pbr_to_paras(e['settings']['credits'])
  return s

def episode_to_html(e):
  """Render an upcoming episode as plain HTML"""
  if len(e['title']):
    s = h(3, f"Episode {e['episode_number']}: {e['title']}")
  else:
    s = h(3, f"Episode {e['episode_number']}")
  s += p(e['release_date'])
  s += h(4, "Artists")
  s += pbr(e['settings']['artists'])
  return s

def venue_to_html(e):
  """Render a venue as plain HTML"""
  s = h(3, e['title'])
  s += p(e['date'])
  s += h(4, "Artists")
  s += pbr(e['settings']['artists'])
  s += pbr_to_paras(e['settings']['credits'])
  return s


######################################################################
# HTML Helper Functions
######################################################################

def table_to_html(db, table, title, fn):
  """Convert a simple table list to HTML"""
  s = h(2, title)
  for e in IterateTable(db[table]):
    s += d(fn(e))
  return d(s)

# Helper functions that wrap stuff in HTML
def pbr_to_paras(s):
  return "".join(list(map(pbr, to_paras(s))))
def to_paras(s):
  return s.replace("# ", "").split("\n\n")
def d(s):
  return f"<div>{s}</div>"
def h(n, s):
  return f"<h{n}>{s}</h{n}>"
def br(s):
  return s.replace("\n","<br>")
def p(s):
  return f"<p>{s}</p>"
def pbr(s):
  return p(br(s))

def write_refresh(url, site_fp_out):
  write_text(f'<meta http-equiv="refresh" content="0; url={home_page}">', join(site_fp_out, 'index.html'))

######################################################################
# DB Iterator Helper
######################################################################

class IterateTable:
  """Iterator for the order-lookup objects we got from the database"""
  def __init__(self, table):
    self.table = table
    self.len = len(table['order'])
    self.index = -1
  def __iter__(self):
    return self
  def __next__(self):
    self.index += 1
    if self.index >= self.len:
      raise StopIteration
    id = self.table['order'][self.index]
    return self.table['lookup'][id]