diff options
Diffstat (limited to 'animism-align/cli/commands/site')
| -rw-r--r-- | animism-align/cli/commands/site/export.py | 105 | ||||
| -rw-r--r-- | animism-align/cli/commands/site/migrate_to_mysql.py | 148 |
2 files changed, 81 insertions, 172 deletions
diff --git a/animism-align/cli/commands/site/export.py b/animism-align/cli/commands/site/export.py index f3f5d3c..d706a55 100644 --- a/animism-align/cli/commands/site/export.py +++ b/animism-align/cli/commands/site/export.py @@ -7,13 +7,23 @@ from functools import reduce from shutil import copyfile import os +MEDIA_ANNOTATION_TYPES = [ + 'image', 'carousel', 'grid', 'gallery', + 'video', + 'vitrine', +] + @click.command('info') # @click.option('-g', '--graph', 'opt_graph_path', required=True, # help='Graph name') @click.option('-o', '--output', 'opt_output_dir', required=False, default="animism", help='Output directory') +@click.option('-s', '--sync/--no-sync', 'opt_sync', required=False, default=True, + help='Whether to sync over FTP') +@click.option('-j', '--js/--no-js', 'opt_js', required=False, default=False, + help='Whether to rebuild the Javascript bundle') @click.pass_context -def cli(ctx, opt_output_dir): +def cli(ctx, opt_output_dir, opt_sync, opt_js): """Export a graph""" # ------------------------------------------------ @@ -28,9 +38,10 @@ def cli(ctx, opt_output_dir): page_title = "Animism: Episode 1" page_name = "episode1" - page_desc = "A Report on Migrating Souls in Museums and Moving Pictures. Animism on e-flux.com is the ninth iteration of the exhibition, which will be released in four episodes starting November 2020." - page_image = "https://animism.e-flux.com/episode1/media/8ca4adc754093cc8578a3033de8f96af96180fbce838c480636ffd5d128d1937.jpg" - site_url = "https://animism.e-flux.com/" + page_name + page_desc = "A Report on Migrating Souls in Museums and Moving Pictures. Animism on e-flux.com is the ninth iteration of the exhibition, presented digitally here." + base_href = "https://animism.e-flux.com" + page_image = base_href + "/episode1/media/8ca4adc754093cc8578a3033de8f96af96180fbce838c480636ffd5d128d1937.jpg" + site_url = base_href + "/" + page_name page_url = "/" + page_name media_url = "/" + page_name + "/media" @@ -58,14 +69,7 @@ def cli(ctx, opt_output_dir): # ------------------------------------------------ # build the json for the e-flux search - search_json = [{ - "text": transcript_to_text(db), - "url": site_url, - "type": "text", - "previewtitle": '<i>Animism</i>, episode 1', - "previewtext": page_desc, - "previewimage": page_image, - }] + search_json = build_search_json(db, base_href, site_url, page_desc, page_image) # ------------------------------------------------ # build the index.html @@ -105,27 +109,80 @@ def cli(ctx, opt_output_dir): copy_tree(join(app_cfg.DIR_STATIC, 'img'), join(site_fp_static, 'img')) copyfile(join(app_cfg.DIR_STATIC, 'favicon.ico'), join(site_fp_root, 'favicon.ico')) + print("Site export complete!") + print(f"Site exported to: {site_fp_out}") + return + # ------------------------------------------------ # build javascript - print("Building javascript...") - print(f'NODE_ENV=production node ./node_modules/webpack-cli/bin/cli.js --config ./webpack.config.site.js -o {site_fp_out}/bundle.js') - os.chdir(app_cfg.DIR_PROJECT_ROOT) - os.system(f'NODE_ENV=production node ./node_modules/webpack-cli/bin/cli.js --config ./webpack.config.site.js -o {site_fp_out}/bundle.js') + if opt_js: + print("Building javascript...") + print(f'NODE_ENV=production node ./node_modules/webpack-cli/bin/cli.js --config ./webpack.config.site.js -o {site_fp_out}/bundle.js') + os.chdir(app_cfg.DIR_PROJECT_ROOT) + os.system(f'NODE_ENV=production node ./node_modules/webpack-cli/bin/cli.js --config ./webpack.config.site.js -o {site_fp_out}/bundle.js') + else: + print("Skipping JS build.") - print("Deploying site...") - os.system(""" - lftp -c "set ssl:verify-certificate false; set ftp:list-options -a; - open ftp://animism:Agkp8#48@93.114.86.205; - lcd ./data_store/exports/animism; - cd /; - mirror --reverse --use-cache --verbose --no-umask --ignore-time --parallel=2" - """) + if opt_sync: + print("Deploying site...") + os.system(""" + lftp -c "set ssl:verify-certificate false; set ftp:list-options -a; + open ftp://animism:Agkp8#48@93.114.86.205; + lcd ./data_store/exports/animism; + cd /; + mirror --reverse --use-cache --verbose --no-umask --ignore-time --parallel=2" + """) + else: + print("Skipping sync.") print("Site export complete!") print(f"Site exported to: {site_fp_out}") ###################################################################### +# Search JSON +###################################################################### + +def build_search_json(db, base_href, site_url, page_desc, page_image): + search_items = [] + search_items.append({ + "text": transcript_to_text(db), + "url": site_url, + "type": "text", + "previewtitle": '<i>Animism</i>, episode 1', + "previewtext": page_desc, + "previewimage": page_image, + }) + seen_media = {} + for annotation in IterateTable(db['annotation']): + if annotation['type'] not in MEDIA_ANNOTATION_TYPES: + continue + if 'media_id' not in annotation['settings']: + continue + media_id = annotation['settings']['media_id'] + if media_id in seen_media: + continue + seen_media[media_id] = True + if media_id not in db['media']['lookup']: + continue + media = db['media']['lookup'][media_id] + if 'hide_in_bibliography' in media['settings'] and media['settings']['hide_in_bibliography']: + continue + if 'thumbnail' not in media['settings']: + continue + start_ts = annotation['start_ts'] + search_items.append({ + "text": media['settings']['bibliography'], + "url": site_url + "/#ts=" + str(start_ts), + "type": annotation['type'], + "previewtitle": media['title'], + "previewtext": page_desc, + "previewimage": base_href + media['settings']['thumbnail']['url'], + }) + + return search_items + +###################################################################### # Database Functions ###################################################################### diff --git a/animism-align/cli/commands/site/migrate_to_mysql.py b/animism-align/cli/commands/site/migrate_to_mysql.py deleted file mode 100644 index e69dd13..0000000 --- a/animism-align/cli/commands/site/migrate_to_mysql.py +++ /dev/null @@ -1,148 +0,0 @@ -import click -import os -import glob -import time - -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker -from sqlalchemy.ext.declarative import declarative_base - -from flask_sqlalchemy import SQLAlchemy - -from app.settings import app_cfg - -@click.command('migrate') -@click.pass_context -def cli(ctx): - """ - - Create connections to both databases - - For each table, for each row, insert from one to the other - """ - mysql_session, mysql_base = make_mysql_base() - sqlite_session, sqlite_base = make_sqlite3_base() - mysql_classes = make_classes(mysql_base) - sqlite_classes = make_classes(sqlite_base) - - for mysql_class, sqlite_class in zip(mysql_classes, sqlite_classes): - sqlite_objs = sqlite_session.query(sqlite_class).order_by(sqlite_class.id).all() - for sqlite_obj in sqlite_objs: - mysql_obj = mysql_class() - for column in sqlite_class.__table__.columns: - table_name, column_name = str(column).split(".") - # print(f"{table_name} => {column_name}") - # if column_name != 'id': - setattr(mysql_obj, column_name, getattr(sqlite_obj, column_name)) - mysql_session.add(mysql_obj) - mysql_session.commit() - -def make_mysql_base(): - """Make a Mysql connection""" - connection_url = "mysql+pymysql://{}:{}@{}/{}?charset=utf8mb4".format( - os.getenv("DB_USER"), - os.getenv("DB_PASS"), - os.getenv("DB_HOST"), - os.getenv("DB_NAME") - ) - return make_base(connection_url) - -def make_sqlite3_base(): - """Make a SQLite3 connection""" - connection_url = "sqlite:///{}".format(os.path.join(app_cfg.DIR_DATABASE, 'animism.sqlite3')) - return make_base(connection_url) - -def make_base(connection_url): - """Make a connection base from a connection URL""" - engine = create_engine(connection_url, encoding="utf-8", pool_recycle=3600) - Session = sessionmaker(bind=engine) - Base = declarative_base() - Base.metadata.bind = engine - db = SQLAlchemy() - return Session(), Base - -def make_classes(Base): - """Make classes from a base""" - - from sqlalchemy import create_engine, Table, Column, Text, String, Integer, \ - Boolean, Float, DateTime, JSON, ForeignKey - from sqlalchemy_utc import UtcDateTime, utcnow - - # from app.sql.common import db, Base, Session - - class Episode(Base): - """Table for storing episodes and their metadata""" - __tablename__ = 'episode' - id = Column(Integer, primary_key=True) - episode_number = Column(Integer) - title = Column(String(256, convert_unicode=True), nullable=False) - release_date = Column(String(256, convert_unicode=True)) - is_live = Column(Boolean, default=False) - settings = Column(JSON, default={}, nullable=True) - - class Annotation(Base): - """Table for storing references to annotations""" - __tablename__ = 'annotation' - id = Column(Integer, primary_key=True) - type = Column(String(16, convert_unicode=True), nullable=False) - paragraph_id = Column(Integer, nullable=True) - start_ts = Column(Float, nullable=False) - end_ts = Column(Float, nullable=True) - text = Column(Text(convert_unicode=True), nullable=True) - settings = Column(JSON, default={}, nullable=True) - - class Media(Base): - """Table for storing references to media""" - __tablename__ = 'media' - id = Column(Integer, primary_key=True) - type = Column(String(16, convert_unicode=True), nullable=False) - tag = Column(String(64, convert_unicode=True), nullable=True) - url = Column(String(256, convert_unicode=True), nullable=True) - title = Column(String(256, convert_unicode=True), nullable=True) - author = Column(String(256, convert_unicode=True), nullable=True) - pre_title = Column(String(256, convert_unicode=True), nullable=True) - post_title = Column(String(256, convert_unicode=True), nullable=True) - translated_title = Column(String(256, convert_unicode=True), nullable=True) - date = Column(String(256, convert_unicode=True), nullable=True) - source = Column(String(256, convert_unicode=True), nullable=True) - medium = Column(String(64, convert_unicode=True), nullable=True) - description = Column(Text(convert_unicode=True), nullable=True) - start_ts = Column(Float, nullable=True) - settings = Column(JSON, default={}, nullable=True) - - class Paragraph(Base): - """Table for storing paragraphs, which contain annotations""" - __tablename__ = 'paragraph' - id = Column(Integer, primary_key=True) - type = Column(String(16, convert_unicode=True), nullable=False) - start_ts = Column(Float, nullable=False) - end_ts = Column(Float, nullable=True) - settings = Column(JSON, default={}, nullable=True) - - class Upload(Base): - """Table for storing references to various media""" - __tablename__ = 'upload' - id = Column(Integer, primary_key=True) - sha256 = Column(String(256), nullable=False) - fn = Column(String(256), nullable=False) - ext = Column(String(4, convert_unicode=True), nullable=False) - tag = Column(String(64, convert_unicode=True), nullable=True) - username = Column(String(16, convert_unicode=True), nullable=False) - created_at = Column(UtcDateTime(), default=utcnow()) - - class User(Base): - """Table for storing the user list""" - __tablename__ = 'user' - id = Column(Integer, primary_key=True) - username = Column(String(256, convert_unicode=True), nullable=False) - password = Column(String(256, convert_unicode=True), nullable=False) - is_admin = Column(Boolean, default=False) - settings = Column(JSON, default={}, nullable=True) - - class Venue(Base): - """Table for storing the venue list""" - __tablename__ = 'venue' - id = Column(Integer, primary_key=True) - title = Column(String(256, convert_unicode=True), nullable=False) - date = Column(String(256, convert_unicode=True), nullable=False) - settings = Column(JSON, default={}, nullable=True) - - return [ Episode, Paragraph, Annotation, Media, Upload, User, Venue ] |