From 1cfe96ca6ef5c54eadd986c951dade0f56d72440 Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Tue, 16 Mar 2021 16:54:28 +0100 Subject: migrating to mysql --- cli/commands/admin/migrate_to_mysql.py | 120 +++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 cli/commands/admin/migrate_to_mysql.py (limited to 'cli/commands') diff --git a/cli/commands/admin/migrate_to_mysql.py b/cli/commands/admin/migrate_to_mysql.py new file mode 100644 index 0000000..32ff7cf --- /dev/null +++ b/cli/commands/admin/migrate_to_mysql.py @@ -0,0 +1,120 @@ +import click +import os +import glob +import time + +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from sqlalchemy.ext.declarative import declarative_base + +from flask_sqlalchemy import SQLAlchemy + +from app.settings import app_cfg + +@click.command('migrate_to_mysql') +@click.pass_context +def cli(ctx): + """ + - Create connections to both databases + - For each table, for each row, insert from one to the other + """ + mysql_session, mysql_base = make_mysql_base() + sqlite_session, sqlite_base = make_sqlite3_base() + mysql_classes = make_classes(mysql_base) + sqlite_classes = make_classes(sqlite_base) + + for mysql_class, sqlite_class in zip(mysql_classes, sqlite_classes): + sqlite_objs = sqlite_session.query(sqlite_class).order_by(sqlite_class.id).all() + for sqlite_obj in sqlite_objs: + mysql_obj = mysql_class() + for column in sqlite_class.__table__.columns: + table_name, column_name = str(column).split(".") + # print(f"{table_name} => {column_name}") + # if column_name != 'id': + setattr(mysql_obj, column_name, getattr(sqlite_obj, column_name)) + mysql_session.add(mysql_obj) + mysql_session.commit() + +def make_mysql_base(): + """Make a Mysql connection""" + connection_url = "mysql+pymysql://{}:{}@{}/{}?charset=utf8mb4".format( + os.getenv("DB_USER"), + os.getenv("DB_PASS"), + os.getenv("DB_HOST"), + os.getenv("DB_NAME") + ) + return make_base(connection_url) + +def make_sqlite3_base(): + """Make a SQLite3 connection""" + connection_url = "sqlite:///{}".format(os.path.join(app_cfg.DIR_DATABASE, 'animism.sqlite3')) + return make_base(connection_url) + +def make_base(connection_url): + """Make a connection base from a connection URL""" + engine = create_engine(connection_url, encoding="utf-8", pool_recycle=3600) + Session = sessionmaker(bind=engine) + Base = declarative_base() + Base.metadata.bind = engine + db = SQLAlchemy() + return Session(), Base + +def make_classes(Base): + """Make classes from a base""" + + from sqlalchemy import create_engine, Table, Column, Text, String, Integer, \ + Boolean, Float, DateTime, JSON, ForeignKey + from sqlalchemy_utc import UtcDateTime, utcnow + + class Upload(Base): + """Table for storing references to various media""" + __tablename__ = 'upload' + id = Column(Integer, primary_key=True) + graph_id = Column(Integer, ForeignKey('graph.id'), nullable=True) + sha256 = Column(String(256), nullable=False) + fn = Column(String(256), nullable=False) + ext = Column(String(4, convert_unicode=True), nullable=False) + tag = Column(String(64, convert_unicode=True), nullable=True) + username = Column(String(16, convert_unicode=True), nullable=False) + created_at = Column(UtcDateTime(), default=utcnow()) + + class Tile(Base): + """Table for storing references to tiles""" + __tablename__ = 'tile' + id = Column(Integer, primary_key=True) + graph_id = Column(Integer, ForeignKey('graph.id'), nullable=True) + page_id = Column(Integer, ForeignKey('page.id'), nullable=True) + target_page_id = Column(Integer, ForeignKey('page.id'), nullable=True) + type = Column(String(16, convert_unicode=True), nullable=False) + sort_order = Column(Integer, default=0) + settings = Column(JSON, default={}, nullable=True) + created_at = Column(UtcDateTime(), default=utcnow()) + updated_at = Column(UtcDateTime(), onupdate=utcnow()) + + class Page(Base): + """Table for storing references to pages""" + __tablename__ = 'page' + id = Column(Integer, primary_key=True) + graph_id = Column(Integer, ForeignKey('graph.id'), nullable=True) + path = Column(String(64, convert_unicode=True), nullable=False) + title = Column(String(64, convert_unicode=True), nullable=False) + username = Column(String(32, convert_unicode=True), nullable=False) + description = Column(Text(convert_unicode=True), nullable=False) + settings = Column(JSON, default={}, nullable=True) + created_at = Column(UtcDateTime(), default=utcnow()) + updated_at = Column(UtcDateTime(), onupdate=utcnow()) + + class Graph(Base): + """Table for storing references to graphs""" + __tablename__ = 'graph' + id = Column(Integer, primary_key=True) + home_page_id = Column(Integer, nullable=True) + path = Column(String(64, convert_unicode=True), nullable=False) + title = Column(String(64, convert_unicode=True), nullable=False) + username = Column(String(32, convert_unicode=True), nullable=False) + description = Column(Text(convert_unicode=True), nullable=False) + settings = Column(JSON, default={}, nullable=True) + created_at = Column(UtcDateTime(), default=utcnow()) + updated_at = Column(UtcDateTime(), onupdate=utcnow()) + + return [ Upload, Graph, Page, Tile ] -- cgit v1.2.3-70-g09d2