lodel2_mirror/DataSource/MySQL/migrationhandler.py

# -*- coding: utf-8 -*-

import copy
import pymysql

import EditorialModel
from DataSource.MySQL.common_utils import MySQL
from DataSource.dummy.migrationhandler import DummyMigrationHandler

# The global MH algorithm is as follow :
# A create_table(table_name, pk_name, pk_opt) method that create a table
# with one pk field
# An add_column(table_name, field_name, field_opt) method that add a column to a table
#
# The create_default_table method will call both methods to create the object and relation tables
#
# Supported operations :
# - EmClass creation
# - EmClass deletion (untested)
# - EmField creation
# - EmField deletion (untested)
# - rel2type attribute creation
# - rel2type attribute deletion (unstested)
#
# Unsupported operations :
# - EmClass rename
# - EmField rename
# - rel2type field rename
# - rel2type attribute rename
# - EmFieldType changes
#
# @todo Unified datasources and migration handlers via utils functions


## @brief Modify a MySQL database given editorial model changes
class MysqlMigrationHandler(DummyMigrationHandler):

    ## @brief Construct a MysqlMigrationHandler
    # @param host str : The db host
    # @param user str : The db user
    # @param password str : The db password
    # @param db str : The db name
    def __init__(self, host, user, passwd, db, module=pymysql, db_engine='InnoDB', foreign_keys=True, debug=False, dryrun=False, drop_if_exists=False):
        self.datasource = MySQL
        self._dbmodule = module
        #Connect to MySQL
        self.db = self._dbmodule.connect(host=host, user=user, passwd=passwd, db=db)
        self.debug = debug
        self.dryrun = dryrun
        self.db_engine = db_engine
        self.foreign_keys = foreign_keys if db_engine == 'InnoDB' else False
        self.drop_if_exists = drop_if_exists
        #Create default tables
        self._create_default_tables(self.drop_if_exists)

    ## @brief Delete all table created by the MH
    # @param model Model : the Editorial model
    def __purge_db(self, model):
        for uid in [c.uid for c in model.components('EmClass')]:
            try:
                self.delete_emclass_table(model, uid)
            except self._dbmodule.err.InternalError as e:
                print(e)

        for tname in [MySQL.get_r2t2table_name(f.em_class.name, model.component(f.rel_to_type_id).name) for f in model.components('EmField') if f.fieldtype == 'rel2type']:
            try:
                self._query("DROP TABLE %s;" % tname)
            except self._dbmodule.err.InternalError as e:
                print(e)

        for tname in [MySQL.relations_table_name, MySQL.objects_table_name]:
            try:
                self._query("DROP TABLE %s;" % tname)
            except self._dbmodule.err.InternalError as e:
                print(e)

    ## @brief Modify the db given an EM change
    # @param em model : The EditorialModel.model object to provide the global context
    # @param uid int : The uid of the change EmComponent
    # @param initial_state dict | None : dict with field name as key and field value as value. Representing the original state. None mean creation of a new component.
    # @param new_state dict | None : dict with field name as key and field value as value. Representing the new state. None mean component deletion
    # @throw EditorialModel.exceptions.MigrationHandlerChangeError if the change was refused
    def register_change(self, em, uid, initial_state, new_state, engine=None):
        if engine is None:
            engine = self.db_engine
        if isinstance(em.component(uid), EditorialModel.classes.EmClass):
            if initial_state is None:
                #EmClass creation
                self.create_emclass_table(em, uid, engine)
            elif new_state is None:
                #EmClass deletion
                self.delete_emclass_table(em, uid)
        elif isinstance(em.component(uid), EditorialModel.fields.EmField):
            emfield = em.component(uid)
            if emfield.rel_field_id is None:
                #non relationnal field
                if initial_state is None:
                    #non relationnal EmField creation
                    if emfield.name not in EditorialModel.classtypes.common_fields.keys():
                        self.add_col_from_emfield(em, uid)
                elif new_state is None:
                    #non relationnal EmField deletion
                    if emfield.name not in EditorialModel.classtypes.common_fields.keys():
                        self.del_col_from_emfield(em, uid)
            else:
                #relationnal field
                if initial_state is None:
                    #Rel2type attr creation
                    self.add_relationnal_field(em, uid)
                elif new_state is None:
                    #Rel2type attr deletion
                    self.del_relationnal_field(em, uid)

    ## @brief dumdumdummy
    # @note implemented to avoid the log message of EditorialModel.migrationhandler.dummy.DummyMigrationHandler
    def register_model_state(self, em, state_hash):
        pass

    ## @brief Exec a query
    # @param query str : SQL query
    def _query(self, query):
        if self.debug:
            print(query + "\n")
        if not self.dryrun:
            with self.db.cursor() as cur:
                cur.execute(query)
        self.db.commit()  # autocommit

    ## @brief Add a relationnal field
    # Add a rel2type attribute
    # @note this function handles the table creation
    # @param em Model : EditorialModel.model.Model instance
    # @param rfuid int : Relationnal field uid
    def add_relationnal_field(self, em, rfuid):
        emfield = em.component(rfuid)
        if not isinstance(emfield, EditorialModel.fields.EmField):
            raise ValueError("The given uid is not an EmField uid")

        r2tf = em.component(emfield.rel_field_id)
        tname = self._r2t2table_name(em, r2tf)
        pkname, pkftype = self._relation_pk

        #If not exists create a relational table
        self._create_table(tname, pkname, pkftype, self.db_engine, if_exists='nothing')
        #Add a foreign key if wanted
        if self.foreign_keys:
            self._add_fk(tname, self.datasource.relations_table_name, pkname, pkname)
        #Add the column
        self._add_column(tname, emfield.name, emfield.fieldtype_instance())
        #Update table triggers
        self._generate_triggers(tname, self._r2type2cols(em, r2tf))

    ## @brief Delete a rel2type attribute
    #
    # Delete a rel2type attribute
    # @note this method handles the table deletion
    # @param em Model : EditorialModel.model.Model instance
    # @param rfuid int : Relationnal field uid
    def del_relationnal_field(self, em, rfuid):
        emfield = em.component(rfuid)
        if not isinstance(emfield, EditorialModel.fields.EmField):
            raise ValueError("The given uid is not an EmField uid")

        r2tf = em.component(emfield.rel_field_id)
        tname = self._r2t2table_name(em, r2tf)

        if len(self._r2type2cols(em, r2tf)) == 1:
            #The table can be deleted (no more attribute for this rel2type)
            self._query("""DROP TABLE {table_name}""".format(table_name=tname))
        else:
            self._del_column(tname, emfield.name)
            #Update table triggers
            self._generate_triggers(tname, self._r2type2cols(em, r2tf))

    ## @brief Given an EmField uid add a column to the corresponding table
    # @param em Model : A Model instance
    # @param uid int : An EmField uid
    def add_col_from_emfield(self, em, uid):
        emfield = em.component(uid)
        if not isinstance(emfield, EditorialModel.fields.EmField):
            raise ValueError("The given uid is not an EmField uid")

        emclass = emfield.em_class
        tname = self._emclass2table_name(emclass)
        self._add_column(tname, emfield.name, emfield.fieldtype_instance())
        # Refresh the table triggers
        cols_l = self._class2cols(emclass)
        self._generate_triggers(tname, cols_l)

    ## @brief Given a class uid create the coressponding table
    # @param em Model : A Model instance
    # @param uid int : An EmField uid
    def create_emclass_table(self, em, uid, engine):
        emclass = em.component(uid)
        if not isinstance(emclass, EditorialModel.classes.EmClass):
            raise ValueError("The given uid is not an EmClass uid")
        pkname, pktype = self._common_field_pk
        table_name = self._emclass2table_name(emclass)
        self._create_table(table_name, pkname, pktype, engine=engine)

        if self.foreign_keys:
            self._add_fk(table_name, self.datasource.objects_table_name, pkname, pkname)

    ## @brief Given an EmClass uid delete the corresponding table
    # @param em Model : A Model instance
    # @param uid int : An EmField uid
    def delete_emclass_table(self, em, uid):
        emclass = em.component(uid)
        if not isinstance(emclass, EditorialModel.classes.EmClass):
            raise ValueError("The give uid is not an EmClass uid")
        tname = self._emclass2table_name(emclass)
        # Delete the table triggers to prevent errors
        self._generate_triggers(tname, dict())

        tname = self.datasource.escape_idname(tname)

        self._query("""DROP TABLE {table_name};""".format(table_name=tname))

    ## @brief Given an EmField delete the corresponding column
    # @param em Model : an @ref EditorialModel.model.Model instance
    # @param uid int : an EmField uid
    def delete_col_from_emfield(self, em, uid):
        emfield = em.component(uid)
        if not isinstance(emfield, EditorialModel.fields.EmField):
            raise ValueError("The given uid is not an EmField uid")

        emclass = emfield.em_class
        tname = self._emclass2table_name(emclass)
        # Delete the table triggers to prevent errors
        self._generate_triggers(tname, dict())

        self._del_column(tname, emfield.name)
        # Refresh the table triggers
        cols_ls = self._class2cols(emclass)
        self._generate_triggers(tname, cols_l)

    ## @brief Delete a column from a table
    # @param tname str : The table name
    # @param fname str : The column name
    def _del_column(self, tname, fname):
        tname = self.datasource.escape_idname(tname)
        fname = self.datasource.escape_idname(fname)

        self._query("""ALTER TABLE {table_name} DROP COLUMN {col_name};""".format(table_name=tname, col_name=fname))

    ## @brief Construct a table name given an EmClass instance
    # @param emclass EmClass : An EmClass instance
    # @return a table name
    def _emclass2table_name(self, emclass):
        return self.datasource.get_table_name_from_class(emclass.name)
        #return "class_%s"%emclass.name

    ## @brief Construct a table name given a rela2type EmField instance
    # @param em Model : A Model instance
    # @param emfield EmField : An EmField instance
    # @return a table name
    def _r2t2table_name(self, em, emfield):
        emclass = emfield.em_class
        emtype = em.component(emfield.rel_to_type_id)
        return self.datasource.get_r2t2table_name(emclass.name, emtype.name)
        #return "%s_%s_%s"%(emclass.name, emtype.name, emfield.name)

    ## @brief Generate a columns_fieldtype dict given a rel2type EmField
    # @param em Model : an @ref EditorialModel.model.Model instance
    # @param emfield EmField : and @ref EditorialModel.fields.EmField instance
    def _r2type2cols(self, em, emfield):
        return {f.name: f.fieldtype_instance() for f in em.components('EmField') if f.rel_field_id == emfield.uid}

    ## @brief Generate a columns_fieldtype dict given an EmClass
    # @param emclass EmClass : An EmClass instance
    # @return A dict with column name as key and EmFieldType instance as value
    def _class2cols(self, emclass):
        if not isinstance(emclass, EditorialModel.classes.EmClass):
            raise ValueError("The given uid is not an EmClass uid")
        return {f.name: f.fieldtype_instance() for f in emclass.fields() if f.name not in EditorialModel.classtypes.common_fields.keys()}

    ## @brief Create object and relations tables
    # @param drop_if_exist bool : If true drop tables if exists
    def _create_default_tables(self, drop_if_exist=False):
        if_exists = 'drop' if drop_if_exist else 'nothing'
        #Object tablea
        tname = self.datasource.objects_table_name
        pk_name, pk_ftype = self._common_field_pk
        self._create_table(tname, pk_name, pk_ftype, engine=self.db_engine, if_exists=if_exists)
        #Adding columns
        cols = {fname: self._common_field_to_ftype(fname) for fname in EditorialModel.classtypes.common_fields}
        for fname, ftype in cols.items():
            if fname != pk_name:
                self._add_column(tname, fname, ftype)
        #Creating triggers
        self._generate_triggers(tname, cols)

        #Relation table
        tname = self.datasource.relations_table_name
        pk_name, pk_ftype = self._relation_pk
        self._create_table(tname, pk_name, pk_ftype, engine=self.db_engine, if_exists=if_exists)
        #Adding columns
        for fname, ftype in self._relation_cols.items():
            self._add_column(tname, fname, ftype)
        #Creating triggers
        self._generate_triggers(tname, self._relation_cols)

    ## @return true if the name changes
    def _name_change(self, initial_state, new_state):
        return 'name' in initial_state and initial_state['name'] != new_state['name']

    ## @brief Create a table with primary key
    # @param table_name str : table name
    # @param pk_name str : pk column name
    # @param pk_specs str : see @ref _field_to_sql()
    # @param engine str : The engine to use with this table
    # @param charset str : The charset of this table
    # @param if_exist str : takes values in ['nothing', 'drop']
    def _create_table(self, table_name, pk_name, pk_ftype, engine, charset='utf8', if_exists='nothing'):
        #Escaped table name
        etname = self.datasource.escape_idname(table_name)
        pk_type = self._field_to_type(pk_ftype)
        pk_specs = self._field_to_specs(pk_ftype)

        if if_exists == 'drop':
            self._query("""DROP TABLE IF EXISTS {table_name};""".format(table_name=etname))
            qres = """
CREATE TABLE {table_name} (
{pk_name} {pk_type} {pk_specs},
PRIMARY KEY({pk_name})
) ENGINE={engine} DEFAULT CHARSET={charset};"""
        elif if_exists == 'nothing':
            qres = """CREATE TABLE IF NOT EXISTS {table_name} (
{pk_name} {pk_type} {pk_specs},
PRIMARY KEY({pk_name})
) ENGINE={engine} DEFAULT CHARSET={charset};"""
        else:
            raise ValueError("Unexpected value for argument if_exists '%s'." % if_exists)

        self._query(qres.format(
            table_name=self.datasource.escape_idname(table_name),
            pk_name=self.datasource.escape_idname(pk_name),
            pk_type=pk_type,
            pk_specs=pk_specs,
            engine=engine,
            charset=charset
        ))

    ## @brief Add a column to a table
    # @param table_name str : The table name
    # @param col_name str : The columns name
    # @param col_fieldtype EmFieldype the fieldtype
    def _add_column(self, table_name, col_name, col_fieldtype, drop_if_exists=False):
        add_col = """ALTER TABLE {table_name}
ADD COLUMN {col_name} {col_type} {col_specs};"""

        etname = self.datasource.escape_idname(table_name)
        ecname = self.datasource.escape_idname(col_name)

        add_col = add_col.format(
            table_name=etname,
            col_name=ecname,
            col_type=self._field_to_type(col_fieldtype),
            col_specs=self._field_to_specs(col_fieldtype),
        )
        try:
            self._query(add_col)
        except self._dbmodule.err.InternalError as e:
            if drop_if_exists:
                self._del_column(table_name, col_name)
                self._add_column(table_name, col_name, col_fieldtype, drop_if_exists)
            else:
                #LOG
                print("Aborded, column `%s` exists" % col_name)

    ## @brief Add a foreign key
    # @param src_table_name str : The name of the table where we will add the FK
    # @param dst_table_name str : The name of the table the FK will point on
    # @param src_col_name str : The name of the concerned column in the src_table
    # @param dst_col_name str : The name of the concerned column in the dst_table
    def _add_fk(self, src_table_name, dst_table_name, src_col_name, dst_col_name):
        stname = self.datasource.escape_idname(src_table_name)
        dtname = self.datasource.escape_idname(dst_table_name)
        scname = self.datasource.escape_idname(src_col_name)
        dcname = self.datasource.escape_idname(dst_col_name)

        fk_name = self.datasource.get_fk_name(src_table_name, dst_table_name)

        self._del_fk(src_table_name, dst_table_name)

        self._query("""ALTER TABLE {src_table}
ADD CONSTRAINT {fk_name}
FOREIGN KEY ({src_col}) references {dst_table}({dst_col});""".format(
            fk_name=self.datasource.escape_idname(fk_name),
            src_table=stname,
            src_col=scname,
            dst_table=dtname,
            dst_col=dcname
        ))

    ## @brief Given a source and a destination table, delete the corresponding FK
    # @param src_table_name str : The name of the table where the FK is
    # @param dst_table_name str : The name of the table the FK point on
    # @warning fails silently
    def _del_fk(self, src_table_name, dst_table_name):
        try:
            self._query("""ALTER TABLE {src_table}
DROP FOREIGN KEY {fk_name}""".format(
                src_table=self.datasource.escape_idname(src_table_name),
                fk_name=self.datasource.escape_idname(self.datasource.get_fk_name(src_table_name, dst_table_name))
            ))
        except self._dbmodule.err.InternalError:
            # If the FK don't exists we do not care
            pass

    ## @brief Generate triggers given a table_name and its columns fieldtypes
    # @param table_name str : Table name
    # @param cols_ftype dict : with col name as key and column fieldtype as value
    def _generate_triggers(self, table_name, cols_ftype):
        colval_l_upd = dict()  # param for update trigger
        colval_l_ins = dict()  # param for insert trigger

        for cname, cftype in cols_ftype.items():
            if cftype.ftype == 'datetime':
                if cftype.now_on_update:
                    colval_l_upd[cname] = 'NOW()'
                if cftype.now_on_create:
                    colval_l_ins[cname] = 'NOW()'

        self._table_trigger(table_name, 'UPDATE', colval_l_upd)
        self._table_trigger(table_name, 'INSERT', colval_l_ins)

    ## @brief Create trigger for a table
    #
    # Primarly designed to create trigger for DATETIME types
    # The method generates triggers of the form
    #
    # CREATE TRIGGER BEFORE <moment> ON <table_name>
    # FOR EACH ROW SET <for colname, colval in cols_val>
    # NEW.<colname> = <colval>,
    # <endfor>;
    # @param table_name str : The table name
    # @param moment str : can be 'update' or 'insert'
    # @param cols_val dict : Dict with column name as key and column value as value
    def _table_trigger(self, table_name, moment, cols_val):
        trigger_name = self.datasource.escape_idname("%s_%s_trig" % (table_name, moment))
        #Try to delete the trigger
        drop_trig = """DROP TRIGGER IF EXISTS {trigger_name};""".format(trigger_name=trigger_name)
        self._query(drop_trig)

        col_val_l = ', '.join(["NEW.%s = %s" % (self.datasource.escape_idname(cname), cval)for cname, cval in cols_val.items()])
        #Create a trigger if needed
        if len(col_val_l) > 0:
            trig_q = """CREATE TRIGGER {trigger_name} BEFORE {moment} ON {table_name}
FOR EACH ROW SET {col_val_list};""".format(
                trigger_name=trigger_name,
                table_name=self.datasource.escape_idname(table_name),
                moment=moment, col_val_list=col_val_l
            )
            self._query(trig_q)

    ## @brief Identifier escaping
    # @param idname str : An SQL identifier
    #def _idname_escape(self, idname):
    #    if '`' in idname:
    #        raise ValueError("Invalid name : '%s'"%idname)
    #    return '`%s`'%idname

    ## @brief Returns column specs from fieldtype
    # @param emfieldtype EmFieldType : An EmFieldType insance
    # @todo escape default value
    def _field_to_specs(self, emfieldtype):
        colspec = ''
        if not emfieldtype.nullable:
            colspec = 'NOT NULL'
        if hasattr(emfieldtype, 'default'):
            colspec += ' DEFAULT '
            if emfieldtype.default is None:
                colspec += 'NULL '
            else:
                colspec += emfieldtype.default  # ESCAPE VALUE HERE !!!!

        if emfieldtype.name == 'pk':
            colspec += ' AUTO_INCREMENT'

        return colspec

    ## @brief Given a fieldtype return a MySQL type specifier
    # @param emfieldtype EmFieldType : A fieldtype
    # @return the corresponding MySQL type
    def _field_to_type(self, emfieldtype):
        ftype = emfieldtype.ftype

        if ftype == 'char' or ftype == 'str':
            res = "VARCHAR(%d)" % emfieldtype.max_length
        elif ftype == 'text':
            res = "TEXT"
        elif ftype == 'datetime':
            res = "DATETIME"
            # client side workaround for only one column with CURRENT_TIMESTAMP : giving NULL to timestamp that don't allows NULL
            # cf. https://dev.mysql.com/doc/refman/5.0/en/timestamp-initialization.html#idm139961275230400
            # The solution for the migration handler is to create triggers :
            # CREATE TRIGGER trigger_name BEFORE INSERT ON `my_super_table`
            # FOR EACH ROW SET NEW.my_date_column = NOW();
            # and
            # CREATE TRIGGER trigger_name BEFORE UPDATE ON

        elif ftype == 'bool':
            res = "BOOL"
        elif ftype == 'int':
            res = "INT"
        elif ftype == 'rel2type':
            res = "INT"
        else:
            raise ValueError("Unsuported fieldtype ftype : %s" % ftype)

        return res

    ## @brief Returns a tuple (pkname, pk_ftype)
    @property
    def _common_field_pk(self):
        for fname, fta in EditorialModel.classtypes.common_fields.items():
            if fta['fieldtype'] == 'pk':
                return (fname, self._common_field_to_ftype(fname))
        return (None, None)

    ## @brief Returns a tuple (rel_pkname, rel_ftype)
    # @todo do it
    @property
    def _relation_pk(self):
        return (MySQL.relations_pkname, EditorialModel.fieldtypes.pk.EmFieldType())

    ## @brief Returns a dict { colname:fieldtype } of relation table columns
    @property
    def _relation_cols(self):
        from_name = EditorialModel.fieldtypes.generic.GenericFieldType.from_name
        return {
            'id_sup': from_name('integer')(),
            'id_sub': from_name('integer')(),
            'rank': from_name('integer')(nullable=True),
            'depth': from_name('integer')(nullable=True),
            'nature': from_name('char')(max_lenght=10, nullable=True),
        }

    ## @brief Given a common field name return an EmFieldType instance
    # @param cname str : Common field name
    # @return An EmFieldType instance
    def _common_field_to_ftype(self, cname):
        fta = copy.copy(EditorialModel.classtypes.common_fields[cname])
        fto = EditorialModel.fieldtypes.generic.GenericFieldType.from_name(fta['fieldtype'])
        del fta['fieldtype']
        return fto(**fta)