# -*- coding: utf-8 -*- import copy import pymysql from Lodel.settings import Settings import EditorialModel import EditorialModel.classtypes import EditorialModel.fieldtypes import EditorialModel.fieldtypes.generic from DataSource.MySQL import utils from DataSource.dummy.migrationhandler import DummyMigrationHandler # The global MH algorithm is as follow : # A create_table(table_name, pk_name, pk_opt) method that create a table # with one pk field # An add_column(table_name, field_name, field_opt) method that add a column to a table # # The create_default_table method will call both methods to create the object and relation tables # # Supported operations : # - EmClass creation # - EmClass deletion # - EmField creation # - EmField deletion # - rel2type attribute creation # - rel2type attribute deletion # # Unsupported operations : # - EmClass rename # - EmField rename # - rel2type field rename # - rel2type attribute rename # - EmFieldType changes # # @todo Unified datasources and migration handlers via utils functions ## @brief Modify a MySQL database given editorial model changes class MysqlMigrationHandler(DummyMigrationHandler): ## @brief Construct a MysqlMigrationHandler # @param module : sql module # @param conn_args dict : A dict containing connection options def __init__(self, module=pymysql, conn_args=None, db_engine='InnoDB', **kwargs): # Database connection self._dbmodule = module if conn_args is None: conn_args = copy.copy(Settings.get('datasource')['default']) self._dbmodule = conn_args['module'] del conn_args['module'] self.db_conn = self._dbmodule.connect(**conn_args) # Fetch options mh_settings = Settings.migration_options self.debug = kwargs['debug'] if 'debug' in kwargs else Settings.debug_sql self.dryrun = kwargs['dryrun'] if 'dryrun' in kwargs else mh_settings['dryrun'] self.foreign_keys = kwargs['foreign_keys'] if 'foreign_keys' in kwargs else mh_settings['foreign_keys'] self.drop_if_exists = kwargs['drop_if_exists'] if 'drop_if_exists' in kwargs else mh_settings['drop_if_exists'] self.db_engine = db_engine #Create default tables self._create_default_tables(self.drop_if_exists) ## @brief Modify the db given an EM change # @param em model : The EditorialModel.model object to provide the global context # @param uid int : The uid of the change EmComponent # @param initial_state dict | None : dict with field name as key and field value as value. Representing the original state. None mean creation of a new component. # @param new_state dict | None : dict with field name as key and field value as value. Representing the new state. None mean component deletion # @throw EditorialModel.exceptions.MigrationHandlerChangeError if the change was refused def register_change(self, em, uid, initial_state, new_state, engine=None): if engine is None: engine = self.db_engine if isinstance(em.component(uid), EditorialModel.classes.EmClass): if initial_state is None: #EmClass creation self.create_emclass_table(em, uid, engine) elif new_state is None: #EmClass deletion self.delete_emclass_table(em, uid) elif isinstance(em.component(uid), EditorialModel.fields.EmField): emfield = em.component(uid) if emfield.rel_field_id is None: #non relationnal field if initial_state is None: #non relationnal EmField creation if emfield.name not in EditorialModel.classtypes.common_fields.keys(): self.add_col_from_emfield(em, uid) elif new_state is None: #non relationnal EmField deletion if emfield.name not in EditorialModel.classtypes.common_fields.keys(): self.delete_col_from_emfield(em, uid) else: #relationnal field if initial_state is None: #Rel2type attr creation self.add_relationnal_field(em, uid) elif new_state is None: #Rel2type attr deletion self.del_relationnal_field(em, uid) ## @brief dumdumdummy # @note implemented to avoid the log message of EditorialModel.migrationhandler.dummy.DummyMigrationHandler def register_model_state(self, em, state_hash): pass ## @brief Add a relationnal field # Add a rel2type attribute # @note this function handles the table creation # @param em Model : EditorialModel.model.Model instance # @param rfuid int : Relationnal field uid def add_relationnal_field(self, edmod, rfuid): emfield = edmod.component(rfuid) if not isinstance(emfield, EditorialModel.fields.EmField): raise ValueError("The given uid is not an EmField uid") r2tf = edmod.component(emfield.rel_field_id) tname = self._r2t2table_name(edmod, r2tf) pkname, pkftype = self._relation_pk #If not exists create a relational table self._create_table(tname, pkname, pkftype, self.db_engine, if_exists='nothing') #Add a foreign key if wanted if self.foreign_keys: self._add_fk(tname, utils.common_tables['relation'], pkname, pkname) #Add the column self._add_column(tname, emfield.name, emfield.fieldtype_instance()) #Update table triggers self._generate_triggers(tname, self._r2type2cols(edmod, r2tf)) ## @brief Delete a rel2type attribute # # Delete a rel2type attribute # @note this method handles the table deletion # @param em Model : EditorialModel.model.Model instance # @param rfuid int : Relationnal field uid def del_relationnal_field(self, edmod, rfuid): emfield = edmod.component(rfuid) if not isinstance(emfield, EditorialModel.fields.EmField): raise ValueError("The given uid is not an EmField uid") r2tf = edmod.component(emfield.rel_field_id) tname = self._r2t2table_name(edmod, r2tf) if len(self._r2type2cols(edmod, r2tf)) == 1: #The table can be deleted (no more attribute for this rel2type) self._query("""DROP TABLE {table_name}""".format(table_name=tname)) else: self._del_column(tname, emfield.name) #Update table triggers self._generate_triggers(tname, self._r2type2cols(edmod, r2tf)) ## @brief Given an EmField uid add a column to the corresponding table # @param em Model : A Model instance # @param uid int : An EmField uid def add_col_from_emfield(self, edmod, uid): emfield = edmod.component(uid) if not isinstance(emfield, EditorialModel.fields.EmField): raise ValueError("The given uid is not an EmField uid") emclass = emfield.em_class tname = utils.object_table_name(emclass.name) self._add_column(tname, emfield.name, emfield.fieldtype_instance()) # Refresh the table triggers cols_l = self._class2cols(emclass) self._generate_triggers(tname, cols_l) ## @brief Given a class uid create the coressponding table # @param em Model : A Model instance # @param uid int : An EmField uid def create_emclass_table(self, edmod, uid, engine): emclass = edmod.component(uid) if not isinstance(emclass, EditorialModel.classes.EmClass): raise ValueError("The given uid is not an EmClass uid") pkname, pktype = self._object_pk table_name = utils.object_table_name(emclass.name) self._create_table(table_name, pkname, pktype, engine=engine) if self.foreign_keys: self._add_fk(table_name, utils.common_tables['object'], pkname, pkname) ## @brief Given an EmClass uid delete the corresponding table # @param em Model : A Model instance # @param uid int : An EmField uid def delete_emclass_table(self, edmod, uid): emclass = edmod.component(uid) if not isinstance(emclass, EditorialModel.classes.EmClass): raise ValueError("The give uid is not an EmClass uid") tname = utils.object_table_name(emclass.name) # Delete the table triggers to prevent errors self._generate_triggers(tname, dict()) tname = utils.escape_idname(tname) self._query("""DROP TABLE {table_name};""".format(table_name=tname)) ## @brief Given an EmField delete the corresponding column # @param em Model : an @ref EditorialModel.model.Model instance # @param uid int : an EmField uid def delete_col_from_emfield(self, edmod, uid): emfield = edmod.component(uid) if not isinstance(emfield, EditorialModel.fields.EmField): raise ValueError("The given uid is not an EmField uid") emclass = emfield.em_class tname = utils.object_table_name(emclass.name) # Delete the table triggers to prevent errors self._generate_triggers(tname, dict()) self._del_column(tname, emfield.name) # Refresh the table triggers cols_l = self._class2cols(emclass) self._generate_triggers(tname, cols_l) ## @brief Delete a column from a table # @param tname str : The table name # @param fname str : The column name def _del_column(self, tname, fname): tname = utils.escape_idname(tname) fname = utils.escape_idname(fname) self._query("""ALTER TABLE {table_name} DROP COLUMN {col_name};""".format(table_name=tname, col_name=fname)) ## @brief Construct a table name given a rela2type EmField instance # @param em Model : A Model instance # @param emfield EmField : An EmField instance # @return a table name def _r2t2table_name(self, edmod, emfield): emclass = emfield.em_class emtype = edmod.component(emfield.rel_to_type_id) return utils.r2t_table_name(emclass.name, emtype.name) ## @brief Generate a columns_fieldtype dict given a rel2type EmField # @param em Model : an @ref EditorialModel.model.Model instance # @param emfield EmField : and @ref EditorialModel.fields.EmField instance def _r2type2cols(self, edmod, emfield): return {f.name: f.fieldtype_instance() for f in edmod.components('EmField') if f.rel_field_id == emfield.uid} ## @brief Generate a columns_fieldtype dict given an EmClass # @param emclass EmClass : An EmClass instance # @return A dict with column name as key and EmFieldType instance as value def _class2cols(self, emclass): if not isinstance(emclass, EditorialModel.classes.EmClass): raise ValueError("The given uid is not an EmClass uid") return {f.name: f.fieldtype_instance() for f in emclass.fields() if f.name not in EditorialModel.classtypes.common_fields.keys()} ## @brief Create object and relations tables # @param drop_if_exist bool : If true drop tables if exists def _create_default_tables(self, drop_if_exist=False): if_exists = 'drop' if drop_if_exist else 'nothing' #Object table tname = utils.common_tables['object'] pk_name, pk_ftype = self._object_pk self._create_table(tname, pk_name, pk_ftype, engine=self.db_engine, if_exists=if_exists) #Adding columns cols = {fname: self._common_field_to_ftype(fname) for fname in EditorialModel.classtypes.common_fields} for fname, ftype in cols.items(): if fname != pk_name: self._add_column(tname, fname, ftype) #Creating triggers self._generate_triggers(tname, cols) object_tname = tname #Relation table tname = utils.common_tables['relation'] pk_name, pk_ftype = self._relation_pk self._create_table(tname, pk_name, pk_ftype, engine=self.db_engine, if_exists=if_exists) #Adding columns for fname, ftype in self._relation_cols.items(): self._add_column(tname, fname, ftype) #Creating triggers self._generate_triggers(tname, self._relation_cols) # Creating foreign keys between relation and object table sup_cname, sub_cname = self.get_sup_and_sub_cols() self._add_fk(tname, object_tname, sup_cname, self._object_pk[0], 'fk_relations_superiors') self._add_fk(tname, object_tname, sub_cname, self._object_pk[0], 'fk_relations_subordinate') ## @brief Returns the fieldname for superior and subordinate in relation table # @return a tuple (superior_name, subordinate_name) @classmethod def get_sup_and_sub_cols(cls): sup = None sub = None for fname, finfo in EditorialModel.classtypes.relations_common_fields.items(): if finfo['fieldtype'] == 'leo': if finfo['superior']: sup = fname else: sub = fname return utils.column_name(sup), utils.column_name(sub) ## @brief Create a table with primary key # @param table_name str : table name # @param pk_name str : pk column name # @param pk_specs str : see @ref _field_to_sql() # @param engine str : The engine to use with this table # @param charset str : The charset of this table # @param if_exist str : takes values in ['nothing', 'drop'] def _create_table(self, table_name, pk_name, pk_ftype, engine, charset='utf8', if_exists='nothing'): #Escaped table name etname = utils.escape_idname(table_name) pk_type = self._field_to_type(pk_ftype) pk_specs = self._field_to_specs(pk_ftype) if if_exists == 'drop': self._query("""DROP TABLE IF EXISTS {table_name};""".format(table_name=etname)) qres = """ CREATE TABLE {table_name} ( {pk_name} {pk_type} {pk_specs}, PRIMARY KEY({pk_name}) ) ENGINE={engine} DEFAULT CHARSET={charset};""" elif if_exists == 'nothing': qres = """CREATE TABLE IF NOT EXISTS {table_name} ( {pk_name} {pk_type} {pk_specs}, PRIMARY KEY({pk_name}) ) ENGINE={engine} DEFAULT CHARSET={charset};""" else: raise ValueError("Unexpected value for argument if_exists '%s'." % if_exists) self._query(qres.format( table_name=utils.escape_idname(table_name), pk_name=utils.escape_idname(pk_name), pk_type=pk_type, pk_specs=pk_specs, engine=engine, charset=charset )) ## @brief Add a column to a table # @param table_name str : The table name # @param col_name str : The columns name # @param col_fieldtype EmFieldype the fieldtype # @return True if the column was added else return False def _add_column(self, table_name, col_name, col_fieldtype, drop_if_exists=False): col_name = utils.column_name(col_name) add_col = """ALTER TABLE {table_name} ADD COLUMN {col_name} {col_type} {col_specs};""" etname = utils.escape_idname(table_name) ecname = utils.escape_idname(col_name) add_col = add_col.format( table_name=etname, col_name=ecname, col_type=self._field_to_type(col_fieldtype), col_specs=self._field_to_specs(col_fieldtype), ) try: self._query(add_col) except self._dbmodule.err.InternalError: if drop_if_exists: self._del_column(table_name, col_name) self._add_column(table_name, col_name, col_fieldtype, drop_if_exists) else: #LOG print("Aborded, column `%s` exists" % col_name) return False return True ## @brief Add a foreign key # @param src_table_name str : The name of the table where we will add the FK # @param dst_table_name str : The name of the table the FK will point on # @param src_col_name str : The name of the concerned column in the src_table # @param dst_col_name str : The name of the concerned column in the dst_table def _add_fk(self, src_table_name, dst_table_name, src_col_name, dst_col_name, fk_name=None): stname = utils.escape_idname(src_table_name) dtname = utils.escape_idname(dst_table_name) scname = utils.escape_idname(src_col_name) dcname = utils.escape_idname(dst_col_name) if fk_name is None: fk_name = utils.get_fk_name(src_table_name, dst_table_name) self._del_fk(src_table_name, dst_table_name, fk_name) self._query("""ALTER TABLE {src_table} ADD CONSTRAINT {fk_name} FOREIGN KEY ({src_col}) references {dst_table}({dst_col});""".format( fk_name=utils.escape_idname(fk_name), src_table=stname, src_col=scname, dst_table=dtname, dst_col=dcname )) ## @brief Given a source and a destination table, delete the corresponding FK # @param src_table_name str : The name of the table where the FK is # @param dst_table_name str : The name of the table the FK point on # @warning fails silently def _del_fk(self, src_table_name, dst_table_name, fk_name=None): if fk_name is None: fk_name = utils.escape_idname(utils.get_fk_name(src_table_name, dst_table_name)) try: self._query("""ALTER TABLE {src_table} DROP FOREIGN KEY {fk_name}""".format( src_table=utils.escape_idname(src_table_name), fk_name=fk_name )) except self._dbmodule.err.InternalError: # If the FK don't exists we do not care pass ## @brief Generate triggers given a table_name and its columns fieldtypes # @param table_name str : Table name # @param cols_ftype dict : with col name as key and column fieldtype as value def _generate_triggers(self, table_name, cols_ftype): colval_l_upd = dict() # param for update trigger colval_l_ins = dict() # param for insert trigger for cname, cftype in cols_ftype.items(): if cftype.ftype == 'datetime': if cftype.now_on_update: colval_l_upd[cname] = 'NOW()' if cftype.now_on_create: colval_l_ins[cname] = 'NOW()' self._table_trigger(table_name, 'UPDATE', colval_l_upd) self._table_trigger(table_name, 'INSERT', colval_l_ins) ## @brief Create trigger for a table # # Primarly designed to create trigger for DATETIME types # The method generates triggers of the form # # CREATE TRIGGER BEFORE ON # FOR EACH ROW SET # NEW. = , # ; # @param table_name str : The table name # @param moment str : can be 'update' or 'insert' # @param cols_val dict : Dict with column name as key and column value as value def _table_trigger(self, table_name, moment, cols_val): trigger_name = utils.escape_idname("%s_%s_trig" % (table_name, moment)) #Try to delete the trigger drop_trig = """DROP TRIGGER IF EXISTS {trigger_name};""".format(trigger_name=trigger_name) self._query(drop_trig) col_val_l = ', '.join(["NEW.%s = %s" % (utils.escape_idname(utils.column_name(cname)), cval)for cname, cval in cols_val.items()]) #Create a trigger if needed if len(col_val_l) > 0: trig_q = """CREATE TRIGGER {trigger_name} BEFORE {moment} ON {table_name} FOR EACH ROW SET {col_val_list};""".format( trigger_name=trigger_name, table_name=utils.escape_idname(table_name), moment=moment, col_val_list=col_val_l ) self._query(trig_q) ## @brief Identifier escaping # @param idname str : An SQL identifier #def _idname_escape(self, idname): # if '`' in idname: # raise ValueError("Invalid name : '%s'"%idname) # return '`%s`'%idname ## @brief Returns column specs from fieldtype # @param emfieldtype EmFieldType : An EmFieldType insance # @todo escape default value def _field_to_specs(self, emfieldtype): colspec = '' if not emfieldtype.nullable: colspec = 'NOT NULL' if hasattr(emfieldtype, 'default'): colspec += ' DEFAULT ' if emfieldtype.default is None: colspec += 'NULL ' else: colspec += emfieldtype.default # ESCAPE VALUE HERE !!!! if emfieldtype.name == 'pk': colspec += ' AUTO_INCREMENT' return colspec ## @brief Given a fieldtype return a MySQL type specifier # @param emfieldtype EmFieldType : A fieldtype # @return the corresponding MySQL type def _field_to_type(self, emfieldtype): ftype = emfieldtype.ftype if ftype == 'char' or ftype == 'str': res = "VARCHAR(%d)" % emfieldtype.max_length elif ftype == 'text': res = "TEXT" elif ftype == 'datetime': res = "DATETIME" # client side workaround for only one column with CURRENT_TIMESTAMP : giving NULL to timestamp that don't allows NULL # cf. https://dev.mysql.com/doc/refman/5.0/en/timestamp-initialization.html#idm139961275230400 # The solution for the migration handler is to create triggers : # CREATE TRIGGER trigger_name BEFORE INSERT ON `my_super_table` # FOR EACH ROW SET NEW.my_date_column = NOW(); # and # CREATE TRIGGER trigger_name BEFORE UPDATE ON elif ftype == 'bool': res = "BOOL" elif ftype == 'int': res = "INT" elif ftype == 'rel2type': res = "INT" elif ftype == 'leobject': res = "INT" else: raise ValueError("Unsuported fieldtype ftype : %s" % ftype) return res ## @brief Delete all table created by the MH # @param model Model : the Editorial model def __purge_db(self, model): for uid in [c.uid for c in model.components('EmClass')]: try: self.delete_emclass_table(model, uid) except self._dbmodule.err.InternalError as expt: print(expt) for tname in [utils.r2t_table_name(f.em_class.name, model.component(f.rel_to_type_id).name) for f in model.components('EmField') if f.fieldtype == 'rel2type']: try: self._query("DROP TABLE %s;" % tname) except self._dbmodule.err.InternalError as expt: print(expt) for tname in [utils.common_tables['relation'], utils.common_tables['relation']]: try: self._query("DROP TABLE %s;" % tname) except self._dbmodule.err.InternalError as expt: print(expt) ## @brief Return primary key name & fieldtype for relation or object @classmethod def extract_pk(cls, common_fields): for fname, finfo in common_fields.items(): if finfo['fieldtype'] == 'pk': fto = EditorialModel.fieldtypes.generic.GenericFieldType.from_name('pk') finfo_cp = copy.copy(finfo) del(finfo_cp['fieldtype']) return (utils.column_name(fname), fto(**finfo_cp)) raise RuntimeError("No primary key found in common fields : %s" % common_fields) ## @brief Exec a query # @param query str : SQL query def _query(self, query): if self.debug: print(query + "\n") if not self.dryrun: with self.db_conn.cursor() as cur: cur.execute(query) self.db_conn.commit() # autocommit ## @brief Given a common field name return an EmFieldType instance # @param cname str : Common field name # @return An EmFieldType instance @classmethod def _common_field_to_ftype(cls, cname): fta = copy.copy(EditorialModel.classtypes.common_fields[cname]) fto = EditorialModel.fieldtypes.generic.GenericFieldType.from_name(fta['fieldtype']) del fta['fieldtype'] return fto(**fta) ## @brief Returns a tuple (pkname, pk_ftype) @property def _object_pk(self): return self.extract_pk(EditorialModel.classtypes.common_fields) ## @brief Returns a tuple (rel_pkname, rel_ftype) @property def _relation_pk(self): return self.extract_pk(EditorialModel.classtypes.relations_common_fields) ## @brief Returns a dict { colname:fieldtype } of relation table columns @property def _relation_cols(self): res = dict() for fieldname, fieldinfo in EditorialModel.classtypes.relations_common_fields.items(): finfo = copy.copy(fieldinfo) fieldtype_name = finfo['fieldtype'] del(finfo['fieldtype']) res[fieldname] = EditorialModel.fieldtypes.generic.GenericFieldType.from_name(fieldtype_name)(**finfo) return res