No Description
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

mysql.py 22KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529
  1. # -*- coding: utf-8 -*-
  2. import copy
  3. import pymysql
  4. import EditorialModel
  5. from DataSource.MySQL.MySQL import MySQL
  6. # The global MH algorithm is as follow :
  7. # A create_table(table_name, pk_name, pk_opt) method that create a table
  8. # with one pk field
  9. # An add_column(table_name, field_name, field_opt) method that add a column to a table
  10. #
  11. # The create_default_table method will call both methods to create the object and relation tables
  12. #
  13. # Supported operations :
  14. # - EmClass creation
  15. # - EmClass deletion (untested)
  16. # - EmField creation
  17. # - EmField deletion (untested)
  18. # - rel2type attribute creation
  19. # - rel2type attribute deletion (unstested)
  20. #
  21. # Unsupported operations :
  22. # - EmClass rename
  23. # - EmField rename
  24. # - rel2type field rename
  25. # - rel2type attribute rename
  26. # - EmFieldType changes
  27. #
  28. # @todo Unified datasources and migration handlers via utils functions
  29. #
  30. ## @brief Modify a MySQL database given editorial model changes
  31. class MysqlMigrationHandler(EditorialModel.migrationhandler.dummy.DummyMigrationHandler):
  32. ## @brief Construct a MysqlMigrationHandler
  33. # @param host str : The db host
  34. # @param user str : The db user
  35. # @param password str : The db password
  36. # @param db str : The db name
  37. def __init__(self, host, user, password, db, db_engine = 'InnoDB', foreign_keys = True, debug = False, dryrun = False, drop_if_exists = False):
  38. self.datasource = MySQL
  39. #Connect to MySQL
  40. self.db = pymysql.connect(host=host, user=user, passwd=password, db=db)
  41. self.debug = debug
  42. self.dryrun = dryrun
  43. self.db_engine = db_engine
  44. self.foreign_keys = foreign_keys if db_engine == 'InnoDB' else False
  45. self.drop_if_exists = drop_if_exists
  46. #Create default tables
  47. self._create_default_tables(self.drop_if_exists)
  48. pass
  49. ## @brief Modify the db given an EM change
  50. #
  51. # @param em model : The EditorialModel.model object to provide the global context
  52. # @param uid int : The uid of the change EmComponent
  53. # @param initial_state dict | None : dict with field name as key and field value as value. Representing the original state. None mean creation of a new component.
  54. # @param new_state dict | None : dict with field name as key and field value as value. Representing the new state. None mean component deletion
  55. # @throw EditorialModel.exceptions.MigrationHandlerChangeError if the change was refused
  56. def register_change(self, em, uid, initial_state, new_state, engine = None):
  57. if engine is None:
  58. engine = self.db_engine
  59. if isinstance(em.component(uid), EditorialModel.classes.EmClass):
  60. if initial_state is None:
  61. #EmClass creation
  62. self.create_emclass_table(em, uid, engine)
  63. elif new_state is None:
  64. #EmClass deletion
  65. self.delete_emclass_table(em, uid)
  66. elif isinstance(em.component(uid), EditorialModel.fields.EmField):
  67. emfield = em.component(uid)
  68. if emfield.rel_field_id is None:
  69. #non rlationnal field
  70. if initial_state is None:
  71. #non relationnal EmField creation
  72. if not(emfield.name in EditorialModel.classtypes.common_fields.keys()):
  73. self.add_col_from_emfield(em,uid)
  74. elif new_state is None:
  75. #non relationnal EmField deletion
  76. if not (emfield.name in EditorialModel.classtypes.common_fields.keys()):
  77. self.del_col_from_emfield(em, uid)
  78. else:
  79. #relationnal field
  80. if initial_state is None:
  81. #Rel2type attr creation
  82. self.add_relationnal_field(em, uid)
  83. elif new_state is None:
  84. #Rel2type attr deletion
  85. self.del_relationnal_field(em, uid)
  86. ## @brief dumdumdummy
  87. # @note implemented to avoid the log message of EditorialModel.migrationhandler.dummy.DummyMigrationHandler
  88. def register_model_state(self, em, state_hash):
  89. pass
  90. ## @brief Exec a query
  91. # @param query str : SQL query
  92. def _query(self, query):
  93. if self.debug:
  94. print(query+"\n")
  95. if not self.dryrun:
  96. with self.db.cursor() as cur:
  97. cur.execute(query)
  98. self.db.commit() #autocommit
  99. ## @brief Add a relationnal field
  100. #
  101. # Add a rel2type attribute
  102. # @note this function handles the table creation
  103. # @param em Model : EditorialModel.model.Model instance
  104. # @param rfuid int : Relationnal field uid
  105. def add_relationnal_field(self, em, rfuid):
  106. emfield = em.component(rfuid)
  107. if not isinstance(emfield, EditorialModel.fields.EmField):
  108. raise ValueError("The given uid is not an EmField uid")
  109. r2tf = em.component(emfield.rel_field_id)
  110. tname = self._r2t2table_name(em, r2tf)
  111. pkname, pkftype = self._relation_pk
  112. #If not exists create a relational table
  113. self._create_table(tname, pkname, pkftype, self.db_engine, if_exists = 'nothing')
  114. #Add a foreign key if wanted
  115. if self.foreign_keys:
  116. self._add_fk(tname, self.datasource.relations_table_name, pkname, pkname)
  117. #Add the column
  118. self._add_column(tname, emfield.name, emfield.fieldtype_instance())
  119. #Update table triggers
  120. self._generate_triggers(tname, self._r2type2cols(em, r2tf))
  121. ## @brief Delete a rel2type attribute
  122. #
  123. # Delete a rel2type attribute
  124. # @note this method handles the table deletion
  125. # @param em Model : EditorialModel.model.Model instance
  126. # @param rfuid int : Relationnal field uid
  127. def del_relationnal_field(self, em, rfuid):
  128. emfield = em.component(rfuid)
  129. if not isinstance(emfield, EditorialModel.fields.EmField):
  130. raise ValueError("The given uid is not an EmField uid")
  131. r2tf = em.component(emfield.rel_field_id)
  132. tname = self._r2t2table_name(em, r2tf)
  133. if len(self._r2type2cols(em, r2tf)) == 1:
  134. #The table can be deleted (no more attribute for this rel2type)
  135. self._query("""DROP TABLE {table_name}""".format(table_name = tname))
  136. else:
  137. self._del_column(tname, emfield.name)
  138. #Update table triggers
  139. self._generate_triggers(tname, self._r2type2cols(em, r2tf))
  140. ## @brief Given an EmField uid add a column to the corresponding table
  141. # @param em Model : A Model instance
  142. # @param uid int : An EmField uid
  143. def add_col_from_emfield(self, em, uid):
  144. emfield = em.component(uid)
  145. if not isinstance(emfield, EditorialModel.fields.EmField):
  146. raise ValueError("The given uid is not an EmField uid")
  147. emclass = emfield.em_class
  148. tname = self._emclass2table_name(emclass)
  149. self._add_column(tname, emfield.name, emfield.fieldtype_instance())
  150. # Refresh the table triggers
  151. cols_l = self._class2cols(emclass)
  152. self._generate_triggers(tname, cols_l)
  153. ## @brief Given a class uid create the coressponding table
  154. # @param em Model : A Model instance
  155. # @param uid int : An EmField uid
  156. def create_emclass_table(self, em, uid, engine):
  157. emclass = em.component(uid)
  158. if not isinstance(emclass, EditorialModel.classes.EmClass):
  159. raise ValueError("The given uid is not an EmClass uid")
  160. pkname, pktype = self._common_field_pk
  161. table_name = self._emclass2table_name(emclass)
  162. self._create_table(table_name, pkname, pktype, engine=engine)
  163. if self.foreign_keys:
  164. self._add_fk(table_name, self.datasource.objects_table_name, pkname, pkname)
  165. ## @brief Given an EmClass uid delete the corresponding table
  166. # @param em Model : A Model instance
  167. # @param uid int : An EmField uid
  168. def delete_emclass_table(self, em, uid):
  169. emclass = emcomponent(uid)
  170. if not isinstance(emclass, EditorialModel.classes.EmClass):
  171. raise ValueError("The give uid is not an EmClass uid")
  172. tname = self.datasource.escape_idname(self._emclass2table_name(emclass))
  173. # Delete the table triggers to prevent errors
  174. self._generate_triggers(tname, dict())
  175. self._query("""DROP TABLE {table_name};""".format(table_name = tname))
  176. ## @brief Given an EmField delete the corresponding column
  177. # @param em Model : an @ref EditorialModel.model.Model instance
  178. # @param uid int : an EmField uid
  179. def delete_col_from_emfield(self, em, uid):
  180. emfield = em.component(uid)
  181. if not isinstance(emfield, EditorialModel.fields.EmField):
  182. raise ValueError("The given uid is not an EmField uid")
  183. emclass = emfield.em_class
  184. tname = self._emclass2table_name(emclass)
  185. # Delete the table triggers to prevent errors
  186. self._generate_triggers(tname, dict())
  187. self._del_column(tname, emfield.name)
  188. # Refresh the table triggers
  189. cols_ls = self._class2cols(emclass)
  190. self._generate_triggers(tname, cols_l)
  191. ## @brief Delete a column from a table
  192. # @param tname str : The table name
  193. # @param fname str : The column name
  194. def _del_column(self, tname, fname):
  195. tname = self.datasource.escape_idname(tname)
  196. fname = self.datasource.escape_idname(fname)
  197. self._query("""ALTER TABLE {table_name} DROP COLUMN {col_name};""".format(table_name = tname, col_name = fname))
  198. ## @brief Construct a table name given an EmClass instance
  199. # @param emclass EmClass : An EmClass instance
  200. # @return a table name
  201. def _emclass2table_name(self, emclass):
  202. return self.datasource.get_table_name_from_class(emclass.name)
  203. #return "class_%s"%emclass.name
  204. ## @brief Construct a table name given a rela2type EmField instance
  205. # @param em Model : A Model instance
  206. # @param emfield EmField : An EmField instance
  207. # @return a table name
  208. def _r2t2table_name(self, em, emfield):
  209. emclass = emfield.em_class
  210. emtype = em.component(emfield.rel_to_type_id)
  211. return self.datasource.get_r2t2table_name(emclass.name, emtype.name)
  212. #return "%s_%s_%s"%(emclass.name, emtype.name, emfield.name)
  213. ## @brief Generate a columns_fieldtype dict given a rel2type EmField
  214. # @param em Model : an @ref EditorialModel.model.Model instance
  215. # @param emfield EmField : and @ref EditorialModel.fields.EmField instance
  216. def _r2type2cols(self, em, emfield):
  217. return { f.name: f.fieldtype_instance() for f in em.components('EmField') if f.rel_field_id == emfield.uid }
  218. ## @brief Generate a columns_fieldtype dict given an EmClass
  219. # @param emclass EmClass : An EmClass instance
  220. # @return A dict with column name as key and EmFieldType instance as value
  221. def _class2cols(self, emclass):
  222. if not isinstance(emclass, EditorialModel.classes.EmClass):
  223. raise ValueError("The given uid is not an EmClass uid")
  224. return { f.name: f.fieldtype_instance() for f in emclass.fields() if f.name not in EditorialModel.classtypes.common_fields.keys() }
  225. ## @brief Create object and relations tables
  226. # @param drop_if_exist bool : If true drop tables if exists
  227. def _create_default_tables(self, drop_if_exist = False):
  228. if_exists = 'drop' if drop_if_exist else 'nothing'
  229. #Object tablea
  230. tname = self.datasource.objects_table_name
  231. pk_name, pk_ftype = self._common_field_pk
  232. self._create_table(tname, pk_name, pk_ftype, engine=self.db_engine, if_exists = if_exists)
  233. #Adding columns
  234. cols = { fname: self._common_field_to_ftype(fname) for fname in EditorialModel.classtypes.common_fields }
  235. for fname, ftype in cols.items():
  236. if fname != pk_name:
  237. self._add_column(tname, fname, ftype)
  238. #Creating triggers
  239. self._generate_triggers(tname, cols)
  240. #Relation table
  241. tname = self.datasource.relations_table_name
  242. pk_name, pk_ftype = self._relation_pk
  243. self._create_table(tname, pk_name, pk_ftype, engine = self.db_engine, if_exists = if_exists)
  244. #Adding columns
  245. for fname, ftype in self._relation_cols.items():
  246. self._add_column(tname, fname, ftype)
  247. #Creating triggers
  248. self._generate_triggers(tname, self._relation_cols)
  249. ## @return true if the name changes
  250. def _name_change(self, initial_state, new_state):
  251. return 'name' in initial_state and initial_state['name'] != new_state['name']
  252. ## @brief Create a table with primary key
  253. # @param table_name str : table name
  254. # @param pk_name str : pk column name
  255. # @param pk_specs str : see @ref _field_to_sql()
  256. # @param engine str : The engine to use with this table
  257. # @param charset str : The charset of this table
  258. # @param if_exist str : takes values in ['nothing', 'drop']
  259. def _create_table(self, table_name, pk_name, pk_ftype, engine, charset = 'utf8', if_exists = 'nothing'):
  260. #Escaped table name
  261. etname = self.datasource.escape_idname(table_name)
  262. pk_type = self._field_to_type(pk_ftype)
  263. pk_specs = self._field_to_specs(pk_ftype)
  264. if if_exists == 'drop':
  265. self._query("""DROP TABLE IF EXISTS {table_name};""".format(table_name = etname))
  266. qres = """
  267. CREATE TABLE {table_name} (
  268. {pk_name} {pk_type} {pk_specs},
  269. PRIMARY KEY({pk_name})
  270. ) ENGINE={engine} DEFAULT CHARSET={charset};"""
  271. elif if_exists == 'nothing':
  272. qres = """CREATE TABLE IF NOT EXISTS {table_name} (
  273. {pk_name} {pk_type} {pk_specs},
  274. PRIMARY KEY({pk_name})
  275. ) ENGINE={engine} DEFAULT CHARSET={charset};"""
  276. else:
  277. raise ValueError("Unexpected value for argument if_exists '%s'."%if_exists)
  278. self._query(qres.format(
  279. table_name = self.datasource.escape_idname(table_name),
  280. pk_name = self.datasource.escape_idname(pk_name),
  281. pk_type = pk_type,
  282. pk_specs = pk_specs,
  283. engine = engine,
  284. charset = charset
  285. ))
  286. ## @brief Add a column to a table
  287. # @param table_name str : The table name
  288. # @param col_name str : The columns name
  289. # @param col_fieldtype EmFieldype the fieldtype
  290. def _add_column(self, table_name, col_name, col_fieldtype, drop_if_exists = False):
  291. add_col = """ALTER TABLE {table_name}
  292. ADD COLUMN {col_name} {col_type} {col_specs};"""
  293. etname = self.datasource.escape_idname(table_name)
  294. ecname = self.datasource.escape_idname(col_name)
  295. add_col = add_col.format(
  296. table_name = etname,
  297. col_name = ecname,
  298. col_type = self._field_to_type(col_fieldtype),
  299. col_specs = self._field_to_specs(col_fieldtype),
  300. )
  301. try:
  302. self._query(add_col)
  303. except pymysql.err.InternalError as e:
  304. if drop_if_exists:
  305. self._del_column(table_name, col_name)
  306. self._add_column(table_name, col_name, col_fieldtype, drop_if_exists)
  307. else:
  308. #LOG
  309. print("Aborded, column `%s` exists"%col_name)
  310. ## @brief Add a foreign key
  311. # @param src_table_name str : The name of the table where we will add the FK
  312. # @param dst_table_name str : The name of the table the FK will point on
  313. # @param src_col_name str : The name of the concerned column in the src_table
  314. # @param dst_col_name str : The name of the concerned column in the dst_table
  315. def _add_fk(self, src_table_name, dst_table_name, src_col_name, dst_col_name):
  316. stname = self.datasource.escape_idname(src_table_name)
  317. dtname = self.datasource.escape_idname(dst_table_name)
  318. scname = self.datasource.escape_idname(src_col_name)
  319. dcname = self.datasource.escape_idname(dst_col_name)
  320. fk_name = self.datasource.get_fk_name(src_table_name, dst_table_name)
  321. self._del_fk(src_table_name, dst_table_name)
  322. self._query("""ALTER TABLE {src_table}
  323. ADD CONSTRAINT {fk_name}
  324. FOREIGN KEY ({src_col}) references {dst_table}({dst_col});""".format(
  325. fk_name = self.datasource.escape_idname(fk_name),
  326. src_table = stname,
  327. src_col = scname,
  328. dst_table = dtname,
  329. dst_col = dcname
  330. ))
  331. ## @brief Given a source and a destination table, delete the corresponding FK
  332. # @param src_table_name str : The name of the table where the FK is
  333. # @param dst_table_name str : The name of the table the FK point on
  334. # @warning fails silently
  335. def _del_fk(self, src_table_name, dst_table_name):
  336. try:
  337. self._query("""ALTER TABLE {src_table}
  338. DROP FOREIGN KEY {fk_name}""".format(
  339. src_table = self.datasource.escape_idname(src_table_name),
  340. fk_name = self.datasource.escape_idname(self.datasource.get_fk_name(src_table_name, dst_table_name))
  341. ))
  342. except pymysql.err.InternalError:
  343. # If the FK don't exists we do not care
  344. pass
  345. ## @brief Generate triggers given a table_name and its columns fieldtypes
  346. # @param table_name str : Table name
  347. # @param cols_ftype dict : with col name as key and column fieldtype as value
  348. def _generate_triggers(self, table_name, cols_ftype):
  349. colval_l_upd = dict() #param for update trigger
  350. colval_l_ins = dict() #param for insert trigger
  351. for cname, cftype in cols_ftype.items():
  352. if cftype.ftype == 'datetime':
  353. if cftype.now_on_update:
  354. colval_l_upd[cname] = 'NOW()'
  355. if cftype.now_on_create:
  356. colval_l_ins[cname] = 'NOW()'
  357. self._table_trigger(table_name, 'UPDATE', colval_l_upd)
  358. self._table_trigger(table_name, 'INSERT', colval_l_ins)
  359. ## @brief Create trigger for a table
  360. #
  361. # Primarly designed to create trigger for DATETIME types
  362. # The method generates triggers of the form
  363. #
  364. # CREATE TRIGGER BEFORE <moment> ON <table_name>
  365. # FOR EACH ROW SET <for colname, colval in cols_val>
  366. # NEW.<colname> = <colval>,
  367. # <endfor>;
  368. # @param table_name str : The table name
  369. # @param moment str : can be 'update' or 'insert'
  370. # @param cols_val dict : Dict with column name as key and column value as value
  371. def _table_trigger(self, table_name, moment, cols_val):
  372. trigger_name = self.datasource.escape_idname("%s_%s_trig"%(table_name, moment))
  373. #Try to delete the trigger
  374. drop_trig = """DROP TRIGGER IF EXISTS {trigger_name};""".format(trigger_name = trigger_name)
  375. self._query(drop_trig)
  376. col_val_l = ', '.join([ "NEW.%s = %s"%(self.datasource.escape_idname(cname), cval)for cname, cval in cols_val.items() ])
  377. #Create a trigger if needed
  378. if len(col_val_l) > 0:
  379. trig_q = """CREATE TRIGGER {trigger_name} BEFORE {moment} ON {table_name}
  380. FOR EACH ROW SET {col_val_list};""".format(
  381. trigger_name = trigger_name,
  382. table_name = self.datasource.escape_idname(table_name),
  383. moment = moment,
  384. col_val_list = col_val_l
  385. )
  386. self._query(trig_q)
  387. ## @brief Identifier escaping
  388. # @param idname str : An SQL identifier
  389. #def _idname_escape(self, idname):
  390. # if '`' in idname:
  391. # raise ValueError("Invalid name : '%s'"%idname)
  392. # return '`%s`'%idname
  393. ## @brief Returns column specs from fieldtype
  394. # @param emfieldtype EmFieldType : An EmFieldType insance
  395. # @todo escape default value
  396. def _field_to_specs(self, emfieldtype):
  397. colspec = ''
  398. if not emfieldtype.nullable:
  399. colspec = 'NOT NULL'
  400. if hasattr(emfieldtype, 'default'):
  401. colspec += ' DEFAULT '
  402. if emfieldtype.default is None:
  403. colspec += 'NULL '
  404. else:
  405. colspec += emfieldtype.default #ESCAPE VALUE HERE !!!!
  406. if emfieldtype.name == 'pk':
  407. colspec += ' AUTO_INCREMENT'
  408. return colspec
  409. ## @brief Given a fieldtype return a MySQL type specifier
  410. # @param emfieldtype EmFieldType : A fieldtype
  411. # @return the corresponding MySQL type
  412. def _field_to_type(self, emfieldtype):
  413. ftype = emfieldtype.ftype
  414. if ftype == 'char' or ftype == 'str':
  415. res = "VARCHAR(%d)"%emfieldtype.max_length
  416. elif ftype == 'text':
  417. res = "TEXT"
  418. elif ftype == 'datetime':
  419. res = "DATETIME"
  420. # client side workaround for only one column with CURRENT_TIMESTAMP : giving NULL to timestamp that don't allows NULL
  421. # cf. https://dev.mysql.com/doc/refman/5.0/en/timestamp-initialization.html#idm139961275230400
  422. # The solution for the migration handler is to create triggers :
  423. # CREATE TRIGGER trigger_name BEFORE INSERT ON `my_super_table`
  424. # FOR EACH ROW SET NEW.my_date_column = NOW();
  425. # and
  426. # CREATE TRIGGER trigger_name BEFORE UPDATE ON
  427. elif ftype == 'bool':
  428. res = "BOOL"
  429. elif ftype == 'int':
  430. res = "INT"
  431. elif ftype == 'rel2type':
  432. res = "INT"
  433. else:
  434. raise ValueError("Unsuported fieldtype ftype : %s"%ftype)
  435. return res
  436. ## @brief Returns a tuple (pkname, pk_ftype)
  437. @property
  438. def _common_field_pk(self):
  439. for fname, fta in EditorialModel.classtypes.common_fields.items():
  440. if fta['fieldtype'] == 'pk':
  441. return (fname, self._common_field_to_ftype(fname))
  442. return (None, None)
  443. ## @brief Returns a tuple (rel_pkname, rel_ftype)
  444. # @todo do it
  445. @property
  446. def _relation_pk(self):
  447. return (MySQL.relation_table_pkname, EditorialModel.fieldtypes.pk.EmFieldType())
  448. ## @brief Returns a dict { colname:fieldtype } of relation table columns
  449. @property
  450. def _relation_cols(self):
  451. from_name = EditorialModel.fieldtypes.generic.GenericFieldType.from_name
  452. return {
  453. 'id_sup': from_name('integer')(),
  454. 'id_sub': from_name('integer')(),
  455. 'rank': from_name('integer')(nullable=True),
  456. 'depth': from_name('integer')(nullable=True),
  457. 'nature': from_name('char')(max_lenght=10, nullable=True),
  458. }
  459. ## @brief Given a common field name return an EmFieldType instance
  460. # @param cname str : Common field name
  461. # @return An EmFieldType instance
  462. def _common_field_to_ftype(self, cname):
  463. fta = copy.copy(EditorialModel.classtypes.common_fields[cname])
  464. fto = EditorialModel.fieldtypes.generic.GenericFieldType.from_name(fta['fieldtype'])
  465. del(fta['fieldtype'])
  466. return fto(**fta)