Databases: get rid of old and unused code

Database/__init__.py

@@ -1 +0,0 @@
-# -*- coding: utf-8 -*- #

Database/sqlalter.py

@@ -1,132 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import os
-import time
-
-import sqlalchemy as sqla
-from sqlalchemy.ext.compiler import compiles
-
-## @file sqlalter.py
-# This file defines all DDL (data definition langage) for the ALTER TABLE instructions
-#
-# It uses the SqlAlchemy compilation and quoting methos to generate SQL
-
-
-class AddColumn(sqla.schema.DDLElement):
-    """ Defines the ddl for adding a column to a table """
-    def __init__(self, table, column):
-        """ Instanciate the DDL
-            @param table sqlalchemy.Table: A sqlalchemy table object
-            @param column sqlalchemy.Column: A sqlalchemy column object
-        """
-        self.col = column
-        self.table = table
-
-
-@compiles(AddColumn, 'mysql')
-@compiles(AddColumn, 'postgresql')
-@compiles(AddColumn, 'sqlite')
-def visit_add_column(element, ddlcompiler, **kw):
-    """ Compiles the AddColumn DDL for mysql, postgresql and sqlite"""
-    prep = ddlcompiler.sql_compiler.preparer
-    tname = prep.format_table(element.table)
-    colname = prep.format_column(element.col)
-    return 'ALTER TABLE %s ADD COLUMN %s %s' % (tname, colname, element.col.type)
-
-
-@compiles(AddColumn)
-def visit_add_column(element, ddlcompiler, **kw):
-    raise NotImplementedError('Add column not yet implemented for ' + str(ddlcompiler.dialect.name))
-
-
-class DropColumn(sqla.schema.DDLElement):
-    """ Defines the DDL for droping a column from a table """
-    def __init__(self, table, column):
-        """ Instanciate the DDL
-            @param table sqlalchemy.Table: A sqlalchemy table object
-            @param column sqlalchemy.Column: A sqlalchemy column object representing the column to drop
-        """
-        self.col = column
-        self.table = table
-
-
-@compiles(DropColumn, 'mysql')
-@compiles(DropColumn, 'postgresql')
-def visit_drop_column(element, ddlcompiler, **kw):
-    """ Compiles the DropColumn DDL for mysql & postgresql """
-    prep = ddlcompiler.sql_compiler.preparer
-    tname = prep.format_table(element.table)
-    colname = prep.format_column(element.col)
-    return 'ALTER TABLE %s DROP COLUMN %s' % (tname, colname)
-
-
-@compiles(DropColumn)
-##
-# @warning Returns more than one query @ref Database.sqlutils.ddl_execute
-def visit_drop_column(element, ddlcompiler, **kw):
-    prep = ddlcompiler.sql_compiler.preparer
-    tname = prep.format_table(element.table)
-
-    #Temporary table
-    tmpname = str(element.table) + '_copydropcol'
-    tmpTable = sqla.Table(tmpname, sqla.MetaData())
-    tmptname = prep.format_table(tmpTable)
-
-    query = 'ALTER TABLE %s RENAME TO %s; ' % (tname, tmptname)
-
-    colname = prep.format_column(element.col)
-
-    meta = sqla.MetaData()
-    newtable = sqla.Table(element.table, meta)
-    clist = element.table.columns
-
-    cols = []
-    for col in clist:
-        if col.name != element.col.name:
-            newtable.append_column(col.copy())
-            cols.append(prep.format_column(col))
-    cols = ', '.join(cols)
-
-    query += str(sqla.schema.CreateTable(newtable).compile(dialect=ddlcompiler.dialect)) + ';'
-
-    query += 'INSERT INTO %s ( %s ) SELECT %s FROM %s;' % (newtable, cols, cols, tmptname)
-
-    query += 'DROP TABLE %s' % (tmpname)
-
-    return query
-
-
-    #raise NotImplementedError('Drop column not yet implemented for '+str(ddlcompiler.dialect.name))
-
-class AlterColumn(sqla.schema.DDLElement):
-    """ Defines the DDL for changing the type of a column """
-    def __init__(self, table, column):
-        """ Instanciate the DDL
-            @param table sqlalchemy.Table: A sqlalchemy Table object
-            @param column sqlalchemy.Column: A sqlalchemy Column object representing the new column
-        """
-        self.col = column
-        self.table = table
-
-
-@compiles(AlterColumn, 'mysql')
-def visit_alter_column(element, ddlcompiler, **kw):
-    """ Compiles the AlterColumn DDL for mysql """
-    prep = ddlcompiler.sql_compiler.preparer
-    tname = prep.format_table(element.table)
-    colname = prep.format_column(element.col)
-    return 'ALTER TABLE %s ALTER COLUMN %s %s' % (tname, colname, element.col.type)
-
-
-@compiles(AlterColumn, 'postgresql')
-def visit_alter_column(element, ddlcompiler, **kw):
-    """ Compiles the AlterColumn DDL for postgresql """
-    prep = ddlcompiler.sql_compiler.preparer
-    tname = prep.format_table(element.table)
-    colname = prep.format_column(element.col)
-    return 'ALTER TABLE %s ALTER COLUMN %s TYPE %s' % (tname, colname, element.col.type)
-
-
-@compiles(AlterColumn)
-def visit_alter_column(element, ddlcompiler, **kw):
-    raise NotImplementedError('Alter column not yet implemented for ' + str(ddlcompiler.dialect.name))

Database/sqlsettings.py

@@ -1,39 +0,0 @@
-# -*- coding: utf-8 -*-
-
-class SQLSettings(object):
-    
-    DEFAULT_HOSTNAME = 'localhost'
-        
-    dbms_list = {
-        'postgresql': {
-            'driver': 'psycopg2',
-            'encoding': 'utf8',
-        },
-        'mysql': {
-            'driver': 'pymysql',
-            'encoding': 'utf8',
-        },
-        'sqlite': {
-            'driver': 'pysqlite',
-            'encoding': 'utf8',
-        },
-    }
-
-    DB_READ_CONNECTION_NAME = 'default'
-    DB_WRITE_CONNECTION_NAME = 'default'
-    
-    querystrings = {
-        'add_column': {
-            'default': 'ALTER TABLE %s ADD COLUMN %s %s'
-        },
-        'alter_column': {
-            'postgresql': 'ALTER TABLE %s ALTER COLUMN %s TYPE %s',
-            'mysql': 'ALTER TABLE %s ALTER COLUMN %s %s'
-        },
-        'drop_column': {
-            'default': 'ALTER TABLE %s DROP COLUMN %s'
-        }
-    }
-    
-    ACTION_TYPE_WRITE = 'write'
-    ACTION_TYPE_READ = 'read'

Database/sqlsetup.py

@@ -1,194 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import sqlalchemy as sql
-import re  # Converting string to sqlalchemy types
-from Database import sqlutils
-
-
-def init_db(dbconfname='default', alchemy_logs=None, schema=None):
-
-    dbe = sqlutils.get_engine(dbconfname, alchemy_logs)
-    meta = sqlutils.meta(dbe)
-    meta.reflect()
-    meta.drop_all(dbe)
-    # refresh meta (maybe useless)
-    meta = sqlutils.meta(dbe)
-    meta.reflect()
-
-    if schema is None:
-        schema = get_schema()
-
-    for table in schema:
-        topt = table.copy()
-        del topt['columns']
-        name = topt['name']
-        del topt['name']
-        cur_table = sql.Table(name, meta, **topt)
-        for col in table['columns']:
-            cur_col = create_column(**col)
-            cur_table.append_column(cur_col)
-
-    meta.create_all(bind=dbe)
-
-
-def get_schema():
-    tables = []
-
-    default_columns = [
-        {"name": "uid", "type": "INTEGER", "extra": {"foreignkey": "uids.uid", "nullable": False, "primarykey": True}},
-        {"name": "name", "type": "VARCHAR(50)", "extra": {"nullable": False, "unique": True}},
-        {"name": "string", "type": "TEXT"},
-        {"name": "help", "type": "TEXT"},
-        {"name": "rank", "type": "INTEGER"},
-        {"name": "date_create", "type": "DATETIME"},
-        {"name": "date_update", "type": "DATETIME"},
-    ]
-
-    # Table listing all objects created by lodel, giving them an unique id
-    uids = {
-        "name": "uids",
-        "columns": [
-            {"name": "uid", "type": "INTEGER", "extra": {"nullable": False, "primarykey": True, 'autoincrement': True}},
-            {"name": "table", "type": "VARCHAR(50)"}
-        ]
-    }
-    tables.append(uids)
-
-    # Table listing the classes
-    em_class = {"name": "em_class"}
-    em_class['columns'] = default_columns + [
-        {"name": "classtype", "type": "VARCHAR(50)"},
-        {"name": "sortcolumn", "type": "VARCHAR(50)", "extra": {"default": "rank"}},
-        {"name": "icon", "type": "INTEGER"},
-    ]
-    tables.append(em_class)
-
-    # Table listing the types
-    em_type = {"name": "em_type"}
-    em_type['columns'] = default_columns + [
-        {"name": "class_id", "type": "INTEGER", "extra": {"foreignkey": "em_class.uid", "nullable": False}},
-        {"name": "sortcolumn", "type": "VARCHAR(50)", "extra": {"default": "rank"}},
-        {"name": "icon", "type": "INTEGER"},
-    ]
-    tables.append(em_type)
-
-    # relation between types: which type can be a child of another
-    em_type_hierarchy = {"name": "em_type_hierarchy"}
-    em_type_hierarchy['columns'] = [
-        {"name": "superior_id", "type": "INTEGER", "extra": {"foreignkey": "em_type.uid", "nullable": False, "primarykey": True}},
-        {"name": "subordinate_id", "type": "INTEGER", "extra": {"foreignkey": "em_type.uid", "nullable": False, "primarykey": True}},
-        {"name": "nature", "type": "VARCHAR(50)", "extra": {"primarykey": True}},
-    ]
-    tables.append(em_type_hierarchy)
-
-   # Table listing the fieldgroups of a class
-    em_fieldgroup = {"name": "em_fieldgroup"}
-    em_fieldgroup['columns'] = default_columns + [
-        {"name": "class_id", "type": "INTEGER", "extra": {"foreignkey": "em_class.uid", "nullable": False}},
-    ]
-    tables.append(em_fieldgroup)
-
-    # Table listing the fields of a fieldgroup
-    em_field = {"name": "em_field"}
-    em_field['columns'] = default_columns + [
-        {"name": "fieldtype", "type": "VARCHAR(50)", "extra": {"nullable": False}},
-        {"name": "fieldgroup_id", "type": "INTEGER", "extra": {"foreignkey": "em_fieldgroup.uid", "nullable": False}},
-        {"name": "rel_to_type_id", "type": "INTEGER", "extra": {"foreignkey": "em_type.uid", "nullable": True, "server_default": sql.text('NULL')}},  # if relational: type this field refer to
-        {"name": "rel_field_id", "type": "INTEGER", "extra": {"foreignkey": "em_type.uid", "nullable": True, "server_default": sql.text('NULL')}},  # if relational: field that specify the rel_to_type_id
-        {"name": "optional", "type": "BOOLEAN"},
-        {"name": "internal", "type": "BOOLEAN"},
-        {"name": "icon", "type": "INTEGER"},
-    ]
-    tables.append(em_field)
-
-    # selected field for each type
-    em_field_type = {"name": "em_field_type"}
-    em_field_type['columns'] = [
-        {"name": "type_id", "type": "INTEGER", "extra": {"foreignkey": "em_type.uid", "nullable": False, "primarykey": True}},
-        {"name": "field_id", "type": "INTEGER", "extra": {"foreignkey": "em_field.uid", "nullable": False, "primarykey": True}},
-    ]
-    tables.append(em_field_type)
-
-    # Table of the objects created by the user (instance of the types)
-    objects = {
-        "name": "objects",
-        "columns": [
-            {"name": "uid", "type": "INTEGER", "extra": {"foreignkey": "uids.uid", "nullable": False, "primarykey": True}},
-            {"name": "string", "type": "VARCHAR(50)"},
-            {"name": "class_id", "type": "INTEGER", "extra": {"foreignkey": "em_class.uid"}},
-            {"name": "type_id", "type": "INTEGER", "extra": {"foreignkey": "em_type.uid"}},
-            {"name": "date_create", "type": "DATETIME"},
-            {"name": "date_update", "type": "DATETIME"},
-            {"name": "history", "type": "TEXT"}
-        ]
-    }
-    tables.append(objects)
-
-    # Table listing all files
-    files = {
-        "name": "files",
-        "columns": [
-            {"name": "uid", "type": "INTEGER", "extra": {"foreignkey": "uids.uid", "nullable": False, "primarykey": True}},
-            {"name": "field1", "type": "VARCHAR(50)"}
-        ]
-    }
-    tables.append(files)
-
-    return tables
-
-
-def create_column(**kwargs):
-    #Converting parameters
-    if 'type_' not in kwargs and 'type' in kwargs:
-        kwargs['type_'] = string_to_sqla_type(kwargs['type'])
-        del kwargs['type']
-
-    if 'extra' in kwargs:
-        # put the extra keys in kwargs
-        for exname in kwargs['extra']:
-            kwargs[exname] = kwargs['extra'][exname]
-        del kwargs['extra']
-
-    if 'foreignkey' in kwargs:
-        # Instanciate a fk
-        foreignkey = sql.ForeignKey(kwargs['foreignkey'])
-        del kwargs['foreignkey']
-    else:
-        foreignkey = None
-
-    if 'primarykey' in kwargs:
-        # renaming primary_key in primarykey in kwargs
-        kwargs['primary_key'] = kwargs['primarykey']
-        del kwargs['primarykey']
-
-    col = sql.Column(**kwargs)
-
-    if foreignkey is not None:
-        col.append_foreign_key(foreignkey)
-
-    return col
-
-
-## Converts a string to an sqlAlchemy column type
-#
-# @param strtype str: string describing the type of the column
-# @return SqlAlchemy column type
-# @raise NameError
-def string_to_sqla_type(strtype):
-    if 'VARCHAR' in strtype:
-        return string_to_varchar(strtype)
-    else:
-        try:
-            return getattr(sql, strtype)
-        except AttributeError:
-            raise NameError("Unknown type '" + strtype + "'")
-
-
-## Converts a string like 'VARCHAR(XX)' (with XX an integer) to a SqlAlchemy varchar type
-#
-# @param vstr str: String to convert
-# @return SqlAlchemy.VARCHAR
-def string_to_varchar(vstr):
-    check_length = re.search(re.compile('VARCHAR\(([\d]+)\)', re.IGNORECASE), vstr)
-    column_length = int(check_length.groups()[0]) if check_length else None
-    return sql.VARCHAR(length=column_length)

Database/sqlutils.py

@@ -1,103 +0,0 @@
-# -*- coding: utf-8 -*-
-import os
-import logging as logger
-
-import sqlalchemy as sqla
-from django.conf import settings
-
-import EditorialModel
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "Lodel.settings")
-
-ENGINES = {
-    'mysql': {
-        'driver': 'pymysql',
-        'encoding': 'utf8'
-    },
-    'postgresql': {
-        'driver': 'psycopg2',
-        'encoding': 'utf8',
-    },
-    'sqlite': {
-        'driver': 'pysqlite',
-        'encoding': 'utf8',
-    },
-}
-
-sql_config = settings.LODEL2SQLWRAPPER
-
-
-## Returns an engine given dbconf name
-#
-# @param ename str: name of an item in django.conf.settings.LODEL2SQLWRAPPER['db']
-# @param sqlaloggin None|bool: If None leave default value, if true activate sqlalchemy logging
-# @return SqlAlchemy engine
-def get_engine(ename='default', sqlalogging=None):
-    # Loading confs
-    cfg = sql_config['db'][ename]
-
-    edata = ENGINES[cfg['ENGINE']]  # engine infos
-    conn_str = ""
-
-    if cfg['ENGINE'] == 'sqlite':
-        #SQLite connection string
-        conn_str = '%s+%s:///%s' % (cfg['ENGINE'], edata['driver'], cfg['NAME'])
-    else:
-        #MySQL and PostgreSQL connection string
-        user = cfg['USER']
-        user += (':' + cfg['PASSWORD'] if 'PASSWORD' in cfg else '')
-
-        if 'HOST' not in cfg:
-            logger.info('Not HOST in configuration, using localhost')
-            host = 'localhost'
-        else:
-            host = cfg['HOST']
-
-        host += (':' + cfg['PORT'] if 'PORT' in cfg else '')
-        conn_str = '%s+%s://' % (cfg['ENGINE'], edata['driver'])
-        conn_str += '%s@%s/%s' % (user, host, cfg['NAME'])
-
-    ret = sqla.create_engine(conn_str, encoding=edata['encoding'], echo=sqlalogging)
-    logger.debug("Getting engine :" + str(ret))
-
-    return ret
-
-
-## Return a sqlalchemy.MetaData object
-# @param engine sqlalchemy.engine : A sqlalchemy engine
-# @return an sql alechemy MetaData instance bind to engine
-def meta(engine):
-    res = sqla.MetaData(bind=engine)
-    res.reflect(bind=engine)
-    return res
-
-
-## Return an sqlalchemy table given an EmComponent child class
-# @warning Except a class type not an instance
-# @param cls : An EmComponent child class
-# @return An sqlalchemy table
-# @throw TypeError if em_instance is an EmComponent  or not an EmComponent child class (or an instance)
-# @todo Move this function as an instance method ?
-def get_table(self):
-    if not issubclass(self.__class__, EditorialModel.components.EmComponent) or self.table is None:
-        raise TypeError("Excepting an EmComponent child class not an " + str(self.__class__))
-    engine = self.db_engine
-    return sqla.Table(self.table, meta(engine))
-
-
-## This function is intended to execute ddl defined in sqlalter
-# @warning There is a dirty workaround here, DDL should returns only one query, but DropColumn for sqlite has to return 4 queries (rename, create, insert, drop). There is a split on the compiled SQL to extract and execute one query at a time
-# @param ddl DDLElement: Can be an Database.sqlalter.DropColumn Database.sqlalter.AddColumn or Database.sqlalter.AlterColumn
-# @param db_engine: A database engine
-# @return True if success, else False
-def ddl_execute(ddl, db_engine):
-    conn = db_engine.connect()
-    req = str(ddl.compile(dialect=db_engine.dialect))
-    queries = req.split(';')
-    for query in queries:
-        logger.debug("Executing custom raw SQL query : '" + query + "'")
-        ret = conn.execute(query)
-        if not bool(ret):
-            return False
-    conn.close()
-    return True

EditorialModel/test/test_classes.py

@@ -25,7 +25,6 @@ EM_TEST_OBJECT = None
 
 
 ## run once for this module
-# define the Database for this module (an sqlite database)
 def setUpModule():
     global EM_TEST_OBJECT
     #EM_TEST_OBJECT = Model(EmBackendJson(EM_TEST), migration_handler=DjandoMigrationHandler('LodelTestInstance'))

EditorialModel/test/test_field.py

@@ -12,13 +12,9 @@ EM_TEST_OBJECT = None
 ## SetUpModule
 #
 # This function is called once for this module.
-# It is designed to overwrite the database configurations, and prepare objects for test_case initialization
 def setUpModule():
     global EM_TEST_OBJECT
     EM_TEST_OBJECT = Model(EmBackendJson(EM_TEST))
-    #initTestDb(TEST_FIELD_DBNAME)
-    #setDbConf(TEST_FIELD_DBNAME)
-    #logging.basicConfig(level=logging.CRITICAL)
 
 
 def tearDownModule():

EditorialModel/test/utils.py

@@ -1,87 +0,0 @@
-import os
-import logging
-import shutil
-
-from django.conf import settings
-from Database import sqlsetup
-
-
-_TESTDB_DEFAULT_DIR = '/tmp/'
-_TESTDB_DEFAULT_NAME = 'lodel2_test_db.sqlite'
-_TESTDB_COPY_SUFFIX = '_bck'
-
-## Make a copy of an empty Db created by Database::SQLSetup::initDb()
-#
-# Usage example :
-#   #Using 'default' as dbconfname
-#   def setUpModule():
-#       dbname = 'supertestdb'
-#       initTestDb(dbname)
-#       setDbConf(dbname)
-#       ...
-#
-#   #Using another name as dbconfname
-#   def setUpModule():
-#       dbname = 'superdbtest'
-#       initTestDb(dbname)
-#       setDbConf(dbname, 'mytest_dbconf')
-#
-#       db_engine = sqlutils.getEngine('mytest_dbconf')
-#       EmComponent.dbconf = 'mytest_dbconf'
-#       #WARNING !!! This example as few chances to work due to the bad isolation of data sources in EmComponent... Will be solved with the EM object
-#
-def initTestDb(dbname = _TESTDB_DEFAULT_NAME, dbdir = _TESTDB_DEFAULT_DIR):
-
-    dbname = os.path.join(dbdir, dbname)
-    db_default = os.path.join(_TESTDB_DEFAULT_DIR, _TESTDB_DEFAULT_NAME)
-    db_copy = os.path.join(_TESTDB_DEFAULT_DIR, _TESTDB_DEFAULT_NAME+_TESTDB_COPY_SUFFIX)
-
-    if not os.path.isfile(db_copy):
-        #The 'backup' copy didn't exists yet. Create it.
-        settings.LODEL2SQLWRAPPER['db']['dbtest_default'] = {
-            'ENGINE': 'sqlite',
-            'NAME': db_default,
-        }
-        sqlsetup.init_db(dbconfname = 'dbtest_default')
-        #Make the backup copy
-        shutil.copyfile(db_default, db_copy)
-    
-    #Copy the Db at the wanted location
-    shutil.copyfile(db_copy, dbname)
-    return dbname
-
-def copyDb(dbname_src, dbname_dst, dbdir_src = _TESTDB_DEFAULT_DIR, dbdir_dst = _TESTDB_DEFAULT_DIR):
-    dbname_src = os.path.join(dbdir_src, dbname_src)
-    dbname_dst = os.path.join(dbdir_dst, dbname_dst)
-    shutil.copyfile(dbname_src, dbname_dst)
-    pass
-
-
-def saveDbState(dbname = _TESTDB_DEFAULT_NAME, dbdir = _TESTDB_DEFAULT_DIR):
-    dbname = os.path.join(dbdir, dbname)
-    shutil.copyfile(dbname, dbname+_TESTDB_COPY_SUFFIX)
-    pass
-
-def restoreDbState(dbname = _TESTDB_DEFAULT_NAME, dbdir = _TESTDB_DEFAULT_DIR):
-    dbname = os.path.join(dbdir, dbname)
-    shutil.copyfile(dbname+_TESTDB_COPY_SUFFIX, dbname)
-    pass
-
-def cleanDb(dbname = _TESTDB_DEFAULT_NAME, dbdir = _TESTDB_DEFAULT_DIR):
-    dbname = os.path.join(dbdir, dbname)
-    try: os.unlink(dbname)
-    except:pass
-    try: os.unlink(dbname+_TESTDB_COPY_SUFFIX)
-    except:pass
-    pass
-
-
-def setDbConf(dbname, dbconfname = 'default', dbdir = _TESTDB_DEFAULT_DIR):
-    
-    settings.LODEL2SQLWRAPPER['db'][dbconfname] = {
-        'ENGINE': 'sqlite',
-        'NAME': os.path.join(_TESTDB_DEFAULT_DIR, dbname)
-    }
-    pass
- 
-    

EditorialModel/types.py

@@ -1,8 +1,5 @@
 #-*- coding: utf-8 -*-
 
-# from Database import sqlutils
-# import sqlalchemy as sql
-
 import EditorialModel
 from EditorialModel.components import EmComponent
 from EditorialModel.fields import EmField