[Broken state] started settings implementation

lodel/plugins.py

@@ -0,0 +1,72 @@
+#-*- coding: utf-8 -*-
+
+import os.path
+
+from importlib.machinery import SourceFileLoader, SourcelessFileLoader
+
+## @package lodel.plugins Lodel2 plugins management
+#
+# Lodel2 plugins are stored in directories
+# A typicall lodel2 plugin directory structure looks like :
+# - {{__init__.py}}} containing informations like full_name, authors, licence etc.
+# - main.py containing hooks registration etc
+# - confspec.py containing a configuration specification dictionary named CONFSPEC
+
+VIRTUAL_PACKAGE_NAME = 'lodel.plugins_pkg'
+CONFSPEC_NAME = 'confspec.py'
+
+class Plugins(object):
+    
+    ## @brief Stores plugin directories paths
+    _plugin_directories = None
+    ## @brief Optimisation cache storage for plugin paths
+    _plugin_paths = dict()
+
+    def __init__(self):
+        self.started()
+    
+    ## @brief Given a plugin name returns the plugin path
+    # @param plugin_name str : The plugin name
+    # @return the plugin directory path
+    @classmethod
+    def plugin_path(cls, plugin_name):
+        cls.started()
+        try:
+            return cls._plugin_paths[plugin_name]
+        except KeyError:
+            pass
+        
+        path = None
+        for cur_path in cls._plugin_directories:
+            plugin_path = os.path.join(cur_path, plugin_name)+'/'
+            print(plugin_path)
+            if os.path.isdir(plugin_path):
+                return plugin_path
+        raise NameError("No plugin named '%s'" % plugin_name)
+
+    ## @brief Fetch a confspec given a plugin_name
+    # @param plugin_name str : The plugin name
+    # @return a dict of conf spec
+    @classmethod
+    def get_confspec(cls, plugin_name):
+        cls.started()
+        plugin_path = cls.plugin_path(plugin_name)
+        plugin_module = '%s.%s' % ( VIRTUAL_PACKAGE_NAME,
+                                    plugin_name)
+        conf_spec_module = plugin_module + '.confspec'
+        
+        conf_spec_source = plugin_path + CONFSPEC_NAME
+
+        loader = SourceFileLoader(conf_spec_module, conf_spec_source)
+        confspec_module = loader.load_module()
+        return getattr(confspec_module, 'CONFSPEC')
+
+    @classmethod
+    def bootstrap(cls, plugins_directories):
+        cls._plugin_directories = plugins_directories
+
+    @classmethod
+    def started(cls, raise_if_not = True):
+        res = cls._plugin_directories is not None
+        if raise_if_not and not res:
+            raise RuntimeError("Class Plugins is not initialized")

lodel/settings.py

@@ -1,145 +0,0 @@
-#-*- coding: utf-8 -*-
-
-import types
-import warnings
-from . import settings_format
-
-## @package Lodel.settings
-#
-# @brief Defines stuff to handles Lodel2 configuration (see @ref lodel_settings )
-#
-# To access the confs use the Lodel.settings.Settings SettingsHandler instance
-
-## @brief A class designed to handles Lodel2 settings
-#
-# When instanciating a SettingsHandler, the new instance is filled with the content of settings.py (in the root directory of lodel2
-#
-# @warning You don't have to instanciate this class, you can access to the global instance with the Settings variable in this module
-# @todo broken stuff... Rewrite it
-# @todo Forbid module assignement in settings ! and disable tests about this
-# @todo Implements a type checking of config value
-# @todo Implements default values for config keys
-class SettingsHandler(object):
-    
-    ## @brief Shortcut
-    _allowed = settings_format.ALLOWED + settings_format.MANDATORY
-    ## @brief Shortcut
-    _mandatory = settings_format.MANDATORY
-
-    def __init__(self):
-        try:
-            import settings as default_settings
-            self._load_module(default_settings)
-        except ImportError:
-            warnings.warn("Unable to find global default settings")
-
-        ## @brief A flag set to True when the instance is fully loaded
-        self._set_loaded(False if len(self._missings()) > 0 else True)
-    
-    ## @brief Compat wrapper for getattr
-    def get(self, name):
-        return getattr(self, name)
-    
-    ## @brief Compat wrapper for setattr
-    def set(self, name, value):
-        return setattr(self, name, value)
-
-    ## @brief Load every module properties in the settings instance
-    #
-    # Load a module content into a SettingsHandler instance and checks that no mandatory settings are missing
-    # @note Example : <pre> import my_cool_settings;
-    # Settings._load_module(my_cool_settings);</pre>
-    # @param module module|None: a loaded module (if None just check for missing settings)
-    # @throw LookupError if invalid settings found or if mandatory settings are missing
-    def load_module(self, module = None):
-        if not(module is None):
-            self._load_module(module)
-        missings = self._missings()
-        if len(missings) > 0:
-            self._loaded = False
-            raise LookupError("Mandatory settings are missing : %s"%missings)
-        self._set_loaded(True)
-    
-    ## @brief supersede of default __setattr__ method
-    def __setattr__(self, name, value):
-        if not hasattr(self, name):
-            if name not in self._allowed:
-                raise LookupError("Invalid setting : %s"%name)
-        super().__setattr__(name, value)
-
-    ## @brief This method do the job for SettingsHandler.load_module()
-    #
-    # @note The difference with SettingsHandler.load_module() is that it didn't check if some settings are missing
-    # @throw LokkupError if an invalid settings is given
-    # @param module : a loaded module
-    def _load_module(self, module):
-        errors = []
-        fatal_errors = []
-        conf_dict = {
-            name: getattr(module, name)
-            for name in dir(module) 
-            if not name.startswith('__') and not isinstance(getattr(module, name), types.ModuleType)
-        }
-        for name, value in conf_dict.items():
-            try:
-                setattr(self, name, value)
-            except LookupError:
-                errors.append(name)
-        if len(errors) > 0:
-            err_msg = "Found invalid settings in %s : %s"%(module.__name__, errors)
-            raise LookupError(err_msg)
-
-    ## @brief Refresh the allowed and mandatory settings list
-    @classmethod
-    def _refresh_format(cls):
-        ## @brief Shortcut
-        cls._allowed = settings_format.ALLOWED + settings_format.MANDATORY
-        ## @brief Shortcut
-        cls._mandatory = settings_format.MANDATORY
-
-    ## @brief If some settings are missings return their names
-    # @return an array of string
-    def _missings(self):
-        return [ confname for confname in self._mandatory if not hasattr(self, confname) ]
-
-    def _set_loaded(self, value):
-        super().__setattr__('_loaded', bool(value))
-
-Settings = SettingsHandler()
-
-## @page lodel_settings Lodel SettingsHandler
-#
-# This page describe the way settings are handled in Lodel2.
-#
-# @section lodel_settings_files Lodel settings files
-#
-# - Lodel/settings.py defines the Lodel.settings package, the SettingsHandler class and the Lodel.settings.Settings instance
-# - Lodel/settings_format.py defines the mandatory and allowed configurations keys lists
-# - install/instance_settings.py is a model of the file that will be deployed in Lodel2 instances directories
-#
-# @section Using Lodel.settings.Settings SettingsHandler instance
-#
-# @subsection lodel_settings_without_loader Without loader
-#
-# Without any loader you can import Lodel.settings.Settings and acces its property with getattr (or . ) or with SettingsHandler.get() method.
-# In the same way you can set a settings by standart affectation of a propery or with SettingsHandler.set() method.
-#
-# @subsection lodel_settings_loader With a loader in a lodel2 instance
-#
-# The loader will import Lodel.settings.Settings and then calls the SettingsHandler.load_module() method to load the content of the instance_settings.py file into the SettingsHandler instance
-#
-# @subsection lodel_settings_example Examples
-#
-# <pre>
-# #!/usr/bin/python
-# from Lodel.settings import Settings
-# if Settings.debug:
-#   print("DEBUG")
-# # or
-# if Settings.get('debug'):
-#   print("DEBUG")
-# Settings.debug = False
-# # or
-# Settings.set('debug', False)
-# </pre>
-# 

lodel/settings/settings.py

@@ -0,0 +1,102 @@
+#-*- coding: utf-8 -*-
+
+import sys
+import os
+import configparser
+
+from lodel.plugins import Plugins
+from lodel.settings.utils import SettingsError, SettingsErrors
+from lodel.settings.validator import SettingValidator
+from lodel.settings.settings_loader import SettingsLoader
+
+## @package lodel.settings Lodel2 settings package
+#
+# Contains all module that help handling settings
+
+## @package lodel.settings.settings Lodel2 settings module
+#
+# Handles configuration load/parse/check.
+#
+# @subsection Configuration load process
+#
+# The configuration load process is not trivial. In fact loaded plugins are able to add their own options.
+# But the list of plugins to load and the plugins options are in the same file, the instance configuration file.
+#
+# @subsection Configuration specification
+#
+# Configuration specification is divided in 2 parts :
+# - default values
+# - value validation/cast (see @ref Lodel.settings.validator.ConfValidator )
+# 
+
+PYTHON_SYS_LIB_PATH = '/usr/local/lib/python{major}.{minor}/'.format(
+
+                                                                        major = sys.version_info.major,
+                                                                        minor = sys.version_info.minor)
+## @brief Handles configuration load etc.
+class Settings(object):
+    
+    ## @brief global conf specsification (default_value + validator)
+    _conf_preload = {
+            'lib_path': (   PYTHON_SYS_LIB_PATH+'/lodel2/',
+                            SettingValidator('directory')),
+            'plugins_path': (   PYTHON_SYS_LIB_PATH+'lodel2/plugins/',
+                                SettingValidator('directory_list')),
+    }
+    
+    def __init__(self, conf_file = '/etc/lodel2/lodel2.conf', conf_dir = 'conf.d'):
+        self.__confs = dict()
+        
+        self.__load_bootstrap_conf(conf_file)
+        # now we should have the self.__confs['lodel2']['plugins_paths'] and
+        # self.__confs['lodel2']['lib_path'] set
+        self.__bootstrap()
+    
+    ## @brief This method handlers Settings instance bootstraping
+    def __bootstrap(self):
+        #loader = SettingsLoader(self.__conf_dir)
+
+        # Starting the Plugins class
+        Plugins.bootstrap(self.__confs['lodel2']['plugins_path'])
+        specs = Plugins.get_confspec('dummy')
+        print("Got specs : %s " % specs)
+        
+        # then fetch options values from conf specs
+    
+    ## @brief Load base global configurations keys
+    #
+    # Base configurations keys are :
+    # - lodel2 lib path
+    # - lodel2 plugins path
+    #
+    # @note return nothing but set the __confs attribute
+    # @see Settings._conf_preload
+    def __load_bootstrap_conf(self, conf_file):
+        config = configparser.ConfigParser()
+        config.read(conf_file)
+        sections = config.sections()
+        if len(sections) != 1 or sections[0].lower() != 'lodel2':
+            raise SettingsError("Global conf error, expected lodel2 section not found")
+        
+        #Load default values in result
+        res = dict()
+        for keyname, (keyvalue, validator) in self._conf_preload.items():
+            res[keyname] = keyvalue
+
+        confs = config[sections[0]]
+        errors = []
+        for name in confs:
+            if name not in res:
+                errors.append(  SettingsError(
+                                    "Unknow field",
+                                    "lodel2.%s" % name,
+                                    conf_file))
+            try:
+                res[name] = self._conf_preload[name][1](confs[name])
+            except Exception as e:
+                errors.append(SettingsError(str(e), name, conf_file))
+        if len(errors) > 0:
+            raise SettingsErrors(errors)
+        
+        self.__confs['lodel2'] = res
+

lodel/settings/utils.py

@@ -23,3 +23,22 @@ class SettingsError(Exception):
         res += ": %s" % (self.__msg)
         return res
 
+## @brief Designed to handles mutliple SettingsError
+class SettingsErrors(Exception):
+    
+    ## @brief Instanciate an SettingsErrors
+    # @param exceptions list : list of SettingsError instance
+    def __init__(self, exceptions):
+        for expt in exceptions: 
+            if not isinstance(expt, SettingsError):
+                raise ValueError("The 'exceptions' argument has to be an array of <class SettingsError>, but a %s was found in the list" % type(expt))
+        self.__exceptions = exceptions
+        
+
+    def __repr__(self): return str(self)
+
+    def __str__(self):
+        res = "Errors :\n"
+        for expt in self.__exceptions:
+            res += "\t%s\n" % str(expt)
+        return res

lodel/settings/validator.py

@@ -0,0 +1,181 @@
+#-*- coding: utf-8 -*-
+
+import sys
+import os.path
+import re
+import inspect
+import copy
+
+## @package lodel.settings.validator Lodel2 settings validators/cast module
+#
+# Validator are registered in the SettingValidator class.
+
+class SettingsValidationError(Exception):
+    pass
+
+## @brief Handles settings validators
+#
+# Class instance are callable objects that takes a value argument (the value to validate). It raises
+# a SettingsValidationError if validation fails, else it returns a properly
+# casted value.
+class SettingValidator(object):
+    
+    _validators = dict()
+    _description = dict()
+    
+    ## @brief Instanciate a validator
+    def __init__(self, name, none_is_valid = False):
+        if name is not None and name not in self._validators:
+            raise NameError("No validator named '%s'" % name)
+        self.__name = name
+
+    ## @brief Call the validator
+    # @param value *
+    # @return properly casted value
+    # @throw SettingsValidationError
+    def __call__(self, value):
+        if self.__name is None:
+            return value
+        try:
+            return self._validators[self.__name](value)
+        except Exception as e:
+            raise SettingsValidationError(e)
+    
+    ## @brief Register a new validator
+    # @param name str : validator name
+    # @param callback callable : the function that will validate a value
+    @classmethod
+    def register_validator(cls, name, callback, description=None):
+        if name in cls._validators:
+            raise NameError("A validator named '%s' allready exists" % name)
+        # Broken test for callable
+        if not inspect.isfunction(callback) and not inspect.ismethod(callback) and not hasattr(callback, '__call__'):
+            raise TypeError("Callable expected but got %s" % type(callback))
+        cls._validators[name] = callback
+        cls._description[name] = description
+    
+    ## @brief Get the validator list associated with description
+    @classmethod
+    def validators_list(cls):
+        return copy.copy(cls._description)
+
+    ## @brief Create and register a list validator
+    # @param elt_validator callable : The validator that will be used for validate each elt value
+    # @param validator_name str
+    # @param description None | str
+    # @param separator str : The element separator
+    # @return A SettingValidator instance
+    @classmethod
+    def create_list_validator(cls, validator_name, elt_validator, description = None, separator = ','):
+        def list_validator(value):
+            res = list()
+            errors = list()
+            for elt in value.split(separator):
+                res.append(elt_validator(elt))
+            return res
+        description = "Convert value to an array" if description is None else description
+        cls.register_validator(
+                                validator_name,
+                                list_validator,
+                                description)
+        return cls(validator_name)
+                
+    ## @brief Create and register a regular expression validator
+    # @param pattern str : regex pattern
+    # @param validator_name str : The validator name
+    # @param description str : Validator description
+    # @return a SettingValidator instance
+    @classmethod
+    def create_re_validator(cls, pattern, validator_name, description = None):
+        def re_validator(value):
+            if not re.match(pattern, value):
+                raise SettingsValidationError("The value '%s' doesn't match the following pattern '%s'" % pattern)
+            return value
+        #registering the validator
+        cls.register_validator(
+                                validator_name,
+                                re_validator,
+                                ("Match value to '%s'" % pattern) if description is None else description)
+        return cls(validator_name)
+
+    
+    ## @return a list of registered validators
+    def validators_list_str(cls):
+        result = ''
+        for name in cls._validators:
+            result += "\t%s" % name
+            if name in self._description and self._description[name] is not None:
+                result += "\t: %s" % self._description[name]
+            result += "\n"
+        return result
+
+## @brief Integer value validator callback
+def int_val(value):
+    return int(value)
+
+## @brief Output file validator callback
+# @return A file object (if filename is '-' return sys.stderr)
+def file_err_output(value):
+    if not isinstance(value, str):
+        raise SettingsValidationError("A string was expected but got '%s' " % value)
+    if value == '-':
+        return sys.stderr
+    return value
+
+## @brief Boolean value validator callback
+def boolean_val(value):
+    if not (value is True) and not (value is False):
+        raise SettingsValidationError("A boolean was expected but got '%s' " % value)
+    return bool(value)
+
+def directory_val(value):
+    res = SettingValidator('strip')(value)
+    if not os.path.isdir(res):
+        raise SettingsValidationError("Folowing path don't exists or is not a directory : '%s'"%res)
+    return res
+
+#
+#   Default validators registration
+#
+
+SettingValidator.register_validator(
+                                        'strip',
+                                        str.strip,
+                                        'String trim')
+
+SettingValidator.register_validator(
+                                        'int',
+                                        int_val,
+                                        'Integer value validator')
+
+SettingValidator.register_validator(
+                                        'bool',
+                                        boolean_val,
+                                        'Boolean value validator')
+
+SettingValidator.register_validator(
+                                        'errfile',
+                                        file_err_output,
+                                        'Error output file validator (return stderr if filename is "-")')
+
+SettingValidator.register_validator(
+                                        'directory',
+                                        directory_val,
+                                        'Directory path validator')
+
+SettingValidator.create_list_validator(
+                                            'list',
+                                            SettingValidator('strip'),
+                                            description = "Simple list validator. Validate a list of values separated by ','",
+                                            separator = ',')
+
+SettingValidator.create_list_validator(
+                                            'directory_list',
+                                            SettingValidator('directory'),
+                                            description = "Validator for a list of directory path separated with ','",
+                                            separator = ',')
+
+SettingValidator.create_re_validator(
+                                        r'^https?://[^\./]+.[^\./]+/?.*$',
+                                        'http_url',
+                                        'Url validator')

lodel/settings_format.py

@@ -1,26 +0,0 @@
-#-*- coding: utf-8 -*-
-## @package Lodel.settings_format Rules for settings
-
-## @brief List mandatory configurations keys
-MANDATORY = [
-    'debug',
-    'debug_sql',
-    'sitename',
-    'lodel2_lib_path',
-    'em_file',
-    'dynamic_code_file',
-    'ds_package',
-    'datasource',
-    'mh_classname',
-    'migration_options',
-    'base_path',
-    'plugins',
-    'logging',
-]
-
-## @brief List allowed (but not mandatory) configurations keys
-ALLOWED = [
-    'em_graph_output',
-    'em_graph_format',
-    'templates_base_dir'
-]

plugins/dummy/confspec.py

@@ -0,0 +1,7 @@
+#-*- coding: utf-8 -*-
+
+CONFSPEC = {
+    'section1': {
+        'key1': None
+    }
+}

settings.ini

@@ -0,0 +1,3 @@
+[lodel2]
+lib_path = /home/yannweb/dev/lodel2/lodel2-git
+plugins_path = /home/yannweb/dev/lodel2/lodel2-git/plugins