From 68a27ff5dd36dd92612d2101d1ac566c05612461 Mon Sep 17 00:00:00 2001 From: Yann Date: Fri, 8 Apr 2016 17:33:48 +0200 Subject: [PATCH] [Broken state] started settings implementation --- lodel/plugins.py | 72 ++++++++++++++ lodel/settings.py | 145 ----------------------------- lodel/settings/settings.py | 102 ++++++++++++++++++++ lodel/settings/utils.py | 19 ++++ lodel/settings/validator.py | 181 ++++++++++++++++++++++++++++++++++++ lodel/settings_format.py | 26 ------ plugins/dummy/confspec.py | 7 ++ settings.ini | 3 + 8 files changed, 384 insertions(+), 171 deletions(-) create mode 100644 lodel/plugins.py delete mode 100644 lodel/settings.py create mode 100644 lodel/settings/settings.py create mode 100644 lodel/settings/validator.py delete mode 100644 lodel/settings_format.py create mode 100644 plugins/dummy/confspec.py create mode 100644 settings.ini diff --git a/lodel/plugins.py b/lodel/plugins.py new file mode 100644 index 0000000..65f7f32 --- /dev/null +++ b/lodel/plugins.py @@ -0,0 +1,72 @@ +#-*- coding: utf-8 -*- + +import os.path + +from importlib.machinery import SourceFileLoader, SourcelessFileLoader + +## @package lodel.plugins Lodel2 plugins management +# +# Lodel2 plugins are stored in directories +# A typicall lodel2 plugin directory structure looks like : +# - {{__init__.py}}} containing informations like full_name, authors, licence etc. +# - main.py containing hooks registration etc +# - confspec.py containing a configuration specification dictionary named CONFSPEC + +VIRTUAL_PACKAGE_NAME = 'lodel.plugins_pkg' +CONFSPEC_NAME = 'confspec.py' + +class Plugins(object): + + ## @brief Stores plugin directories paths + _plugin_directories = None + ## @brief Optimisation cache storage for plugin paths + _plugin_paths = dict() + + def __init__(self): + self.started() + + ## @brief Given a plugin name returns the plugin path + # @param plugin_name str : The plugin name + # @return the plugin directory path + @classmethod + def plugin_path(cls, plugin_name): + cls.started() + try: + return cls._plugin_paths[plugin_name] + except KeyError: + pass + + path = None + for cur_path in cls._plugin_directories: + plugin_path = os.path.join(cur_path, plugin_name)+'/' + print(plugin_path) + if os.path.isdir(plugin_path): + return plugin_path + raise NameError("No plugin named '%s'" % plugin_name) + + ## @brief Fetch a confspec given a plugin_name + # @param plugin_name str : The plugin name + # @return a dict of conf spec + @classmethod + def get_confspec(cls, plugin_name): + cls.started() + plugin_path = cls.plugin_path(plugin_name) + plugin_module = '%s.%s' % ( VIRTUAL_PACKAGE_NAME, + plugin_name) + conf_spec_module = plugin_module + '.confspec' + + conf_spec_source = plugin_path + CONFSPEC_NAME + + loader = SourceFileLoader(conf_spec_module, conf_spec_source) + confspec_module = loader.load_module() + return getattr(confspec_module, 'CONFSPEC') + + @classmethod + def bootstrap(cls, plugins_directories): + cls._plugin_directories = plugins_directories + + @classmethod + def started(cls, raise_if_not = True): + res = cls._plugin_directories is not None + if raise_if_not and not res: + raise RuntimeError("Class Plugins is not initialized") diff --git a/lodel/settings.py b/lodel/settings.py deleted file mode 100644 index 204bddb..0000000 --- a/lodel/settings.py +++ /dev/null @@ -1,145 +0,0 @@ -#-*- coding: utf-8 -*- - -import types -import warnings -from . import settings_format - -## @package Lodel.settings -# -# @brief Defines stuff to handles Lodel2 configuration (see @ref lodel_settings ) -# -# To access the confs use the Lodel.settings.Settings SettingsHandler instance - -## @brief A class designed to handles Lodel2 settings -# -# When instanciating a SettingsHandler, the new instance is filled with the content of settings.py (in the root directory of lodel2 -# -# @warning You don't have to instanciate this class, you can access to the global instance with the Settings variable in this module -# @todo broken stuff... Rewrite it -# @todo Forbid module assignement in settings ! and disable tests about this -# @todo Implements a type checking of config value -# @todo Implements default values for config keys -class SettingsHandler(object): - - ## @brief Shortcut - _allowed = settings_format.ALLOWED + settings_format.MANDATORY - ## @brief Shortcut - _mandatory = settings_format.MANDATORY - - def __init__(self): - try: - import settings as default_settings - self._load_module(default_settings) - except ImportError: - warnings.warn("Unable to find global default settings") - - ## @brief A flag set to True when the instance is fully loaded - self._set_loaded(False if len(self._missings()) > 0 else True) - - ## @brief Compat wrapper for getattr - def get(self, name): - return getattr(self, name) - - ## @brief Compat wrapper for setattr - def set(self, name, value): - return setattr(self, name, value) - - ## @brief Load every module properties in the settings instance - # - # Load a module content into a SettingsHandler instance and checks that no mandatory settings are missing - # @note Example : 
 import my_cool_settings;
-    # Settings._load_module(my_cool_settings);
- # @param module module|None: a loaded module (if None just check for missing settings) - # @throw LookupError if invalid settings found or if mandatory settings are missing - def load_module(self, module = None): - if not(module is None): - self._load_module(module) - missings = self._missings() - if len(missings) > 0: - self._loaded = False - raise LookupError("Mandatory settings are missing : %s"%missings) - self._set_loaded(True) - - ## @brief supersede of default __setattr__ method - def __setattr__(self, name, value): - if not hasattr(self, name): - if name not in self._allowed: - raise LookupError("Invalid setting : %s"%name) - super().__setattr__(name, value) - - ## @brief This method do the job for SettingsHandler.load_module() - # - # @note The difference with SettingsHandler.load_module() is that it didn't check if some settings are missing - # @throw LokkupError if an invalid settings is given - # @param module : a loaded module - def _load_module(self, module): - errors = [] - fatal_errors = [] - conf_dict = { - name: getattr(module, name) - for name in dir(module) - if not name.startswith('__') and not isinstance(getattr(module, name), types.ModuleType) - } - for name, value in conf_dict.items(): - try: - setattr(self, name, value) - except LookupError: - errors.append(name) - if len(errors) > 0: - err_msg = "Found invalid settings in %s : %s"%(module.__name__, errors) - raise LookupError(err_msg) - - ## @brief Refresh the allowed and mandatory settings list - @classmethod - def _refresh_format(cls): - ## @brief Shortcut - cls._allowed = settings_format.ALLOWED + settings_format.MANDATORY - ## @brief Shortcut - cls._mandatory = settings_format.MANDATORY - - ## @brief If some settings are missings return their names - # @return an array of string - def _missings(self): - return [ confname for confname in self._mandatory if not hasattr(self, confname) ] - - def _set_loaded(self, value): - super().__setattr__('_loaded', bool(value)) - -Settings = SettingsHandler() - -## @page lodel_settings Lodel SettingsHandler -# -# This page describe the way settings are handled in Lodel2. -# -# @section lodel_settings_files Lodel settings files -# -# - Lodel/settings.py defines the Lodel.settings package, the SettingsHandler class and the Lodel.settings.Settings instance -# - Lodel/settings_format.py defines the mandatory and allowed configurations keys lists -# - install/instance_settings.py is a model of the file that will be deployed in Lodel2 instances directories -# -# @section Using Lodel.settings.Settings SettingsHandler instance -# -# @subsection lodel_settings_without_loader Without loader -# -# Without any loader you can import Lodel.settings.Settings and acces its property with getattr (or . ) or with SettingsHandler.get() method. -# In the same way you can set a settings by standart affectation of a propery or with SettingsHandler.set() method. -# -# @subsection lodel_settings_loader With a loader in a lodel2 instance -# -# The loader will import Lodel.settings.Settings and then calls the SettingsHandler.load_module() method to load the content of the instance_settings.py file into the SettingsHandler instance -# -# @subsection lodel_settings_example Examples -# -# 
-# #!/usr/bin/python
-# from Lodel.settings import Settings
-# if Settings.debug:
-#   print("DEBUG")
-# # or
-# if Settings.get('debug'):
-#   print("DEBUG")
-# Settings.debug = False
-# # or
-# Settings.set('debug', False)
-#
-# diff --git a/lodel/settings/settings.py b/lodel/settings/settings.py new file mode 100644 index 0000000..007bbd0 --- /dev/null +++ b/lodel/settings/settings.py @@ -0,0 +1,102 @@ +#-*- coding: utf-8 -*- + +import sys +import os +import configparser + +from lodel.plugins import Plugins +from lodel.settings.utils import SettingsError, SettingsErrors +from lodel.settings.validator import SettingValidator +from lodel.settings.settings_loader import SettingsLoader + +## @package lodel.settings Lodel2 settings package +# +# Contains all module that help handling settings + +## @package lodel.settings.settings Lodel2 settings module +# +# Handles configuration load/parse/check. +# +# @subsection Configuration load process +# +# The configuration load process is not trivial. In fact loaded plugins are able to add their own options. +# But the list of plugins to load and the plugins options are in the same file, the instance configuration file. +# +# @subsection Configuration specification +# +# Configuration specification is divided in 2 parts : +# - default values +# - value validation/cast (see @ref Lodel.settings.validator.ConfValidator ) +#  + +PYTHON_SYS_LIB_PATH = '/usr/local/lib/python{major}.{minor}/'.format( + + major = sys.version_info.major, + minor = sys.version_info.minor) +## @brief Handles configuration load etc. +class Settings(object): + + ## @brief global conf specsification (default_value + validator) + _conf_preload = { + 'lib_path': ( PYTHON_SYS_LIB_PATH+'/lodel2/', + SettingValidator('directory')), + 'plugins_path': ( PYTHON_SYS_LIB_PATH+'lodel2/plugins/', + SettingValidator('directory_list')), + } + + def __init__(self, conf_file = '/etc/lodel2/lodel2.conf', conf_dir = 'conf.d'): + self.__confs = dict() + + self.__load_bootstrap_conf(conf_file) + # now we should have the self.__confs['lodel2']['plugins_paths'] and + # self.__confs['lodel2']['lib_path'] set + self.__bootstrap() + + ## @brief This method handlers Settings instance bootstraping + def __bootstrap(self): + #loader = SettingsLoader(self.__conf_dir) + + # Starting the Plugins class + Plugins.bootstrap(self.__confs['lodel2']['plugins_path']) + specs = Plugins.get_confspec('dummy') + print("Got specs : %s " % specs) + + # then fetch options values from conf specs + + ## @brief Load base global configurations keys + # + # Base configurations keys are : + # - lodel2 lib path + # - lodel2 plugins path + # + # @note return nothing but set the __confs attribute + # @see Settings._conf_preload + def __load_bootstrap_conf(self, conf_file): + config = configparser.ConfigParser() + config.read(conf_file) + sections = config.sections() + if len(sections) != 1 or sections[0].lower() != 'lodel2': + raise SettingsError("Global conf error, expected lodel2 section not found") + + #Load default values in result + res = dict() + for keyname, (keyvalue, validator) in self._conf_preload.items(): + res[keyname] = keyvalue + + confs = config[sections[0]] + errors = [] + for name in confs: + if name not in res: + errors.append( SettingsError( + "Unknow field", + "lodel2.%s" % name, + conf_file)) + try: + res[name] = self._conf_preload[name][1](confs[name]) + except Exception as e: + errors.append(SettingsError(str(e), name, conf_file)) + if len(errors) > 0: + raise SettingsErrors(errors) + + self.__confs['lodel2'] = res + diff --git a/lodel/settings/utils.py b/lodel/settings/utils.py index 37847da..c641a3d 100644 --- a/lodel/settings/utils.py +++ b/lodel/settings/utils.py @@ -22,4 +22,23 @@ class SettingsError(Exception): res += ": %s" % (self.__msg) return res + +## @brief Designed to handles mutliple SettingsError +class SettingsErrors(Exception): + + ## @brief Instanciate an SettingsErrors + # @param exceptions list : list of SettingsError instance + def __init__(self, exceptions): + for expt in exceptions: + if not isinstance(expt, SettingsError): + raise ValueError("The 'exceptions' argument has to be an array of , but a %s was found in the list" % type(expt)) + self.__exceptions = exceptions + + def __repr__(self): return str(self) + + def __str__(self): + res = "Errors :\n" + for expt in self.__exceptions: + res += "\t%s\n" % str(expt) + return res diff --git a/lodel/settings/validator.py b/lodel/settings/validator.py new file mode 100644 index 0000000..25dfef9 --- /dev/null +++ b/lodel/settings/validator.py @@ -0,0 +1,181 @@ +#-*- coding: utf-8 -*- + +import sys +import os.path +import re +import inspect +import copy + +## @package lodel.settings.validator Lodel2 settings validators/cast module +# +# Validator are registered in the SettingValidator class. + +class SettingsValidationError(Exception): + pass + +## @brief Handles settings validators +# +# Class instance are callable objects that takes a value argument (the value to validate). It raises +# a SettingsValidationError if validation fails, else it returns a properly +# casted value. +class SettingValidator(object): + + _validators = dict() + _description = dict() + + ## @brief Instanciate a validator + def __init__(self, name, none_is_valid = False): + if name is not None and name not in self._validators: + raise NameError("No validator named '%s'" % name) + self.__name = name + + ## @brief Call the validator + # @param value * + # @return properly casted value + # @throw SettingsValidationError + def __call__(self, value): + if self.__name is None: + return value + try: + return self._validators[self.__name](value) + except Exception as e: + raise SettingsValidationError(e) + + ## @brief Register a new validator + # @param name str : validator name + # @param callback callable : the function that will validate a value + @classmethod + def register_validator(cls, name, callback, description=None): + if name in cls._validators: + raise NameError("A validator named '%s' allready exists" % name) + # Broken test for callable + if not inspect.isfunction(callback) and not inspect.ismethod(callback) and not hasattr(callback, '__call__'): + raise TypeError("Callable expected but got %s" % type(callback)) + cls._validators[name] = callback + cls._description[name] = description + + ## @brief Get the validator list associated with description + @classmethod + def validators_list(cls): + return copy.copy(cls._description) + + ## @brief Create and register a list validator + # @param elt_validator callable : The validator that will be used for validate each elt value + # @param validator_name str + # @param description None | str + # @param separator str : The element separator + # @return A SettingValidator instance + @classmethod + def create_list_validator(cls, validator_name, elt_validator, description = None, separator = ','): + def list_validator(value): + res = list() + errors = list() + for elt in value.split(separator): + res.append(elt_validator(elt)) + return res + description = "Convert value to an array" if description is None else description + cls.register_validator( + validator_name, + list_validator, + description) + return cls(validator_name) + + ## @brief Create and register a regular expression validator + # @param pattern str : regex pattern + # @param validator_name str : The validator name + # @param description str : Validator description + # @return a SettingValidator instance + @classmethod + def create_re_validator(cls, pattern, validator_name, description = None): + def re_validator(value): + if not re.match(pattern, value): + raise SettingsValidationError("The value '%s' doesn't match the following pattern '%s'" % pattern) + return value + #registering the validator + cls.register_validator( + validator_name, + re_validator, + ("Match value to '%s'" % pattern) if description is None else description) + return cls(validator_name) + + + ## @return a list of registered validators + def validators_list_str(cls): + result = '' + for name in cls._validators: + result += "\t%s" % name + if name in self._description and self._description[name] is not None: + result += "\t: %s" % self._description[name] + result += "\n" + return result + +## @brief Integer value validator callback +def int_val(value): + return int(value) + +## @brief Output file validator callback +# @return A file object (if filename is '-' return sys.stderr) +def file_err_output(value): + if not isinstance(value, str): + raise SettingsValidationError("A string was expected but got '%s' " % value) + if value == '-': + return sys.stderr + return value + +## @brief Boolean value validator callback +def boolean_val(value): + if not (value is True) and not (value is False): + raise SettingsValidationError("A boolean was expected but got '%s' " % value) + return bool(value) + +def directory_val(value): + res = SettingValidator('strip')(value) + if not os.path.isdir(res): + raise SettingsValidationError("Folowing path don't exists or is not a directory : '%s'"%res) + return res + +# +# Default validators registration +# + +SettingValidator.register_validator( + 'strip', + str.strip, + 'String trim') + +SettingValidator.register_validator( + 'int', + int_val, + 'Integer value validator') + +SettingValidator.register_validator( + 'bool', + boolean_val, + 'Boolean value validator') + +SettingValidator.register_validator( + 'errfile', + file_err_output, + 'Error output file validator (return stderr if filename is "-")') + +SettingValidator.register_validator( + 'directory', + directory_val, + 'Directory path validator') + +SettingValidator.create_list_validator( + 'list', + SettingValidator('strip'), + description = "Simple list validator. Validate a list of values separated by ','", + separator = ',') + +SettingValidator.create_list_validator( + 'directory_list', + SettingValidator('directory'), + description = "Validator for a list of directory path separated with ','", + separator = ',') + +SettingValidator.create_re_validator( + r'^https?://[^\./]+.[^\./]+/?.*$', + 'http_url', + 'Url validator') diff --git a/lodel/settings_format.py b/lodel/settings_format.py deleted file mode 100644 index 6f184fb..0000000 --- a/lodel/settings_format.py +++ /dev/null @@ -1,26 +0,0 @@ -#-*- coding: utf-8 -*- -## @package Lodel.settings_format Rules for settings - -## @brief List mandatory configurations keys -MANDATORY = [ - 'debug', - 'debug_sql', - 'sitename', - 'lodel2_lib_path', - 'em_file', - 'dynamic_code_file', - 'ds_package', - 'datasource', - 'mh_classname', - 'migration_options', - 'base_path', - 'plugins', - 'logging', -] - -## @brief List allowed (but not mandatory) configurations keys -ALLOWED = [ - 'em_graph_output', - 'em_graph_format', - 'templates_base_dir' -] diff --git a/plugins/dummy/confspec.py b/plugins/dummy/confspec.py new file mode 100644 index 0000000..c3e1897 --- /dev/null +++ b/plugins/dummy/confspec.py @@ -0,0 +1,7 @@ +#-*- coding: utf-8 -*- + +CONFSPEC = { + 'section1': { + 'key1': None + } +} diff --git a/settings.ini b/settings.ini new file mode 100644 index 0000000..82d2756 --- /dev/null +++ b/settings.ini @@ -0,0 +1,3 @@ +[lodel2] +lib_path = /home/yannweb/dev/lodel2/lodel2-git +plugins_path = /home/yannweb/dev/lodel2/lodel2-git/plugins