Add settings & logger module

lodel/leapi/datahandlers/base_classes.py

@@ -8,6 +8,7 @@ import copy
 import importlib
 import inspect
 
+from lodel import logger
 
 class FieldValidationError(Exception):
     pass
@@ -121,7 +122,7 @@ class DataHandler(object):
                 module = importlib.import_module('lodel.leapi.datahandlers.%s' % module_name)
                 for name, obj in inspect.getmembers(module):
                     if inspect.isclass(obj):
-                        print("Data handler found : %s in %s" % (name, module_name))
+                        logger.debug("Load data handler %s.%s" % (obj.__module__, obj.__name__))
                         cls.__base_handlers[name.lower()] = obj
         return copy.copy(cls.__base_handlers)
 

lodel/logger.py

@@ -0,0 +1,113 @@
+#-*- coding: utf-8 -*-
+
+import logging, logging.handlers
+import os.path
+from lodel.settings import Settings
+
+# Variables & constants definitions
+default_format = '%(asctime)-15s %(levelname)s %(_pathname)s:%(_lineno)s:%(_funcName)s() : %(message)s'
+simple_format = '%(asctime)-15s %(levelname)s : %(message)s'
+SECURITY_LOGLEVEL = 35
+logging.addLevelName(SECURITY_LOGLEVEL, 'SECURITY')
+handlers = dict() # Handlers list (generated from settings)
+
+# Fetching default root logger
+logger = logging.getLogger()
+
+# Setting options from Lodel.settings.Settings.logging
+def __init_from_settings():
+    # Disabled, because the custom format raises error (enable to give the _ preffixed arguments to logger resulting into a KeyError exception )
+    #logging.captureWarnings(True) # Log warnings
+
+    logger.setLevel(logging.DEBUG)
+    for name, logging_opt in Settings.logging.items():
+        add_handler(name, logging_opt)
+
+## @brief Add an handler, identified by a name, to a given logger 
+#
+# logging_opt is a dict with logger option. Allowed keys are : 
+# - filename : take a filepath as value and cause the use of a logging.handlers.RotatingFileHandler
+# - level : the minimum logging level for a logger, takes values [ 'DEBUG', 'INFO', 'WARNING', 'SECURITY', 'ERROR', 'CRITICAL' ]
+# - format : DONT USE THIS OPTION (or if you use it be sure to includes %(_pathname)s %(_lineno)s %(_funcName)s format variables in format string
+# - context : boolean, if True include the context (module:lineno:function_name) in the log format
+# @todo Move the logging_opt documentation somewhere related with settings
+# 
+# @param name str : The handler name
+# @param logging_opt dict : dict containing options ( see above )
+def add_handler(name, logging_opt):
+    logger = logging.getLogger()
+    if name in handlers:
+        raise KeyError("A handler named '%s' allready exists")
+
+    if 'filename' in logging_opt:
+        maxBytes = (1024 * 10) if 'maxBytes' not in logging_opt else logging_opt['maxBytes']
+        backupCount = 10 if 'backupCount' not in logging_opt else logging_opt['backupCount']
+
+        handler = logging.handlers.RotatingFileHandler(
+                                        logging_opt['filename'],
+                                        maxBytes = maxBytes,
+                                        backupCount = backupCount,
+                                        encoding = 'utf-8')
+    else:
+        handler = logging.StreamHandler()
+    
+    if 'level' in logging_opt:
+        handler.setLevel(getattr(logging, logging_opt['level'].upper()))
+
+    if 'format' in logging_opt:
+        formatter = logging.Formatter(logging_opt['format'])
+    else:
+        if 'context' in logging_opt and not logging_opt['context']:
+            formatter = logging.Formatter(simple_format)
+        else:
+            formatter = logging.Formatter(default_format)
+
+    handler.setFormatter(formatter)
+    handlers[name] = handler
+    logger.addHandler(handler)
+    
+
+## @brief Remove an handler generated from configuration (runtime logger configuration)
+# @param name str : handler name
+def remove_handler(name):
+    if name in handlers:
+        logger.removeHandler(handlers[name])
+    # else: can we do anything ?
+
+## @brief Utility function that disable unconditionnaly handlers that implies console output
+# @note In fact, this function disables handlers generated from settings wich are instances of logging.StreamHandler
+def remove_console_handlers():
+    for name, handler in handlers.items():
+        if isinstance(handler, logging.StreamHandler):
+            remove_handler(name)
+    
+
+# Utility functions
+
+## @brief Generic logging function
+# @param lvl int : Log severity
+# @param msg str : log message
+# @param *args : additional positionnal arguments
+# @param **kwargs : additional named arguments
+def log(lvl, msg, *args, **kwargs):
+    caller = logger.findCaller() # Opti warning : small overhead
+    extra = {
+        '_pathname': os.path.relpath(
+                                        caller[0],
+                                        start=Settings.lodel2_lib_path
+        ), # os.path.relpath add another small overhead
+        '_lineno': caller[1],
+        '_funcName': caller[2],
+    }
+    logger.log(lvl, msg, extra = extra, *args, **kwargs)
+
+def debug(msg, *args, **kwargs): log(logging.DEBUG, msg, *args, **kwargs)
+def info(msg, *args, **kwargs): log(logging.INFO, msg, *args, **kwargs)
+def warning(msg, *args, **kwargs): log(logging.WARNING, msg, *args, **kwargs)
+def security(msg, *args, **kwargs): log(SECURITY_LOGLEVEL, msg, *args, **kwargs)
+def error(msg, *args, **kwargs): log(logging.ERROR, msg, *args, **kwargs)
+def critical(msg, *args, **kwargs): log(logging.CRITICAL, msg, *args, **kwargs)
+
+# Initialisation triggering
+if len(handlers) == 0:
+    __init_from_settings()

lodel/settings.py

@@ -0,0 +1,145 @@
+#-*- coding: utf-8 -*-
+
+import types
+import warnings
+from . import settings_format
+
+## @package Lodel.settings
+#
+# @brief Defines stuff to handles Lodel2 configuration (see @ref lodel_settings )
+#
+# To access the confs use the Lodel.settings.Settings SettingsHandler instance
+
+## @brief A class designed to handles Lodel2 settings
+#
+# When instanciating a SettingsHandler, the new instance is filled with the content of settings.py (in the root directory of lodel2
+#
+# @warning You don't have to instanciate this class, you can access to the global instance with the Settings variable in this module
+# @todo broken stuff... Rewrite it
+# @todo Forbid module assignement in settings ! and disable tests about this
+# @todo Implements a type checking of config value
+# @todo Implements default values for config keys
+class SettingsHandler(object):
+    
+    ## @brief Shortcut
+    _allowed = settings_format.ALLOWED + settings_format.MANDATORY
+    ## @brief Shortcut
+    _mandatory = settings_format.MANDATORY
+
+    def __init__(self):
+        try:
+            import settings as default_settings
+            self._load_module(default_settings)
+        except ImportError:
+            warnings.warn("Unable to find global default settings")
+
+        ## @brief A flag set to True when the instance is fully loaded
+        self._set_loaded(False if len(self._missings()) > 0 else True)
+    
+    ## @brief Compat wrapper for getattr
+    def get(self, name):
+        return getattr(self, name)
+    
+    ## @brief Compat wrapper for setattr
+    def set(self, name, value):
+        return setattr(self, name, value)
+
+    ## @brief Load every module properties in the settings instance
+    #
+    # Load a module content into a SettingsHandler instance and checks that no mandatory settings are missing
+    # @note Example : <pre> import my_cool_settings;
+    # Settings._load_module(my_cool_settings);</pre>
+    # @param module module|None: a loaded module (if None just check for missing settings)
+    # @throw LookupError if invalid settings found or if mandatory settings are missing
+    def load_module(self, module = None):
+        if not(module is None):
+            self._load_module(module)
+        missings = self._missings()
+        if len(missings) > 0:
+            self._loaded = False
+            raise LookupError("Mandatory settings are missing : %s"%missings)
+        self._set_loaded(True)
+    
+    ## @brief supersede of default __setattr__ method
+    def __setattr__(self, name, value):
+        if not hasattr(self, name):
+            if name not in self._allowed:
+                raise LookupError("Invalid setting : %s"%name)
+        super().__setattr__(name, value)
+
+    ## @brief This method do the job for SettingsHandler.load_module()
+    #
+    # @note The difference with SettingsHandler.load_module() is that it didn't check if some settings are missing
+    # @throw LokkupError if an invalid settings is given
+    # @param module : a loaded module
+    def _load_module(self, module):
+        errors = []
+        fatal_errors = []
+        conf_dict = {
+            name: getattr(module, name)
+            for name in dir(module) 
+            if not name.startswith('__') and not isinstance(getattr(module, name), types.ModuleType)
+        }
+        for name, value in conf_dict.items():
+            try:
+                setattr(self, name, value)
+            except LookupError:
+                errors.append(name)
+        if len(errors) > 0:
+            err_msg = "Found invalid settings in %s : %s"%(module.__name__, errors)
+            raise LookupError(err_msg)
+
+    ## @brief Refresh the allowed and mandatory settings list
+    @classmethod
+    def _refresh_format(cls):
+        ## @brief Shortcut
+        cls._allowed = settings_format.ALLOWED + settings_format.MANDATORY
+        ## @brief Shortcut
+        cls._mandatory = settings_format.MANDATORY
+
+    ## @brief If some settings are missings return their names
+    # @return an array of string
+    def _missings(self):
+        return [ confname for confname in self._mandatory if not hasattr(self, confname) ]
+
+    def _set_loaded(self, value):
+        super().__setattr__('_loaded', bool(value))
+
+Settings = SettingsHandler()
+
+## @page lodel_settings Lodel SettingsHandler
+#
+# This page describe the way settings are handled in Lodel2.
+#
+# @section lodel_settings_files Lodel settings files
+#
+# - Lodel/settings.py defines the Lodel.settings package, the SettingsHandler class and the Lodel.settings.Settings instance
+# - Lodel/settings_format.py defines the mandatory and allowed configurations keys lists
+# - install/instance_settings.py is a model of the file that will be deployed in Lodel2 instances directories
+#
+# @section Using Lodel.settings.Settings SettingsHandler instance
+#
+# @subsection lodel_settings_without_loader Without loader
+#
+# Without any loader you can import Lodel.settings.Settings and acces its property with getattr (or . ) or with SettingsHandler.get() method.
+# In the same way you can set a settings by standart affectation of a propery or with SettingsHandler.set() method.
+#
+# @subsection lodel_settings_loader With a loader in a lodel2 instance
+#
+# The loader will import Lodel.settings.Settings and then calls the SettingsHandler.load_module() method to load the content of the instance_settings.py file into the SettingsHandler instance
+#
+# @subsection lodel_settings_example Examples
+#
+# <pre>
+# #!/usr/bin/python
+# from Lodel.settings import Settings
+# if Settings.debug:
+#   print("DEBUG")
+# # or
+# if Settings.get('debug'):
+#   print("DEBUG")
+# Settings.debug = False
+# # or
+# Settings.set('debug', False)
+# </pre>
+# 

lodel/settings_format.py

@@ -0,0 +1,26 @@
+#-*- coding: utf-8 -*-
+## @package Lodel.settings_format Rules for settings
+
+## @brief List mandatory configurations keys
+MANDATORY = [
+    'debug',
+    'debug_sql',
+    'sitename',
+    'lodel2_lib_path',
+    'em_file',
+    'dynamic_code_file',
+    'ds_package',
+    'datasource',
+    'mh_classname',
+    'migration_options',
+    'base_path',
+    'plugins',
+    'logging',
+]
+
+## @brief List allowed (but not mandatory) configurations keys
+ALLOWED = [
+    'em_graph_output',
+    'em_graph_format',
+    'templates_base_dir'
+]

settings.py

@@ -0,0 +1,46 @@
+#-*- coding:utf8 -*-
+## @package settings Configuration file
+
+import pymysql
+import os
+import os.path
+
+lodel2_lib_path = os.path.dirname(os.path.abspath(__file__))
+base_path = os.path.dirname(os.path.abspath(__file__))
+debug = False
+debug_sql = False
+
+plugins = ['dummy', 'dummy_auth']
+
+datasource = {
+    'default': {
+        'module':pymysql,
+        'host': None,
+        'user': None,
+        'passwd': None,
+        'db': None,
+    }
+}
+
+migration_options = {
+    'dryrun': False,
+    'foreign_keys': True,
+    'drop_if_exists': False,
+}
+
+em_graph_format = 'png'
+em_graph_output = '/tmp/em_%s_graph.png'
+
+logging = {
+    'stderr': {
+        'level': 'DEBUG',
+        'context': False,
+    },
+    'logfile': {
+        'level': 'DEBUG',
+        'filename': '/tmp/lodel2.log',
+        'maxBytes': 1024 * 50, # rotate at 50MB
+        'backupCount': 10, # keep at most 10 backup
+        'context': True, # if false use a simpler format string
+    }
+}