New way to handles plugin

- renamed Plugins class to Plugin
- an instance represent a loaded plugin
- classmethod allows to preload & load plugins

install/conf.d/datasources.ini

@@ -1,5 +1,5 @@
 [lodel2.datasources.default]
-identifier = dummy.example
+identifier = dummy_datasource.example
 
 [lodel2.datasource.dummy_datasource.example]
 dummy =

install/loader.py

@@ -24,8 +24,8 @@ from lodel.settings import Settings
 
 
 #Load plugins
-from lodel.plugin import Plugins
+from lodel.plugin import Plugin
-Plugins.bootstrap()
+Plugin.load_all()
 
 from lodel.plugin import LodelHook
 LodelHook.call_hook('lodel2_bootstraped', '__main__', None)

lodel/__init__.py

@@ -1 +1,14 @@
 #-*- coding: utf-8 -*-
+
+##@page lodel2_start Lodel2 boot mechanism
+#
+# @par Lodel2 boot sequence
+# see @ref install/loader.py
+# 1. lodel package is imported
+# 2. settings are started 
+# 3. plugins are pre-loaded from conf to load plugins configuration specs
+# 4. settings are loaded from conf and checked
+# 3. plugins are loaded (hooks are registered etc)
+# 4. "lodel2_bootstraped" hooks are called
+# 5. "lodel2_loader_main" hooks are called (if runned from loader.py as main executable)
+#

lodel/leapi/leobject.py

@@ -2,7 +2,7 @@
 
 import importlib
 
-from lodel.plugin import Plugins
+from lodel.plugin import Plugin
 from lodel import logger
 from lodel.settings import Settings
 from lodel.settings.utils import SettingsError
@@ -221,7 +221,7 @@ class LeObject(object):
 
         ds_conf = getattr(ds_conf, ds_name)
         #Checks that the datasource plugin exists
-        ds_plugin_module = Plugins.plugin_module(ds_plugin)
+        ds_plugin_module = Plugin.get(ds_plugin).module
         try:
             datasource_class = getattr(ds_plugin_module, "Datasource")
         except AttributeError as e:

lodel/plugin/__init__.py

@@ -40,4 +40,4 @@
 # 
 
 from .hooks import LodelHook
-from .plugins import Plugins
+from .plugins import Plugin

lodel/plugin/plugins.py

@@ -1,9 +1,11 @@
 #-*- coding: utf-8 -*-
 
+import sys
 import os.path
-
 import importlib
+import copy
 from importlib.machinery import SourceFileLoader, SourcelessFileLoader
+
 import plugins
 
 ## @package lodel.plugins Lodel2 plugins management
@@ -15,23 +17,160 @@ import plugins
 # - confspec.py containing a configuration specification dictionary named CONFSPEC
 
 ##@brief The package in wich we will load plugins modules
-VIRTUAL_PACKAGE_NAME = 'lodel.plugins_pkg'
+VIRTUAL_PACKAGE_NAME = 'lodel.plugins'
-CONFSPEC_FILENAME = 'confspec.py'
+INIT_FILENAME = '__init__.py' # Loaded with settings
-MAIN_FILENAME = '__init__.py'
+CONFSPEC_FILENAME_VARNAME = '__confspec__'
 CONFSPEC_VARNAME = 'CONFSPEC'
+LOADER_FILENAME_VARNAME = '__loader__'
 
 class PluginError(Exception):
     pass
 
-class Plugins(object):
+class Plugin(object):
     
     ##@brief Stores plugin directories paths
     _plugin_directories = None
-    ##@brief Optimisation cache storage for plugin paths
-    _plugin_paths = dict()
 
-    def __init__(self): # may be useless
+    _plugin_instances = dict()
+
+    ##@brief Plugin class constructor
+    #
+    # Called by setting in early stage of lodel2 boot sequence using classmethod
+    # register
+    #
+    # @param plugin_name str : plugin name
+    # @throw PluginError
+    def __init__(self, plugin_name):
         self.started()
+        
+        self.name = plugin_name
+        self.path = self.plugin_path(plugin_name)
+        self.module = None
+        self.__confspecs = dict()
+        
+        # Importing __init__.py
+        plugin_module = '%s.%s' % ( VIRTUAL_PACKAGE_NAME,
+                                    plugin_name)
+        init_source = self.path + INIT_FILENAME
+        try:
+            loader = SourceFileLoader(plugin_module, init_source)
+            self.module = loader.load_module()
+        except ImportError as e:
+            raise PluginError("Failed to load plugin '%s'. It seems that the plugin name is not valid" % plugin_name)
+
+        # loading confspecs
+        try:
+            # Loading confspec directly from __init__.py
+            self.__confspecs = getattr(self.module, CONFSPEC_VARNAME)
+        except AttributeError:
+            # Loading file in __confspec__ var in __init__.py
+            try:
+                module = self._import_from_init_var(CONFSPEC_FILENAME_VARNAME)
+            except AttributeError:
+                msg = "Malformed plugin {plugin} . No {varname} not {filevar} found in __init__.py"
+                msg = msg.format(
+                    plugin = self.name,
+                    varname = CONFSPEC_VARNAME,
+                    filevar = CONFSPEC_FILENAME_VARNAME)
+                raise PluginError(msg)
+            except ImportError as e:
+                msg = "Broken plugin {plugin} : {expt}"
+                msg = msg.format(
+                    plugin = self.name,
+                    expt = str(e))
+                raise PluginError(msg)
+
+            try:
+                # loading confpsecs from file
+                self.__confspecs = getattr(module, CONFSPEC_VARNAME)
+            except AttributeError:
+                msg = "Broken plugin. {varname} not found in '{filename}'"
+                msg = msg.format(
+                    varname = CONFSPEC_VARNAME,
+                    filename = confspec_filename)
+                raise PluginError(msg)
+
+    ##@brief Try to import a file from a variable in __init__.py
+    #@param varname str : The variable name
+    #@throw AttributeError if varname not found
+    #@throw ImportError if the file fails to be imported
+    #@throw PluginError if the filename was not valid
+    def _import_from_init_var(self, varname):
+        # Read varname
+        filename = getattr(self.module, varname)
+        #Path are not allowed
+        if filename != os.path.basename(filename):
+            msg = "Invalid {varname} content : '{fname}' for plugin {name}"
+            msg = msg.format(
+                varname = varname,
+                fname = filename,
+                name = self.name)
+            raise PluginError(msg)
+        # importing the file in varname
+        module_name = self.module.__name__+"."+varname
+        filename = self.path + filename
+        loader = SourceFileLoader(module_name, filename)
+        return loader.load_module()
+
+    ##@brief Register hooks etc
+    def load(self):
+        from lodel import logger
+        try:
+            return self._import_from_init_var(LOADER_FILENAME_VARNAME)
+        except AttributeError:
+            msg = "Malformed plugin {plugin}. No {varname} found in __init__.py"
+            msg = msg.format(
+                plugin = self.name,
+                varname = LOADER_FILENAME_VARNAME)
+            raise PluginError(msg)
+        except ImportError as e:
+            msg = "Broken plugin {plugin} : {expt}"
+            msg = msg.format(
+                plugin = self.name,
+                expt = str(e))
+            raise PluginError(msg)
+        logger.debug("Plugin '%s' loaded" % self.name)
+                
+    @classmethod
+    def load_all(cls):
+        errors = dict()
+        for name, plugin in cls._plugin_instances.items():
+            try:
+                plugin.load()
+            except PluginError as e:
+                errors[name] = e
+        if len(errors) > 0:
+            msg = "Errors while loading plugins :"
+            for name, e in errors.items():
+                msg += "\n\t%20s : %s" % (name,e)
+            msg += "\n"
+            raise PluginError(msg)
+
+    @property
+    def confspecs(self):
+        return copy.copy(self.__confspecs)
+
+    ##@brief Register a new plugin
+    # 
+    # preload
+    @classmethod
+    def register(cls, plugin_name):
+        if plugin_name in cls._plugin_instances:
+            msg = "Plugin allready registered with same name %s"
+            msg %= plugin_name
+            raise PluginError(msg)
+        plugin = cls(plugin_name)
+        cls._plugin_instances[plugin_name] = plugin
+        return plugin
+
+    @classmethod
+    def get(cls, plugin_name):
+        try:
+            return cls._plugin_instances[plugin_name]
+        except KeyError:
+            msg = "No plugin named '%s' loaded"
+            msg %= plugin_name
+            raise PluginError(msg)
     
     ##@brief Given a plugin name returns the plugin path
     # @param plugin_name str : The plugin name
@@ -40,8 +179,8 @@ class Plugins(object):
     def plugin_path(cls, plugin_name):
         cls.started()
         try:
-            return cls._plugin_paths[plugin_name]
+            return cls.get(plugin_name).path
-        except KeyError:
+        except PluginError:
             pass
         
         path = None
@@ -51,71 +190,41 @@ class Plugins(object):
                 return plugin_path
         raise NameError("No plugin named '%s'" % plugin_name)
 
-    ##@brief Fetch a confspec given a plugin_name
-    # @param plugin_name str : The plugin name
-    # @return a dict of conf spec
-    # @throw PluginError if plugin_name is not valid
     @classmethod
-    def get_confspec(cls, plugin_name):
+    def plugin_module_name(cls, plugin_name):
-        cls.started()
+        return "%s.%s" % (VIRTUAL_PACKAGE_NAME, plugin_name)
-        plugin_path = cls.plugin_path(plugin_name)
-        plugin_module = '%s.%s' % ( VIRTUAL_PACKAGE_NAME,
-                                    plugin_name)
-        conf_spec_module = plugin_module + '.confspec'
-        
-        conf_spec_source = plugin_path + CONFSPEC_FILENAME
-        try:
-            loader = SourceFileLoader(conf_spec_module, conf_spec_source)
-            confspec_module = loader.load_module()
-        except ImportError:
-            raise PluginError("Failed to load plugin '%s'. It seems that the plugin name is not valid" % plugin_name)
-        return getattr(confspec_module, CONFSPEC_VARNAME)
- 
-    ##@brief Load a module to register plugin's hooks
-    # @param plugin_name str : The plugin name
-    @classmethod
-    def load_plugin(cls, plugin_name):
-        cls.plugin_module(plugin_name)
-    
-    ##@brief Load a plugin module and return it
-    #@return the plugin module
-    @classmethod
-    def plugin_module(cls, plugin_name):
-        cls.started()
-        plugin_path = cls.plugin_path(plugin_name)
-        plugin_module = '%s.%s' % ( VIRTUAL_PACKAGE_NAME,
-                                    plugin_name)
-        main_module = plugin_module
-        main_source = plugin_path + MAIN_FILENAME
-        try:
-            loader = SourceFileLoader(main_module, main_source)
-            module = loader.load_module()
-            return module
-        except ImportError as e:
-            raise e
-            raise PluginError("Failed to load plugin '%s'. It seems that the plugin name is not valid" % plugin_name)
 
-
+    ##@brief Start the Plugin class
-    ##@brief Bootstrap the Plugins class
+    # 
+    # Called by Settings.__bootstrap()
+    #
+    # This method load path and preload plugins
     @classmethod
-    def bootstrap(cls):
+    def start(cls, plugins_directories, plugins):
-        from lodel.settings import Settings
+        if cls._plugin_directories is not None:
-        for plugin_name in Settings.plugins:
+            return
-            cls.load_plugin(plugin_name)
-    
-    @classmethod
-    def start(cls, plugins_directories):
         import inspect
-        self_path = inspect.getsourcefile(Plugins)
+        self_path = inspect.getsourcefile(Plugin)
         default_plugin_path = os.path.abspath(self_path + '../../../../plugins')
         if plugins_directories is None:
             plugins_directories = list()
         plugins_directories += [ default_plugin_path ]
         cls._plugin_directories = list(set(plugins_directories))
+        for plugin_name in plugins:
+            cls.register(plugin_name)
         
-
     @classmethod
     def started(cls, raise_if_not = True):
         res = cls._plugin_directories is not None
         if raise_if_not and not res:
             raise RuntimeError("Class Plugins is not initialized")
+
+##@page lodel2_plugins Lodel2 plugins system
+#
+# @par Plugin structure
+#A plugin is  a package (a folder containing, at least, an __init__.py file.
+#This file should expose multiple things :
+# - a CONFSPEC variable containing configuration specifications
+# - an _activate() method that returns True if the plugin can be activated (
+# optionnal)
+#

lodel/settings/settings.py

@@ -8,7 +8,7 @@ import warnings
 import types # for dynamic bindings
 from collections import namedtuple
 
-from lodel.plugin.plugins import Plugins, PluginError
+from lodel.plugin.plugins import Plugin, PluginError
 from lodel.settings.utils import SettingsError, SettingsErrors
 from lodel.settings.validator import SettingValidator, LODEL2_CONF_SPECS
 from lodel.settings.settings_loader import SettingsLoader
@@ -153,15 +153,15 @@ class Settings(object, metaclass=MetaSettings):
                                             plugins_path_opt_specs[0],
                                             False)
         # Starting the Plugins class
-        Plugins.start(plugins_path)
+        Plugin.start(plugins_path, plugins_list)
         # Fetching conf specs from plugins
         specs = [lodel2_specs]
         errors = list()
         for plugin_name in plugins_list:
             try:
-                specs.append(Plugins.get_confspec(plugin_name))
+                specs.append(Plugin.get(plugin_name).confspecs)
             except PluginError as e:
-                errors.append(e)
+                errors.append(SettingsError(msg=str(e)))
         if len(errors) > 0: #Raise all plugins import errors
             raise SettingsErrors(errors)
         self.__conf_specs = self.__merge_specs(specs)

plugins/datasources/__init__.py

@@ -0,0 +1,8 @@
+from lodel.settings.validator import SettingValidator
+
+__loader__ = 'main.py'
+
+CONFSPEC = {
+                'lodel2.datasources.*': {
+                        'identifier': ( None,
+                                        SettingValidator('string'))}}

plugins/datasources/confspec.py

@@ -1,8 +0,0 @@
-#-*- coding: utf-8 -*-
-
-from lodel.settings.validator import SettingValidator
-
-CONFSPEC = {
-                'lodel2.datasources.*': {
-                        'identifier': ( None,
-                                        SettingValidator('string'))}}

plugins/dummy/__init__.py

@@ -1,2 +1,10 @@
+from lodel.settings.validator import SettingValidator
+
+__loader__ = "main.py"
+__confspec__ = "confspec.py"
 __author__ = "Lodel2 dev team"
 __fullname__ = "Dummy plugin"
+
+
+def _activate():
+    return True

plugins/dummy/main.py

@@ -2,6 +2,10 @@
 
 from lodel.plugin import LodelHook
 
+def _activate():
+    return True
+    
+
 ##@brief Hook's callback example
 @LodelHook('leapi_get_pre')
 @LodelHook('leapi_get_post')

plugins/dummy_datasource/__init__.py

@@ -1 +1,86 @@
+from lodel.settings.validator import SettingValidator
 from .main import DummyDatasource as Datasource
+
+__loader__ = 'main.py'
+
+CONFSPEC = {
+    'lodel2.datasource.dummy_datasource.*' : {
+        'dummy': (  None,
+                    SettingValidator('dummy'))}
+}
+
+##@page lodel2_datasources Lodel2 datasources
+#
+#@par lodel2_datasources_intro Intro
+# A single lodel2 website can interact with multiple datasources. This page
+# aims to describe configuration & organisation of datasources in lodel2.
+# Each object is attached to a datasource. This association is done in the
+# editorial model, the datasource is identified by a name.
+#
+#@par Datasources declaration
+# To define a datasource you have to write something like this in confs file :
+#<pre>
+#[lodel2.datasources.DATASOURCE_NAME]
+#identifier = DATASOURCE_FAMILY.SOURCE_NAME
+#</pre>
+# See below for DATASOURCE_FAMILY & SOURCE_NAME
+#
+#@par Datasources plugins
+# Each datasource family is a plugin. For example mysql or a mongodb plugins.
+# Here is the CONFSPEC variable templates for datasources plugins
+#<pre>
+#CONFSPEC = {
+#                'lodel2.datasource.example.*' : {
+#                    'conf1' : VALIDATOR_OPTS,
+#                    'conf2' : VALIDATOR_OPTS,
+#                    ...
+#                }
+#}
+#</pre>
+#MySQL example
+#<pre>
+#CONFSPEC = {
+#                'lodel2.datasource.mysql.*' : {
+#                    'host': (   'localhost',
+#                                SettingValidator('host')),
+#                    'db_name': (    'lodel',
+#                                    SettingValidator('string')),
+#                    'username': (   None,
+#                                    SettingValidator('string')),
+#                    'password': (   None,
+#                                    SettingValidator('string')),
+#                }
+#}
+#</pre>
+#
+#@par Configuration example
+#<pre>
+# [lodel2.datasources.main]
+# identifier = mysql.Core
+# [lodel2.datasources.revues_write]
+# identifier = mysql.Revues
+# [lodel2.datasources.revues_read]
+# identifier = mysql.Revues
+# [lodel2.datasources.annuaire_persons]
+# identifier = persons_web_api.example
+# ;
+# ; Then, in the editorial model you are able to use "main", "revues_write", 
+# ; etc as datasource
+# ;
+# ; Here comes the datasources declarations
+# [lodel2.datasource.mysql.Core]
+# host = db.core.labocleo.org
+# db_name = core
+# username = foo
+# password = bar
+# ;
+# [lodel2.datasource.mysql.Revues]
+# host = revues.org
+# db_name = RO
+# username = foo
+# password = bar
+# ;
+# [lodel2.datasource.persons_web_api.example]
+# host = foo.bar
+# username = cleo
+#</pre>

plugins/dummy_datasource/confspec.py

@@ -1,85 +0,0 @@
-#-*- coding: utf-8 -*-
-
-from lodel.settings.validator import SettingValidator
-
-CONFSPEC = {
-    'lodel2.datasource.dummy_datasource.*' : {
-        'dummy': (  None,
-                    SettingValidator('dummy'))}
-}
-
-##@page lodel2_datasources Lodel2 datasources
-#
-#@par lodel2_datasources_intro Intro
-# A single lodel2 website can interact with multiple datasources. This page
-# aims to describe configuration & organisation of datasources in lodel2.
-# Each object is attached to a datasource. This association is done in the
-# editorial model, the datasource is identified by a name.
-#
-#@par Datasources declaration
-# To define a datasource you have to write something like this in confs file :
-#<pre>
-#[lodel2.datasources.DATASOURCE_NAME]
-#identifier = DATASOURCE_FAMILY.SOURCE_NAME
-#</pre>
-# See below for DATASOURCE_FAMILY & SOURCE_NAME
-#
-#@par Datasources plugins
-# Each datasource family is a plugin. For example mysql or a mongodb plugins.
-# Here is the CONFSPEC variable templates for datasources plugins
-#<pre>
-#CONFSPEC = {
-#                'lodel2.datasource.example.*' : {
-#                    'conf1' : VALIDATOR_OPTS,
-#                    'conf2' : VALIDATOR_OPTS,
-#                    ...
-#                }
-#}
-#</pre>
-#MySQL example
-#<pre>
-#CONFSPEC = {
-#                'lodel2.datasource.mysql.*' : {
-#                    'host': (   'localhost',
-#                                SettingValidator('host')),
-#                    'db_name': (    'lodel',
-#                                    SettingValidator('string')),
-#                    'username': (   None,
-#                                    SettingValidator('string')),
-#                    'password': (   None,
-#                                    SettingValidator('string')),
-#                }
-#}
-#</pre>
-#
-#@par Configuration example
-#<pre>
-# [lodel2.datasources.main]
-# identifier = mysql.Core
-# [lodel2.datasources.revues_write]
-# identifier = mysql.Revues
-# [lodel2.datasources.revues_read]
-# identifier = mysql.Revues
-# [lodel2.datasources.annuaire_persons]
-# identifier = persons_web_api.example
-# ;
-# ; Then, in the editorial model you are able to use "main", "revues_write", 
-# ; etc as datasource
-# ;
-# ; Here comes the datasources declarations
-# [lodel2.datasource.mysql.Core]
-# host = db.core.labocleo.org
-# db_name = core
-# username = foo
-# password = bar
-# ;
-# [lodel2.datasource.mysql.Revues]
-# host = revues.org
-# db_name = RO
-# username = foo
-# password = bar
-# ;
-# [lodel2.datasource.persons_web_api.example]
-# host = foo.bar
-# username = cleo
-#</pre>

plugins/webui/__init__.py

@@ -0,0 +1,22 @@
+from lodel.settings.validator import SettingValidator
+
+__loader__ = 'main.py'
+
+CONFSPEC = {
+    'lodel2.webui': {
+        'standalone': ( False,
+                        SettingValidator('bool')),
+        'listen_address': ( '127.0.0.1',
+                            SettingValidator('dummy')),
+        'listen_port': (    '9090',
+                            SettingValidator('int')),
+    },
+    'lodel2.webui.sessions': {
+        'directory': (  '/tmp/lodel2_session',
+                        SettingValidator('path')),
+        'expiration': ( 900,
+                        SettingValidator('int')),
+        'file_template': (  'lodel2_%s.sess',
+                            SettingValidator('dummy')),
+    }
+}

plugins/webui/confspec.py

@@ -1,22 +0,0 @@
-#-*- coding: utf-8 -*-
-
-from lodel.settings.validator import SettingValidator
-
-CONFSPEC = {
-    'lodel2.webui': {
-        'standalone': ( False,
-                        SettingValidator('bool')),
-        'listen_address': ( '127.0.0.1',
-                            SettingValidator('dummy')),
-        'listen_port': (    '9090',
-                            SettingValidator('int')),
-    },
-    'lodel2.webui.sessions': {
-        'directory': (  '/tmp/lodel2_session',
-                        SettingValidator('path')),
-        'expiration': ( 900,
-                        SettingValidator('int')),
-        'file_template': (  'lodel2_%s.sess',
-                            SettingValidator('dummy')),
-    }
-}