From 61f19772fb1392a364b3d4672b1708552dc7d931 Mon Sep 17 00:00:00 2001 From: Yann Date: Wed, 1 Jun 2016 16:35:46 +0200 Subject: [PATCH] New way to handles plugin - renamed Plugins class to Plugin - an instance represent a loaded plugin - classmethod allows to preload & load plugins --- install/conf.d/datasources.ini | 2 +- install/loader.py | 4 +- lodel/__init__.py | 13 ++ lodel/leapi/leobject.py | 4 +- lodel/plugin/__init__.py | 2 +- lodel/plugin/plugins.py | 235 ++++++++++++++++++++------- lodel/settings/settings.py | 8 +- plugins/datasources/__init__.py | 8 + plugins/datasources/confspec.py | 8 - plugins/dummy/__init__.py | 8 + plugins/dummy/main.py | 4 + plugins/dummy_datasource/__init__.py | 85 ++++++++++ plugins/dummy_datasource/confspec.py | 85 ---------- plugins/webui/__init__.py | 22 +++ plugins/webui/confspec.py | 22 --- 15 files changed, 322 insertions(+), 188 deletions(-) delete mode 100644 plugins/datasources/confspec.py delete mode 100644 plugins/dummy_datasource/confspec.py delete mode 100644 plugins/webui/confspec.py diff --git a/install/conf.d/datasources.ini b/install/conf.d/datasources.ini index da1928e..f227379 100644 --- a/install/conf.d/datasources.ini +++ b/install/conf.d/datasources.ini @@ -1,5 +1,5 @@ [lodel2.datasources.default] -identifier = dummy.example +identifier = dummy_datasource.example [lodel2.datasource.dummy_datasource.example] dummy = diff --git a/install/loader.py b/install/loader.py index 348c175..0441ae0 100644 --- a/install/loader.py +++ b/install/loader.py @@ -24,8 +24,8 @@ from lodel.settings import Settings #Load plugins -from lodel.plugin import Plugins -Plugins.bootstrap() +from lodel.plugin import Plugin +Plugin.load_all() from lodel.plugin import LodelHook LodelHook.call_hook('lodel2_bootstraped', '__main__', None) diff --git a/lodel/__init__.py b/lodel/__init__.py index 41e364b..eaadac8 100644 --- a/lodel/__init__.py +++ b/lodel/__init__.py @@ -1 +1,14 @@ #-*- coding: utf-8 -*- + +##@page lodel2_start Lodel2 boot mechanism +# +# @par Lodel2 boot sequence +# see @ref install/loader.py +# 1. lodel package is imported +# 2. settings are started +# 3. plugins are pre-loaded from conf to load plugins configuration specs +# 4. settings are loaded from conf and checked +# 3. plugins are loaded (hooks are registered etc) +# 4. "lodel2_bootstraped" hooks are called +# 5. "lodel2_loader_main" hooks are called (if runned from loader.py as main executable) +# diff --git a/lodel/leapi/leobject.py b/lodel/leapi/leobject.py index a5a2018..21e23dc 100644 --- a/lodel/leapi/leobject.py +++ b/lodel/leapi/leobject.py @@ -2,7 +2,7 @@ import importlib -from lodel.plugin import Plugins +from lodel.plugin import Plugin from lodel import logger from lodel.settings import Settings from lodel.settings.utils import SettingsError @@ -221,7 +221,7 @@ class LeObject(object): ds_conf = getattr(ds_conf, ds_name) #Checks that the datasource plugin exists - ds_plugin_module = Plugins.plugin_module(ds_plugin) + ds_plugin_module = Plugin.get(ds_plugin).module try: datasource_class = getattr(ds_plugin_module, "Datasource") except AttributeError as e: diff --git a/lodel/plugin/__init__.py b/lodel/plugin/__init__.py index 63e511d..54e5df4 100644 --- a/lodel/plugin/__init__.py +++ b/lodel/plugin/__init__.py @@ -40,4 +40,4 @@ #  from .hooks import LodelHook -from .plugins import Plugins +from .plugins import Plugin diff --git a/lodel/plugin/plugins.py b/lodel/plugin/plugins.py index 1dd209b..8a5f752 100644 --- a/lodel/plugin/plugins.py +++ b/lodel/plugin/plugins.py @@ -1,9 +1,11 @@ #-*- coding: utf-8 -*- +import sys import os.path - import importlib +import copy from importlib.machinery import SourceFileLoader, SourcelessFileLoader + import plugins ## @package lodel.plugins Lodel2 plugins management @@ -15,23 +17,160 @@ import plugins # - confspec.py containing a configuration specification dictionary named CONFSPEC ##@brief The package in wich we will load plugins modules -VIRTUAL_PACKAGE_NAME = 'lodel.plugins_pkg' -CONFSPEC_FILENAME = 'confspec.py' -MAIN_FILENAME = '__init__.py' +VIRTUAL_PACKAGE_NAME = 'lodel.plugins' +INIT_FILENAME = '__init__.py' # Loaded with settings +CONFSPEC_FILENAME_VARNAME = '__confspec__' CONFSPEC_VARNAME = 'CONFSPEC' +LOADER_FILENAME_VARNAME = '__loader__' class PluginError(Exception): pass -class Plugins(object): +class Plugin(object): ##@brief Stores plugin directories paths _plugin_directories = None - ##@brief Optimisation cache storage for plugin paths - _plugin_paths = dict() - def __init__(self): # may be useless + _plugin_instances = dict() + + ##@brief Plugin class constructor + # + # Called by setting in early stage of lodel2 boot sequence using classmethod + # register + # + # @param plugin_name str : plugin name + # @throw PluginError + def __init__(self, plugin_name): self.started() + + self.name = plugin_name + self.path = self.plugin_path(plugin_name) + self.module = None + self.__confspecs = dict() + + # Importing __init__.py + plugin_module = '%s.%s' % ( VIRTUAL_PACKAGE_NAME, + plugin_name) + init_source = self.path + INIT_FILENAME + try: + loader = SourceFileLoader(plugin_module, init_source) + self.module = loader.load_module() + except ImportError as e: + raise PluginError("Failed to load plugin '%s'. It seems that the plugin name is not valid" % plugin_name) + + # loading confspecs + try: + # Loading confspec directly from __init__.py + self.__confspecs = getattr(self.module, CONFSPEC_VARNAME) + except AttributeError: + # Loading file in __confspec__ var in __init__.py + try: + module = self._import_from_init_var(CONFSPEC_FILENAME_VARNAME) + except AttributeError: + msg = "Malformed plugin {plugin} . No {varname} not {filevar} found in __init__.py" + msg = msg.format( + plugin = self.name, + varname = CONFSPEC_VARNAME, + filevar = CONFSPEC_FILENAME_VARNAME) + raise PluginError(msg) + except ImportError as e: + msg = "Broken plugin {plugin} : {expt}" + msg = msg.format( + plugin = self.name, + expt = str(e)) + raise PluginError(msg) + + try: + # loading confpsecs from file + self.__confspecs = getattr(module, CONFSPEC_VARNAME) + except AttributeError: + msg = "Broken plugin. {varname} not found in '{filename}'" + msg = msg.format( + varname = CONFSPEC_VARNAME, + filename = confspec_filename) + raise PluginError(msg) + + ##@brief Try to import a file from a variable in __init__.py + #@param varname str : The variable name + #@throw AttributeError if varname not found + #@throw ImportError if the file fails to be imported + #@throw PluginError if the filename was not valid + def _import_from_init_var(self, varname): + # Read varname + filename = getattr(self.module, varname) + #Path are not allowed + if filename != os.path.basename(filename): + msg = "Invalid {varname} content : '{fname}' for plugin {name}" + msg = msg.format( + varname = varname, + fname = filename, + name = self.name) + raise PluginError(msg) + # importing the file in varname + module_name = self.module.__name__+"."+varname + filename = self.path + filename + loader = SourceFileLoader(module_name, filename) + return loader.load_module() + + ##@brief Register hooks etc + def load(self): + from lodel import logger + try: + return self._import_from_init_var(LOADER_FILENAME_VARNAME) + except AttributeError: + msg = "Malformed plugin {plugin}. No {varname} found in __init__.py" + msg = msg.format( + plugin = self.name, + varname = LOADER_FILENAME_VARNAME) + raise PluginError(msg) + except ImportError as e: + msg = "Broken plugin {plugin} : {expt}" + msg = msg.format( + plugin = self.name, + expt = str(e)) + raise PluginError(msg) + logger.debug("Plugin '%s' loaded" % self.name) + + @classmethod + def load_all(cls): + errors = dict() + for name, plugin in cls._plugin_instances.items(): + try: + plugin.load() + except PluginError as e: + errors[name] = e + if len(errors) > 0: + msg = "Errors while loading plugins :" + for name, e in errors.items(): + msg += "\n\t%20s : %s" % (name,e) + msg += "\n" + raise PluginError(msg) + + @property + def confspecs(self): + return copy.copy(self.__confspecs) + + ##@brief Register a new plugin + # + # preload + @classmethod + def register(cls, plugin_name): + if plugin_name in cls._plugin_instances: + msg = "Plugin allready registered with same name %s" + msg %= plugin_name + raise PluginError(msg) + plugin = cls(plugin_name) + cls._plugin_instances[plugin_name] = plugin + return plugin + + @classmethod + def get(cls, plugin_name): + try: + return cls._plugin_instances[plugin_name] + except KeyError: + msg = "No plugin named '%s' loaded" + msg %= plugin_name + raise PluginError(msg) ##@brief Given a plugin name returns the plugin path # @param plugin_name str : The plugin name @@ -40,8 +179,8 @@ class Plugins(object): def plugin_path(cls, plugin_name): cls.started() try: - return cls._plugin_paths[plugin_name] - except KeyError: + return cls.get(plugin_name).path + except PluginError: pass path = None @@ -51,71 +190,41 @@ class Plugins(object): return plugin_path raise NameError("No plugin named '%s'" % plugin_name) - ##@brief Fetch a confspec given a plugin_name - # @param plugin_name str : The plugin name - # @return a dict of conf spec - # @throw PluginError if plugin_name is not valid @classmethod - def get_confspec(cls, plugin_name): - cls.started() - plugin_path = cls.plugin_path(plugin_name) - plugin_module = '%s.%s' % ( VIRTUAL_PACKAGE_NAME, - plugin_name) - conf_spec_module = plugin_module + '.confspec' - - conf_spec_source = plugin_path + CONFSPEC_FILENAME - try: - loader = SourceFileLoader(conf_spec_module, conf_spec_source) - confspec_module = loader.load_module() - except ImportError: - raise PluginError("Failed to load plugin '%s'. It seems that the plugin name is not valid" % plugin_name) - return getattr(confspec_module, CONFSPEC_VARNAME) - - ##@brief Load a module to register plugin's hooks - # @param plugin_name str : The plugin name - @classmethod - def load_plugin(cls, plugin_name): - cls.plugin_module(plugin_name) - - ##@brief Load a plugin module and return it - #@return the plugin module - @classmethod - def plugin_module(cls, plugin_name): - cls.started() - plugin_path = cls.plugin_path(plugin_name) - plugin_module = '%s.%s' % ( VIRTUAL_PACKAGE_NAME, - plugin_name) - main_module = plugin_module - main_source = plugin_path + MAIN_FILENAME - try: - loader = SourceFileLoader(main_module, main_source) - module = loader.load_module() - return module - except ImportError as e: - raise e - raise PluginError("Failed to load plugin '%s'. It seems that the plugin name is not valid" % plugin_name) + def plugin_module_name(cls, plugin_name): + return "%s.%s" % (VIRTUAL_PACKAGE_NAME, plugin_name) - - ##@brief Bootstrap the Plugins class + ##@brief Start the Plugin class + # + # Called by Settings.__bootstrap() + # + # This method load path and preload plugins @classmethod - def bootstrap(cls): - from lodel.settings import Settings - for plugin_name in Settings.plugins: - cls.load_plugin(plugin_name) - - @classmethod - def start(cls, plugins_directories): + def start(cls, plugins_directories, plugins): + if cls._plugin_directories is not None: + return import inspect - self_path = inspect.getsourcefile(Plugins) + self_path = inspect.getsourcefile(Plugin) default_plugin_path = os.path.abspath(self_path + '../../../../plugins') if plugins_directories is None: plugins_directories = list() plugins_directories += [ default_plugin_path ] cls._plugin_directories = list(set(plugins_directories)) + for plugin_name in plugins: + cls.register(plugin_name) - @classmethod def started(cls, raise_if_not = True): res = cls._plugin_directories is not None if raise_if_not and not res: raise RuntimeError("Class Plugins is not initialized") + +##@page lodel2_plugins Lodel2 plugins system +# +# @par Plugin structure +#A plugin is a package (a folder containing, at least, an __init__.py file. +#This file should expose multiple things : +# - a CONFSPEC variable containing configuration specifications +# - an _activate() method that returns True if the plugin can be activated ( +# optionnal) +# diff --git a/lodel/settings/settings.py b/lodel/settings/settings.py index a9fee5c..17c8b51 100644 --- a/lodel/settings/settings.py +++ b/lodel/settings/settings.py @@ -8,7 +8,7 @@ import warnings import types # for dynamic bindings from collections import namedtuple -from lodel.plugin.plugins import Plugins, PluginError +from lodel.plugin.plugins import Plugin, PluginError from lodel.settings.utils import SettingsError, SettingsErrors from lodel.settings.validator import SettingValidator, LODEL2_CONF_SPECS from lodel.settings.settings_loader import SettingsLoader @@ -153,15 +153,15 @@ class Settings(object, metaclass=MetaSettings): plugins_path_opt_specs[0], False) # Starting the Plugins class - Plugins.start(plugins_path) + Plugin.start(plugins_path, plugins_list) # Fetching conf specs from plugins specs = [lodel2_specs] errors = list() for plugin_name in plugins_list: try: - specs.append(Plugins.get_confspec(plugin_name)) + specs.append(Plugin.get(plugin_name).confspecs) except PluginError as e: - errors.append(e) + errors.append(SettingsError(msg=str(e))) if len(errors) > 0: #Raise all plugins import errors raise SettingsErrors(errors) self.__conf_specs = self.__merge_specs(specs) diff --git a/plugins/datasources/__init__.py b/plugins/datasources/__init__.py index e69de29..c3720aa 100644 --- a/plugins/datasources/__init__.py +++ b/plugins/datasources/__init__.py @@ -0,0 +1,8 @@ +from lodel.settings.validator import SettingValidator + +__loader__ = 'main.py' + +CONFSPEC = { + 'lodel2.datasources.*': { + 'identifier': ( None, + SettingValidator('string'))}} diff --git a/plugins/datasources/confspec.py b/plugins/datasources/confspec.py deleted file mode 100644 index 4956ccd..0000000 --- a/plugins/datasources/confspec.py +++ /dev/null @@ -1,8 +0,0 @@ -#-*- coding: utf-8 -*- - -from lodel.settings.validator import SettingValidator - -CONFSPEC = { - 'lodel2.datasources.*': { - 'identifier': ( None, - SettingValidator('string'))}} diff --git a/plugins/dummy/__init__.py b/plugins/dummy/__init__.py index 39be57c..6b3f446 100644 --- a/plugins/dummy/__init__.py +++ b/plugins/dummy/__init__.py @@ -1,2 +1,10 @@ +from lodel.settings.validator import SettingValidator + +__loader__ = "main.py" +__confspec__ = "confspec.py" __author__ = "Lodel2 dev team" __fullname__ = "Dummy plugin" + + +def _activate(): + return True diff --git a/plugins/dummy/main.py b/plugins/dummy/main.py index 083317f..d1d7000 100644 --- a/plugins/dummy/main.py +++ b/plugins/dummy/main.py @@ -2,6 +2,10 @@ from lodel.plugin import LodelHook +def _activate(): + return True + + ##@brief Hook's callback example @LodelHook('leapi_get_pre') @LodelHook('leapi_get_post') diff --git a/plugins/dummy_datasource/__init__.py b/plugins/dummy_datasource/__init__.py index d5123c8..f8c11df 100644 --- a/plugins/dummy_datasource/__init__.py +++ b/plugins/dummy_datasource/__init__.py @@ -1 +1,86 @@ +from lodel.settings.validator import SettingValidator from .main import DummyDatasource as Datasource + +__loader__ = 'main.py' + +CONFSPEC = { + 'lodel2.datasource.dummy_datasource.*' : { + 'dummy': ( None, + SettingValidator('dummy'))} +} + +##@page lodel2_datasources Lodel2 datasources +# +#@par lodel2_datasources_intro Intro +# A single lodel2 website can interact with multiple datasources. This page +# aims to describe configuration & organisation of datasources in lodel2. +# Each object is attached to a datasource. This association is done in the +# editorial model, the datasource is identified by a name. +# +#@par Datasources declaration +# To define a datasource you have to write something like this in confs file : +#
+#[lodel2.datasources.DATASOURCE_NAME]
+#identifier = DATASOURCE_FAMILY.SOURCE_NAME
+#
+# See below for DATASOURCE_FAMILY & SOURCE_NAME +# +#@par Datasources plugins +# Each datasource family is a plugin. For example mysql or a mongodb plugins. +# Here is the CONFSPEC variable templates for datasources plugins +#
+#CONFSPEC = {
+#                'lodel2.datasource.example.*' : {
+#                    'conf1' : VALIDATOR_OPTS,
+#                    'conf2' : VALIDATOR_OPTS,
+#                    ...
+#                }
+#}
+#
+#MySQL example +#
+#CONFSPEC = {
+#                'lodel2.datasource.mysql.*' : {
+#                    'host': (   'localhost',
+#                                SettingValidator('host')),
+#                    'db_name': (    'lodel',
+#                                    SettingValidator('string')),
+#                    'username': (   None,
+#                                    SettingValidator('string')),
+#                    'password': (   None,
+#                                    SettingValidator('string')),
+#                }
+#}
+#
+# +#@par Configuration example +#
+# [lodel2.datasources.main]
+# identifier = mysql.Core
+# [lodel2.datasources.revues_write]
+# identifier = mysql.Revues
+# [lodel2.datasources.revues_read]
+# identifier = mysql.Revues
+# [lodel2.datasources.annuaire_persons]
+# identifier = persons_web_api.example
+# ;
+# ; Then, in the editorial model you are able to use "main", "revues_write", 
+# ; etc as datasource
+# ;
+# ; Here comes the datasources declarations
+# [lodel2.datasource.mysql.Core]
+# host = db.core.labocleo.org
+# db_name = core
+# username = foo
+# password = bar
+# ;
+# [lodel2.datasource.mysql.Revues]
+# host = revues.org
+# db_name = RO
+# username = foo
+# password = bar
+# ;
+# [lodel2.datasource.persons_web_api.example]
+# host = foo.bar
+# username = cleo
+#
diff --git a/plugins/dummy_datasource/confspec.py b/plugins/dummy_datasource/confspec.py deleted file mode 100644 index 44172b0..0000000 --- a/plugins/dummy_datasource/confspec.py +++ /dev/null @@ -1,85 +0,0 @@ -#-*- coding: utf-8 -*- - -from lodel.settings.validator import SettingValidator - -CONFSPEC = { - 'lodel2.datasource.dummy_datasource.*' : { - 'dummy': ( None, - SettingValidator('dummy'))} -} - -##@page lodel2_datasources Lodel2 datasources -# -#@par lodel2_datasources_intro Intro -# A single lodel2 website can interact with multiple datasources. This page -# aims to describe configuration & organisation of datasources in lodel2. -# Each object is attached to a datasource. This association is done in the -# editorial model, the datasource is identified by a name. -# -#@par Datasources declaration -# To define a datasource you have to write something like this in confs file : -#
-#[lodel2.datasources.DATASOURCE_NAME]
-#identifier = DATASOURCE_FAMILY.SOURCE_NAME
-#
-# See below for DATASOURCE_FAMILY & SOURCE_NAME -# -#@par Datasources plugins -# Each datasource family is a plugin. For example mysql or a mongodb plugins. -# Here is the CONFSPEC variable templates for datasources plugins -#
-#CONFSPEC = {
-#                'lodel2.datasource.example.*' : {
-#                    'conf1' : VALIDATOR_OPTS,
-#                    'conf2' : VALIDATOR_OPTS,
-#                    ...
-#                }
-#}
-#
-#MySQL example -#
-#CONFSPEC = {
-#                'lodel2.datasource.mysql.*' : {
-#                    'host': (   'localhost',
-#                                SettingValidator('host')),
-#                    'db_name': (    'lodel',
-#                                    SettingValidator('string')),
-#                    'username': (   None,
-#                                    SettingValidator('string')),
-#                    'password': (   None,
-#                                    SettingValidator('string')),
-#                }
-#}
-#
-# -#@par Configuration example -#
-# [lodel2.datasources.main]
-# identifier = mysql.Core
-# [lodel2.datasources.revues_write]
-# identifier = mysql.Revues
-# [lodel2.datasources.revues_read]
-# identifier = mysql.Revues
-# [lodel2.datasources.annuaire_persons]
-# identifier = persons_web_api.example
-# ;
-# ; Then, in the editorial model you are able to use "main", "revues_write", 
-# ; etc as datasource
-# ;
-# ; Here comes the datasources declarations
-# [lodel2.datasource.mysql.Core]
-# host = db.core.labocleo.org
-# db_name = core
-# username = foo
-# password = bar
-# ;
-# [lodel2.datasource.mysql.Revues]
-# host = revues.org
-# db_name = RO
-# username = foo
-# password = bar
-# ;
-# [lodel2.datasource.persons_web_api.example]
-# host = foo.bar
-# username = cleo
-#
diff --git a/plugins/webui/__init__.py b/plugins/webui/__init__.py index e69de29..7217043 100644 --- a/plugins/webui/__init__.py +++ b/plugins/webui/__init__.py @@ -0,0 +1,22 @@ +from lodel.settings.validator import SettingValidator + +__loader__ = 'main.py' + +CONFSPEC = { + 'lodel2.webui': { + 'standalone': ( False, + SettingValidator('bool')), + 'listen_address': ( '127.0.0.1', + SettingValidator('dummy')), + 'listen_port': ( '9090', + SettingValidator('int')), + }, + 'lodel2.webui.sessions': { + 'directory': ( '/tmp/lodel2_session', + SettingValidator('path')), + 'expiration': ( 900, + SettingValidator('int')), + 'file_template': ( 'lodel2_%s.sess', + SettingValidator('dummy')), + } +} diff --git a/plugins/webui/confspec.py b/plugins/webui/confspec.py deleted file mode 100644 index 9959226..0000000 --- a/plugins/webui/confspec.py +++ /dev/null @@ -1,22 +0,0 @@ -#-*- coding: utf-8 -*- - -from lodel.settings.validator import SettingValidator - -CONFSPEC = { - 'lodel2.webui': { - 'standalone': ( False, - SettingValidator('bool')), - 'listen_address': ( '127.0.0.1', - SettingValidator('dummy')), - 'listen_port': ( '9090', - SettingValidator('int')), - }, - 'lodel2.webui.sessions': { - 'directory': ( '/tmp/lodel2_session', - SettingValidator('path')), - 'expiration': ( 900, - SettingValidator('int')), - 'file_template': ( 'lodel2_%s.sess', - SettingValidator('dummy')), - } -}