[broken] Starting multisite loader rewriting refs #235

At this point the loader in lodel/plugins/multisite/run.py is theorically
complete & correct. But some features are missing in order to be able
to test & debug it ( refs #275 ).

lodel/plugins/multisite/__init__.py

@@ -1,3 +1,22 @@
+##@brief 
+#
+#@par Notes on FS organisation
+#
+#The plan is as follow :
+#
+#The autotools deploiment chain or the debian package will create a var 
+#folder dedicated for lodel2 ( for example /var/lodel2 ). This folder will
+#contains folder named as the lodesites instances (lodelsites is a lodel site
+#that handles lodel site as content). We will call this folder multisite
+#folder or lodelsites folder.
+#The multisite folder contains 2 settings folders :
+#- lodelsites.conf.d : the lodelsites configuration
+#- server.conf.d : the multisite process configuration
+#- datas : a folder containing datas for each site handled by the lodelsites
+#- .contexts : a folder containing context stuff for each site handlers by
+#the lodelsites ( the lodel package symlink + dyncode.py)
+#
+
 from lodel.context import LodelContext, ContextError
 try:
     LodelContext.expose_modules(globals(), {

lodel/plugins/multisite/confspecs.py

@@ -5,7 +5,9 @@ LodelContext.expose_modules(globals(), {
 #Define a minimal confspec used by multisite loader
 LODEL2_CONFSPECS = {
     'lodel2': {
-        'debug': (True, SettingValidator('bool'))
+        'debug': (True, SettingValidator('bool')),
+        'sites_handler_name': (None,
+            SettingValidator('string', none_is_valid = False)),
     },
     'lodel2.server': {
         'listen_address': ('127.0.0.1', SettingValidator('dummy')),

lodel/plugins/multisite/loader.py

@@ -4,11 +4,24 @@ import sys
 import shlex
 import warnings
 
+##@brief Preloader for a multisite process
+#
+#This loader is a kind of fake loader. In fact it only read configurations
+#for the multisite instance and then run a UWSGI process that will run
+#the run.py file.
+#
+#If you want to see the "real" multisite loading process see
+#@ref lodel/plugins/multisite/run.py file
+#
+#@par Implementation details 
 #Here we have to bootstrap a minimal __loader__ context in order
 #to be able to load the settings
 #
 #This file (once bootstraped) start a new process for uWSGI. uWSGI then
 #run lodel.plugins.multisite.run.application function
+#@note the uwsgi process in started using the execl function when UWSGI
+#will exit this process will stop too
+#
 try:
     from lodel.context import LodelContext
 except ImportError:

lodel/plugins/multisite/run.py

@@ -3,15 +3,49 @@ import os
 import os.path
 import warnings
 
-#This file expose common function to process a wsgi request and the
-#uWSGI application callback
+##@brief File designed to be executed by an UWSGI process in order to
+#run a multisite instance process
+#
+#@warning In this module we have to be VERY carrefull about module exposure !
+#in fact here we will go through every context at least once and the globals
+#will be shared all among the module. So ANY exposed module will be accessible
+#for a short time outside its context !! A good way be protected about that
+#is to del(globals()[exposed_module_name]) after use. But it's really not
+#sure that this way of doing is a real safe protection !
+#
+#@par Expected context when called
+#- cwd has to be the lodelsites directory (the directory containing the 
+#conf.d folder)
+#
+#This file is divided in two blocks :
+#
+#@par Run at load
+#The main function will be called at import.
+#This piece of code handles :
+#- loading the lodelsites site (the site that handles sites ;) )
+#- fetch the list of handled lodel sites
+#- loading the whole list of lodel sites
+#
+#@par WSGI processing
+#The other functions are here to handles WSGI request. The entry function
+#is application(), following the PEP 3333 specifications
 
+#
+#   Constants declarations
+#
 
-#preloading all instances
-FAST_APP_EXPOSAL_CACHE = dict()
+##@brief basename of multisite process conf folder
+#@todo find a better place to declare it
+SERVER_CONFD = 'server.conf.d' #Should be accessible elsewhere
+##@brief basename of lodelsites site conf folder
+#@todo find a better place to declare it
+LODELSITES_CONFD = 'lodelsites.conf.d' #Should be accessible elsewhere
 
-LODEL2_INSTANCES_DIR = '.'
-EXCLUDE_DIR = {'conf.d', '__pycache__'}
+##@brief A cache allowing a fast application exposure
+#
+#This dict contains reference on interface module of each handled site in
+#order to quickly call the application (PEP 3333) function of concerned site
+FAST_APP_EXPOSAL_CACHE = dict()
 
 try:
     from lodel.context import LodelContext
@@ -20,71 +54,147 @@ except ImportError:
         os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
     from lodel.context import LodelContext, ContextError
 
-LodelContext.init(LodelContext.MULTISITE)
-LodelContext.set(None) #Loading context creation
-
-#Multisite instance settings loading
-CONFDIR = os.path.join(os.getcwd(), 'conf.d')
-if not os.path.isdir(CONFDIR):
-    warnings.warn('%s do not exists, default settings used' % CONFDIR)
-LodelContext.expose_modules(globals(), {
-    'lodel.settings.settings': [('Settings', 'settings')],
-    'lodel.plugins.multisite.confspecs': 'multisite_confspecs'})
-if not settings.started():
-    settings('./conf.d', multisite_confspecs.LODEL2_CONFSPECS)
-
-#Fetching insrtance list from subdirectories
-lodelsites_list = [ os.path.realpath(os.path.join(LODEL2_INSTANCES_DIR,sitename)) 
-    for sitename in os.listdir(LODEL2_INSTANCES_DIR)
-    if os.path.isdir(sitename) and sitename not in EXCLUDE_DIR]
-
-#Bootstraping instances
-for lodelsite_path in lodelsites_list:
-    ctx_name = LodelContext.from_path(lodelsite_path)
-    #Switch to new context
+import os.path
+import lodel.buildconf #safe even outside contexts
+
+
+##@brief Function that will run at import time
+#
+#Handles lodelsites site loading, handled site list fecth & load
+#@note called at end of file
+#
+#@todo evaluate if it is safe to assume that lodelsites_datapath = os.getcwd()
+#@todo get rid of hardcoded stuff (like shortname fieldname)
+#@todo use the dyncode getter when it will be available (replaced by
+#the string SUPERDYNCODE_ACCESSOR.Lodelsite for the moment)
+def main():
+    LodelContext.init(LodelContext.MULTISITE)
+    #Set current context to reserved loader context
+    LodelContext.set(None)
+    LodelContext.expose_modules(globals(), {
+        'lodel.logger': 'logger',
+        'lodel.exceptions': ['LodelFatalError'],
+    })
+    
+    CONFDIR = os.path.join(os.getcwd(), SERVER_CONFD)
+    if not os.path.isdir(CONFDIR):
+        logger.critical('Multisite process bootstraping fails : unable to \
+find the %s folder' % SERVER_CONFD)
+    
+    #Settings bootstraping for mutlisite process
+    LodelContext.expose_modules(globals(), {
+        'lodel.settings.settings': [('Settings', 'settings')],
+        'lodel.plugins.multisite.confspecs': 'multisite_confspecs'})
+    settings(CONFDIR, mutisite_confspecs.LODEL2_CONFSPECS)
+    #Loading settings
+    del(globals()['settings']) #useless but may be safer
+    #Exposing "real" settings object in loader context
+    LodelContext.expose_modules(globals(), {
+        'lodel.settings': ['Settings']})
+    #Fetching lodelsites informations
+    lodelsites_name = Settings.sites_handler_name
+    #Following path construction is kind of dirty ! We should be able
+    #to assume that the lodelsites_datapath == os.getcwd()....
+    lodelsites_datapath = os.path.join(LODEL2VARDIR, lodelsites_name)
+    #loading lodelsites site
+    load_site(lodelsites_datapath, LODELSITES_CONFD)
+
+    #Fetching handled sites list 
+    #WARNING ! Here we assert that context name == basename(lodelsites_datapath)
+    LodelContext.set(lodelsites_name)
+    #in lodelsites context
+    Lodelsite_leo = SUPERDYNCODE_ACCESSOR.Lodelsite #hardcoded leo name
+    LodelContext.expose_modules(globals(), {
+        'lodel.leapi.query': ['LeGetQuery'],
+    })
+    #the line bellow you will find another harcoded thing : the shortname
+    #fieldname for a lodelsite
+    handled_sites = LeGetQuery(lodelsite_leo, query_filters = [],
+        field_list = ['shortname'])
+    #Now that we have the handled sitenames list we can go back to
+    #loader context and clean it
+    LodelContext.set(None)
+    for mname in ['LeGetQuery', 'Settings']:
+        del(globals()[mname])
+    #Loading handled sites
+    for handled_sitename in [s['shortname'] for s in handled_sites]:
+        datapath = os.path.join(lodelsites_datapath, handled_sitename)
+        site_load(datapath) #using default conf.d configuration dirname
+    
+
+##@brief Load a site
+#
+#Apply a common (as MONOSITE) loading process to a site :
+#1. Conf preload
+#2. Plugins preload
+#3. Conf loading
+#4. starting plugins & hooks
+#@warning At this point we need a uniq identifier for the site (using it
+#as key for contexts & FAST_APP_EXPOSAL_CACHE). To achieve this we use
+#the data_path basename. It should works for handled sites and for the 
+#lodelsites instance
+#@param data_path str : path to the datas directory (containing the confdir)
+#@param confdir_basename str : the basename of the site confdir
+#
+#@todo For now the interface plugin name for sites is hardcoded (set to
+#webui). It HAS TO be loaded from settings. But it is a bit complicated, 
+#we have to get the plugin's module name abstracted from context :
+#lodel.something but if we ask directly to Plugin class the module name
+#it will return something like : lodelsites.sitename.something...
+#
+#@todo there is a quick & dirty workarround with comments saying that it
+#avoid context escape via hooks. We have to understand why and how and then
+#replace the workarround by a real solution !
+def site_load(data_path, confdir_basename = 'conf.d'):
+    #args check
+    if confdir_basename != os.path.basename(confdir_basename):
+        LodelFatalError('Bad argument given to site_load(). This really \
+sux !')
+    #Determining uniq sitename from data_path
+    data_path = data_path.rstrip('/') #else basename returns ''
+    ctx_name = os.path.basename(data_path)
+    #Immediately switching to the context
     LodelContext.set(ctx_name)
-    os.chdir(lodelsite_path)
-    # Loading settings
+    os.chdir(data_path) #Now the confdir is ./$condir_basename
+    #Loading settings for current site
     LodelContext.expose_modules(globals(), {
-        'lodel.settings.settings': [('Settings', 'settings')]})
-    if not settings.started():
-        settings('./conf.d')
-    LodelContext.expose_modules(globals(), {'lodel.settings': ['Settings']})
-
-    # Loading hooks & plugins
+        'lodel.settings.settings': [('Settings', 'settings_preloader')]})
+    if settings_preloader.started():
+        msg = 'Settings seems to be allready started for "%s". \
+This should not append !' % ctx_name
+        #switch back to loader context in order to log & raise
+        LodelContext.set(None)
+        logger.critical(msg)
+        raise LodelFatalError(msg)
+    settings(os.path.join('./', confdir_basename)
+    #
+    #Loading hooks & plugins
+    #
     LodelContext.expose_modules(globals(), {
-        'lodel.plugin': ['LodelHook'],
+        'lodel.plugin': ['Plugin', 'LodelHook'],
+        'lodel.logger': 'logger',
         'lodel.plugin.core_hooks': 'core_hooks',
         'lodel.plugin.core_scripts': 'core_scripts'
     })
-
-    #Load plugins
-    LodelContext.expose_modules(globals(), {
-        'lodel.logger': 'logger',
-        'lodel.plugin': ['Plugin']})
-    logger.debug("Loader.start() called")
-    Plugin.load_all()
-    #Import & expose dyncode
-    LodelContext.expose_dyncode(globals())
-    #Next hook triggers dyncode datasource instanciations
+    Plugin.load_all() #Then all plugins & hooks are loaded
+    #triggering dyncode datasource instanciations
     LodelHook.call_hook('lodel2_plugins_loaded', '__main__', None)
-    #Next hook triggers call of interface's main loop
+    #triggering boostrapped hook
     LodelHook.call_hook('lodel2_bootstraped', '__main__', None)
-    #FAST_APP_EXPOSAL_CACHE populate
+    #Populating FAST_APP_EXPOSAL_CACHE
+    #
+    #WARNING !!!! Hardcoded interface name ! Here we have to find the 
+    #interface plugin name in order to populate the cache properly
     FAST_APP_EXPOSAL_CACHE[ctx_name] = LodelContext.module(
-    	'lodel.plugins.webui.run')
-    LodelContext
+        'lodel.plugins.webui.run')
     #a dirty & quick attempt to fix context unwanted exite via
     #hooks
     for name in ( 'LodelHook', 'core_hooks', 'core_scripts',
             'Settings', 'settings', 'logger', 'Plugin'):
         del(globals()[name])
-    #switch back to loader context
+    #site fully loaded, switching back to loader context
     LodelContext.set(None)
-
-#
-# From here lodel2 multisite instances are loaded and ready to run
-#
+    #lodel2 multisite instances are loaded and ready to run
 
 
 ##@brief Utility function to return quickly an error
@@ -131,3 +241,5 @@ def application(env, start_response):
     #    'lodel.plugins.webui.run': ['application']})
     #return application(env, start_response)
 
+#calling the main function
+main()