Begin to fix some boostraping problems

Moved some function from plugin specific code to the bootstrap module
+ some bugfixes

lodel/bootstrap.py

@@ -104,3 +104,124 @@ MONOSITE instance")
     LodelContext.expose_modules(globals(), {
         'lodel.settings': ['Settings']})
     
+##@brief Preload a site
+#
+#Apply a common (as MONOSITE) loading process to a site :
+#1. Conf preload
+#2. Plugins preload
+#3. Conf loading
+#
+#4. starting plugins & hooks
+#@warning At this point we need a uniq identifier for the site (using it
+#as key for contexts & FAST_APP_EXPOSAL_CACHE). To achieve this we use
+#the data_path basename. It should works for handled sites and for the 
+#lodelsites instance
+#@warning may only work for handled sites in a multisite context
+#@param data_path str : path to the datas directory (containing the confdir)
+#@param confdir_basename str : the basename of the site confdir
+#@param lodelsites_instance bool : if true we are loading the lodelsites
+#instance of the multisite (allow to load the good confspecs)
+#
+#@todo For now the interface plugin name for sites is hardcoded (set to
+#webui). It HAS TO be loaded from settings. But it is a bit complicated, 
+#we have to get the plugin's module name abstracted from context :
+#lodel.something but if we ask directly to Plugin class the module name
+#it will return something like : lodelsites.sitename.something...
+#
+#@todo there is a quick & dirty workarround with comments saying that it
+#avoid context escape via hooks. We have to understand why and how and then
+#replace the workarround by a real solution !
+#@todo check if it works with monosite context !
+#@todo change data_path argument to sitename and determine datapath from it
+def site_preload(data_path, confdir_basename = 'conf.d', lodelsites_instance = False):
+    #args check
+    if confdir_basename != os.path.basename(confdir_basename):
+        LodelFatalError('Bad argument given to site_load(). This really \
+sux !')
+    #Determining uniq sitename from data_path
+    data_path = data_path.rstrip('/') #else basename returns ''
+    ctx_name = os.path.basename(data_path)
+    if not os.path.exists(data_path) or not os.path.isdir(data_path):
+        LodelContext.expose_modules(globals(), {
+            'lodel.exceptions': ['LodelFatalError']})
+        raise LodelFatalError("A site named '%s' was found in the DB but not on the FS (expected to found it in '%s'!!!" % (os.path.basename(data_path), data_path))
+    #Immediately switching to the context
+    LodelContext.new(ctx_name)
+    LodelContext.set(ctx_name)
+    os.chdir(data_path) #Now the confdir is ./$condir_basename
+    #Loading settings for current site
+    LodelContext.expose_modules(globals(), {
+        'lodel.settings.settings': [('Settings', 'settings_preloader')]})
+    if settings_preloader.started():
+        msg = 'Settings seems to be allready started for "%s". \
+This should not append !' % ctx_name
+        #switch back to loader context in order to log & raise
+        LodelContext.set(None)
+        logger.critical(msg)
+        raise LodelFatalError(msg)
+    if lodelsites_instance:
+        settings_preloader(os.path.join('./', confdir_basename), )
+    else:
+        settings_preloader(os.path.join('./', confdir_basename))
+    LodelContext.set(None)
+    return
+
+##@brief End a site loading process (load plugins & hooks)
+#@param data_path str : site data path (used to extract the sitename !!)
+#@todo change data_path argument to sitename
+def site_load(data_path):
+    ctx_name = os.path.basename(data_path)
+    LodelContext.set(ctx_name)
+    #
+    #Loading hooks & plugins
+    #
+    LodelContext.expose_modules(globals(), {
+        'lodel.plugin': ['Plugin', 'LodelHook'],
+        'lodel.logger': 'logger',
+        'lodel.plugin.core_hooks': 'core_hooks',
+        'lodel.plugin.core_scripts': 'core_scripts'
+    })
+    Plugin.load_all() #Then all plugins & hooks are loaded
+    #triggering dyncode datasource instanciations
+    LodelHook.call_hook('lodel2_plugins_loaded', '__main__', None)
+    #triggering boostrapped hook
+    LodelHook.call_hook('lodel2_bootstraped', '__main__', None)
+    #Populating FAST_APP_EXPOSAL_CACHE
+    #
+    #WARNING !!!! Hardcoded interface name ! Here we have to find the 
+    #interface plugin name in order to populate the cache properly
+    FAST_APP_EXPOSAL_CACHE[ctx_name] = LodelContext.module(
+        'lodel.plugins.webui.run')
+    #a dirty & quick attempt to fix context unwanted exite via
+    #hooks
+    for name in ( 'LodelHook', 'core_hooks', 'core_scripts',
+            'Settings', 'settings', 'logger', 'Plugin'):
+        del(globals()[name])
+    #site fully loaded, switching back to loader context
+    LodelContext.set(None)
+
+##@brief Fetch handled sites name
+#@warning assert that a full __loader__ context is ready and that the
+#multisite context is preloaded too
+#@warning hardcoded Lodelsite leo name and shortname fieldname
+#@todo attempt to delete hardcoded leo name
+#@todo attempt to delete hardcoded fieldname
+def get_handled_sites_name():
+    LodelContext.expose_modules(globals(), {
+        'lodel.settings': ['Settings']})
+    lodelsites_name = Settings.sitename
+    LodelContext.set(lodelsites_name)
+    lodelsite_leo = leapi_dyncode.Lodelsite #hardcoded leo name
+    LodelContext.expose_modules(globals(), {
+        'lodel.leapi.query': ['LeGetQuery'],
+    })
+    handled_sites = LeGetQuery(lodelsite_leo, query_filters = [],
+        field_list = ['shortname']).execute()
+    if handled_sites is None:
+        return []
+    res = [ s['shortname'] for s in handled_sites]
+    del(globals()['LeGetQuery'])
+    del(globals()['Settings'])
+    LodelContext.set(None)
+    return res
+    

lodel/plugin/core_scripts.py

@@ -3,6 +3,7 @@ import shutil
 import tempfile
 import os, os.path
 from lodel.context import LodelContext
+from lodel import buildconf
 LodelContext.expose_modules(globals(), {
     'lodel.plugin.scripts': ['LodelScript'],
     'lodel.logger': 'logger'})
@@ -302,6 +303,8 @@ class ListHooks(LodelScript):
             print("\n")
 
 
+##@brief Implements lodel_admin **dyncode** action
+#@ingroup lodel2_script
 class RefreshDyncode(LodelScript):
     _action = 'dyncode'
     _description = 'Update the dynamic code according to EM and conf'
@@ -314,10 +317,16 @@ class RefreshDyncode(LodelScript):
         parser.add_argument('-o', '--dyncode',
             help='Specify the filename where the dyncode should be written',
             type=str, default='')
+        if LodelContext.multisite():
+            parser.add_argument('-a', '--all', action='store_true',
+                help="ONLY VALID FOR MULtisites ! Refresh lodelsites dyncode \
++ all handled sites dyncode")
         return
 
     ##@todo think of a better method to determine if we are in mono or
     #multisite instance
+    #@todo code factorisation to fetch handled sites list
+    #@todo fetch & use correct em_translator for handled sites
     @classmethod
     def run(cls, args):
         LodelContext.expose_modules(globals(), {
@@ -329,15 +338,61 @@ class RefreshDyncode(LodelScript):
         if len(args.em.strip()) == 0:
             #using the default em_file
             model_file = Settings.editorialmodel.emfile
-        #Model loaded
-        model = EditorialModel.load(em_translator, filename = model_file)
+        elif LodelContext.multisite() and args.all:
+            raise 
         #Creating dyncode
         dyncode_file = args.dyncode
         if len(args.dyncode.strip()) == 0:
             #using dyncode filename from conf
             dyncode_file = Settings.editorialmodel.dyncode
+        elif LodelContext.multisite() and args.all:
+            raise 
+        if LodelContext.multisite() and args.all:
+            #NOTE the code bellow can be factorised with 
+            #plugins/multisite/loader_utils.py
+
+            #Get the lodelsites instance name and the emfile path
+            LodelContext.set(None)
+            LodelContext.expose_modules(globals(), {
+                'lodel.settings':['Settings']})
+            lodelsites_name = Settings.sitename
+            emfile_path = Settings.lodelsites.sites_emfile
+            del(globals()['Settings']) #should be useless
+            #Get the list of handled sites name
+            LodelContext.set(lodelsites_name)
+            LodelContext.expose_modules(globals(), {
+                'lodel.leapi.query': ['LeGetQuery'],
+            })
+            handled_sites = LeGetQuery(lodelsite_leo, query_filters = [],
+                field_list = ['shortname']).execute()
+            del(globals()['LeGetQuery']) #should be useless
+            lodlesites_path = os.path.join(buildconf.LODEL2VARDIR,
+                MULTISITE_CONTEXTDIR)
+            if handled_sites is not None:
+                for sitename in handled_sites:
+                    LodelContext.set(None)
+                    #construct dyncode filename
+                    dyncode_path = os.path.join(
+                        os.path.join(lodlesites_path, sitename),
+                        'leapi_dyncode') #BOO hardcoded dyncode file name
+                    cls.refresh_dyncode(emfile_path, dyncode_path,
+                        em_translator)
+        #Refresh only one dyncode
+        #if multisite it's the lodelsites dyncode
+        LodelContext.set(None)
+        cls.refresh_dyncode(model_file, dyncode_file, em_translator)
+
+    
+    ##@brief Refresh dyncode
+    #@param model_file str : EM filename
+    #@param dyncode_file str : dyncode output filename
+    #@param em_translator str : translator name
+    @classmethod
+    def refresh_dyncode(cls, model_file, dyncode_file, em_translator):
+        #Model loaded
+        model = EditorialModel.load(em_translator, filename = model_file)
         dyncode_content = lefactory.dyncode_from_em(model)
         with open(dyncode_file, 'w+') as dfp:
             dfp.write(dyncode_content)
         print("Dyncode written in %s from em %s" % (dyncode_file, model_file))
-
+        

lodel/plugins/multisite/loader_utils.py

@@ -30,8 +30,10 @@ LODELSITES_CONFD = lodel.buildconf.LODELSITE_CONFDIR
 #@todo get rid of hardcoded stuff (like shortname fieldname)
 #@todo use the dyncode getter when it will be available (replaced by
 #the string SUPERDYNCODE_ACCESSOR.Lodelsite for the moment)
+#@todo remove hardcoded app name (need a better abstraction or a really
+#generic multiinstance runner)
 #@return lodelsites instance name
-def main():
+def main(handled_sites_may_not_load = False):
     #Set current context to reserved loader context
     from lodel import bootstrap
     bootstrap.bootstrap('__loader__')
@@ -79,20 +81,27 @@ def main():
     #loader context and clean it
     if handled_sites is not None:
         LodelContext.set(None)
+        LodelContext.expose_modules(globals(), {
+            'lodel.bootstrap': ['site_preload', 'site_load']})
         for mname in ['LeGetQuery', 'Settings', 'LodelHook', 'Plugin', 'logger']:
             del(globals()[mname])
         #Loading handled sites
         for handled_sitename in [s['shortname'] for s in handled_sites]:
             datapath = os.path.join(lodelsites_datapath, handled_sitename)
             try:
-                site_load(datapath) #using default conf.d configuration dirname
+                site_preload(datapath) #using default conf.d configuration dirname
+                site_load(datapath)
+                #
+                # HARDCODED APP NAME
+                #
+                populate_fast_app_cache(datapath, 'lodel.plugins.webui.run')
             except Exception as e:
                 LodelContext.set(None)
                 LodelContext.set(lodelsites_name)
                 LodelContext.expose_modules(globals(), {
                     'lodel.settings': ['Settings'],
                     'lodel.logger': 'logger'})
-                if Settings.debug:
+                if Settings.debug or handled_sites_may_not_load:
                     logger.critical("Unable to load site %s : %s" % (
                         e, handled_sitename))
                 else:
@@ -102,89 +111,12 @@ def main():
     LodelContext.set(None)
     return lodelsites_name
     
-
-##@brief Load a site
-#
-#Apply a common (as MONOSITE) loading process to a site :
-#1. Conf preload
-#2. Plugins preload
-#3. Conf loading
-#4. starting plugins & hooks
-#@warning At this point we need a uniq identifier for the site (using it
-#as key for contexts & FAST_APP_EXPOSAL_CACHE). To achieve this we use
-#the data_path basename. It should works for handled sites and for the 
-#lodelsites instance
-#@param data_path str : path to the datas directory (containing the confdir)
-#@param confdir_basename str : the basename of the site confdir
-#@param lodelsites_instance bool : if true we are loading the lodelsites
-#instance of the multisite (allow to load the good confspecs)
-#
-#@todo For now the interface plugin name for sites is hardcoded (set to
-#webui). It HAS TO be loaded from settings. But it is a bit complicated, 
-#we have to get the plugin's module name abstracted from context :
-#lodel.something but if we ask directly to Plugin class the module name
-#it will return something like : lodelsites.sitename.something...
-#
-#@todo there is a quick & dirty workarround with comments saying that it
-#avoid context escape via hooks. We have to understand why and how and then
-#replace the workarround by a real solution !
-def site_load(data_path, confdir_basename = 'conf.d', lodelsites_instance = False):
-    #args check
-    if confdir_basename != os.path.basename(confdir_basename):
-        LodelFatalError('Bad argument given to site_load(). This really \
-sux !')
-    #Determining uniq sitename from data_path
-    data_path = data_path.rstrip('/') #else basename returns ''
+##@brief Add an app to FAST_APP_EXPOSAL_CACHE
+#@param data_path str : instance data_path (used to extract the sitename !)
+#@param app_name str : application name (like lodel.plugins.webui.run)
+def populate_fast_app_cache(data_path, app_name):
     ctx_name = os.path.basename(data_path)
-    if not os.path.exists(data_path) or not os.path.isdir(data_path):
-        LodelContext.expose_modules(globals(), {
-            'lodel.exceptions': ['LodelFatalError']})
-        raise LodelFatalError("A site named '%s' was found in the DB but not on the FS (expected to found it in '%s'!!!" % (os.path.basename(data_path), data_path))
-    #Immediately switching to the context
-    LodelContext.new(ctx_name)
     LodelContext.set(ctx_name)
-    os.chdir(data_path) #Now the confdir is ./$condir_basename
-    #Loading settings for current site
-    LodelContext.expose_modules(globals(), {
-        'lodel.settings.settings': [('Settings', 'settings_preloader')]})
-    if settings_preloader.started():
-        msg = 'Settings seems to be allready started for "%s". \
-This should not append !' % ctx_name
-        #switch back to loader context in order to log & raise
-        LodelContext.set(None)
-        logger.critical(msg)
-        raise LodelFatalError(msg)
-    if lodelsites_instance:
-        settings_preloader(os.path.join('./', confdir_basename), )
-    else:
-        settings_preloader(os.path.join('./', confdir_basename))
-    #
-    #Loading hooks & plugins
-    #
-    LodelContext.expose_modules(globals(), {
-        'lodel.plugin': ['Plugin', 'LodelHook'],
-        'lodel.logger': 'logger',
-        'lodel.plugin.core_hooks': 'core_hooks',
-        'lodel.plugin.core_scripts': 'core_scripts'
-    })
-    Plugin.load_all() #Then all plugins & hooks are loaded
-    #triggering dyncode datasource instanciations
-    LodelHook.call_hook('lodel2_plugins_loaded', '__main__', None)
-    #triggering boostrapped hook
-    LodelHook.call_hook('lodel2_bootstraped', '__main__', None)
-    #Populating FAST_APP_EXPOSAL_CACHE
-    #
-    #WARNING !!!! Hardcoded interface name ! Here we have to find the 
-    #interface plugin name in order to populate the cache properly
-    FAST_APP_EXPOSAL_CACHE[ctx_name] = LodelContext.module(
-        'lodel.plugins.webui.run')
-    #a dirty & quick attempt to fix context unwanted exite via
-    #hooks
-    for name in ( 'LodelHook', 'core_hooks', 'core_scripts',
-            'Settings', 'settings', 'logger', 'Plugin'):
-        del(globals()[name])
-    #site fully loaded, switching back to loader context
+    FAST_APP_EXPOSAL_CACHE[ctx_name] = LodelContext.module(app_name)
     LodelContext.set(None)
-    #lodel2 multisite instances are loaded and ready to run
-
 

progs/slim/install_model/lodel_admin.py

@@ -139,15 +139,36 @@ def update_plugin_discover_cache(path_list = None):
 
 if __name__ == '__main__':
     _simlink_hack()
+    import lodel.buildconf
     from lodel import bootstrap
+    from lodel.context import LodelContext
     #to deleted when we known wich action need what kind of MULTISITE API
     if not bootstrap._monosite_test():
         bootstrap.bootstrap('lodelsites')
+        LodelContext.set(None)
+        #lodelsites instance preloading
+        #THIS CODE SHOULD BE IN AN UTIL FUNCTION
+        LodelContext.expose_modules(globals(), {
+            'lodel.bootstrap': ['get_handled_sites_name', 'site_preload'],
+            'lodel.settings': ['Settings']})
+        lodelsites_name = Settings.sitename
+        lodelsites_datapath = os.path.join(
+            os.path.join(lodel.buildconf.LODEL2VARDIR, lodelsites_name),
+            lodel.buildconf.MULTISITE_DATADIR)
+        del(globals()['Settings'])
+        site_preload(lodelsites_datapath, 'lodelsites.conf.d', True)
+
+        #preloading all handled sites
+        for sitename in get_handled_sites_name():
+            datapath = os.path.join(
+                os.path.join(lodel.buildconf.LODEL2VARDIR, sitename),
+                lodel.buildconf.MULTISITE_DATADIR)
+            site_preload(datapath)
     else:
         bootstrap.bootstrap()
-    from lodel.context import LodelContext
     #from lodel.plugin.scripts import main_run
     LodelContext.expose_modules(globals(),
         {'lodel.plugin.scripts': ['main_run'],
          'lodel.plugin.core_scripts': 'core_scripts'})
     main_run()
+    exit()