First SLIM version

Able to :
- create/delete instances
- list instances
- run make %target% in all instances
- edit configuration given an instance name

refs #145

progs/instances.json

@@ -0,0 +1 @@
+{"test1": {"path": "/tmp/lodel2_instances/test1"}}

progs/slim.py

@@ -0,0 +1,219 @@
+#!/usr/bin/env python3
+#-*- coding: utf-8 -*-
+
+
+INSTANCES_ABSPATH="/tmp/lodel2_instances"
+STORE_FILE='./instances.json'
+CREATION_SCRIPT='../scripts/create_instance.sh'
+INSTALL_TPL = './slim_ressources/slim_install_model'
+EMFILE = './slim_ressources/emfile.pickle'
+
+import os, os.path
+import sys
+import shutil
+import argparse
+import logging
+import json
+
+logging.basicConfig(level=logging.INFO)
+
+CREATION_SCRIPT=os.path.join(os.path.dirname(__file__), CREATION_SCRIPT)
+STORE_FILE=os.path.join(os.path.dirname(__file__), STORE_FILE)
+INSTALL_TPL=os.path.join(os.path.dirname(__file__), INSTALL_TPL)
+EMFILE=os.path.join(os.path.dirname(__file__), EMFILE)
+
+
+#STORE_FILE syntax :
+#
+#First level keys are instances names, their values are dict with following
+#informations : 
+# - path
+#
+
+##@brief Run 'make %target%' for each instances given in names
+#@param target str : make target
+#@param names list : list of instance name
+def run_make(target, names):
+    validate_names(names)
+    store_datas = get_store_datas()
+    cwd = os.getcwd()
+    for name in [n for n in store_datas if n in names]:
+        datas = store_datas[name]
+        logging.info("Running 'make %s' for '%s' in %s" % (
+            target, name, datas['path']))
+        os.chdir(datas['path'])
+        os.system('make %s' % target)
+    os.chdir(cwd)
+            
+
+##@brief If the name is not valid raise
+def name_is_valid(name):
+    allowed_chars = [chr(i) for i in range(ord('a'), ord('z')+1)]
+    allowed_chars += [chr(i) for i in range(ord('A'), ord('Z')+1)]
+    allowed_chars += [chr(i) for i in range(ord('0'), ord('9')+1)]
+    allowed_chars += ['_']
+    for c in name:
+        if c not in allowed_chars:
+            raise RuntimeError("Allowed characters for instance name are \
+lower&upper alphanum and '_'. Name '%s' is invalid" % name)
+
+##@brief Create a new instance
+#@param name str : the instance name
+def new_instance(name):
+    name_is_valid(name)
+    store_datas = get_store_datas()
+    if name in store_datas:
+        logging.error("An instance named '%s' already exists" % name)
+        exit(1)
+    if not os.path.isdir(INSTANCES_ABSPATH):
+        logging.info("Instances directory '%s' don't exists, creating it")
+        os.mkdir(INSTANCES_ABSPATH)
+    instance_path = os.path.join(INSTANCES_ABSPATH, name)
+    creation_cmd = '{script} "{name}" "{path}" "{install_tpl}" \
+"{emfile}"'.format(
+        script = CREATION_SCRIPT,
+        name = name,
+        path = instance_path,
+        install_tpl = INSTALL_TPL,
+        emfile = EMFILE)
+    res = os.system(creation_cmd)
+    if res != 0:
+        logging.error("Creation script fails")
+        exit(res)
+    #storing new instance
+    store_datas[name] = {'path': instance_path}
+    save_datas(store_datas)
+
+##@brief Delete an instance
+#@param name str : the instance name
+def delete_instance(name):
+    store_datas = get_store_datas()
+    logging.warning("Deleting instance %s" % name)
+    logging.info("Deleting instance folder %s" % store_datas[name]['path'])
+    shutil.rmtree(store_datas[name]['path'])
+    logging.info("Deleting instance from json store file")
+    del(store_datas[name])
+    save_datas(store_datas)
+
+##@brief returns stored datas
+def get_store_datas():
+    if not os.path.isfile(STORE_FILE):
+        return dict()
+    else:
+        with open(STORE_FILE, 'r') as sfp:
+            datas = json.load(sfp)
+    return datas
+
+##@brief Checks names validity and exit if fails
+def validate_names(names):
+    store_datas = get_store_datas()
+    invalid = [ n for n in names if n not in store_datas]
+    if len(invalid) > 0:
+        print("Following names are not existing instance :", file=sys.stderr)
+        for name in invalid:
+            print("\t%s" % name, file=sys.stderr)
+        exit(1)
+
+##@brief Check if instance are specified
+def get_specified(args):
+    if args.all:
+        names = list(get_store_datas().keys())
+    elif args.name is not None:
+        names = args.name
+    else:
+        names = None
+    return names
+
+##@brief Saves store datas
+def save_datas(datas):
+    with open(STORE_FILE, 'w+') as sfp:
+        json.dump(datas, sfp)
+
+##@brief Print the list of instances and exit
+def list_instances(verbosity):
+    if not os.path.isfile(STORE_FILE):
+        print("No store file, no instances are existing. Exiting...",
+            file=sys.stderr)
+        exit(0)
+    print('Instances list :')
+    exit(0)
+
+##@brief Returns instanciated parser
+def get_parser():
+    parser = argparse.ArgumentParser(
+        description='SLIM (Simple Lodel Instance Manager.)')
+    selector = parser.add_argument_group('Instances selectors')
+    actions = parser.add_argument_group('Instances actions')
+    parser.add_argument('-l', '--list', 
+        help='list existing instances and exit', action='store_const',
+        const=True, default=False)
+    parser.add_argument('-v', '--verbose', action='count')
+    actions.add_argument('-c', '--create', action='store_const',
+        default=False, const=True,
+        help="Create a new instance with given name (see -n --name)")
+    actions.add_argument('-d', '--delete', action='store_const',
+        default=False, const=True,
+        help="Delete an instance with given name (see -n --name)")
+    actions.add_argument('-e', '--edit-config', action='store_const',
+        default=False, const=True,
+        help='Edit configuration of specified instance')
+    selector.add_argument('-a', '--all', action='store_const',
+        default=False, const=True,
+        help='Select all instances')
+    selector.add_argument('-n', '--name', metavar='NAME', type=str, nargs='*',
+        help="Specify an instance name")
+    actions.add_argument('--stop', action='store_const', 
+        default=False, const=True, help="Stop instances")
+    actions.add_argument('--start', action='store_const', 
+        default=False, const=True, help="Start instances")
+    actions.add_argument('-m', '--make', metavar='TARGET', type=str,
+        nargs="?", default='not',
+        help='Run make for selected instances')
+    return parser
+
+if __name__ == '__main__':
+    parser = get_parser()
+    args = parser.parse_args()
+    if args.list:
+        list_instances(args.verbose)
+    elif args.create:
+        if args.name is None:
+            parser.print_help()
+            print("\nAn instance name expected when creating an instance !",
+                file=sys.stderr)
+            exit(1)
+        for name in args.name:
+            new_instance(name)
+    elif args.delete:
+        if args.name is None:
+            parser.print_help()
+            print("\nAn instance name expected when creating an instance !",
+                file=sys.stderr)
+            exit(1)
+        validate_names(args.name)
+        for name in args.name:
+            delete_instance(name)
+    elif args.make != 'not':
+        if args.make is None:
+            target = 'all'
+        else:
+            target = args.make
+        names = get_specified(args)
+        if names is None:
+            parser.print_help()
+            print("\nWhen using -m --make options you have to select \
+instances, either by name using -n or all using -a")
+            exit(1)
+        run_make(target, names)
+    elif args.edit_config:
+        names = get_specified(args)
+        if len(names) > 1:
+            print("\n-e --edit-config option works only when 1 instance is \
+specified")
+        validate_names(names)
+        name = names[0]
+        store_datas = get_store_datas()
+        conffile = os.path.join(store_datas[name]['path'], 'conf.d')
+        conffile = os.path.join(conffile, 'lodel2.ini')
+        os.system('editor "%s"' % conffile)
+        exit(0)

progs/slim_ressources/slim_install_model/Makefile

@@ -0,0 +1,15 @@
+python=python3
+
+all: dyncode
+
+dyncode:
+	$(python) -c 'import lodel_admin; lodel_admin.refresh_dyncode()'
+
+init_db: 
+	$(python) -c 'import lodel_admin; lodel_admin.init_all_dbs()'
+
+list_hooks: dyncode
+	$(python) -c 'import lodel_admin; lodel_admin.list_registered_hooks()'
+
+refresh_plugins:
+	$(python) -c 'import lodel_admin; lodel_admin.update_plugin_discover_cache()'

progs/slim_ressources/slim_install_model/conf.d/lodel2.ini

@@ -0,0 +1,34 @@
+[lodel2.datasources.default]
+identifier = dummy_datasource.example
+
+[lodel2.datasources.dummy2]
+identifier = dummy_datasource.dummy2
+
+[lodel2.datasource.dummy_datasource.example]
+dummy =
+[lodel2.datasource.dummy_datasource.dummy2]
+dummy =
+[lodel2.section1]
+key1 = 42
+[lodel2]
+debug = False
+sitename = noname
+
+[lodel2.logging.stderr]
+level = DEBUG
+filename = -
+context = True
+
+[lodel2.editorialmodel]
+groups = base_group, editorial_abstract, editorial_person
+emfile = editorial_model.pickle
+dyncode = leapi_dyncode.py
+
+[lodel2.webui]
+standalone=False
+#listen_address=127.0.0.1
+#listen_port=9090
+[lodel2.webui.sessions]
+directory=./sessions
+expiration=900
+file_template=lodel2_%s.sess

progs/slim_ressources/slim_install_model/loader.py

@@ -0,0 +1,77 @@
+#-*- coding: utf-8 -*-
+
+##@brief Lodel2 loader script
+#
+#@note If you want to avoid settings loading you can set the environment
+#variable LODEL2_NO_SETTINGS_LOAD (see @ref install.lodel_admin.update_plugin_discover_cache()
+#
+
+import sys, os, os.path
+#
+# Bootstraping
+#
+LODEL2_LIB_ABS_PATH = None
+if LODEL2_LIB_ABS_PATH is not None:
+    if not os.path.isdir(LODEL2_LIB_ABS_PATH):
+        print("FATAL ERROR : the LODEL2_LIB_ABS_PATH variable in loader.py is \
+not correct : '%s'" % LODEL2_LIB_ABS_PATH, file=sys.stderr)
+    sys.path.append(os.path.dirname(LODEL2_LIB_ABS_PATH))
+
+try:
+    import lodel
+except ImportError as e:
+    print("Unable to load lodel module. exiting...")
+    print(e)
+    exit(1)
+
+if 'LODEL2_NO_SETTINGS_LOAD' not in os.environ:
+    #
+    # Loading settings
+    #
+    from lodel.settings.settings import Settings as settings
+    if not settings.started():
+        settings('conf.d')
+    from lodel.settings import Settings
+    
+    #Starts hooks
+    from lodel.plugin import LodelHook
+    from lodel.plugin import core_hooks
+    from lodel.plugin import core_scripts
+
+def start():
+    #Load plugins
+    from lodel import logger
+    from lodel.plugin import Plugin
+    logger.debug("Loader.start() called")
+    Plugin.load_all()
+    LodelHook.call_hook('lodel2_bootstraped', '__main__', None)
+
+
+if __name__ == '__main__':
+
+    start()
+    if Settings.runtest:
+        import unittest
+        import tests
+        loader = unittest.TestLoader()
+        test_dir = os.path.join(LODEL2_LIB_ABS_PATH, 'tests')
+        suite = loader.discover(test_dir)
+        runner = unittest.TextTestRunner(
+            failfast = '-f' in sys.argv,
+            verbosity = 2 if '-v' in sys.argv else 1)
+        runner.run(suite)
+        exit()
+
+    import lodel
+    import leapi_dyncode as dyncode
+    lodel.dyncode = dyncode
+    LodelHook.call_hook('lodel2_dyncode_bootstraped', '__main__', None)
+    LodelHook.call_hook('lodel2_loader_main', '__main__', None)
+
+    #Run interative python
+    import code
+    print("""
+     Running interactive python in Lodel2 %s instance environment
+
+"""%Settings.sitename)
+    code.interact(local=locals())

progs/slim_ressources/slim_install_model/lodel_admin.py

@@ -0,0 +1,132 @@
+#-*- coding: utf-8 -*-
+
+import sys
+import os, os.path
+import argparse
+
+##@brief Dirty hack to avoid problems with simlink to lodel2 lib folder
+#
+#In instance folder we got a loader.py (the one we want to import here when
+#writing "import loader". The problem is that lodel_admin.py is a simlink to
+#LODEL2LIB_FOLDER/install/lodel_admin.py . In this folder there is the 
+#generic loader.py template. And when writing "import loader" its 
+#LODEL2LIB_FOLDER/install/loader.py that gets imported.
+#
+#In order to solve this problem the _simlink_hack() function delete the
+#LODEL2LIB_FOLDER/install entry from sys.path
+#@note This problem will be solved once lodel lib dir will be in 
+#/usr/lib/python3/lodel
+def _simlink_hack():
+    sys.path[0] = os.getcwd()
+
+## @brief Utility method to generate python code given an emfile and a
+# translator
+# @param model_file str : An em file
+# @param translator str : a translator name
+# @return python code as string
+def generate_dyncode(model_file, translator):
+    from lodel.editorial_model.model import EditorialModel
+    from lodel.leapi import lefactory
+
+    model = EditorialModel.load(translator, filename  = model_file)
+    dyncode = lefactory.dyncode_from_em(model)
+    return dyncode
+
+## @brief Utility method to generate a python file representing leapi dyncode
+# given an em file and the associated translator name
+#
+# @param model_file str : An em file
+# @param translator str : a translator name
+# @param output_filename str : the output file
+def create_dyncode(model_file, translator, output_filename):
+    from lodel import logger
+    dyncode = generate_dyncode(model_file, translator)
+    with open(output_filename, 'w+') as out_fd:
+        out_fd.write(dyncode)
+    out_fd.close()
+    logger.info("Dynamic leapi code written in %s", output_filename)
+
+
+## @brief Refresh dynamic leapi code from settings
+def refresh_dyncode():
+    import loader
+    from lodel.settings import Settings
+    # EditorialModel update/refresh
+    
+    # TODO
+
+    # Dyncode refresh
+    create_dyncode( Settings.editorialmodel.emfile,
+                    Settings.editorialmodel.emtranslator,
+                    Settings.editorialmodel.dyncode)
+
+
+def init_all_dbs():
+    import loader
+    loader.start()
+    import leapi_dyncode as dyncode
+    from lodel.plugin.datasource_plugin import DatasourcePlugin
+    from lodel.settings.utils import SettingsError
+    from lodel.leapi.leobject import LeObject
+    from lodel.plugin import Plugin
+    from lodel import logger
+
+    ds_cls = dict() # EmClass indexed by rw_datasource
+    for cls in dyncode.dynclasses:
+        ds = cls._datasource_name
+        if ds not in ds_cls:
+            ds_cls[ds] = [cls]
+        else:
+            ds_cls[ds].append(cls)
+    
+    for ds_name in ds_cls:
+        mh = DatasourcePlugin.init_migration_handler(ds_name)
+        #Retrieve plugin_name & ds_identifier for logging purpose
+        plugin_name, ds_identifier = DatasourcePlugin.plugin_name(
+            ds_name, False)
+        try:
+            mh.init_db(ds_cls[ds_name])
+        except Exception as e:
+            msg = "Migration failed for datasource %s(%s.%s) when running \
+init_db method: %s"
+            msg %= (ds_name, plugin_name, ds_identifier, e)
+        logger.info("Database initialisation done for %s(%s.%s)" % (
+            ds_name, plugin_name, ds_identifier))
+
+def list_registered_hooks():
+    import loader
+    loader.start()
+    from lodel.plugin.hooks import LodelHook
+    hlist = LodelHook.hook_list()
+    print("Registered hooks are : ")
+    for name in sorted(hlist.keys()):
+        print("\t- %s is registered by : " % name)
+        for reg_hook in hlist[name]:
+            hook, priority = reg_hook
+            msg = "\t\t- {modname}.{funname} with priority : {priority}"
+            msg = msg.format(
+                modname = hook.__module__,
+                funname = hook.__name__,
+                priority = priority)
+            print(msg)
+        print("\n")
+
+##@brief update plugin's discover cache
+#@note impossible to give arguments from a Makefile...
+#@todo write a __main__ to be able to run ./lodel_admin
+def update_plugin_discover_cache(path_list = None):
+    os.environ['LODEL2_NO_SETTINGS_LOAD'] = 'True'
+    import loader
+    from lodel.plugin.plugins import Plugin
+    res = Plugin.discover(path_list)
+    print("Plugin discover result in %s :\n" % res['path_list'])
+    for pname, pinfos in res['plugins'].items():
+        print("\t- %s %s -> %s" % (
+            pname, pinfos['version'], pinfos['path']))
+
+if __name__ == '__main__':
+    _simlink_hack()
+    import loader
+    loader.start()
+    from lodel.plugin.scripts import main_run
+    main_run()