Enhancement in deployment with autotools and in README file

Remove duplicate install templates directories (the same directories can be used by SLIM & runtest)
Addapted runtest so it can fetch the install template from the lodel2 datadir
Make lodel_admin.py runnable (add the python3 shebang + chmod +x at install-data target)
And a beautifull ASCII formating for the README

Makefile.am

@@ -1,10 +1,13 @@
 SUBDIRS=lodel progs plugins
-EXTRA_DIST=doc plugins runtest tplinstall examples tests
+EXTRA_DIST=doc plugins runtest examples tests README
+CLEANFILES=runtest
 
-lodel2datadir=$(datadir)/lodel2
 lodel2_localstate_DATA =
 lodel2_localstatedir=$(localstatedir)/lodel2
 
+lodel2datadir=$(datadir)/lodel2
+install_model_dir = $(lodel2datadir)/install_model
+
 #python=/usr/bin/env python3
 python=@PYTHON@
 dyncode_filename='lodel/leapi/dyncode.py'
@@ -19,6 +22,14 @@ generate-doc: clean doc_graphviz
 doc_graphviz:
 	cd doc/img/graphviz; make
 
+
+do_subst = sed -e 's,\[@\]INSTALLMODEL_DIR\[@\],$(install_model_dir),g' 
+
+runtest: runtest.sh
+	$(do_subst) < $(srcdir)/runtest.sh > runtest
+	chmod +x runtest
+	
+
 # Test em update ( examples/em_test.pickle )
 em_test: em_test.py
 	$(python) em_test.py

README

@@ -1,5 +1,8 @@
 Dependencies :
+==============
+
 	lodel2 components :
+	-------------------
 		core :
 			python3 python3-lxml
 
@@ -13,10 +16,13 @@ Dependencies :
 			python3-pymongo
 
 	lodel2 utils :
+	--------------
 		mass_deploy dependencies :
 			pwgen wamerican mongodb-clients
 
 Build & install :
+=================
+
 	./bootstrap.sh #generate the configure, refresh the Makefile.in files
 	#--prefix=/usr is mandatory on debian because of a python3.4.2 bug
 	./configure --sysconfdir=/etc --localstatedir=/var --prefix=/usr
@@ -28,55 +34,96 @@ Build & install :
 	make uninstall
 
 Other make targets :
+====================
+
 	Doxygen documentation generation :
+	----------------------------------
 		make doc #or simply run doxygen in the source dir
+
 	Run unit tests :
+	----------------
+
 		make check
 		#or
 		make unittest
 
 Lodel2 instance management utilities :
+======================================
+
 	SLIM : Simple Lodel Instance Manager
-		create a new instance named foo :
+	------------------------------------
+
+		- create a new instance named foo :
 			slim -n foo -c
-		list instances : 
+
+		- list instances : 
 			slim -l
-		list instances with details :
+
+		- list instances with details :
 			slim -lv[v[v]]
-		list instances with output formated for bash processing :
+
+		- list instances with output formated for bash processing :
 			slim -l -b [-v[v[v]]
-		run make all for instance foo :
+
+		- run make all for instance foo :
 			slim -n foo -m
-		run make for all existing instances :
+
+		- run make for all existing instances :
 			slim --all -m
-		start an instance in forground
+
+		- start an instance in forground
 			slim -n foo --start -f
-		start all instances :
+
+		- start all instances :
 			slim -a --start
-		generate a nginx conf for all instances :
+
+		- generate a nginx conf for all instances :
 			slim -a --nginx-conf
-		get some help for more options & actions :
+
+		- get some help for more options & actions :
 			slim -h
 
-	mass_deploy.sh : use for easy mass deployment of instances with webui & uwsgi
-		the script can be found in $prefix/share/lodel2/scripts/mass_deploy.sh
+	mass_deploy.sh :
+	----------------
+		Use for easy mass deployment of instances with webui & uwsgi
+		the script at $prefix/share/lodel2/scripts/mass_deploy.sh
 
 		Usage : mass_deploy.sh N #with N the number of instances to deploy
+		-------
 		
-		- configuration can be found in /etc/lodel2/mass_deploy.cfg (
-		  to indicate mongodb admin credentials)
+		Notes :
+		-------
+
+		- configuration can be found in /etc/lodel2/mass_deploy.cfg 
+		  to indicate mongodb admin credentials
+
 		- mass deploy create instances directories 
 		  in /tmp/lodel2_instances
 
 Instance operations :
-	A Makefile is written to allow running most of operations. Existing targets are for the moment :
+=====================
 
-	make dyncode # Leapi dynamic code creation ( in leapi_dyncode.py in lodel2 instance root dir)
-	make init_db # Call migration handlers to tell them to init all needed databases. (note : this target has dyncode as dependencie)
-    make list_hooks # List all the hooks registered
+	- Two utils are available for instances operations :
+		* Make for op without parameters
+		* lodel_admin.py for op with parameters
 
+	- Makefile available target :
+		# Leapi dynamic code creation ( in leapi_dyncode.py in lodel2 
+		#instance root dir)
+		make dyncode 
+
+		# Call migration handlers to tell them to init all needed 
+		#databases. (note : this target has dyncode as dependencie)
+		make init_db 
+		make list_hooks # List all the hooks registered
+
+	- lodel_admin.py : see lodel_admin.py -h
+
+
+##########################################
+# Following informations can be obsolete #
+##########################################
 
-#Following informations can be obsolete
 Dynamic code generation :
 	python3 scripts/refreshdyn.py examples/em_test.pickle OUTPUTFILE.py
 
@@ -89,3 +136,4 @@ Instance creation :
 Instance loader uppdate :
 	If the install/loader.py is updated you can update instance's loader.py using
 		scripts/create_instance.sh -u INSTANCE_PATH
+

progs/mass_deploy.sh

@@ -64,7 +64,7 @@ then
 		echo -e "MONGODB_DB_PREFIX not indicated in $conffile. Using purgedb without prefix is really a bad idea... It will result in a deletion of ALL db.\nexiting."
 		exit 3
 	fi
-	echo -n "Are you sure you want to delete all mongo database with name begining with '$MONGODB_DB_PREFIX' ? Y/n"
+	echo -n "Are you sure you want to delete all mongo database with name begining with '$MONGODB_DB_PREFIX' ? Y/n "
 	read rep
 	if [ "$rep" = "Y" ]
 	then

progs/slim/Makefile.am

@@ -4,15 +4,17 @@ EXTRA_DIST = slim.py install_model emfile.pickle
 
 lodel2_scripts_dir = $(datadir)/lodel2/scripts
 
-slimdatadir = $(datadir)/lodel2/slim/
+lodel2datadir = $(datadir)/lodel2
+
+slimdatadir = $(lodel2datadir)/slim
 slimdata_DATA = emfile.pickle
 
-slim_install_model_dir = $(slimdatadir)/install_model
-slim_install_model__DATA = 	install_model/loader.py \
-				install_model/lodel_admin.py \
-				install_model/Makefile 
+install_model_dir = $(lodel2datadir)/install_model
+install_model__DATA = 	install_model/loader.py \
+			install_model/lodel_admin.py \
+			install_model/Makefile 
 
-slim_conf_model_dir = $(slim_install_model_dir)/conf.d/
+slim_conf_model_dir = $(install_model_dir)/conf.d/
 slim_conf_model__DATA = install_model/conf.d/lodel2.ini
 
 slim_var__DATA = slim_instances.json slim_instances_pid.json
@@ -25,7 +27,7 @@ slim_instances_pid.json:
 
 do_subst = sed -e 's,\[@\]LODEL2_PROGSDIR\[@\],$(lodel2_scripts_dir),g' \
 	-e 's,\[@\]SLIM_DATADIR\[@\],$(slimdatadir),g' \
-	-e 's,\[@\]SLIM_INSTALLMODEL_DIR\[@\],$(slim_install_model_dir),g' \
+	-e 's,\[@\]INSTALLMODEL_DIR\[@\],$(install_model_dir),g' \
 	-e 's,\[@\]SLIM_VAR_DIR\[@\],$(slim_var_dir),g' \
 	-e 's,\[@\]PKGPYTHONDIR\[@\],$(pkgpythondir),g'
 
@@ -33,3 +35,6 @@ slim: slim.py
 	$(do_subst) < $(srcdir)/slim.py > slim
 	chmod +x slim
 
+install-data-hook:
+	chmod +x $(install_model_dir)/lodel_admin.py
+

progs/slim/install_model/lodel_admin.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 #-*- coding: utf-8 -*-
 
 import sys

progs/slim/slim.py

@@ -22,7 +22,7 @@ try:
     STORE_FILE = os.path.join("[@]SLIM_VAR_DIR[@]", 'slim_instances.json')
     PID_FILE = os.path.join("[@]SLIM_VAR_DIR[@]", 'slim_instances_pid.json')
     CREATION_SCRIPT = os.path.join("[@]LODEL2_PROGSDIR[@]", 'create_instance')
-    INSTALL_TPL = "[@]SLIM_INSTALLMODEL_DIR[@]"
+    INSTALL_TPL = "[@]INSTALLMODEL_DIR[@]"
     EMFILE = os.path.join("[@]SLIM_DATADIR[@]", 'emfile.pickle')
 
 except SyntaxError:

runtest → runtest.sh

@@ -55,12 +55,11 @@ fi
 PYTHON='env python3'
 testdir=$(mktemp -d)
 rmdir $testdir
-./progs/create_instance test_instance $testdir ./tplinstall ./examples/em_file.pickle $(dirname $0)
+./progs/create_instance test_instance $testdir "[@]INSTALLMODEL_DIR[@]" ./examples/em_file.pickle $(dirname $0)
 cp -R examples $testdir
 cp -R tests $testdir
 cd $testdir
-ls -la
-pwd
+chmod +x lodel_admin.py
 rm -R conf.d && mv tests/tests_conf.d conf.d
 make
 make refresh_plugins

tplinstall/Makefile

@@ -1,15 +0,0 @@
-python=python3
-
-all: dyncode
-
-dyncode:
-	$(python) -c 'import lodel_admin; lodel_admin.refresh_dyncode()'
-
-init_db: 
-	$(python) -c 'import lodel_admin; lodel_admin.init_all_dbs()'
-
-list_hooks: dyncode
-	$(python) -c 'import lodel_admin; lodel_admin.list_registered_hooks()'
-
-refresh_plugins:
-	$(python) -c 'import lodel_admin; lodel_admin.update_plugin_discover_cache()'

tplinstall/conf.d/datasources.ini

@@ -1,10 +0,0 @@
-[lodel2.datasources.default]
-identifier = dummy_datasource.example
-
-[lodel2.datasources.dummy2]
-identifier = dummy_datasource.dummy2
-
-[lodel2.datasource.dummy_datasource.example]
-dummy =
-[lodel2.datasource.dummy_datasource.dummy2]
-dummy =

tplinstall/conf.d/dummy_plugin.ini

@@ -1,2 +0,0 @@
-[lodel2.section1]
-key1 = 42

tplinstall/conf.d/lodel2.ini

@@ -1,14 +0,0 @@
-[lodel2]
-debug = False
-sitename = noname
-
-[lodel2.logging.stderr]
-level = DEBUG
-filename = -
-context = True
-
-[lodel2.editorialmodel]
-groups = base_group, editorial_abstract, editorial_person
-emfile = editorial_model.pickle
-dyncode = leapi_dyncode.py
-

tplinstall/conf.d/webui_plugin.ini

@@ -1,8 +0,0 @@
-[lodel2.webui]
-standalone=False
-#listen_address=127.0.0.1
-#listen_port=9090
-[lodel2.webui.sessions]
-directory=./sessions
-expiration=900
-file_template=lodel2_%s.sess

tplinstall/loader.py

@@ -1,82 +0,0 @@
-#-*- coding: utf-8 -*-
-
-##@brief Lodel2 loader script
-#
-#@note If you want to avoid settings loading you can set the environment
-#variable LODEL2_NO_SETTINGS_LOAD (see @ref install.lodel_admin.update_plugin_discover_cache()
-#
-
-import sys, os, os.path
-#
-# Bootstraping
-#
-LODEL2_LIB_ABS_PATH = None
-if LODEL2_LIB_ABS_PATH is not None:
-    if not os.path.isdir(LODEL2_LIB_ABS_PATH):
-        print("FATAL ERROR : the LODEL2_LIB_ABS_PATH variable in loader.py is \
-not correct : '%s'" % LODEL2_LIB_ABS_PATH, file=sys.stderr)
-    sys.path.append(LODEL2_LIB_ABS_PATH)
-
-#Set the cwd to the instance dir
-instance_dir = os.path.dirname(sys.argv[0])
-if len(instance_dir) > 0:
-    os.chdir(instance_dir)
-
-try:
-    import lodel
-except ImportError as e:
-    print("Unable to load lodel module. exiting...")
-    print(e)
-    exit(1)
-
-if 'LODEL2_NO_SETTINGS_LOAD' not in os.environ:
-    #
-    # Loading settings
-    #
-    from lodel.settings.settings import Settings as settings
-    if not settings.started():
-        settings('conf.d')
-    from lodel.settings import Settings
-    
-    #Starts hooks
-    from lodel.plugin import LodelHook
-    from lodel.plugin import core_hooks
-    from lodel.plugin import core_scripts
-
-def start():
-    #Load plugins
-    from lodel import logger
-    from lodel.plugin import Plugin
-    logger.debug("Loader.start() called")
-    Plugin.load_all()
-    LodelHook.call_hook('lodel2_bootstraped', '__main__', None)
-
-
-if __name__ == '__main__':
-
-    start()
-    if Settings.runtest:
-        import unittest
-        import tests
-        loader = unittest.TestLoader()
-        test_dir = os.path.join(LODEL2_LIB_ABS_PATH, 'tests')
-        suite = loader.discover(test_dir)
-        runner = unittest.TextTestRunner(
-            failfast = '-f' in sys.argv,
-            verbosity = 2 if '-v' in sys.argv else 1)
-        runner.run(suite)
-        exit()
-
-    import lodel
-    import leapi_dyncode as dyncode
-    lodel.dyncode = dyncode
-    LodelHook.call_hook('lodel2_dyncode_bootstraped', '__main__', None)
-    LodelHook.call_hook('lodel2_loader_main', '__main__', None)
-
-    #Run interative python
-    import code
-    print("""
-     Running interactive python in Lodel2 %s instance environment
-
-"""%Settings.sitename)
-    code.interact(local=locals())

tplinstall/lodel_admin.py

@@ -1,132 +0,0 @@
-#-*- coding: utf-8 -*-
-
-import sys
-import os, os.path
-import argparse
-
-##@brief Dirty hack to avoid problems with simlink to lodel2 lib folder
-#
-#In instance folder we got a loader.py (the one we want to import here when
-#writing "import loader". The problem is that lodel_admin.py is a simlink to
-#LODEL2LIB_FOLDER/install/lodel_admin.py . In this folder there is the 
-#generic loader.py template. And when writing "import loader" its 
-#LODEL2LIB_FOLDER/install/loader.py that gets imported.
-#
-#In order to solve this problem the _simlink_hack() function delete the
-#LODEL2LIB_FOLDER/install entry from sys.path
-#@note This problem will be solved once lodel lib dir will be in 
-#/usr/lib/python3/lodel
-def _simlink_hack():
-    sys.path[0] = os.getcwd()
-
-## @brief Utility method to generate python code given an emfile and a
-# translator
-# @param model_file str : An em file
-# @param translator str : a translator name
-# @return python code as string
-def generate_dyncode(model_file, translator):
-    from lodel.editorial_model.model import EditorialModel
-    from lodel.leapi import lefactory
-
-    model = EditorialModel.load(translator, filename  = model_file)
-    dyncode = lefactory.dyncode_from_em(model)
-    return dyncode
-
-## @brief Utility method to generate a python file representing leapi dyncode
-# given an em file and the associated translator name
-#
-# @param model_file str : An em file
-# @param translator str : a translator name
-# @param output_filename str : the output file
-def create_dyncode(model_file, translator, output_filename):
-    from lodel import logger
-    dyncode = generate_dyncode(model_file, translator)
-    with open(output_filename, 'w+') as out_fd:
-        out_fd.write(dyncode)
-    out_fd.close()
-    logger.info("Dynamic leapi code written in %s", output_filename)
-
-
-## @brief Refresh dynamic leapi code from settings
-def refresh_dyncode():
-    import loader
-    from lodel.settings import Settings
-    # EditorialModel update/refresh
-    
-    # TODO
-
-    # Dyncode refresh
-    create_dyncode( Settings.editorialmodel.emfile,
-                    Settings.editorialmodel.emtranslator,
-                    Settings.editorialmodel.dyncode)
-
-
-def init_all_dbs():
-    import loader
-    loader.start()
-    import leapi_dyncode as dyncode
-    from lodel.plugin.datasource_plugin import DatasourcePlugin
-    from lodel.settings.utils import SettingsError
-    from lodel.leapi.leobject import LeObject
-    from lodel.plugin import Plugin
-    from lodel import logger
-
-    ds_cls = dict() # EmClass indexed by rw_datasource
-    for cls in dyncode.dynclasses:
-        ds = cls._datasource_name
-        if ds not in ds_cls:
-            ds_cls[ds] = [cls]
-        else:
-            ds_cls[ds].append(cls)
-    
-    for ds_name in ds_cls:
-        mh = DatasourcePlugin.init_migration_handler(ds_name)
-        #Retrieve plugin_name & ds_identifier for logging purpose
-        plugin_name, ds_identifier = DatasourcePlugin.plugin_name(
-            ds_name, False)
-        try:
-            mh.init_db(ds_cls[ds_name])
-        except Exception as e:
-            msg = "Migration failed for datasource %s(%s.%s) when running \
-init_db method: %s"
-            msg %= (ds_name, plugin_name, ds_identifier, e)
-        logger.info("Database initialisation done for %s(%s.%s)" % (
-            ds_name, plugin_name, ds_identifier))
-
-def list_registered_hooks():
-    import loader
-    loader.start()
-    from lodel.plugin.hooks import LodelHook
-    hlist = LodelHook.hook_list()
-    print("Registered hooks are : ")
-    for name in sorted(hlist.keys()):
-        print("\t- %s is registered by : " % name)
-        for reg_hook in hlist[name]:
-            hook, priority = reg_hook
-            msg = "\t\t- {modname}.{funname} with priority : {priority}"
-            msg = msg.format(
-                modname = hook.__module__,
-                funname = hook.__name__,
-                priority = priority)
-            print(msg)
-        print("\n")
-
-##@brief update plugin's discover cache
-#@note impossible to give arguments from a Makefile...
-#@todo write a __main__ to be able to run ./lodel_admin
-def update_plugin_discover_cache(path_list = None):
-    os.environ['LODEL2_NO_SETTINGS_LOAD'] = 'True'
-    import loader
-    from lodel.plugin.plugins import Plugin
-    res = Plugin.discover(path_list)
-    print("Plugin discover result in %s :\n" % res['path_list'])
-    for pname, pinfos in res['plugins'].items():
-        print("\t- %s %s -> %s" % (
-            pname, pinfos['version'], pinfos['path']))
-
-if __name__ == '__main__':
-    _simlink_hack()
-    import loader
-    loader.start()
-    from lodel.plugin.scripts import main_run
-    main_run()