Merge branch 'newlodel' of git.labocleo.org:lodel2 into newlodel

Makefile

@@ -1,3 +1,4 @@
+python=python3
 dyncode_filename='lodel/leapi/dyncode.py'
 
 all: tests doc dyncode
@@ -11,11 +12,11 @@ doc_graphviz:
 
 # Test em update ( examples/em_test.pickle )
 em_test:
-	python3 em_test.py
+	$(python) em_test.py
 
 # generate leapi dynamic code
 dyncode: clean_dyn em_test
-	python3 scripts/refreshdyn.py examples/em_test.pickle $(dyncode_filename) && echo -e "\n\nCode generated in $(dyncode_filename)"
+	$(python) scripts/refreshdyn.py examples/em_test.pickle $(dyncode_filename) && echo -e "\n\nCode generated in $(dyncode_filename)"
 
 # run tests
 tests:

README.txt

@@ -1,6 +1,10 @@
 Local configuration :
 	First of all copy the settings.ini to settings_local.ini and replace values by correct path
 
+Dependencies :
+	with pip : see requierments.txt
+	debian stable : python3 python3-lxml python3-jinja2 python3-werkzeug python3-pymongo doxygen graphviz
+
 Doxygen documentation generation :
 	doxygen
 

globconf.d/global.ini

@@ -8,3 +8,4 @@ emtranslator = picklefile
 dyncode = lodel/leapi/dyncode.py
 editormode = True
 groups = 
+

install/Makefile

@@ -1,4 +1,6 @@
+python=python3
+
 all: dyncode
 
 dyncode:
-	python -c 'import lodel_admin; lodel_admin.refresh_dyncode()'
+	$(python) -c 'import lodel_admin; lodel_admin.refresh_dyncode()'

install/conf.d/lodel2.ini

@@ -1,7 +1,8 @@
 [lodel2]
 debug = False
-plugins = dummy
 sitename = noname
+plugins_path = /foo/plugins
+plugins = dummy, webui
 
 [lodel2.logging.stderr]
 level = DEBUG

install/conf.d/webui_plugin.ini

@@ -0,0 +1,8 @@
+[lodel2.webui]
+standalone=False
+#listen_address=127.0.0.1
+#listen_port=9090
+[lodel2.webui.sessions]
+directory=./sessions
+expiration=900
+file_template=lodel2_%s.sess

install/loader.py

@@ -22,8 +22,14 @@ from lodel.settings.settings import Settings as settings
 settings('conf.d')
 from lodel.settings import Settings
 
+#Load plugins
+from lodel.plugin import Plugins
+Plugins.bootstrap()
 
-if __name__ == '__main__': # To allow running interactive python
+if __name__ == '__main__':
+    from lodel.plugin import LodelHook
+    LodelHook.call_hook('lodel2_loader_main', '__main__', None)
+    #Run interative python
     import code
     print("""
      Running interactive python in Lodel2 %s instance environment

run.py → install/run.py

@@ -1,15 +1,17 @@
 # -*- coding: utf-8 -*-
+import loader # Lodel2 loader
+
 import os
 from werkzeug.contrib.sessions import FilesystemSessionStore
 
+from lodel.settings import Settings
 from lodel.interface.web.router import get_controller
 from lodel.interface.web.lodelrequest import LodelRequest
 from lodel.utils.datetime import get_utc_timestamp
 
-# TODO Déplacer ces trois paramètres dans les settings
-SESSION_FILES_TEMPLATE = 'lodel_%s.sess'
-SESSION_FILES_BASE_DIR = 'tmp/sessions'
-SESSION_EXPIRATION_LIMIT = 900 # 15 min
+SESSION_FILES_BASE_DIR = Settings.webui.sessions.directory
+SESSION_FILES_TEMPLATE = Settings.webui.sessions.file_template
+SESSION_EXPIRATION_LIMIT = Settings.webui.sessions.expiration
 
 session_store = FilesystemSessionStore(path=SESSION_FILES_BASE_DIR, filename_template=SESSION_FILES_TEMPLATE)
 

lodel/leapi/datahandlers/base_classes.py

@@ -203,6 +203,10 @@ class Reference(DataHandler):
     def back_reference(self):
         return copy.copy(self.__back_reference)
 
+    @property
+    def linked_classes(self):
+        return copy.copy(self.__allowed_classes)
+
     ##@brief Set the back reference for this field.
     def _set_back_reference(self, back_reference):
         self.__back_reference = back_reference

lodel/leapi/leobject.py

@@ -118,6 +118,11 @@ class LeObject(object):
     @property
     def initialized(self):
         return not isinstance(self.__initialized, list)
+    
+    ##@return The uid field name
+    @classmethod
+    def uid_fieldname(cls):
+        return cls._uid
 
     ##@brief Return a list of fieldnames
     # @param include_ro bool : if True include read only field names
@@ -160,6 +165,18 @@ class LeObject(object):
     @classmethod
     def is_abstract(cls):
         return cls._abstract
+    
+    ##@brief Field data handler gettet
+    #@param fieldname str : The field name
+    #@return A datahandler instance
+    #@throw NameError if the field doesn't exist
+    @classmethod
+    def field(cls, fieldname):
+        try:
+            return cls._fields[field_uid]
+        except KeyError:
+            raise NameError("No field named '%s' in %s" % ( field_uid,
+                                                            cls.__name__))
 
     ##@brief Read only access to all datas
     # @note for fancy data accessor use @ref LeObject.g attribute @ref LeObjectValues instance

lodel/leapi/query.py

@@ -3,11 +3,13 @@
 import re
 from .leobject import LeObject, LeApiErrors, LeApiDataCheckError
 from lodel.plugin.hooks import LodelHook
+from lodel import logger
 
 class LeQueryError(Exception):
     ##@brief Instanciate a new exceptions handling multiple exceptions
-    # @param msg str : Exception message
-    # @param exceptions dict : A list of data check Exception with concerned field (or stuff) as key
+    #@param msg str : Exception message
+    #@param exceptions dict : A list of data check Exception with concerned
+    # field (or stuff) as key
     def __init__(self, msg = "Unknow error", exceptions = None):
         self._msg = msg
         self._exceptions = dict() if exceptions is None else exceptions
@@ -17,7 +19,10 @@ class LeQueryError(Exception):
 
     def __str__(self):
         msg = self._msg
-        for_iter = self._exceptions.items() if isinstance(self._exceptions, dict) else enumerate(self.__exceptions)
+        if isinstance(self._exceptions, dict):
+            for_iter = self._exceptions.items()
+        else:
+            for_iter = enumerate(self.__exceptions)
         for obj, expt in for_iter:
             msg += "\n\t{expt_obj} : ({expt_name}) {expt_msg}; ".format(
                     expt_obj = obj,
@@ -36,7 +41,7 @@ class LeQuery(object):
     ##@brief Abstract constructor
     # @param target_class LeObject : class of object the query is about
     def __init__(self, target_class):
-        if hook_prefix is None:
+        if self._hook_prefix is None:
             raise NotImplementedError("Abstract class")
         if not issubclass(target_class, LeObject):
             raise TypeError("target class has to be a child class of LeObject")
@@ -49,9 +54,11 @@ class LeQuery(object):
     #
     # @note maybe the datasource in not an argument but should be determined
     #elsewhere
-    def execute(self, datasource, **datas = None):
+    def execute(self, datasource, datas = None):
         if len(datas) > 0:
-            self.__target_class.check_datas_value(datas, **self._data_check_args)
+            self.__target_class.check_datas_value(
+                                                    datas,
+                                                    **self._data_check_args)
             self.__target_class.prepare_datas() #not yet implemented
         if self._hook_prefix is None:
             raise NotImplementedError("Abstract method")
@@ -90,48 +97,196 @@ class LeFilteredQuery(LeQuery):
     # @param query_filters list : with a tuple (only one filter) or a list of tuple
     #   or a dict: {OP,list(filters)} with OP = 'OR' or 'AND
     #   For tuple (FIELD,OPERATOR,VALUE)
-    def __init__(self, target_class, query_filter):
+    def __init__(self, target_class, query_filters = None):
         super().__init__(target_class)
         ##@brief The query filter
         self.__query_filter = None
         self.set_query_filter(query_filter)
     
-    ##@brief Set the query filter for a query
-    def set_query_filter(self, query_filter):
-        #
-        #   Query filter check & prepare 
-        #   query_filters can be a tuple (only one filter), a list of tuple
-        #   or a dict: {OP,list(filters)} with OP = 'OR' or 'AND
-        #   For tuple (FIELD,OPERATOR,VALUE)
-        #   FIELD has to be in the field_names list of target class
-        #   OPERATOR in query_operator attribute
-        #   VALUE has to be a correct value for FIELD
+    ##@brief Add filter(s) to the query
+    #@param query_filter list|tuple|str : A single filter or a list of filters
+    #@see LeFilteredQuery._prepare_filters()
+    def filter(self, query_filter):
+        pass
+
+    ## @brief Prepare filters for datasource
+    # 
+    #A filter can be a string or a tuple with len = 3.
+    #
+    #This method divide filters in two categories :
+    #
+    #@par Simple filters
+    #
+    #Those filters concerns fields that represent object values (a title,
+    #the content, etc.) They are composed of three elements : FIELDNAME OP
+    # VALUE . Where :
+    #- FIELDNAME is the name of the field
+    #- OP is one of the authorized comparison operands ( see 
+    #@ref LeFilteredQuery.query_operators )
+    #- VALUE is... a value
+    #
+    #@par Relational filters
+    #
+    #Those filters concerns on reference fields ( see the corresponding
+    #abstract datahandler @ref lodel.leapi.datahandlers.base_classes.Reference)
+    #The filter as quite the same composition than simple filters :
+    # FIELDNAME[.REF_FIELD] OP VALUE . Where :
+    #- FIELDNAME is the name of the reference field
+    #- REF_FIELD is an optionnal addon to the base field. It indicate on wich
+    #field of the referenced object the comparison as to be done. If no
+    #REF_FIELD is indicated the comparison will be done on identifier.
+    #
+    #@param cls
+    #@param filters_l list : This list of str or tuple (or both)
+    #@return a tuple(FILTERS, RELATIONNAL_FILTERS
+    #@todo move this doc in another place (a dedicated page ?)
+    @classmethod
+    def _prepare_filters(cls, filters_l):
+        filters = list()
+        res_filters = list()
+        rel_filters = list()
+        err_l = dict()
+        #Splitting in tuple if necessary
+        for fil in filters_l:
+            if len(fil) == 3 and not isinstance(fil, str):
+                filters.append(tuple(fil))
+            else:
+                filters.append(cls.split_filter(fil))
 
-        fieldnames = self.__target_class.fieldnames()
-        # Recursive method which checks filters
-        def check_tuple(tupl, fieldnames, target_class):
-            if isinstance(tupl, tuple):
-                if tupl[0] not in fieldnames:
-                    return False
-                if tupl[1] not in self.query_operators:
-                    return False
-                if not isinstance(tupl[2], target_class.datahandler(tupl[0])):
-                    return False
-                return True
-            elif isinstance(tupl,dict):
-                return check_tuple(tupl[1])
-            elif isinstance(tupl,list):
-                for tup in tupl:
-                    return check_tuple(tup)
-            else: 
-                raise TypeError("Wrong filters for query")
+        for field, operator, value in filters:
+            # Spliting field name to be able to detect a relational field
+            field_spl = field.split('.')
+            if len(field_spl) == 2:
+                field, ref_field = field_spl
+            elif len(field_spl) == 1:
+                ref_field = None
+            else:
+                err_l[field] = NameError(   "'%s' is not a valid relational \
+field name" % fieldname)
+                continue   
+            # Checking field against target_class
+            ret = self.__check_field(self.__target_class, field)
+            if isinstance(ret, Exception):
+                err_l[field] = ret
+                continue
+            # Check that the field is relational if ref_field given
+            if ref_field is not None and not cls.field(field).is_reference():
+                # inconsistency
+                err_l[field] = NameError(   "The field '%s' in %s is not\
+a relational field, but %s.%s was present in the filter"
+                                            % ( field,
+                                                field,
+                                                ref_field))
+            # Prepare relational field
+            if cls.field(field).is_reference():
+                ret = cls._prepare_relational_fields(field, ref_field)
+                if isinstance(ret, Exception):
+                    err_l[field] = ret
+                else:
+                    rel_filters.append((ret, operator, value))
+            else:
+                res_filters.append((field,operator, value))
+        
+        if len(err_l) > 0:
+            raise LeApiDataCheckError(
+                                        "Error while preparing filters : ",
+                                        err_l)
+        return (res_filters, rel_filters)
+
+    ## @brief Check and split a query filter
+    # @note The query_filter format is "FIELD OPERATOR VALUE"
+    # @param query_filter str : A query_filter string
+    # @param cls
+    # @return a tuple (FIELD, OPERATOR, VALUE)
+    @classmethod
+    def split_filter(cls, query_filter):
+        if cls._query_re is None:
+            cls.__compile_query_re()
+        matches = cls._query_re.match(query_filter)
+        if not matches:
+            raise ValueError("The query_filter '%s' seems to be invalid"%query_filter)
+        result = (matches.group('field'), re.sub(r'\s', ' ', matches.group('operator'), count=0), matches.group('value').strip())
+        for r in result:
+            if len(r) == 0:
+                raise ValueError("The query_filter '%s' seems to be invalid"%query_filter)
+        return result
+
+    ## @brief Compile the regex for query_filter processing
+    # @note Set _LeObject._query_re
+    @classmethod
+    def __compile_query_re(cls):
+        op_re_piece = '(?P<operator>(%s)'%cls._query_operators[0].replace(' ', '\s')
+        for operator in cls._query_operators[1:]:
+            op_re_piece += '|(%s)'%operator.replace(' ', '\s')
+        op_re_piece += ')'
+        cls._query_re = re.compile('^\s*(?P<field>(((superior)|(subordinate))\.)?[a-z_][a-z0-9\-_]*)\s*'+op_re_piece+'\s*(?P<value>[^<>=!].*)\s*$', flags=re.IGNORECASE)
+        pass
 
-        check_ok=check_tuple(query_filter, fieldnames, self.__target_class)
-        if check_ok:            
-            self.__query_filter = query_filter
-            
-		def execute(self, datasource, **datas = None):
-			super().execute(datasource, **datas)
+    @classmethod
+    def __check_field(cls, target_class, fieldname):
+        try:
+            target_class.field(fieldname)
+        except NameError:
+            tc_name = target_class.__name__
+            return ValueError("No such field '%s' in %s" % (    fieldname,
+                                                                tc_name))
+
+    ##@brief Prepare a relational filter
+    #
+    #Relational filters are composed of a tuple like the simple filters
+    #but the first element of this tuple is a tuple to :
+    #
+    #<code>( (FIELDNAME, {REF_CLASS: REF_FIELD}), OP, VALUE)</code>
+    # Where :
+    #- FIELDNAME is the field name is the target class
+    #- the second element is a dict with :
+    # - REF_CLASS as key. It's a LeObject child class
+    # - REF_FIELD as value. The name of the referenced field in the REF_CLASS
+    #
+    #Visibly the REF_FIELD value of the dict will vary only when
+    #no REF_FIELD is explicitly given in the filter string notation
+    #and REF_CLASSES has differents uid
+    #
+    #@par String notation examples
+    #<pre>contributeur IN (1,2,3,5)</pre> will be transformed into :
+    #<pre>(
+    #       (
+    #           contributeur, 
+    #           {
+    #               auteur: 'lodel_id',
+    #               traducteur: 'lodel_id'
+    #           } 
+    #       ),
+    #       ' IN ',
+    #       [ 1,2,3,5 ])</pre>
+    #@todo move the documentation to another place
+    #
+    #@param fieldname str : The relational field name
+    #@param ref_field str|None : The referenced field name (if None use
+    #uniq identifiers as referenced field
+    #@return a well formed relational filter tuple or an Exception instance
+    @classmethod
+    def __prepare_relational_fields(cls, fieldname, ref_field = None):
+        datahandler = self.__target_class.field(fieldname)
+        # now we are going to fetch the referenced class to see if the
+        # reference field is valid
+        ref_classes = datahandler.linked_classes
+        ref_dict = dict()
+        if ref_field is None:
+            for ref_class in ref_classes:
+                ref_dict[ref_class] = ref_class.uid_fieldname
+        else:
+            for ref_class in ref_classes:
+                if ref_field in ref_class.fieldnames(True):
+                    ref_dict[ref_class] = ref_field
+                else:
+                    logger.debug("Warning the class %s is not considered in \
+the relational filter %s" % ref_class.__name__)
+        if len(ref_dict) == 0:
+            return NameError(   "No field named '%s' in referenced objects"
+                                % ref_field)
+        return ( (fieldname, ref_dict), op, value)
+ 
 
 ##@brief A query to insert a new object
 class LeInsertQuery(LeQuery):
@@ -178,7 +333,7 @@ class LeUpdateQuery(LeFilteredQuery):
         l_uids=datasource.select(self.__target_class,list(self.__target_class.getuid()),query_filter,None, None, None, None, 0, False)
         # list of dict l_uids : _uid(s) of the objects to be updated, corresponding datas
         nb_updated = datasource.update(self.__target_class,l_uids, **datas)
-        if (nb_updated != len(l_uids):
+        if nb_updated != len(l_uids):
             raise LeQueryError("Number of updated items: %d is not as expected: %d " % (nb_updated, len(l_uids)))
         return nb_updated
     
@@ -206,7 +361,7 @@ class LeDeleteQuery(LeFilteredQuery):
         l_uids=datasource.select(self.__target_class,list(self.__target_class.getuid()),query_filter,None, None, None, None, 0, False)
         # list of dict l_uids : _uid(s) of the objects to be deleted
         nb_deleted = datasource.update(self.__target_class,l_uids, **datas)
-        if (nb_deleted != len(l_uids):
+        if nb_deleted != len(l_uids):
             raise LeQueryError("Number of deleted items %d is not as expected %d " % (nb_deleted, len(l_uids)))
         return nb_deleted
 

lodel/plugin/plugins.py

@@ -87,12 +87,11 @@ class Plugins(object):
         
     ##@brief Bootstrap the Plugins class
     @classmethod
-    def bootstrap(cls, plugins_directories):
+    def bootstrap(cls):
         from lodel.settings import Settings
-        cls.start(Settings.plugins_path)
+        for plugin_name in Settings.plugins:
+            cls.load_plugin(plugin_name)
     
-    ##@brief Start the Plugins class by explicitly giving a plugin directory path
-    # @param plugins_directories list : List of path
     @classmethod
     def start(cls, plugins_directories):
         import inspect

lodel/settings/settings_loader.py

@@ -5,6 +5,7 @@ import glob
 import copy
 
 from lodel.settings.utils import *
+from lodel.settings.validator import SettingsValidationError
 
    
 ##@brief Merges and loads configuration files
@@ -60,7 +61,14 @@ class SettingsLoader(object):
             sec=conf[section]
             if keyname in sec:
                 optionstr=sec[keyname]['value']
-                option= validator(sec[keyname]['value'])
+                try:
+                    option= validator(sec[keyname]['value'])
+                except Exception as e:
+                    raise SettingsValidationError(
+                                                    "For %s.%s : %s" % 
+                                                    (section, keyname,e)
+                    )
+
                 try:
                     del self.__conf_sv[section + ':' + keyname]
                 except KeyError: #allready fetched

lodel/settings/validator.py

@@ -168,15 +168,20 @@ def directory_val(value):
 def loglevel_val(value):
     valids = ['DEBUG', 'INFO', 'SECURITY', 'ERROR', 'CRITICAL']
     if value.upper() not in valids:
-        raise SettingsValidationError("The value '%s' is not a valid loglevel")
+        raise SettingsValidationError(
+                "The value '%s' is not a valid loglevel" % value)
     return value.upper()
 
 def path_val(value):
     if not os.path.exists(value):
-        raise SettingsValidationError("The value '%s' is not a valid path")
+        raise SettingsValidationError(
+                "path '%s' doesn't exists" % value)
     return value
 
-def dummy_val(value): return value
+def none_val(value):
+    if value is None:
+        return None
+    raise SettingsValidationError("This settings cannot be set in configuration file")
 
 #
 #   Default validators registration
@@ -184,9 +189,14 @@ def dummy_val(value): return value
 
 SettingValidator.register_validator(
                                         'dummy',
-                                        dummy_val,
+                                        lambda value:value,
                                         'Validate anything')
 
+SettingValidator.register_validator(
+                                        'none',
+                                        none_val,
+                                        'Validate None')
+
 SettingValidator.register_validator(
                                         'strip',
                                         str.strip,

lodel/template/loader.py

@@ -29,7 +29,7 @@ class TemplateLoader(object):
     #
     # @return str. String containing the HTML output of the processed templated
     def render_to_html(self, template_file, template_vars={}, template_extra=None):
-        loader = jinja2.FileSystemLoader(searchpath=self.search_path, followlinks=self.follow_links)
+        loader = jinja2.FileSystemLoader(searchpath=self.search_path)
         environment = jinja2.Environment(loader=loader) if self.is_cache_active else jinja2.Environment(loader=loader,
                                                                                                         cache_size=0)
         template = environment.get_template(template_file)
@@ -68,4 +68,4 @@ class TemplateLoader(object):
     # @param key str
     # @return bool
     def _is_allowed_template_key(self, key):
-        return False if key in self.__class__.__reserved_template_keys else True
+        return False if key in self.__class__.__reserved_template_keys else True

plugins/webui/confspec.py

@@ -0,0 +1,22 @@
+#-*- coding: utf-8 -*-
+
+from lodel.settings.validator import SettingValidator
+
+CONFSPEC = {
+    'lodel2.webui': {
+        'standalone': ( False,
+                        SettingValidator('bool')),
+        'listen_address': ( '127.0.0.1',
+                            SettingValidator('dummy')),
+        'listen_port': (    '9090',
+                            SettingValidator('int')),
+    },
+    'lodel2.webui.sessions': {
+        'directory': (  '/tmp/lodel2_session',
+                        SettingValidator('path')),
+        'expiration': ( 900,
+                        SettingValidator('int')),
+        'file_template': (  'lodel2_%s.sess',
+                            SettingValidator('dummy')),
+    }
+}

plugins/webui/main.py

@@ -0,0 +1,15 @@
+#-*- coding: utf-8 -*-
+
+import os
+from lodel.plugin import LodelHook
+from lodel.settings import Settings
+
+##@brief uwsgi startup demo
+@LodelHook('lodel2_loader_main')
+def uwsgi_fork(hook_name, caller, payload):
+    if Settings.webui.standalone:
+        cmd='uwsgi_python3 --http-socket {addr}:{port} --module run'
+        cmd = cmd.format(
+                    addr = Settings.webui.listen_address,
+                    port = Settings.webui.listen_port)
+        exit(os.system(cmd))

requirements.txt

@@ -1,12 +1,13 @@
 astroid==1.4.4
 itsdangerous==0.24
-Jinja2==2.8
+Jinja2==2.7.3
 lazy-object-proxy==1.2.1
 logilab-common==1.2.0
 MarkupSafe==0.23
 pep8==1.6.2
 pylint==1.4.4
-pymongo==3.2.2
+pymongo==3.0.3
 six==1.10.0
 Werkzeug==0.11.4
 wrapt==1.10.6
+lxml==3.6.0

scripts/create_instance.sh

@@ -29,8 +29,19 @@ fi
 
 echo "Creating lodel instance directory '$instdir'"
 mkdir -pv "$instdir"
+mkdir -pv "$instdir/sessions"
+chmod 700 "$instdir/sessions"
+
+#cp -Rv $libdir/install/* $instdir
+cp -Rv $libdir/install/conf.d $instdir/
+cp -Rv $libdir/install/loader.py $instdir/
+cp -Rv $libdir/install/editorial_model.pickle $instdir/
+ln -sv $libdir/install/Makefile $instdir/Makefile
+ln -sv $libdir/install/run.py $instdir/run.py
+ln -sv $libdir/install/lodel_admin.py $instdir/lodel_admin.py
+ln -sv $libdir/plugins $instdir/plugins
+
 
-cp -Rv $libdir/install/* $instdir
 
 # Adding lib path to loader
 sed -i -E "s#^(LODEL2_LIB_ABS_PATH = )None#\1'$libdir'#" "$loader"