lodel2_mirror/lodel/plugins/mongodb_datasource/datasource.py

# -*- coding: utf-8 -*-

## @package plugins.mongodb_datasource.datasource Main datasource module
# 
# In this class, there is the MongoDbDatasource class, that handles the basic 
# operations that can be done (CRUD ones).

import re
import warnings
import copy
import functools
from bson.son import SON
from collections import OrderedDict
import pymongo
from pymongo.errors import BulkWriteError

from lodel.context import LodelContext
LodelContext.expose_modules(globals(), {
    'lodel.logger': 'logger',
    'lodel.leapi.leobject': ['CLASS_ID_FIELDNAME'],
    'lodel.leapi.datahandlers.base_classes': ['Reference', 'MultipleRef'],
    'lodel.exceptions': ['LodelException', 'LodelFatalError'],
    'lodel.plugin.datasource_plugin': ['AbstractDatasource']})

from . import utils
from .exceptions import *
from .utils import object_collection_name, collection_name, \
    MONGODB_SORT_OPERATORS_MAP, connection_string, mongo_fieldname


## @brief Datasource class
# @ingroup plugin_mongodb_datasource
class MongoDbDatasource(AbstractDatasource):

    ## @brief Stores existing connections
    #
    # The key of this dict is a hash built upon the connection string and the 
    # ro (read-only) parameter.
    # 
    # The value is a dict with 2 keys :
    # - conn_count : the number of instanciated datasource that use this
    # connection
    # - db : the pymongo database object instance
    _connections = dict()

    ## @brief Mapping from lodel2 operators to mongodb operators
    lodel2mongo_op_map = {
        '=':'$eq', '<=':'$lte', '>=':'$gte', '!=':'$ne', '<':'$lt',
        '>':'$gt', 'in':'$in', 'not in':'$nin' }
    
    ## @brief List of mongodb operators that expect re as value
    mongo_op_re = ['$in', '$nin']
    wildcard_re = re.compile('[^\\\\]\*')

    ## @brief instanciates a database object given a connection name
    # @param host str : hostname or IP
    # @param port int : mongodb listening port
    # @param db_name str
    # @param username str
    # @param password str
    # @param read_only bool : If True the Datasource is for read only, else the
    # Datasource is write only !
    def __init__(self, host, port, db_name, username, password, read_only = False):
        ## @brief Connections infos that can be kept securly
        self.__db_infos = {'host': host, 'port': port, 'db_name': db_name}
        ## @brief Is the instance read only ? (if not it's write only)
        self.__read_only = bool(read_only)
        ## @brief Uniq ID for mongodb connection
        self.__conn_hash= None
        ## @brief Stores the database cursor
        self.database = self.__connect(
            username, password, db_name, self.__read_only)

    ## @brief Destructor that attempt to close connection to DB
    #
    # Decrease the conn_count of associated MongoDbDatasource::_connections
    # item. If it reach 0 close the connection to the db
    # @see MongoDbDatasource::__connect()
    def __del__(self):
        self._connections[self.__conn_hash]['conn_count'] -= 1
        if self._connections[self.__conn_hash]['conn_count'] <= 0:
            self._connections[self.__conn_hash]['db'].close()
            del(self._connections[self.__conn_hash])
            logger.info("Closing connection to database")

    ## @brief Provides a new uniq numeric ID
    # @param emcomp LeObject subclass (not instance) : To know on wich things we
    # have to be uniq
    # @warning multiple UID broken by this method
    # @return an integer
    def new_numeric_id(self, emcomp):
        target = emcomp.uid_source()
        tuid = target._uid[0] # Multiple UID broken here
        results = self.select(
            target, field_list = [tuid], filters = [],
            order=[(tuid, 'DESC')], limit = 1)
        if len(results) == 0:
            return 1
        return results[0][tuid]+1

    ## @brief returns a selection of documents from the datasource
    # @param target Emclass
    # @param field_list list
    # @param filters list : List of filters
    # @param relational_filters list : List of relational filters
    # @param order list : List of column to order. ex: order = [('title', 'ASC'),]
    # @param group list : List of tupple representing the column used as
    # "group by" fields. ex: group = [('title', 'ASC'),]
    # @param limit int : Number of records to be returned
    # @param offset int: used with limit to choose the start record
    # @return list
    # @todo Implement group for abstract LeObject childs
    def select(self, target, field_list, filters = None,
            relational_filters=None, order=None, group=None, limit=None,
            offset=0):
        if target.is_abstract():
            # Reccursive calls for abstract LeObject child
            results =  self.__act_on_abstract(target, filters,
                relational_filters, self.select, field_list = field_list,
                order = order, group = group, limit = limit)

            # Here we may implement the group
            # If sorted query we have to sort again
            if order is not None:
                key_fun = functools.cmp_to_key(
                    self.__generate_lambda_cmp_order(order))
                results = sorted(results, key=key_fun)
            # If limit given apply limit again
            if offset > len(results):
                results = list()
            else:
                if limit is not None:
                    if limit + offset > len(results):
                        limit = len(results)-offset-1
                    results = results[offset:offset+limit]
            return results
        # Default behavior
        if filters is None:
            filters = list()
        if relational_filters is None:
            relational_filters = list()

        collection_name = object_collection_name(target)
        collection = self.database[collection_name]

        query_filters = self.__process_filters(
            target, filters, relational_filters)

        query_result_ordering = None
        if order is not None:
            query_result_ordering = utils.parse_query_order(order)

        if group is None:
            if field_list is None:
                field_list = dict()
            else:
                f_list=dict()
                for fl in field_list:
                    f_list[fl] = 1
                field_list = f_list
            field_list['_id'] = 0
            cursor = collection.find(
                spec = query_filters,
                fields=field_list,
                skip=offset,
                limit=limit if limit != None else 0,
                sort=query_result_ordering)
        else:
            pipeline = list()
            unwinding_list = list()
            grouping_dict = OrderedDict()
            sorting_list = list()
            for group_param in group:
                field_name = group_param[0]
                field_sort_option = group_param[1]
                sort_option = MONGODB_SORT_OPERATORS_MAP[field_sort_option]
                unwinding_list.append({'$unwind': '$%s' % field_name})
                grouping_dict[field_name] = '$%s' % field_name
                sorting_list.append((field_name, sort_option))

            sorting_list.extends(query_result_ordering)

            pipeline.append({'$match': query_filters})
            if field_list is not None:
                pipeline.append({
                    '$project': SON([{field_name: 1}
                    for field_name in field_list])})
            pipeline.extend(unwinding_list)
            pipeline.append({'$group': grouping_dict})
            pipeline.extend({'$sort': SON(sorting_list)})
            if offset > 0:
                pipeline.append({'$skip': offset})
            if limit is not None:
                pipeline.append({'$limit': limit})

        results = list()
        for document in cursor:
            results.append(document)

        return results

    ## @brief Deletes records according to given filters
    # @param target Emclass : class of the record to delete
    # @param filters list : List of filters
    # @param relational_filters list : List of relational filters
    # @return int : number of deleted records
    def delete(self, target, filters, relational_filters):
        if target.is_abstract():
            logger.debug("Delete called on %s filtered by (%s,%s). Target is \
abstract, preparing reccursiv calls" % (target, filters, relational_filters))
            # Deletion with abstract LeObject as target (reccursiv calls)
            return self.__act_on_abstract(target, filters,
                relational_filters, self.delete)
        logger.debug("Delete called on %s filtered by (%s,%s)." % (
            target, filters, relational_filters))
        # Non abstract beahavior
        mongo_filters = self.__process_filters(
            target, filters, relational_filters)
        # Updating backref before deletion
        self.__update_backref_filtered(target, filters, relational_filters,
            None)
        res = self.__collection(target).remove(mongo_filters)
        return res['n']

    ## @brief updates records according to given filters
    # @param target Emclass : class of the object to insert
    # @param filters list : List of filters
    # @param relational_filters list : List of relational filters
    # @param upd_datas dict : datas to update (new values)
    # @return int : Number of updated records
    def update(self, target, filters, relational_filters, upd_datas):
        self._data_cast(upd_datas)
        #fetching current datas state
        mongo_filters = self.__process_filters(
            target, filters, relational_filters)
        old_datas_l = self.__collection(target).find(
            mongo_filters)
        old_datas_l = list(old_datas_l)
        #Running update
        res = self.__update_no_backref(target, filters, relational_filters,
            upd_datas)
        #updating backref
        self.__update_backref_filtered(target, filters, relational_filters,
            upd_datas, old_datas_l)
        return res

    ## @brief Designed to be called by backref update in order to avoid
    # infinite updates between back references
    # @see update()
    def __update_no_backref(self, target, filters, relational_filters,
            upd_datas):
        logger.debug("Update called on %s filtered by (%s,%s) with datas \
%s" % (target, filters, relational_filters, upd_datas))
        if target.is_abstract():
            #Update using abstract LeObject as target (reccursiv calls)
            return self.__act_on_abstract(target, filters,
                relational_filters, self.update, upd_datas = upd_datas)
        #Non abstract beahavior
        mongo_filters = self.__process_filters(
            target, filters, relational_filters)
        self._data_cast(upd_datas)
        mongo_arg = {'$set': upd_datas }
        res = self.__collection(target).update(mongo_filters, mongo_arg)
        return res['n']

    ## @brief Inserts a record in a given collection
    # @param target Emclass : class of the object to insert
    # @param new_datas dict : datas to insert
    # @return the inserted uid
    def insert(self, target, new_datas):
        self._data_cast(new_datas)
        logger.debug("Insert called on %s with datas : %s"% (
            target, new_datas))
        uidname = target.uid_fieldname()[0] #MULTIPLE UID BROKEN HERE
        if uidname not in new_datas:
            raise MongoDataSourceError("Missing UID data will inserting a new \
%s" % target.__class__)
        res = self.__collection(target).insert(new_datas)
        self.__update_backref(target, new_datas[uidname], None, new_datas)
        return str(res)

    ## @brief Inserts a list of records in a given collection
    # @param target Emclass : class of the objects inserted
    # @param datas_list list : list of dict
    # @return list : list of the inserted records' ids
    def insert_multi(self, target, datas_list):
        for datas in datas_list:
            self._data_cast(datas)
        res = self.__collection(target).insert_many(datas_list)
        for new_datas in datas_list:
            self.__update_backref(target, None, new_datas)
            target.make_consistency(datas=new_datas)
        return list(res.inserted_ids)

    ## @brief Update backref giving an action
    # @param target leObject child class
    # @param filters
    # @param relational_filters,
    # @param new_datas None | dict : optional new datas if None mean we are deleting
    # @param old_datas_l None | list : if None fetch old datas from db (usefull
    # when modifications are made on instance before updating backrefs)
    # @return nothing (for the moment
    def __update_backref_filtered(self, target,
            filters, relational_filters, new_datas = None, old_datas_l = None):
        # Getting all the UID of the object that will be deleted in order
        # to update back_references
        if old_datas_l is None:
            mongo_filters = self.__process_filters(
                target, filters, relational_filters)
            old_datas_l = self.__collection(target).find(
                mongo_filters)
            old_datas_l = list(old_datas_l)

        uidname = target.uid_fieldname()[0]  # MULTIPLE UID BROKEN HERE
        for old_datas in old_datas_l:
            self.__update_backref(
                target, old_datas[uidname], old_datas, new_datas)

    ## @brief Update back references of an object
    # @ingroup plugin_mongodb_bref_op
    #
    # old_datas and new_datas arguments are set to None to indicate
    # insertion or deletion. Calls examples :
    # @par LeObject insert __update backref call
    # <pre>
    # Insert(datas):
    #   self.make_insert(datas)
    #   self.__update_backref(self.__class__, None, datas)
    # </pre>
    # @par LeObject delete __update backref call
    # Delete()
    #   old_datas = self.datas()
    #   self.make_delete()
    #   self.__update_backref(self.__class__, old_datas, None)
    # @par LeObject update __update_backref call
    # <pre>
    # Update(new_datas):
    #   old_datas = self.datas()
    #   self.make_udpdate(new_datas)
    #   self.__update_backref(self.__class__, old_datas, new_datas)
    # </pre>
    #
    # @param target LeObject child classa
    # @param tuid mixed : The target UID (the value that will be inserted in
    # back references)
    # @param old_datas dict : datas state before update
    # @param new_datas dict : datas state after the update process
    def __update_backref(self, target, tuid, old_datas, new_datas):
        #upd_dict is the dict that will allow to run updates in an optimized
        #way (or try to help doing it)
        #
        #Its structure looks like :
        # { LeoCLASS : {
        #       UID1: (
        #           LeoINSTANCE,
        #           { fname1 : value, fname2: value }),
        #       UID2 (LeoINSTANCE, {fname...}),
        #       },
        #   LeoClass2: {...
        #
        upd_dict = {}
        for fname, fdh in target.reference_handlers().items():
            oldd = old_datas is not None and fname in old_datas and \
                (not hasattr(fdh, 'default') or old_datas[fname] != fdh.default) \
                and not old_datas[fname] is None
            newd = new_datas is not None and fname in new_datas and \
                (not hasattr(fdh, 'default') or new_datas[fname] != fdh.default) \
                and not new_datas[fname] is None
            if (oldd and newd and old_datas[fname] == new_datas[fname])\
                    or not(oldd or newd):
                # No changes or not concerned
                continue
            bref_cls = fdh.back_reference[0]
            bref_fname = fdh.back_reference[1]
            if not fdh.is_singlereference():
                # fdh is a multiple reference. So the update preparation will be
                # divided into two loops :
                # - one loop for deleting old datas
                # - one loop for inserting updated datas
                #
                # Preparing the list of values to delete or to add
                if newd and oldd:
                    old_values = old_datas[fname]
                    new_values = new_datas[fname]
                    to_del = [  val
                                for val in old_values
                                if val not in new_values]
                    to_add = [  val
                                for val in new_values
                                if val not in old_values]
                elif oldd and not newd:
                    to_del = old_datas[fname]
                    to_add = []
                elif not oldd and newd:
                    to_del = []
                    to_add = new_datas[fname]
                # Calling __back_ref_upd_one_value() with good arguments
                for vtype, vlist in [('old',to_del), ('new', to_add)]:
                    for value in vlist:
                        # fetching backref infos
                        bref_infos = self.__bref_get_check(
                            bref_cls, value, bref_fname)
                        # preparing the upd_dict
                        upd_dict = self.__update_backref_upd_dict_prepare(
                            upd_dict, bref_infos, bref_fname, value)
                        # preparing updated bref_infos
                        bref_cls, bref_leo, bref_dh, bref_value = bref_infos
                        bref_infos = (bref_cls, bref_leo, bref_dh,
                            upd_dict[bref_cls][value][1][bref_fname])
                        vdict = {vtype: value}
                        # fetch and store updated value
                        new_bref_val = self.__back_ref_upd_one_value(
                            fname, fdh, tuid, bref_infos, **vdict)
                        upd_dict[bref_cls][value][1][bref_fname] = new_bref_val
            else:
                # fdh is a single ref so the process is simpler, we do not have
                # to loop and we may do an update in only one
                # __back_ref_upd_one_value() call by giving both old and new
                # value
                vdict = {}
                if oldd:
                    vdict['old'] = old_datas[fname]
                    uid_val = vdict['old']
                if newd:
                    vdict['new'] = new_datas[fname]
                    if not oldd:
                        uid_val = vdict['new']
                # Fetching back ref infos
                bref_infos = self.__bref_get_check(
                    bref_cls, uid_val, bref_fname)
                # prepare the upd_dict
                upd_dict = self.__update_backref_upd_dict_prepare(
                    upd_dict, bref_infos, bref_fname, uid_val)
                # forging update bref_infos
                bref_cls, bref_leo, bref_dh, bref_value = bref_infos
                bref_infos = (bref_cls, bref_leo, bref_dh,
                        upd_dict[bref_cls][uid_val][1][bref_fname])
                # fetch and store updated value
                new_bref_val = self.__back_ref_upd_one_value(
                    fname, fdh, tuid, bref_infos, **vdict)
                upd_dict[bref_cls][uid_val][1][bref_fname] = new_bref_val
        # Now we've got our upd_dict ready.
        # running the updates
        for bref_cls, uid_dict in upd_dict.items():
            for uidval, (leo, datas) in uid_dict.items():
                # MULTIPLE UID BROKEN 2 LINES BELOW
                self.__update_no_backref(
                    leo.__class__, [(leo.uid_fieldname()[0], '=', uidval)],
                    [], datas)

    ## @brief Utility function designed to handle the upd_dict of __update_backref()
    #
    # Basically checks if a key exists at some level, if not create it with
    # the good default value (in most case dict())
    # @param upd_dict dict : in & out args modified by reference
    # @param bref_infos tuple : as returned by __bref_get_check()
    # @param bref_fname str : name of the field in referenced class
    # @param uid_val mixed : the UID of the referenced object
    # @return the updated version of upd_dict
    @staticmethod
    def __update_backref_upd_dict_prepare(upd_dict,bref_infos, bref_fname,
            uid_val):
        bref_cls, bref_leo, bref_dh, bref_value = bref_infos
        if bref_cls not in upd_dict:
            upd_dict[bref_cls] = {}
        if uid_val not in upd_dict[bref_cls]:
            upd_dict[bref_cls][uid_val] = (bref_leo, {})
        if bref_fname not in upd_dict[bref_cls][uid_val]:
            upd_dict[bref_cls][uid_val][1][bref_fname] = bref_value
        return upd_dict


    ## @brief Prepare a one value back reference update
    # @param fname str : the source Reference field name
    # @param fdh DataHandler : the source Reference DataHandler
    # @param tuid mixed : the uid of the Leo that make reference to the backref
    # @param bref_infos tuple : as returned by __bref_get_check() method
    # @param old mixed : (optional **values) the old value
    # @param new mixed : (optional **values) the new value
    # @return the new back reference field value
    def __back_ref_upd_one_value(self, fname, fdh, tuid, bref_infos, **values):
        bref_cls, bref_leo, bref_dh, bref_val = bref_infos
        oldd = 'old' in values
        newdd = 'new' in values
        if bref_val is None:
            bref_val = bref_dh.empty()
        if not bref_dh.is_singlereference():
            if oldd and newdd:
                if tuid not in bref_val:
                    raise MongoDbConsistencyError("The value we want to \
delete in this back reference update was not found in the back referenced \
object : %s. Value was : '%s'" % (bref_leo, tuid))
                return bref_val
            elif oldd and not newdd:
                # deletion
                old_value = values['old']
                if tuid not in bref_val:
                    raise MongoDbConsistencyError("The value we want to \
delete in this back reference update was not found in the back referenced \
object : %s. Value was : '%s'" % (bref_leo, tuid))
                if isinstance(bref_val, tuple):
                    bref_val = set(bref_val)
                if isinstance(bref_val, set):
                    bref_val -= set([tuid])
                else:
                    del(bref_val[bref_val.index(tuid)])
            elif not oldd and newdd:
                if tuid in bref_val:
                    raise MongoDbConsistencyError("The value we want to \
add in this back reference update was found in the back referenced \
object : %s. Value was : '%s'" % (bref_leo, tuid))
                if isinstance(bref_val, tuple):
                    bref_val = set(bref_val)
                if isinstance(bref_val, set):
                    bref_val |= set([tuid])
                else:
                    bref_val.append(tuid)
        else:
            # Single value backref
            if oldd and newdd:
                if bref_val != tuid:
                    raise MongoDbConsistencyError("The backreference doesn't \
have expected value. Expected was %s but found %s in %s" % (
                        tuid, bref_val, bref_leo))
                return bref_val
            elif oldd and not newdd:
                # deletion
                if not hasattr(bref_dh, "default"):
                    raise MongoDbConsistencyError("Unable to delete a \
value for a back reference update. The concerned field don't have a default \
value : in %s field %s" % (bref_leo,fname))
                bref_val = getattr(bref_dh, "default")
            elif not oldd and newdd:
                bref_val = tuid
        return bref_val

    ## @brief Fetch back reference informations
    # @warning thank's to __update_backref_act() this method is useless
    # @param bref_cls LeObject child class : __back_reference[0]
    # @param uidv mixed : UID value (the content of the reference field)
    # @param bref_fname str : the name of the back_reference field
    # @return tuple(bref_class, bref_LeObect_instance, bref_datahandler, bref_value)
    # @throw MongoDbConsistencyError when LeObject instance not found given uidv
    # @throw LodelFatalError if the back reference field is not a Reference subclass (major failure)
    def __bref_get_check(self, bref_cls, uidv, bref_fname):
        bref_leo = bref_cls.get_from_uid(uidv)
        if bref_leo is None:
            raise MongoDbConsistencyError("Unable to get the object we make \
reference to : %s with uid = %s" % (bref_cls, repr(uidv)))
        bref_dh = bref_leo.data_handler(bref_fname)
        if not bref_dh.is_reference():
            raise LodelFatalError("Found a back reference field that \
is not a reference : '%s' field '%s'" % (bref_leo, bref_fname))
        bref_val = bref_leo.data(bref_fname)
        return (bref_leo.__class__, bref_leo, bref_dh, bref_val)

    ## @brief Act on abstract LeObject child
    #
    # This method is designed to be called by insert, select and delete method
    # when they encounter an abtract class
    # @param target LeObject child class
    # @param filters
    # @param relational_filters
    # @param act function : the caller method
    # @param **kwargs other arguments
    # @return sum of results (if it's an array it will result in a concat)
    # @todo optimization implementing a cache for __bref_get_check()
    def __act_on_abstract(self,
        target, filters, relational_filters, act, **kwargs):

        logger.debug("Abstract %s, running reccursiv select \
on non abstract childs" % act.__name__)
        result = list() if act == self.select else 0
        if not target.is_abstract():
            target_childs = [target]
        else:
            target_childs = [tc for tc in target.child_classes()
                if not tc.is_abstract()]
        for target_child in target_childs:
            logger.debug(
                "Abstract %s on %s" % (act.__name__, target_child.__name__))
            # Add target_child to filter
            new_filters = copy.copy(filters)
            for i in range(len(filters)):
                fname, op, val = filters[i]
                if fname == CLASS_ID_FIELDNAME:
                    logger.warning("Dirty drop of filter : '%s %s %s'" % (
                        fname, op, val))
                    del(new_filters[i])
            new_filters.append(
                (CLASS_ID_FIELDNAME, '=',
                    collection_name(target_child.__name__)))
            result += act(
                target = target_child,
                filters = new_filters,
                relational_filters = relational_filters,
                **kwargs)
        return result

    ## @brief Connect to database
    # @note this method avoid opening two times the same connection using MongoDbDatasource::_connections static attribute
    # @param username str
    # @param password str
    # @param ro bool : If True the Datasource is for read only, else it will be write only
    def __connect(self, username, password, db_name, ro):
        conn_string = connection_string(
            username = username, password = password,
            host = self.__db_infos['host'],
            port = self.__db_infos['port'],
            db_name = db_name,
            ro = ro)

        self.__conn_hash = conn_h = hash(conn_string)
        if conn_h in self._connections:
            self._connections[conn_h]['conn_count'] += 1
            return self._connections[conn_h]['db'][self.__db_infos['db_name']]
        else:
            logger.info("Opening a new connection to database")
            self._connections[conn_h] = {
                'conn_count': 1,
                'db': utils.connect(conn_string)}
            return self._connections[conn_h]['db'][self.__db_infos['db_name']]


    ## @brief Return a pymongo collection given a LeObject child class
    # @param leobject LeObject child class (no instance)
    # @return a pymongo.collection instance
    def __collection(self, leobject):
        return self.database[object_collection_name(leobject)]

    ## @brief Perform subqueries implies by relational filters and append the
    # result to existing filters
    #
    # The processing is divided in multiple steps :
    #  - determine (for each relational field of the target)  every collection that are involved
    #  - generate subqueries for relational_filters that concerns a different collection than target collection filters
    #  - execute subqueries
    #  - transform subqueries results in filters
    #  - merge subqueries generated filters with existing filters
    #
    # @param target LeObject subclass (no instance) : Target class
    # @param filters list : List of tuple(FIELDNAME, OP, VALUE)
    # @param relational_filters : same composition thant filters except that FIELD is represented by a tuple(FIELDNAME, {CLASS1:RFIELD1, CLASS2:RFIELD2})
    # @return a list of pymongo filters ( dict {FIELD:{OPERATOR:VALUE}} )
    def __process_filters(self,target, filters, relational_filters):
        # Simple filters lodel2 -> pymongo converting
        res = self.__filters2mongo(filters, target)
        rfilters = self.__prepare_relational_filters(target, relational_filters)
        #Now that everything is well organized, begin to forge subquerie
        #filters
        self.__subqueries_from_relational_filters(target, rfilters)
        # Executing subqueries, creating filters from result, and injecting
        # them in original filters of the query
        if len(rfilters) > 0:
            logger.debug("Begining subquery execution")
        for fname in rfilters:
            if fname not in res:
                res[fname] = dict()
            subq_results = set()
            for leobject, sq_filters in rfilters[fname].items():
                uid_fname = mongo_fieldname(leobject._uid)
                log_msg = "Subquery running on collection {coll} with filters \
'{filters}'"
                logger.debug(log_msg.format(
                    coll=object_collection_name(leobject),
                    filters=sq_filters))

                cursor = self.__collection(leobject).find(
                    filter=sq_filters,
                    projection=uid_fname)
                subq_results |= set(doc[uid_fname] for doc in cursor)
            #generating new filter from result
            if '$in' in res[fname]:
                #WARNING we allready have a IN on this field, doing dedup
                #from result
                deduped = set(res[fname]['$in']) & subq_results
                if len(deduped) == 0:
                    del(res[fname]['$in'])
                else:
                    res[fname]['$in'] = list(deduped)
            else:
                res[fname]['$in'] = list(subq_results)
        if len(rfilters) > 0:
            logger.debug("End of subquery execution")
        return res

    ## @brief Generate subqueries from rfilters tree
    #
    # Returned struct organization :
    #  - 1st level keys : relational field name of target
    #  - 2nd level keys : referenced leobject
    #  - 3th level values : pymongo filters (dict)
    #
    # @note The only caller of this method is __process_filters
    # @warning No return value, the rfilters arguement is modified by reference
    #
    # @param target LeObject subclass (no instance) : Target class
    # @param rfilters dict : A struct as returned by MongoDbDatasource.__prepare_relational_filters()
    # @return None, the rfilters argument is modified by reference
    @classmethod
    def __subqueries_from_relational_filters(cls, target, rfilters):
        for fname in rfilters:
            for leobject in rfilters[fname]:
                for rfield in rfilters[fname][leobject]:
                    #This way of doing is not optimized but allows to trigger
                    #warnings in some case (2 different values for a same op
                    #on a same field on a same collection)
                    mongofilters = cls.__op_value_listconv(
                        rfilters[fname][leobject][rfield], target.field(fname))
                    rfilters[fname][leobject][rfield] = mongofilters

    ## @brief Generate a tree from relational_filters
    #
    # The generated struct is a dict with :
    #  - 1st level keys : relational field name of target
    #  - 2nd level keys : referenced leobject
    #  - 3th level keys : referenced field in referenced class
    #  - 4th level values : list of tuple(op, value)
    #
    # @note The only caller of this method is __process_filters
    # @warning An assertion is done : if two leobject are stored in the same collection they share the same uid
    #
    # @param target LeObject subclass (no instance) : Target class
    # @param relational_filters : same composition thant filters except that
    # @return a struct as described above
    @classmethod
    def __prepare_relational_filters(cls, target, relational_filters):
        # We are going to regroup relationnal filters by reference field
        # then by collection
        rfilters = dict()
        if relational_filters is None:
            relational_filters = []
        for (fname, rfields), op, value in relational_filters:
            if fname not in rfilters:
                rfilters[fname] = dict()
            rfilters[fname] = dict()
            # Stores the representative leobject for associated to a collection
            # name
            leo_collname = dict()
            # WARNING ! Here we assert that all leobject that are stored
            # in a same collection are identified by the same field
            for leobject, rfield in rfields.items():
                #here we are filling a dict with leobject as index but
                #we are doing a UNIQ on collection name
                cur_collname = object_collection_name(leobject)
                if cur_collname not in leo_collname:
                    leo_collname[cur_collname] = leobject
                    rfilters[fname][leobject] = dict()
                #Fecthing the collection's representative leobject
                repr_leo = leo_collname[cur_collname]

                if rfield not in rfilters[fname][repr_leo]:
                    rfilters[fname][repr_leo][rfield] = list()
                rfilters[fname][repr_leo][rfield].append((op, value))
        return rfilters

    ## @brief Convert lodel2 filters to pymongo conditions
    # @param filters list : list of lodel filters
    # @return dict representing pymongo conditions
    @classmethod
    def __filters2mongo(cls, filters, target):
        res = dict()
        eq_fieldname = [] #Stores field with equal comparison OP
        for fieldname, op, value in filters:
            oop = op
            ovalue = value
            op, value = cls.__op_value_conv(op, value, target.field(fieldname))
            if op == '=':
                eq_fieldname.append(fieldname)
                if fieldname in res:
                    logger.warning("Dropping previous condition. Overwritten \
by an equality filter")
                res[fieldname] = value
                continue
            if fieldname in eq_fieldname:
                logger.warning("Dropping condition : '%s %s %s'" % (
                    fieldname, op, value))
                continue

            if fieldname not in res:
                res[fieldname] = dict()
            if op in res[fieldname]:
                logger.warning("Dropping condition : '%s %s %s'" % (
                    fieldname, op, value))
            else:
                if op not in cls.lodel2mongo_op_map:
                    raise ValueError("Invalid operator : '%s'" % op)
                new_op = cls.lodel2mongo_op_map[op]
                res[fieldname][new_op] = value
        return res


    ## @brief Convert lodel2 operator and value to pymongo struct
    #
    # Convertion is done using MongoDbDatasource::lodel2mongo_op_map
    # @param op str : take value in LeFilteredQuery::_query_operators
    # @param value mixed : the value
    # @param dhdl
    # @return a tuple(mongo_op, mongo_value)
    @classmethod
    def __op_value_conv(cls, op, value, dhdl):
        if op not in cls.lodel2mongo_op_map:
            msg = "Invalid operator '%s' found" % op
            raise MongoDbDataSourceError(msg)
        mongop = cls.lodel2mongo_op_map[op]
        mongoval = value
        # Converting lodel2 wildcarded string into a case insensitive
        # mongodb re
        if mongop in cls.mongo_op_re:
            if value.startswith('(') and value.endswith(')'):
                if (dhdl.cast_type is not None):
                    mongoval = [ dhdl.cast_type(item) for item in mongoval[1:-1].split(',') ]
                else:
                    mongoval = [ item for item in mongoval[1:-1].split(',') ]
        elif mongop == 'like':
            #unescaping \
            mongoval = value.replace('\\\\','\\')
            if not mongoval.startswith('*'):
                mongoval = '^'+mongoval
            #For the end of the string it's harder to detect escaped *
            if not (mongoval[-1] == '*' and mongoval[-2] != '\\'):
                mongoval += '$'
            #Replacing every other unescaped wildcard char
            mongoval = cls.wildcard_re.sub('.*', mongoval)
            mongoval = {'$regex': mongoval, '$options': 'i'}
        return (op, mongoval)

    ## @brief Convert a list of tuple(OP, VALUE) into a pymongo filter dict
    # @param op_value_list list
    # @param dhdl
    # @return a dict with mongo op as key and value as value...
    @classmethod
    def __op_value_listconv(cls, op_value_list, dhdl):
        result = dict()
        for op, value in op_value_list:
            mongop, mongoval = cls.__op_value_conv(op, value, dhdl)
            if mongop in result:
                warnings.warn("Duplicated value given for a single \
field/operator couple in a query. We will keep only the first one")
            else:
                result[mongop] = mongoval
        return result

    ##@brief Generate a comparison function for post reccursion sorting in
    #select
    #@return a lambda function that take 2 dict as arguement
    @classmethod
    def __generate_lambda_cmp_order(cls, order):
        if len(order) == 0:
            return lambda a,b: 0
        glco = cls.__generate_lambda_cmp_order
        fname, cmpdir = order[0]
        order = order[1:]
        return lambda a,b: glco(order)(a,b) if a[fname] == b[fname] else (\
            1 if (a[fname]>b[fname] if cmpdir == 'ASC' else a[fname]<b[fname])\
            else -1)


    ##@brief Correct some datas before giving them to pymongo
    #
    #For example sets has to be casted to lise
    #@param datas
    #@return datas
    @classmethod
    def _data_cast(cls, datas):
        for dname in datas:
            if isinstance(datas[dname], set):
                #pymongo raises :
                #bson.errors.InvalidDocument: Cannot encode object: {...}
                #with sets
                datas[dname] = list(datas[dname])
        return datas