# -*- coding: utf-8 -*-
import re
import warnings
import copy
import functools
from bson.son import SON
from collections import OrderedDict
import pymongo
from pymongo.errors import BulkWriteError
from lodel.context import LodelContext
LodelContext.expose_modules(globals(), {
'lodel.logger': 'logger',
'lodel.leapi.leobject': ['CLASS_ID_FIELDNAME'],
'lodel.leapi.datahandlers.base_classes': ['Reference', 'MultipleRef'],
'lodel.exceptions': ['LodelException', 'LodelFatalError'],
'lodel.plugin.datasource_plugin': ['AbstractDatasource']})
from . import utils
from .exceptions import *
from .utils import object_collection_name, collection_name, \
MONGODB_SORT_OPERATORS_MAP, connection_string, mongo_fieldname
##@brief Datasource class
#@ingroup plugin_mongodb_datasource
class MongoDbDatasource(AbstractDatasource):
##@brief Stores existing connections
#
#The key of this dict is a hash of the connection string + ro parameter.
#The value is a dict with 2 keys :
# - conn_count : the number of instanciated datasource that use this
#connection
# - db : the pymongo database object instance
_connections = dict()
##@brief Mapping from lodel2 operators to mongodb operator
lodel2mongo_op_map = {
'=':'$eq', '<=':'$lte', '>=':'$gte', '!=':'$ne', '<':'$lt',
'>':'$gt', 'in':'$in', 'not in':'$nin' }
##@brief List of mongodb operators that expect re as value
mongo_op_re = ['$in', '$nin']
wildcard_re = re.compile('[^\\\\]\*')
##@brief instanciates a database object given a connection name
#@param host str : hostname or IP
#@param port int : mongodb listening port
#@param db_name str
#@param username str
#@param password str
#@param read_only bool : If True the Datasource is for read only, else the
#Datasource is write only !
def __init__(self, host, port, db_name, username, password, read_only = False):
##@brief Connections infos that can be kept securly
self.__db_infos = {'host': host, 'port': port, 'db_name': db_name}
##@brief Is the instance read only ? (if not it's write only)
self.__read_only = bool(read_only)
##@brief Uniq ID for mongodb connection
self.__conn_hash= None
##@brief Stores the database cursor
self.database = self.__connect(
username, password, db_name, self.__read_only)
##@brief Destructor that attempt to close connection to DB
#
#Decrease the conn_count of associated MongoDbDatasource::_connections
#item. If it reach 0 close the connection to the db
#@see MongoDbDatasource::__connect()
def __del__(self):
self._connections[self.__conn_hash]['conn_count'] -= 1
if self._connections[self.__conn_hash]['conn_count'] <= 0:
self._connections[self.__conn_hash]['db'].close()
del(self._connections[self.__conn_hash])
logger.info("Closing connection to database")
##@brief Provide a new uniq numeric ID
#@param emcomp LeObject subclass (not instance) : To know on wich things we
#have to be uniq
#@warning multiple UID broken by this method
#@return an integer
def new_numeric_id(self, emcomp):
target = emcomp.uid_source()
tuid = target._uid[0] # Multiple UID broken here
results = self.select(
target, field_list = [tuid], filters = [],
order=[(tuid, 'DESC')], limit = 1)
if len(results) == 0:
return 1
return results[0][tuid]+1
##@brief returns a selection of documents from the datasource
#@param target Emclass
#@param field_list list
#@param filters list : List of filters
#@param relational_filters list : List of relational filters
#@param order list : List of column to order. ex: order =
#[('title', 'ASC'),]
#@param group list : List of tupple representing the column used as
#"group by" fields. ex: group = [('title', 'ASC'),]
#@param limit int : Number of records to be returned
#@param offset int: used with limit to choose the start record
#@return list
#@todo Implement group for abstract LeObject childs
def select(self, target, field_list, filters = None,
relational_filters=None, order=None, group=None, limit=None,
offset=0):
if target.is_abstract():
#Reccursiv calls for abstract LeObject child
results = self.__act_on_abstract(target, filters,
relational_filters, self.select, field_list = field_list,
order = order, group = group, limit = limit)
#Here we may implement the group
#If sorted query we have to sort again
if order is not None:
results = sorted(results,
key=functools.cmp_to_key(
self.__generate_lambda_cmp_order(order)))
#If limit given apply limit again
if offset > len(results):
results = list()
else:
if limit is not None:
if limit + offset > len(results):
limit = len(results)-offset-1
results = results[offset:offset+limit]
return results
# Default behavior
if filters is None:
filters = list()
if relational_filters is None:
relational_filters = list()
collection_name = object_collection_name(target)
collection = self.database[collection_name]
query_filters = self.__process_filters(
target, filters, relational_filters)
query_result_ordering = None
if order is not None:
query_result_ordering = utils.parse_query_order(order)
if group is None:
if field_list is None:
field_list = dict()
else:
f_list=dict()
for fl in field_list:
f_list[fl] = 1
field_list = f_list
field_list['_id'] = 0
cursor = collection.find(
spec = query_filters,
fields=field_list,
skip=offset,
limit=limit if limit != None else 0,
sort=query_result_ordering)
else:
pipeline = list()
unwinding_list = list()
grouping_dict = OrderedDict()
sorting_list = list()
for group_param in group:
field_name = group_param[0]
field_sort_option = group_param[1]
sort_option = MONGODB_SORT_OPERATORS_MAP[field_sort_option]
unwinding_list.append({'$unwind': '$%s' % field_name})
grouping_dict[field_name] = '$%s' % field_name
sorting_list.append((field_name, sort_option))
sorting_list.extends(query_result_ordering)
pipeline.append({'$match': query_filters})
if field_list is not None:
pipeline.append({
'$project': SON([{field_name: 1}
for field_name in field_list])})
pipeline.extend(unwinding_list)
pipeline.append({'$group': grouping_dict})
pipeline.extend({'$sort': SON(sorting_list)})
if offset > 0:
pipeline.append({'$skip': offset})
if limit is not None:
pipeline.append({'$limit': limit})
results = list()
for document in cursor:
results.append(document)
return results
##@brief Deletes records according to given filters
#@param target Emclass : class of the record to delete
#@param filters list : List of filters
#@param relational_filters list : List of relational filters
#@return int : number of deleted records
def delete(self, target, filters, relational_filters):
if target.is_abstract():
logger.debug("Delete called on %s filtered by (%s,%s). Target is \
abstract, preparing reccursiv calls" % (target, filters, relational_filters))
#Deletion with abstract LeObject as target (reccursiv calls)
return self.__act_on_abstract(target, filters,
relational_filters, self.delete)
logger.debug("Delete called on %s filtered by (%s,%s)." % (
target, filters, relational_filters))
#Non abstract beahavior
mongo_filters = self.__process_filters(
target, filters, relational_filters)
#Updating backref before deletion
self.__update_backref_filtered(target, filters, relational_filters,
None)
res = self.__collection(target).remove(mongo_filters)
return res['n']
##@brief updates records according to given filters
#@param target Emclass : class of the object to insert
#@param filters list : List of filters
#@param relational_filters list : List of relational filters
#@param upd_datas dict : datas to update (new values)
#@return int : Number of updated records
def update(self, target, filters, relational_filters, upd_datas):
self._data_cast(upd_datas)
#fetching current datas state
mongo_filters = self.__process_filters(
target, filters, relational_filters)
old_datas_l = self.__collection(target).find(
mongo_filters)
old_datas_l = list(old_datas_l)
#Running update
res = self.__update_no_backref(target, filters, relational_filters,
upd_datas)
#updating backref
self.__update_backref_filtered(target, filters, relational_filters,
upd_datas, old_datas_l)
return res
##@brief Designed to be called by backref update in order to avoid
#infinite updates between back references
#@see update()
def __update_no_backref(self, target, filters, relational_filters,
upd_datas):
logger.debug("Update called on %s filtered by (%s,%s) with datas \
%s" % (target, filters, relational_filters, upd_datas))
if target.is_abstract():
#Update using abstract LeObject as target (reccursiv calls)
return self.__act_on_abstract(target, filters,
relational_filters, self.update, upd_datas = upd_datas)
#Non abstract beahavior
mongo_filters = self.__process_filters(
target, filters, relational_filters)
self._data_cast(upd_datas)
mongo_arg = {'$set': upd_datas }
res = self.__collection(target).update(mongo_filters, mongo_arg)
return res['n']
## @brief Inserts a record in a given collection
# @param target Emclass : class of the object to insert
# @param new_datas dict : datas to insert
# @return the inserted uid
def insert(self, target, new_datas):
self._data_cast(new_datas)
logger.debug("Insert called on %s with datas : %s"% (
target, new_datas))
uidname = target.uid_fieldname()[0] #MULTIPLE UID BROKEN HERE
if uidname not in new_datas:
raise MongoDataSourceError("Missing UID data will inserting a new \
%s" % target.__class__)
res = self.__collection(target).insert(new_datas)
self.__update_backref(target, new_datas[uidname], None, new_datas)
return str(res)
## @brief Inserts a list of records in a given collection
# @param target Emclass : class of the objects inserted
# @param datas_list list : list of dict
# @return list : list of the inserted records' ids
def insert_multi(self, target, datas_list):
for datas in datas_list:
self._data_cast(datas)
res = self.__collection(target).insert_many(datas_list)
for new_datas in datas_list:
self.__update_backref(target, None, new_datas)
target.make_consistency(datas=new_datas)
return list(res.inserted_ids)
##@brief Update backref giving an action
#@param target leObject child class
#@param filters
#@param relational_filters,
#@param new_datas None | dict : optional new datas if None mean we are deleting
#@param old_datas_l None | list : if None fetch old datas from db (usefull
#when modifications are made on instance before updating backrefs)
#@return nothing (for the moment
def __update_backref_filtered(self, target,
filters, relational_filters, new_datas = None, old_datas_l = None):
#Getting all the UID of the object that will be deleted in order
#to update back_references
if old_datas_l is None:
mongo_filters = self.__process_filters(
target, filters, relational_filters)
old_datas_l = self.__collection(target).find(
mongo_filters)
old_datas_l = list(old_datas_l)
uidname = target.uid_fieldname()[0] #MULTIPLE UID BROKEN HERE
for old_datas in old_datas_l:
self.__update_backref(
target, old_datas[uidname], old_datas, new_datas)
##@brief Update back references of an object
#@ingroup plugin_mongodb_bref_op
#
#old_datas and new_datas arguments are set to None to indicate
#insertion or deletion. Calls examples :
#@par LeObject insert __update backref call
#
#Insert(datas):
# self.make_insert(datas)
# self.__update_backref(self.__class__, None, datas)
#
#@par LeObject delete __update backref call
#Delete()
# old_datas = self.datas()
# self.make_delete()
# self.__update_backref(self.__class__, old_datas, None)
#@par LeObject update __update_backref call
#
#Update(new_datas):
# old_datas = self.datas()
# self.make_udpdate(new_datas)
# self.__update_backref(self.__class__, old_datas, new_datas)
#
#
#@param target LeObject child classa
#@param tuid mixed : The target UID (the value that will be inserted in
#back references)
#@param old_datas dict : datas state before update
#@param new_datas dict : datas state after the update process
#retun None
def __update_backref(self, target, tuid, old_datas, new_datas):
#upd_dict is the dict that will allow to run updates in an optimized
#way (or try to help doing it)
#
#It's struct looks like :
# { LeoCLASS : {
# UID1: (
# LeoINSTANCE,
# { fname1 : value, fname2: value }),
# UID2 (LeoINSTANCE, {fname...}),
# },
# LeoClass2: {...
#
upd_dict = {}
for fname, fdh in target.reference_handlers().items():
oldd = old_datas is not None and fname in old_datas and \
(not hasattr(fdh, 'default') or old_datas[fname] != fdh.default) \
and not old_datas[fname] is None
newd = new_datas is not None and fname in new_datas and \
(not hasattr(fdh, 'default') or new_datas[fname] != fdh.default) \
and not new_datas[fname] is None
if (oldd and newd and old_datas[fname] == new_datas[fname])\
or not(oldd or newd):
#No changes or not concerned
continue
bref_cls = fdh.back_reference[0]
bref_fname = fdh.back_reference[1]
if issubclass(fdh.__class__, MultipleRef):
#fdh is a multiple ref. So the update preparation will be
#divided into two loops :
#- one loop for deleting old datas
#- one loop for inserting updated datas
#
#Preparing the list of values to delete or to add
if newd and oldd:
old_values = old_datas[fname]
new_values = new_datas[fname]
to_del = [ val
for val in old_values
if val not in new_values]
to_add = [ val
for val in new_values
if val not in old_values]
elif oldd and not newd:
to_del = old_datas[fname]
to_add = []
elif not oldd and newd:
to_del = []
to_add = new_datas[fname]
#Calling __back_ref_upd_one_value() with good arguments
for vtype, vlist in [('old',to_del), ('new', to_add)]:
for value in vlist:
#fetching backref infos
bref_infos = self.__bref_get_check(
bref_cls, value, bref_fname)
#preparing the upd_dict
upd_dict = self.__update_backref_upd_dict_prepare(
upd_dict, bref_infos, bref_fname, value)
#preparing updated bref_infos
bref_cls, bref_leo, bref_dh, bref_value = bref_infos
bref_infos = (bref_cls, bref_leo, bref_dh,
upd_dict[bref_cls][value][1][bref_fname])
vdict = {vtype: value}
#fetch and store updated value
new_bref_val = self.__back_ref_upd_one_value(
fname, fdh, tuid, bref_infos, **vdict)
upd_dict[bref_cls][value][1][bref_fname] = new_bref_val
else:
#fdh is a single ref so the process is simpler, we do not have
#to loop and we may do an update in only one
#__back_ref_upd_one_value() call by giving both old and new
#value
vdict = {}
if oldd:
vdict['old'] = old_datas[fname]
uid_val = vdict['old']
if newd:
vdict['new'] = new_datas[fname]
if not oldd:
uid_val = vdict['new']
#Fetching back ref infos
bref_infos = self.__bref_get_check(
bref_cls, uid_val, bref_fname)
#prepare the upd_dict
upd_dict = self.__update_backref_upd_dict_prepare(
upd_dict, bref_infos, bref_fname, uid_val)
#forging update bref_infos
bref_cls, bref_leo, bref_dh, bref_value = bref_infos
bref_infos = (bref_cls, bref_leo, bref_dh,
upd_dict[bref_cls][uid_val][1][bref_fname])
#fetche and store updated value
new_bref_val = self.__back_ref_upd_one_value(
fname, fdh, tuid, bref_infos, **vdict)
upd_dict[bref_cls][uid_val][1][bref_fname] = new_bref_val
#Now we've got our upd_dict ready.
#running the updates
for bref_cls, uid_dict in upd_dict.items():
for uidval, (leo, datas) in uid_dict.items():
#MULTIPLE UID BROKEN 2 LINES BELOW
self.__update_no_backref(
leo.__class__, [(leo.uid_fieldname()[0], '=', uidval)],
[], datas)
##@brief Utility function designed to handle the upd_dict of
#__update_backref()
#
#Basically checks if a key exists at some level, if not create it with
#the good default value (in most case dict())
#@param upd_dict dict : in & out args modified by reference
#@param bref_infos tuple : as returned by __bref_get_check()
#@param bref_fname str : name of the field in referenced class
#@param uid_val mixed : the UID of the referenced object
#@return the updated version of upd_dict
@staticmethod
def __update_backref_upd_dict_prepare(upd_dict,bref_infos, bref_fname,
uid_val):
bref_cls, bref_leo, bref_dh, bref_value = bref_infos
if bref_cls not in upd_dict:
upd_dict[bref_cls] = {}
if uid_val not in upd_dict[bref_cls]:
upd_dict[bref_cls][uid_val] = (bref_leo, {})
if bref_fname not in upd_dict[bref_cls][uid_val]:
upd_dict[bref_cls][uid_val][1][bref_fname] = bref_value
return upd_dict
##@brief Prepare a one value back reference update
#@param fname str : the source Reference field name
#@param fdh DataHandler : the source Reference DataHandler
#@param tuid mixed : the uid of the Leo that make reference to the backref
#@param bref_infos tuple : as returned by __bref_get_check() method
#@param old mixed : (optional **values) the old value
#@param new mixed : (optional **values) the new value
#@return the new back reference field value
def __back_ref_upd_one_value(self, fname, fdh, tuid, bref_infos, **values):
bref_cls, bref_leo, bref_dh, bref_val = bref_infos
oldd = 'old' in values
newdd = 'new' in values
if bref_val is None:
bref_val = bref_dh.empty()
if issubclass(bref_dh.__class__, MultipleRef):
if oldd and newdd:
if tuid not in bref_val:
raise MongoDbConsistencyError("The value we want to \
delete in this back reference update was not found in the back referenced \
object : %s. Value was : '%s'" % (bref_leo, tuid))
return bref_val
elif oldd and not newdd:
#deletion
old_value = values['old']
if tuid not in bref_val:
raise MongoDbConsistencyError("The value we want to \
delete in this back reference update was not found in the back referenced \
object : %s. Value was : '%s'" % (bref_leo, tuid))
if isinstance(bref_val, tuple):
bref_val = set(bref_val)
if isinstance(bref_val, set):
bref_val -= set([tuid])
else:
del(bref_val[bref_val.index(tuid)])
elif not oldd and newdd:
if tuid in bref_val:
raise MongoDbConsistencyError("The value we want to \
add in this back reference update was found in the back referenced \
object : %s. Value was : '%s'" % (bref_leo, tuid))
if isinstance(bref_val, tuple):
bref_val = set(bref_val)
if isinstance(bref_val, set):
bref_val |= set([tuid])
else:
bref_val.append(tuid)
else:
#Single value backref
if oldd and newdd:
if bref_val != tuid:
raise MongoDbConsistencyError("The backreference doesn't \
have expected value. Expected was %s but found %s in %s" % (
tuid, bref_val, bref_leo))
return bref_val
elif oldd and not newdd:
#deletion
if not hasattr(bref_dh, "default"):
raise MongoDbConsistencyError("Unable to delete a \
value for a back reference update. The concerned field don't have a default \
value : in %s field %s" % (bref_leo,fname))
bref_val = getattr(bref_dh, "default")
elif not oldd and newdd:
bref_val = tuid
return bref_val
##@brief Fetch back reference informations
#@warning thank's to __update_backref_act() this method is useless
#@param bref_cls LeObject child class : __back_reference[0]
#@param uidv mixed : UID value (the content of the reference field)
#@param bref_fname str : the name of the back_reference field
#@return tuple(bref_class, bref_LeObect_instance, bref_datahandler,
#bref_value)
#@throw MongoDbConsistencyError when LeObject instance not found given
#uidv
#@throw LodelFatalError if the back reference field is not a Reference
#subclass (major failure)
def __bref_get_check(self, bref_cls, uidv, bref_fname):
bref_leo = bref_cls.get_from_uid(uidv)
if bref_leo is None:
raise MongoDbConsistencyError("Unable to get the object we make \
reference to : %s with uid = %s" % (bref_cls, repr(uidv)))
bref_dh = bref_leo.data_handler(bref_fname)
if not isinstance(bref_dh, Reference):
raise LodelFatalError("Found a back reference field that \
is not a reference : '%s' field '%s'" % (bref_leo, bref_fname))
bref_val = bref_leo.data(bref_fname)
return (bref_leo.__class__, bref_leo, bref_dh, bref_val)
##@brief Act on abstract LeObject child
#
#This method is designed to be called by insert, select and delete method
#when they encounter an abtract class
#@param target LeObject child class
#@param filters
#@param relational_filters
#@param act function : the caller method
#@param **kwargs other arguments
#@return sum of results (if it's an array it will result in a concat)
#@todo optimization implementing a cache for __bref_get_check()
def __act_on_abstract(self,
target, filters, relational_filters, act, **kwargs):
result = list() if act == self.select else 0
if not target.is_abstract():
target_childs = target
else:
target_childs = [tc for tc in target.child_classes()
if not tc.is_abstract()]
for target_child in target_childs:
#Add target_child to filter
new_filters = copy.copy(filters)
for i in range(len(filters)):
fname, op, val = filters[i]
if fname == CLASS_ID_FIELDNAME:
logger.warning("Dirty drop of filter : '%s %s %s'" % (
fname, op, val))
del(new_filters[i])
new_filters.append(
(CLASS_ID_FIELDNAME, '=',
collection_name(target_child.__name__)))
result += act(
target = target_child,
filters = new_filters,
relational_filters = relational_filters,
**kwargs)
return result
##@brief Connect to database
#@note this method avoid opening two times the same connection using
#MongoDbDatasource::_connections static attribute
#@param username str
#@param password str
#@param ro bool : If True the Datasource is for read only, else the
def __connect(self, username, password, db_name, ro):
conn_string = connection_string(
username = username, password = password,
host = self.__db_infos['host'],
port = self.__db_infos['port'],
db_name = db_name,
ro = ro)
self.__conn_hash = conn_h = hash(conn_string)
if conn_h in self._connections:
self._connections[conn_h]['conn_count'] += 1
return self._connections[conn_h]['db'][self.__db_infos['db_name']]
else:
logger.info("Opening a new connection to database")
self._connections[conn_h] = {
'conn_count': 1,
'db': utils.connect(conn_string)}
return self._connections[conn_h]['db'][self.__db_infos['db_name']]
##@brief Return a pymongo collection given a LeObject child class
#@param leobject LeObject child class (no instance)
#return a pymongo.collection instance
def __collection(self, leobject):
return self.database[object_collection_name(leobject)]
##@brief Perform subqueries implies by relational filters and append the
# result to existing filters
#
#The processing is divided in multiple steps :
# - determine (for each relational field of the target) every collection
#that are involved
# - generate subqueries for relational_filters that concerns a different
#collection than target collection
#filters
# - execute subqueries
# - transform subqueries results in filters
# - merge subqueries generated filters with existing filters
#
#@param target LeObject subclass (no instance) : Target class
#@param filters list : List of tuple(FIELDNAME, OP, VALUE)
#@param relational_filters : same composition thant filters except that
# FIELD is represented by a tuple(FIELDNAME, {CLASS1:RFIELD1,
# CLASS2:RFIELD2})
#@return a list of pymongo filters ( dict {FIELD:{OPERATOR:VALUE}} )
def __process_filters(self,target, filters, relational_filters):
# Simple filters lodel2 -> pymongo converting
res = self.__filters2mongo(filters, target)
rfilters = self.__prepare_relational_filters(target, relational_filters)
#Now that everything is well organized, begin to forge subquerie
#filters
self.__subqueries_from_relational_filters(target, rfilters)
# Executing subqueries, creating filters from result, and injecting
# them in original filters of the query
if len(rfilters) > 0:
logger.debug("Begining subquery execution")
for fname in rfilters:
if fname not in res:
res[fname] = dict()
subq_results = set()
for leobject, sq_filters in rfilters[fname].items():
uid_fname = mongo_fieldname(leobject._uid)
log_msg = "Subquery running on collection {coll} with filters \
'{filters}'"
logger.debug(log_msg.format(
coll=object_collection_name(leobject),
filters=sq_filters))
cursor = self.__collection(leobject).find(
filter=sq_filters,
projection=uid_fname)
subq_results |= set(doc[uid_fname] for doc in cursor)
#generating new filter from result
if '$in' in res[fname]:
#WARNING we allready have a IN on this field, doing dedup
#from result
deduped = set(res[fname]['$in']) & subq_results
if len(deduped) == 0:
del(res[fname]['$in'])
else:
res[fname]['$in'] = list(deduped)
else:
res[fname]['$in'] = list(subq_results)
if len(rfilters) > 0:
logger.debug("End of subquery execution")
return res
##@brief Generate subqueries from rfilters tree
#
#Returned struct organization :
# - 1st level keys : relational field name of target
# - 2nd level keys : referenced leobject
# - 3th level values : pymongo filters (dict)
#
#@note The only caller of this method is __process_filters
#@warning No return value, the rfilters arguement is modified by
#reference
#
#@param target LeObject subclass (no instance) : Target class
#@param rfilters dict : A struct as returned by
#MongoDbDatasource.__prepare_relational_filters()
#@return None, the rfilters argument is modified by reference
@classmethod
def __subqueries_from_relational_filters(cls, target, rfilters):
for fname in rfilters:
for leobject in rfilters[fname]:
for rfield in rfilters[fname][leobject]:
#This way of doing is not optimized but allows to trigger
#warnings in some case (2 different values for a same op
#on a same field on a same collection)
mongofilters = cls.__op_value_listconv(
rfilters[fname][leobject][rfield], target.field(fname))
rfilters[fname][leobject][rfield] = mongofilters
##@brief Generate a tree from relational_filters
#
#The generated struct is a dict with :
# - 1st level keys : relational field name of target
# - 2nd level keys : referenced leobject
# - 3th level keys : referenced field in referenced class
# - 4th level values : list of tuple(op, value)
#
#@note The only caller of this method is __process_filters
#@warning An assertion is done : if two leobject are stored in the same
#collection they share the same uid
#
#@param target LeObject subclass (no instance) : Target class
#@param relational_filters : same composition thant filters except that
#@return a struct as described above
@classmethod
def __prepare_relational_filters(cls, target, relational_filters):
# We are going to regroup relationnal filters by reference field
# then by collection
rfilters = dict()
if relational_filters is None:
relational_filters = []
for (fname, rfields), op, value in relational_filters:
if fname not in rfilters:
rfilters[fname] = dict()
rfilters[fname] = dict()
# Stores the representative leobject for associated to a collection
# name
leo_collname = dict()
# WARNING ! Here we assert that all leobject that are stored
# in a same collection are identified by the same field
for leobject, rfield in rfields.items():
#here we are filling a dict with leobject as index but
#we are doing a UNIQ on collection name
cur_collname = object_collection_name(leobject)
if cur_collname not in leo_collname:
leo_collname[cur_collname] = leobject
rfilters[fname][leobject] = dict()
#Fecthing the collection's representative leobject
repr_leo = leo_collname[cur_collname]
if rfield not in rfilters[fname][repr_leo]:
rfilters[fname][repr_leo][rfield] = list()
rfilters[fname][repr_leo][rfield].append((op, value))
return rfilters
##@brief Convert lodel2 filters to pymongo conditions
#@param filters list : list of lodel filters
#@return dict representing pymongo conditions
@classmethod
def __filters2mongo(cls, filters, target):
res = dict()
eq_fieldname = [] #Stores field with equal comparison OP
for fieldname, op, value in filters:
oop = op
ovalue = value
op, value = cls.__op_value_conv(op, value, target.field(fieldname))
if op == '=':
eq_fieldname.append(fieldname)
if fieldname in res:
logger.warning("Dropping previous condition. Overwritten \
by an equality filter")
res[fieldname] = value
continue
if fieldname in eq_fieldname:
logger.warning("Dropping condition : '%s %s %s'" % (
fieldname, op, value))
continue
if fieldname not in res:
res[fieldname] = dict()
if op in res[fieldname]:
logger.warning("Dropping condition : '%s %s %s'" % (
fieldname, op, value))
else:
if op not in cls.lodel2mongo_op_map:
raise ValueError("Invalid operator : '%s'" % op)
new_op = cls.lodel2mongo_op_map[op]
res[fieldname][new_op] = value
return res
##@brief Convert lodel2 operator and value to pymongo struct
#
#Convertion is done using MongoDbDatasource::lodel2mongo_op_map
#@param op str : take value in LeFilteredQuery::_query_operators
#@param value mixed : the value
#@return a tuple(mongo_op, mongo_value)
@classmethod
def __op_value_conv(cls, op, value, dhdl):
if op not in cls.lodel2mongo_op_map:
msg = "Invalid operator '%s' found" % op
raise MongoDbDataSourceError(msg)
mongop = cls.lodel2mongo_op_map[op]
mongoval = value
#Converting lodel2 wildcarded string into a case insensitive
#mongodb re
if mongop in cls.mongo_op_re:
if value.startswith('(') and value.endswith(')') and ',' in value:
if (dhdl.cast_type is not None):
mongoval = [ dhdl.cast_type(item) for item in mongoval[1:-1].split(',') ]
else:
mongoval = [ item for item in mongoval[1:-1].split(',') ]
elif mongop == 'like':
#unescaping \
mongoval = value.replace('\\\\','\\')
if not mongoval.startswith('*'):
mongoval = '^'+mongoval
#For the end of the string it's harder to detect escaped *
if not (mongoval[-1] == '*' and mongoval[-2] != '\\'):
mongoval += '$'
#Replacing every other unescaped wildcard char
mongoval = cls.wildcard_re.sub('.*', mongoval)
mongoval = {'$regex': mongoval, '$options': 'i'}
return (op, mongoval)
##@brief Convert a list of tuple(OP, VALUE) into a pymongo filter dict
#@return a dict with mongo op as key and value as value...
@classmethod
def __op_value_listconv(cls, op_value_list, dhdl):
result = dict()
for op, value in op_value_list:
mongop, mongoval = cls.__op_value_conv(op, value, dhdl)
if mongop in result:
warnings.warn("Duplicated value given for a single \
field/operator couple in a query. We will keep only the first one")
else:
result[mongop] = mongoval
return result
##@brief Generate a comparison function for post reccursion sorting in
#select
#@return a lambda function that take 2 dict as arguement
@classmethod
def __generate_lambda_cmp_order(cls, order):
if len(order) == 0:
return lambda a,b: 0
glco = cls.__generate_lambda_cmp_order
fname, cmpdir = order[0]
order = order[1:]
return lambda a,b: glco(order) if a[fname] == b[fname] else (\
1 if (a[fname]>b[fname] if cmpdir == 'ASC' else a[fname]