yannweb/lodel2_mirror
1
0
Fork 0
mirror of https://github.com/yweber/lodel2.git synced 2026-02-03 17:50:12 +01:00
Code Releases Activity

Continuing mongodb backreference implementation

Browse source
This commit is contained in:
Yann 2016-09-05 08:52:42 +02:00
commit 4e90475dbf
4 changed files with 79 additions and 49 deletions
Download patch file Download diff file Expand all files Collapse all files

7
lodel/leapi/datahandlers/base_classes.py
View file

@ -294,10 +294,11 @@ class Reference(DataHandler):
#@return value
#@todo implement the check when we have LeObject uid check value
def _check_data_value(self, value):
value= super()._check_data_value(value)
elt = self.__allowed_classes[0]
from lodel.leapi.leobject import LeObject
value = super()._check_data_value(value)
elt = list(self.__allowed_classes)[0]
uid = elt.uid_fieldname()[0]# TODO multiple uid is broken
if (expt is None and not (isinstance(value, LeObject)) or (value is uid)):
if not (hasattr(value, '__class__') and issubclass(value.__class__, LeObject)) or (value is uid):
raise FieldValidationError("LeObject instance or id expected for a reference field")
return value

6
lodel/leapi/leobject.py
View file

@ -156,9 +156,9 @@ class LeObject(object):
@classmethod
def reference_handlers(cls, with_backref = True):
return { fname: fdh
for fname, fdh in cls.fields(True)
if issubclass(fdh, Reference) and \
(not with_backref or fdh.backreference is not None)}
for fname, fdh in cls.fields(True).items()
if issubclass(fdh.__class__, Reference) and \
(not with_backref or fdh.back_reference is not None)}
##@brief Return a LeObject child class from a name
# @warning This method has to be called from dynamically generated LeObjects

35
plugins/mongodb_datasource/__init__.py
View file

@ -27,3 +27,38 @@ def _activate():
##@defgroup plugin_mongodb_datasource MongoDB datasource plugin
#@brief Doc about mongodb datasource
##@page plugin_mongodb_backref_complexity Reflexion on back reference complexity
#@ingroup plugin_mongodb_bref_op
#
#Their is a huge performance issue in the way we implemented references
#and back references for mongodb datasource :
#
#For each write action (update, delete or insert) we HAVE TO run a select
#on all concerned LeObject. In fact those methods headers looks like
#<pre>def write_action(target_cls, filters, [datas])</pre>
#
#We have no idea if all the modified objects are of the target class (they
#can be of any target's child classes). So that means we have no idea of the
#@ref base_classes.Reference "References" that will be modified by the action.
#
#Another problem is that when we run an update or a delete we have no idea
#of the values that will be updated or deleted (we do not have the concerned
#instances datas). As a result we cannot replace or delete the concerned
#back references.
#
#In term of complexity the number of DB query looks like :
#<pre>
#With n the number of instances to modify :
#queryO(n) ~= 2n ( n * select + n * update )
#</pre>
#But it can go really bad, really fast if we take in consideration that
#query's can be done on mixed classes or abstract classes. With :
#- n : the number of LeObect child classes represented by the abstract class
#- m : the number of LeObject child classes for each n
#- o : the number of concerned back_reference classes for each m
#
#<pre>queryO(n,m,o) ~= n + (n*m) + (n*m*o) => n + n*m select and n*m*o updates</pre>
#
#All of this is really sad especially as the update and the delete will be
#run on LeObject instances....
#

80
plugins/mongodb_datasource/datasource.py
View file

@ -202,6 +202,9 @@ class MongoDbDatasource(object):
#Non abstract beahavior
mongo_filters = self.__process_filters(
target, filters, relational_filters)
#Updating backref before deletion
self.__update_backref_filtered(target, filters, relational_filters,
None)
res = self.__collection(target).delete_many(mongo_filters)
return res.deleted_count
@ -220,7 +223,8 @@ class MongoDbDatasource(object):
mongo_filters = self.__process_filters(
target, filters, relational_filters)
res = self.__collection(target).update(mongo_filters, upd_datas)
target.make_consistency(datas=upd_datas, type_query='update')
self.__update_backref_filtered(target, filters, relational_filters,
upd_datas)
return res['n']
## @brief Inserts a record in a given collection
@ -229,7 +233,7 @@ class MongoDbDatasource(object):
# @return the inserted uid
def insert(self, target, new_datas):
res = self.__collection(target).insert(new_datas)
target.make_consistency(datas=new_datas)
self.__update_backref(target, None, new_datas)
return str(res)
## @brief Inserts a list of records in a given collection
@ -239,9 +243,23 @@ class MongoDbDatasource(object):
def insert_multi(self, target, datas_list):
res = self.__collection(target).insert_many(datas_list)
for new_datas in datas_list:
self.__update_backref(target, None, new_datas)
target.make_consistency(datas=new_datas)
return list(res.inserted_ids)
##@brief Update backref giving an action
#@param target leObject child class
#@param filters
#@param relational_filters,
#@param datas None | dict : optional new datas if None mean we are deleting
#@return nothing (for the moment
def __update_backref_filtered(self, target, act,
filters, relational_filters, datas = None):
#gathering datas
old_datas_l = target.select(target, None, filters, relational_filters)
for old_datas in old_datas_l:
self.__update_backref(target, old_datas, datas)
##@brief Update back references of an object
#@ingroup plugin_mongodb_bref_op
#
@ -284,16 +302,18 @@ class MongoDbDatasource(object):
# LeoClass2: {...
#
upd_dict = {}
for fname, fdh in target.reference_handlers():
oldd = fname in old_datas
newd = fname in new_datas
for fname, fdh in target.reference_handlers().items():
oldd = old_datas is not None and fname in old_datas and \
hasattr(fdh, 'default') and old_datas[fname] != fdh.default
newd = new_datas is not None and fname in new_datas and \
hasattr(fdh, 'default') and new_datas[fname] != fdh.default
if (oldd and newd and old_datas[fname] == new_datas[fname])\
or not(oldd or newd):
#No changes or not concerned
continue
bref_cls = fdh.data_handler[0]
bref_fname = fdh.data_handler[1]
if issubclass(fdh, MultipleRef):
bref_cls = fdh.back_reference[0]
bref_fname = fdh.back_reference[1]
if issubclass(fdh.__class__, MultipleRef):
#fdh is a multiple ref. So the update preparation will be
#divided into two loops :
#- one loop for deleting old datas
@ -309,12 +329,12 @@ class MongoDbDatasource(object):
to_add = [ val
for val in new_values
if val not in old_values]
elif oldd and not newdd:
to_del = old_datas[fname]
elif oldd and not newd:
to_del = [old_datas[fname]]
to_add = []
elif not oldd and newdd:
elif not oldd and newd:
to_del = []
to_add = new_datas[fname]
to_add = [new_datas[fname]]
#Calling __back_ref_upd_one_value() with good arguments
for vtype, vlist in [('old',to_del), ('new', to_add)]:
for value in vlist:
@ -402,7 +422,7 @@ class MongoDbDatasource(object):
old_value = old_datas[fname]
bref_cls, bref_leo, bref_dh, bref_val = self.__bref_get_check(
bref_cls, old_value, bref_fname)
if issubclass(bref_dh, MultipleRef):
if issubclass(bref_dh.__class__, MultipleRef):
#
# Multiple ref update (iterable)
if old_value not in bref_val:
@ -451,7 +471,7 @@ value : in %s field %s" % (bref_leo, ref_fname))
elif newd:
#It's an "insert"
new_value = new_datas[fname]
if issubclass(bref_dh, MultipleRef):
if issubclass(bref_dh.__class__, MultipleRef):
if isinstance(bref_val, set):
bref_val |= set([new_value])
else:
@ -463,6 +483,7 @@ value : in %s field %s" % (bref_leo, ref_fname))
##@brief Fetch back reference informations
#@warning thank's to __update_backref_act() this method is useless
#@param bref_cls LeObject child class : __back_reference[0]
#@param uidv mixed : UID value (the content of the reference field)
#@param bref_fname str : the name of the back_reference field
@ -473,7 +494,7 @@ value : in %s field %s" % (bref_leo, ref_fname))
#@throw LodelFatalError if the back reference field is not a Reference
#subclass (major failure)
def __bref_get_check(self, bref_cls, uidv, bref_fname):
bref_leo = bref_cls.get_from_uid(old_datas[fname])
bref_leo = bref_cls.get_from_uid(uidv)
if len(bref_leo) == 0:
raise MongoDbConsistencyError("Unable to get the object we make \
reference to : %s with uid = %s" % (bref_cls, repr(uidv)))
@ -483,34 +504,6 @@ reference to : %s with uid = %s" % (bref_cls, repr(uidv)))
is not a reference : '%s' field '%s'" % (bref_leo, bref_fname))
bref_val = bref_leo.data(bref_fname)
return (bref_leo.__class__, bref_leo, bref_dh, bref_val)
##@defgroup plugin_mongodb_bref_op BackReferences operations
#@brief Contains back_reference operations
#@ingroup plugin_mongodb_datasource
#
#Back reference operations are implemented by privates functions
# - __bref_op_update()
# - __bref_op_delete()
# - __bref_op_add()
#
#This 3 methods returns the same thing, a dict containing the field
#name to update associated with the new value of this field.
#
#This 3 methods also expect that the following tests was done :
# - the datahandler pointed by the back reference is a child class of
#base_classes.Reference
# - the value of this field contains or is the given old value (not for
#the __bref_op_add() method)
#
#All this methods asserts that we allready checks that the given LeObject
#instance concerned field (the one stored as back reference) is a child
#class of base_classes.Reference
#
#@todo factorise __bref_op_update() and __bref_op_delete() and update this
#comment
##@brief Act on abstract LeObject child
#
@ -522,6 +515,7 @@ is not a reference : '%s' field '%s'" % (bref_leo, bref_fname))
#@param act function : the caller method
#@param **kwargs other arguments
#@return sum of results (if it's an array it will result in a concat)
#@todo optimization implementing a cache for __bref_get_check()
def __act_on_abstract(self,
target, filters, relational_filters, act, **kwargs):