1
0
Fork 0
mirror of https://github.com/yweber/lodel2.git synced 2025-11-02 04:20:55 +01:00
lodel2_mirror/lodel/leapi/query.py
2017-03-23 15:35:10 +01:00

738 lines
29 KiB
Python
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#-*- coding: utf-8 -*-
import re
import copy
import inspect
import warnings
from lodel.context import LodelContext
LodelContext.expose_modules(globals(), {
'lodel.leapi.exceptions': ['LeApiError', 'LeApiErrors',
'LeApiDataCheckError', 'LeApiDataCheckErrors', 'LeApiQueryError',
'LeApiQueryErrors'],
'lodel.plugin.hooks': ['LodelHook'],
'lodel.logger': ['logger']})
# @todo check data when running query
class LeQuery(object):
# @brief Hookname prefix
_hook_prefix = None
# @brief arguments for the LeObject.check_data_value()
_data_check_args = {'complete': False, 'allow_internal': False}
# @brief Abstract constructor
# @param target_class LeObject : class of object the query is about
def __init__(self, target_class):
from .leobject import LeObject
if self._hook_prefix is None:
raise NotImplementedError("Abstract class")
if not inspect.isclass(target_class) or \
not issubclass(target_class, LeObject):
raise TypeError(
"target class has to be a child class of LeObject but %s given" % target_class)
self._target_class = target_class
self._ro_datasource = target_class._ro_datasource
self._rw_datasource = target_class._rw_datasource
# @brief Executes a query and returns the result
#@param **data
#@return the query result
#@see LeQuery._query()
#@todo check that the check_datas_value is not duplicated/useless
def execute(self, data):
if data is not None:
self._target_class.check_datas_value(
data,
**self._data_check_args)
self._target_class.prepare_datas(data) # not yet implemented
if self._hook_prefix is None:
raise NotImplementedError("Abstract method")
LodelHook.call_hook(self._hook_prefix + 'pre',
self._target_class,
data)
ret = self._query(data=data)
ret = LodelHook.call_hook(self._hook_prefix + 'post',
self._target_class,
ret)
return ret
# @brief Child classes implement this method to execute the query
#@param **data
#@return query result
def _query(self, **data):
raise NotImplementedError("Asbtract method")
# @return a dict with query infos
def dump_infos(self):
return {'target_class': self._target_class}
def __repr__(self):
ret = "<{classname} target={target_class}>"
return ret.format(
classname=self.__class__.__name__,
target_class=self._target_class)
# @brief Abstract class handling query with filters
class LeFilteredQuery(LeQuery):
# @brief The available operators used in query definitions
_query_operators = [
' = ',
' <= ',
' >= ',
' != ',
' < ',
' > ',
' in ',
' not in ',
' like ',
' not like ']
# @brief Regular expression to process filters
_query_re = None
# @brief Abtract constructor for queries with filter
#@param target_class LeObject : class of object the query is about
#@param query_filters list : with a tuple (only one filter) or a list of
# tuple or a dict: {OP,list(filters)} with OP = 'OR' or 'AND for tuple
# (FIELD,OPERATOR,VALUE)
def __init__(self, target_class, query_filters=None):
super().__init__(target_class)
# @brief The query filter tuple(std_filter, relational_filters)
self._query_filter = None
# @brief Stores potential subqueries (used when a query implies
# more than one datasource.
#
# Subqueries are tuple(target_class_ref_field, LeGetQuery)
self.subqueries = None
query_filters = [] if query_filters is None else query_filters
self.set_query_filter(query_filters)
# @brief Abstract FilteredQuery execution method
#
# This method takes care to execute subqueries before calling super execute
def execute(self, data=None):
# copy originals filters
orig_filters = copy.copy(self._query_filter)
std_filters, rel_filters = self._query_filter
for rfield, subq in self.subqueries:
subq_res = subq.execute()
std_filters.append(
(rfield, ' in ', subq_res))
self._query_filter = (std_filters, rel_filters)
try:
filters, rel_filters = self._query_filter
res = super().execute(data)
except Exception as e:
# restoring filters even if an exception is raised
self.__query_filter = orig_filters
raise e # reraise
# restoring filters
self._query_filter = orig_filters
return res
# @brief Add filter(s) to the query
#
# This method is also able to slice query if different datasources are
# implied in the request
#
#@param query_filter list|tuple|str : A single filter or a list of filters
#@see LeFilteredQuery._prepare_filters()
#@warning Does not support multiple UID
def set_query_filter(self, query_filter):
if isinstance(query_filter, str):
query_filter = [query_filter]
# Query filter prepration
filters_orig, rel_filters = self._prepare_filters(query_filter)
# Here we now that each relational filter concern only one datasource
# thank's to _prepare_relational_fields
# Multiple datasources detection
self_ds_name = self._target_class._datasource_name
result_rel_filters = list() # The filters that will stay in the query
other_ds_filters = dict()
for rfilter in rel_filters:
(rfield, ref_dict), op, value = rfilter
# rfield : the field in self._target_class
tmp_rel_filter = dict() # designed to stores rel_field of same DS
# First step : simplification
# Trying to delete relational filters done on referenced class uid
for tclass, tfield in copy.copy(ref_dict).items():
# tclass : reference target class
# tfield : referenced field from target class
#
# !!!WARNING!!!
# The line below brake multi UID support
#
if tfield == tclass.uid_fieldname()[0]:
# This relational filter can be simplified as
# ref_field, op, value
# Note : we will have to dedup filters_orig
filters_orig.append((rfield, op, value))
del(ref_dict[tclass])
if len(ref_dict) == 0:
continue
# Determine what to do with other relational filters given
# referenced class datasource
# Remember : each class in a relational filter has the same
# datasource
tclass = list(ref_dict.keys())[0]
cur_ds = tclass._datasource_name
if cur_ds == self_ds_name:
# Same datasource, the filter stay is self query
result_rel_filters.append(((rfield, ref_dict), op, value))
else:
# Different datasource, we will have to create a subquery
if cur_ds not in other_ds_filters:
other_ds_filters[cur_ds] = list()
other_ds_filters[cur_ds].append(
((rfield, ref_dict), op, value))
# deduplication of std filters
filters_cp = set()
if not isinstance(filters_orig, set):
for i, cfilt in enumerate(filters_orig):
a, b, c = cfilt
if isinstance(c, list): # list are not hashable
newc = tuple(c)
else:
newc = c
old_len = len(filters_cp)
filters_cp |= set((a, b, newc))
if len(filters_cp) == old_len:
del(filters_orig[i])
# Sets _query_filter attribute of self query
self._query_filter = (filters_orig, result_rel_filters)
# Sub queries creation
subq = list()
for ds, rfilters in other_ds_filters.items():
for rfilter in rfilters:
(rfield, ref_dict), op, value = rfilter
for tclass, tfield in ref_dict.items():
query = LeGetQuery(
target_class=tclass,
query_filters=[(tfield, op, value)],
field_list=[tfield])
subq.append((rfield, query))
self.subqueries = subq
# @return informations
def dump_infos(self):
ret = super().dump_infos()
ret['query_filter'] = self._query_filter
ret['subqueries'] = self.subqueries
return ret
def __repr__(self):
res = "<{classname} target={target_class} query_filter={query_filter}"
res = res.format(
classname=self.__class__.__name__,
query_filter=self._query_filter,
target_class=self._target_class)
if len(self.subqueries) > 0:
for n, subq in enumerate(self.subqueries):
res += "\n\tSubquerie %d : %s"
res %= (n, subq)
res += '>'
return res
# @brief Prepare filters for datasource
#
# A filter can be a string or a tuple with len = 3.
#
# This method divide filters in two categories :
#
#@par Simple filters
#
# Those filters concerns fields that represent object values (a title,
# the content, etc.) They are composed of three elements : FIELDNAME OP
# VALUE . Where :
#- FIELDNAME is the name of the field
#- OP is one of the authorized comparison operands (see
#@ref LeFilteredQuery.query_operators )
#- VALUE is... a value
#
#@par Relational filters
#
# Those filters concerns on reference fields (see the corresponding
# abstract datahandler @ref lodel.leapi.datahandlers.base_classes.Reference)
# The filter as quite the same composition than simple filters :
# FIELDNAME[.REF_FIELD] OP VALUE . Where :
#- FIELDNAME is the name of the reference field
#- REF_FIELD is an optionnal addon to the base field. It indicate on wich
# field of the referenced object the comparison as to be done. If no
# REF_FIELD is indicated the comparison will be done on identifier.
#
#@param cls
#@param filters_l list : This list of str or tuple (or both)
#@return a tuple(FILTERS, RELATIONNAL_FILTERS
#@todo move this doc in another place (a dedicated page ?)
#@warning Does not supports multiple UID for an EmClass
def _prepare_filters(self, filters_l):
filters = list()
res_filters = list()
rel_filters = list()
err_l = dict()
# Splitting in tuple if necessary
for i, fil in enumerate(filters_l):
if len(fil) == 3 and not isinstance(fil, str):
filters.append(tuple(fil))
else:
try:
filters.append(self.split_filter(fil))
except ValueError as e:
err_l["filter %d" % i] = e
for field, operator, value in filters:
err_key = "%s %s %s" % (field, operator, value) # to push in err_l
# Spliting field name to be able to detect a relational field
field_spl = field.split('.')
if len(field_spl) == 2:
field, ref_field = field_spl
elif len(field_spl) == 1:
ref_field = None
else:
err_l[field] = NameError("'%s' is not a valid relational \
field name" % field)
continue
# Checking field against target_class
ret = self._check_field(self._target_class, field)
if isinstance(ret, Exception):
err_l[field] = ret
continue
field_datahandler = self._target_class.field(field)
if isinstance(field_datahandler, Exception):
err_l[field] = field_datahandler
continue
if ref_field is not None and not field_datahandler.is_reference():
# inconsistency
err_l[field] = NameError("The field '%s' in %s is not \
a relational field, but %s.%s was present in the filter"
% (field,
self._target_class.__name__,
field,
ref_field))
if field_datahandler.is_reference():
# Relationnal field
if ref_field is None:
# ref_field default value
#
# !!! WARNING !!!
# This piece of code does not supports multiple UID for an
# emclass
#
ref_uid = [
lc._uid[0] for lc in field_datahandler.linked_classes]
if len(set(ref_uid)) == 1:
ref_field = ref_uid[0]
else:
if len(ref_uid) > 1:
msg = "The referenced classes are identified by \
fields with different name. Unable to determine wich field to use for the \
reference"
else:
msg = "Unknow error when trying to determine wich \
field to use for the relational filter"
err_l[err_key] = RuntimeError(msg)
continue
# Prepare relational field
ret = self._prepare_relational_fields(field, ref_field)
if isinstance(ret, Exception):
err_l[err_key] = ret
continue
else:
rel_filters.append((ret, operator, value))
else:
value_orig = value
value, error = field_datahandler.check_data_value(value)
if isinstance(error, Exception):
value = value_orig
res_filters.append((field, operator, value))
if len(err_l) > 0:
raise LeApiDataCheckErrors(
"Error while preparing filters : ",
err_l)
return (res_filters, rel_filters)
# @brief Check and split a query filter
# @note The query_filter format is "FIELD OPERATOR VALUE"
# @param query_filter str : A query_filter string
# @param cls
# @return a tuple (FIELD, OPERATOR, VALUE)
@classmethod
def split_filter(cls, query_filter):
if cls._query_re is None:
cls.__compile_query_re()
matches = cls._query_re.match(query_filter)
if not matches:
msg = "The query_filter '%s' seems to be invalid"
raise ValueError(msg % query_filter)
result = (
matches.group('field'),
re.sub(r'\s', ' ', matches.group('operator'), count=0),
matches.group('value').strip())
result = [r.strip() for r in result]
for r in result:
if len(r) == 0:
msg = "The query_filter '%s' seems to be invalid"
raise ValueError(msg % query_filter)
return result
# @brief Compile the regex for query_filter processing
# @note Set _LeObject._query_re
@classmethod
def __compile_query_re(cls):
op_re_piece = '(?P<operator>(%s)'
op_re_piece %= cls._query_operators[0].replace(' ', '\s')
for operator in cls._query_operators[1:]:
op_re_piece += '|(%s)' % operator.replace(' ', '\s')
op_re_piece += ')'
re_full = '^\s*(?P<field>([a-z_][a-z0-9\-_]*\.)?[a-z_][a-z0-9\-_]*)\s*'
re_full += op_re_piece + '\s*(?P<value>.*)\s*$'
cls._query_re = re.compile(re_full, flags=re.IGNORECASE)
pass
@classmethod
def _check_field(cls, target_class, fieldname):
try:
target_class.field(fieldname)
except NameError as e:
msg = "No field named '%s' in %s'"
msg %= (fieldname, target_class.__name__)
return NameError(msg)
# @brief Prepare a relational filter
#
# Relational filters are composed of a tuple like the simple filters
# but the first element of this tuple is a tuple to :
#
#<code>((FIELDNAME, {REF_CLASS: REF_FIELD}), OP, VALUE)</code>
# Where :
#- FIELDNAME is the field name is the target class
#- the second element is a dict with :
# - REF_CLASS as key. It's a LeObject child class
# - REF_FIELD as value. The name of the referenced field in the REF_CLASS
#
# Visibly the REF_FIELD value of the dict will vary only when
# no REF_FIELD is explicitly given in the filter string notation
# and REF_CLASSES has differents uid
#
#@par String notation examples
#<pre>contributeur IN (1,2,3,5)</pre> will be transformed into :
#<pre>(
# (
# contributeur,
# {
# auteur: 'lodel_id',
# traducteur: 'lodel_id'
# }
# ),
# ' IN ',
# [ 1,2,3,5 ])</pre>
#@todo move the documentation to another place
#
#@param fieldname str : The relational field name
#@param ref_field str|None : The referenced field name (if None use
# uniq identifiers as referenced field
#@return a well formed relational filter tuple or an Exception instance
def _prepare_relational_fields(self, fieldname, ref_field=None):
datahandler = self._target_class.field(fieldname)
# now we are going to fetch the referenced class to see if the
# reference field is valid
ref_classes = datahandler.linked_classes
ref_dict = dict()
if ref_field is None:
for ref_class in ref_classes:
ref_dict[ref_class] = ref_class.uid_fieldname
else:
r_ds = None
for ref_class in ref_classes:
if r_ds is None:
r_ds = ref_class._datasource_name
elif ref_class._datasource_name != r_ds:
return RuntimeError("All referenced class doesn't have the\
same datasource. Query not possible")
if ref_field in ref_class.fieldnames(True):
ref_dict[ref_class] = ref_field
else:
msg = "Warning the class %s is not considered in \
the relational filter %s"
msg %= (ref_class.__name__, ref_field)
logger.debug(msg)
if len(ref_dict) == 0:
return NameError("No field named '%s' in referenced objects [%s]"
% (ref_field,
','.join([rc.__name__ for rc in ref_classes])))
return (fieldname, ref_dict)
# @brief A query to insert a new object
class LeInsertQuery(LeQuery):
_hook_prefix = 'leapi_insert_'
_data_check_args = {'complete': True, 'allow_internal': False}
def __init__(self, target_class):
if target_class.is_abstract():
raise LeApiQueryError("Trying to create an insert query on an \
abstract LeObject : %s" % target_class)
super().__init__(target_class)
#  @brief Implements an insert query operation, with only one insertion
# @param data : data to be inserted
def _query(self, data):
data = self._target_class.prepare_datas(data, True, False)
id_inserted = self._rw_datasource.insert(self._target_class, data)
return id_inserted
"""
## @brief Implements an insert query operation, with multiple insertions
# @param data : list of **data to be inserted
def _query(self, data):
nb_inserted = self._datasource.insert_multi(
self._target_class,data_list)
if nb_inserted < 0:
raise LeApiQueryError("Multiple insertions error")
return nb_inserted
"""
#  @brief Execute the insert query
def execute(self, data):
return super().execute(data=data)
# @brief A query to update data for a given object
#
#@todo Change behavior, Huge optimization problem when updating using filters
# and not instance. We have to run a GET and then 1 update by fecthed object...
class LeUpdateQuery(LeFilteredQuery):
_hook_prefix = 'leapi_update_'
_data_check_args = {'complete': False, 'allow_internal': False}
# @brief Instanciate an update query
#
# If a class and not an instance is given, no query_filters are expected
# and the update will be fast and simple. Else we have to run a get query
# before updating (to fetch data, update them and then, construct them
# and check their consistency)
#@param target LeObject clas or instance
#@param query_filters list|None
#@todo change strategy with instance update. We have to accept data for
# the execute method
def __init__(self, target, query_filters=None):
# @brief This attr is set only if the target argument is an
# instance of a LeObject subclass
self.__leobject_instance_datas = None
target_class = target
if not inspect.isclass(target):
if query_filters is not None:
msg = "No query_filters accepted when an instance is given as \
target to LeUpdateQuery constructor"
raise AttributeError(msg)
target_class = target.__class__
if target_class.initialized:
self.__leobject_instance_datas = target.datas(True)
else:
query_filters = [(target._uid[0], '=', target.uid())]
super().__init__(target_class, query_filters)
# @brief Implements an update query
#@param data dict : data to be updated
#@returns the number of updated items
#@todo change stategy for instance update. Data should be allowed
# for execute method (and query)
def _query(self, data):
uid_name = self._target_class._uid[0]
if self.__leobject_instance_datas is not None:
# Instance update
# Building query_filter
filters = [(
uid_name,
'=',
str(self.__leobject_instance_datas[uid_name]))]
res = self._rw_datasource.update(
self._target_class, filters, [],
self.__leobject_instance_datas)
else:
# Update by filters, we have to fetch data before updating
res = self._ro_datasource.select(
self._target_class, self._target_class.fieldnames(True),
self._query_filter[0],
self._query_filter[1])
# Checking and constructing data
upd_data = dict()
for res_data in res:
res_data.update(data)
res_data = self._target_class.prepare_datas(
res_data, True, True)
filters = [(uid_name, '=', res_data[uid_name])]
res = self._rw_datasource.update(
self._target_class, filters, [],
res_data)
return res
#  @brief Execute the update query
def execute(self, data=None):
if self.__leobject_instance_datas is not None and data is not None:
raise LeApiQueryError("No data expected when running an update \
query on an instance")
if self.__leobject_instance_datas is None and data is None:
raise LeApiQueryError("Data are mandatory when running an update \
query on a class with filters")
return super().execute(data=data)
# @brief A query to delete an object
class LeDeleteQuery(LeFilteredQuery):
_hook_prefix = 'leapi_delete_'
def __init__(self, target_class, query_filter):
super().__init__(target_class, query_filter)
#  @brief Execute the delete query
# @param data
def execute(self, data=None):
return super().execute()
# @brief Implements delete query operations
# @param data
#@returns the number of deleted items
def _query(self, data=None):
filters, rel_filters = self._query_filter
nb_deleted = self._rw_datasource.delete(
self._target_class, filters, rel_filters)
return nb_deleted
class LeGetQuery(LeFilteredQuery):
_hook_prefix = 'leapi_get_'
# @brief Instanciate a new get query
#@param target_class LeObject : class of object the query is about
#@param query_filters dict : {OP, list of query filters}
# or tuple (FIELD, OPERATOR, VALUE) )
#@param kwargs dict : other query-related arguments and options
# - field_list list|None : list of string representing fields see @ref leobject_filters
# - order list : A list of field names or tuple (FIELDNAME,[ASC | DESC])
# - group list : A list of field names or tuple (FIELDNAME,[ASC | DESC])
# - limit int : The maximum number of returned results
# - offset int : offset
def __init__(self, target_class, query_filters, **kwargs):
super().__init__(target_class, query_filters)
# @brief The fields to get
self._field_list = None
# @brief An equivalent to the SQL ORDER BY
self._order = None
# @brief An equivalent to the SQL GROUP BY
self._group = None
# @brief An equivalent to the SQL LIMIT x
self._limit = None
# @brief An equivalent to the SQL LIMIT x, OFFSET
self._offset = 0
# Checking kwargs and assigning default values if there is some
for argname in kwargs:
if argname not in (
'field_list', 'order', 'group', 'limit', 'offset'):
raise TypeError("Unexpected argument '%s'" % argname)
if 'field_list' not in kwargs:
self.set_field_list(target_class.fieldnames(include_ro=True))
else:
self.set_field_list(kwargs['field_list'])
if 'order' in kwargs:
# check kwargs['order']
self._order = kwargs['order']
if 'group' in kwargs:
# check kwargs['group']
self._group = kwargs['group']
if 'limit' in kwargs and kwargs['limit'] is not None:
try:
self._limit = int(kwargs['limit'])
if self._limit <= 0:
raise ValueError()
except ValueError:
msg = "limit argument expected to be an interger > 0"
raise ValueError(msg)
if 'offset' in kwargs:
try:
self._offset = int(kwargs['offset'])
if self._offset < 0:
raise ValueError()
except ValueError:
msg = "offset argument expected to be an integer >= 0"
raise ValueError(msg)
# @brief Set the field list
# @param field_list list | None : If None use all fields
# @return None
# @throw LeApiQueryError if unknown field given
def set_field_list(self, field_list):
err_l = dict()
if field_list is not None:
for fieldname in field_list:
ret = self._check_field(self._target_class, fieldname)
if isinstance(ret, Exception):
msg = "No field named '%s' in %s"
msg %= (fieldname, self._target_class.__name__)
expt = NameError(msg)
err_l[fieldname] = expt
if len(err_l) > 0:
msg = "Error while setting field_list in a get query"
raise LeApiQueryErrors(msg=msg, exceptions=err_l)
self._field_list = list(set(field_list))
# @brief Execute the get query
def execute(self, data=None):
return super().execute()
# @brief Implements select query operations
# @returns a list containing the item(s)
def _query(self, data=None):
# select data corresponding to query_filter
fl = list(self._field_list) if self._field_list is not None else None
l_data = self._ro_datasource.select(
target=self._target_class,
field_list=fl,
filters=self._query_filter[0],
relational_filters=self._query_filter[1],
order=self._order,
group=self._group,
limit=self._limit,
offset=self._offset)
return l_data
# @return a dict with query infos
def dump_infos(self):
ret = super().dump_infos()
ret.update({'field_list': self._field_list,
'order': self._order,
'group': self._group,
'limit': self._limit,
'offset': self._offset,
})
return ret
def __repr__(self):
res = "<LeGetQuery target={target_class} filter={query_filter} \
field_list={field_list} order={order} group={group} limit={limit} \
offset={offset}"
res = res.format(**self.dump_infos())
if len(self.subqueries) > 0:
for n, subq in enumerate(self.subqueries):
res += "\n\tSubquerie %d : %s"
res %= (n, subq)
res += ">"
return res