#-*- coding: utf-8 -*- import re import copy import inspect import warnings from lodel.context import LodelContext LodelContext.expose_modules(globals(), { 'lodel.leapi.exceptions': ['LeApiError', 'LeApiErrors', 'LeApiDataCheckError', 'LeApiDataCheckErrors', 'LeApiQueryError', 'LeApiQueryErrors'], 'lodel.plugin.hooks': ['LodelHook'], 'lodel.logger': ['logger']}) ##@todo check datas when running query class LeQuery(object): ##@brief Hookname prefix _hook_prefix = None ##@brief arguments for the LeObject.check_data_value() _data_check_args = {'complete': False, 'allow_internal': False} ##@brief Abstract constructor # @param target_class LeObject : class of object the query is about def __init__(self, target_class): from .leobject import LeObject if self._hook_prefix is None: raise NotImplementedError("Abstract class") if not inspect.isclass(target_class) or \ not issubclass(target_class, LeObject): raise TypeError("target class has to be a child class of LeObject but %s given"% target_class) self._target_class = target_class self._ro_datasource = target_class._ro_datasource self._rw_datasource = target_class._rw_datasource ##@brief Execute a query and return the result #@param **datas #@return the query result #@see LeQuery._query() #@todo check that the check_datas_value is not duplicated/useless def execute(self, datas): if not datas is None: self._target_class.check_datas_value( datas, **self._data_check_args) self._target_class.prepare_datas(datas) #not yet implemented if self._hook_prefix is None: raise NotImplementedError("Abstract method") LodelHook.call_hook(self._hook_prefix+'pre', self._target_class, datas) ret = self._query(datas=datas) ret = LodelHook.call_hook(self._hook_prefix+'post', self._target_class, ret) return ret ##@brief Childs classes implements this method to execute the query #@param **datas #@return query result def _query(self, **datas): raise NotImplementedError("Asbtract method") ##@return a dict with query infos def dump_infos(self): return {'target_class': self._target_class} def __repr__(self): ret = "<{classname} target={target_class}>" return ret.format( classname=self.__class__.__name__, target_class = self._target_class) ##@brief Abstract class handling query with filters class LeFilteredQuery(LeQuery): ##@brief The available operators used in query definitions _query_operators = [ ' = ', ' <= ', ' >= ', ' != ', ' < ', ' > ', ' in ', ' not in ', ' like ', ' not like '] ##@brief Regular expression to process filters _query_re = None ##@brief Abtract constructor for queries with filter #@param target_class LeObject : class of object the query is about #@param query_filters list : with a tuple (only one filter) or a list of # tuple or a dict: {OP,list(filters)} with OP = 'OR' or 'AND for tuple # (FIELD,OPERATOR,VALUE) def __init__(self, target_class, query_filters=None): super().__init__(target_class) ##@brief The query filter tuple(std_filter, relational_filters) self._query_filter = None ##@brief Stores potential subqueries (used when a query implies # more than one datasource. # # Subqueries are tuple(target_class_ref_field, LeGetQuery) self.subqueries = None query_filters = [] if query_filters is None else query_filters self.set_query_filter(query_filters) ##@brief Abstract FilteredQuery execution method # # This method takes care to execute subqueries before calling super execute def execute(self, datas=None): #copy originals filters orig_filters = copy.copy(self._query_filter) std_filters, rel_filters = self._query_filter for rfield, subq in self.subqueries: subq_res = subq.execute() std_filters.append( (rfield, ' in ', subq_res)) self._query_filter = (std_filters, rel_filters) try: filters, rel_filters = self._query_filter res = super().execute(datas) except Exception as e: #restoring filters even if an exception is raised self.__query_filter = orig_filters raise e #reraise #restoring filters self._query_filter = orig_filters return res ##@brief Add filter(s) to the query # # This method is also able to slice query if different datasources are # implied in the request # #@param query_filter list|tuple|str : A single filter or a list of filters #@see LeFilteredQuery._prepare_filters() #@warning Does not support multiple UID def set_query_filter(self, query_filter): if isinstance(query_filter, str): query_filter = [query_filter] #Query filter prepration filters_orig, rel_filters = self._prepare_filters(query_filter) # Here we now that each relational filter concern only one datasource # thank's to _prepare_relational_fields #Multiple datasources detection self_ds_name = self._target_class._datasource_name result_rel_filters = list() # The filters that will stay in the query other_ds_filters = dict() for rfilter in rel_filters: (rfield, ref_dict), op, value = rfilter #rfield : the field in self._target_class tmp_rel_filter = dict() #designed to stores rel_field of same DS # First step : simplification # Trying to delete relational filters done on referenced class uid for tclass, tfield in copy.copy(ref_dict).items(): #tclass : reference target class #tfield : referenced field from target class # # !!!WARNING!!! # The line below brake multi UID support # if tfield == tclass.uid_fieldname()[0]: #This relational filter can be simplified as # ref_field, op, value # Note : we will have to dedup filters_orig filters_orig.append((rfield, op, value)) del(ref_dict[tclass]) if len(ref_dict) == 0: continue #Determine what to do with other relational filters given # referenced class datasource #Remember : each class in a relational filter has the same # datasource tclass = list(ref_dict.keys())[0] cur_ds = tclass._datasource_name if cur_ds == self_ds_name: # Same datasource, the filter stay is self query result_rel_filters.append(((rfield, ref_dict), op, value)) else: # Different datasource, we will have to create a subquery if cur_ds not in other_ds_filters: other_ds_filters[cur_ds] = list() other_ds_filters[cur_ds].append( ((rfield, ref_dict), op, value)) #deduplication of std filters filters_cp = set() if not isinstance(filters_orig, set): for i, cfilt in enumerate(filters_orig): a, b, c = cfilt if isinstance(c, list): #list are not hashable newc = tuple(c) else: newc = c old_len = len(filters_cp) filters_cp |= set((a,b,newc)) if len(filters_cp) == old_len: del(filters_orig[i]) # Sets _query_filter attribute of self query self._query_filter = (filters_orig, result_rel_filters) #Sub queries creation subq = list() for ds, rfilters in other_ds_filters.items(): for rfilter in rfilters: (rfield, ref_dict), op, value = rfilter for tclass, tfield in ref_dict.items(): query = LeGetQuery( target_class=tclass, query_filters=[(tfield, op, value)], field_list=[tfield]) subq.append((rfield, query)) self.subqueries = subq ##@return informations def dump_infos(self): ret = super().dump_infos() ret['query_filter'] = self._query_filter ret['subqueries'] = self.subqueries return ret def __repr__(self): res = "<{classname} target={target_class} query_filter={query_filter}" res = res.format( classname=self.__class__.__name__, query_filter=self._query_filter, target_class=self._target_class) if len(self.subqueries) > 0: for n, subq in enumerate(self.subqueries): res += "\n\tSubquerie %d : %s" res %= (n, subq) res += '>' return res ## @brief Prepare filters for datasource # #A filter can be a string or a tuple with len = 3. # #This method divide filters in two categories : # #@par Simple filters # #Those filters concerns fields that represent object values (a title, #the content, etc.) They are composed of three elements : FIELDNAME OP # VALUE . Where : #- FIELDNAME is the name of the field #- OP is one of the authorized comparison operands (see #@ref LeFilteredQuery.query_operators ) #- VALUE is... a value # #@par Relational filters # #Those filters concerns on reference fields (see the corresponding #abstract datahandler @ref lodel.leapi.datahandlers.base_classes.Reference) #The filter as quite the same composition than simple filters : # FIELDNAME[.REF_FIELD] OP VALUE . Where : #- FIELDNAME is the name of the reference field #- REF_FIELD is an optionnal addon to the base field. It indicate on wich #field of the referenced object the comparison as to be done. If no #REF_FIELD is indicated the comparison will be done on identifier. # #@param cls #@param filters_l list : This list of str or tuple (or both) #@return a tuple(FILTERS, RELATIONNAL_FILTERS #@todo move this doc in another place (a dedicated page ?) #@warning Does not supports multiple UID for an EmClass def _prepare_filters(self, filters_l): filters=list() res_filters = list() rel_filters = list() err_l = dict() #Splitting in tuple if necessary for i, fil in enumerate(filters_l): if len(fil) == 3 and not isinstance(fil, str): filters.append(tuple(fil)) else: try: filters.append(self.split_filter(fil)) except ValueError as e: err_l["filter %d" % i] = e for field, operator, value in filters: err_key = "%s %s %s" % (field, operator, value) #to push in err_l # Spliting field name to be able to detect a relational field field_spl = field.split('.') if len(field_spl) == 2: field, ref_field = field_spl elif len(field_spl) == 1: ref_field = None else: err_l[field] = NameError("'%s' is not a valid relational \ field name" % field) continue # Checking field against target_class ret = self._check_field(self._target_class, field) if isinstance(ret, Exception): err_l[field] = ret continue field_datahandler = self._target_class.field(field) if isinstance(field_datahandler, Exception): err_l[field] = field_datahandler continue if ref_field is not None and not field_datahandler.is_reference(): # inconsistency err_l[field] = NameError("The field '%s' in %s is not \ a relational field, but %s.%s was present in the filter" % (field, self._target_class.__name__, field, ref_field)) if field_datahandler.is_reference(): #Relationnal field if ref_field is None: # ref_field default value # # !!! WARNING !!! # This piece of code does not supports multiple UID for an # emclass # ref_uid = [ lc._uid[0] for lc in field_datahandler.linked_classes] if len(set(ref_uid)) == 1: ref_field = ref_uid[0] else: if len(ref_uid) > 1: msg = "The referenced classes are identified by \ fields with different name. Unable to determine wich field to use for the \ reference" else: msg = "Unknow error when trying to determine wich \ field to use for the relational filter" err_l[err_key] = RuntimeError(msg) continue # Prepare relational field ret = self._prepare_relational_fields(field, ref_field) if isinstance(ret, Exception): err_l[err_key] = ret continue else: rel_filters.append((ret, operator, value)) else: value_orig = value value, error = field_datahandler.check_data_value(value) if isinstance(error, Exception): value = value_orig res_filters.append((field,operator, value)) if len(err_l) > 0: raise LeApiDataCheckErrors( "Error while preparing filters : ", err_l) return (res_filters, rel_filters) ## @brief Check and split a query filter # @note The query_filter format is "FIELD OPERATOR VALUE" # @param query_filter str : A query_filter string # @param cls # @return a tuple (FIELD, OPERATOR, VALUE) @classmethod def split_filter(cls, query_filter): if cls._query_re is None: cls.__compile_query_re() matches = cls._query_re.match(query_filter) if not matches: msg = "The query_filter '%s' seems to be invalid" raise ValueError(msg % query_filter) result = ( matches.group('field'), re.sub(r'\s', ' ', matches.group('operator'), count=0), matches.group('value').strip()) result = [r.strip() for r in result] for r in result: if len(r) == 0: msg = "The query_filter '%s' seems to be invalid" raise ValueError(msg % query_filter) return result ## @brief Compile the regex for query_filter processing # @note Set _LeObject._query_re @classmethod def __compile_query_re(cls): op_re_piece = '(?P(%s)' op_re_piece %= cls._query_operators[0].replace(' ', '\s') for operator in cls._query_operators[1:]: op_re_piece += '|(%s)'%operator.replace(' ', '\s') op_re_piece += ')' re_full = '^\s*(?P([a-z_][a-z0-9\-_]*\.)?[a-z_][a-z0-9\-_]*)\s*' re_full += op_re_piece+'\s*(?P.*)\s*$' cls._query_re = re.compile(re_full, flags=re.IGNORECASE) pass @classmethod def _check_field(cls, target_class, fieldname): try: target_class.field(fieldname) except NameError as e: msg = "No field named '%s' in %s'" msg %= (fieldname, target_class.__name__) return NameError(msg) ##@brief Prepare a relational filter # #Relational filters are composed of a tuple like the simple filters #but the first element of this tuple is a tuple to : # #((FIELDNAME, {REF_CLASS: REF_FIELD}), OP, VALUE) # Where : #- FIELDNAME is the field name is the target class #- the second element is a dict with : # - REF_CLASS as key. It's a LeObject child class # - REF_FIELD as value. The name of the referenced field in the REF_CLASS # #Visibly the REF_FIELD value of the dict will vary only when #no REF_FIELD is explicitly given in the filter string notation #and REF_CLASSES has differents uid # #@par String notation examples #
contributeur IN (1,2,3,5)
will be transformed into : #
(
    #       (
    #           contributeur,
    #           {
    #               auteur: 'lodel_id',
    #               traducteur: 'lodel_id'
    #          }
    #       ),
    #       ' IN ',
    #       [ 1,2,3,5 ])
#@todo move the documentation to another place # #@param fieldname str : The relational field name #@param ref_field str|None : The referenced field name (if None use #uniq identifiers as referenced field #@return a well formed relational filter tuple or an Exception instance def _prepare_relational_fields(self, fieldname, ref_field=None): datahandler = self._target_class.field(fieldname) # now we are going to fetch the referenced class to see if the # reference field is valid ref_classes = datahandler.linked_classes ref_dict = dict() if ref_field is None: for ref_class in ref_classes: ref_dict[ref_class] = ref_class.uid_fieldname else: r_ds = None for ref_class in ref_classes: if r_ds is None: r_ds = ref_class._datasource_name elif ref_class._datasource_name != r_ds: return RuntimeError("All referenced class doesn't have the\ same datasource. Query not possible") if ref_field in ref_class.fieldnames(True): ref_dict[ref_class] = ref_field else: msg = "Warning the class %s is not considered in \ the relational filter %s" msg %= (ref_class.__name__, ref_field) logger.debug(msg) if len(ref_dict) == 0: return NameError("No field named '%s' in referenced objects [%s]" % (ref_field, ','.join([rc.__name__ for rc in ref_classes]))) return (fieldname, ref_dict) ##@brief A query to insert a new object class LeInsertQuery(LeQuery): _hook_prefix = 'leapi_insert_' _data_check_args = {'complete': True, 'allow_internal': False} def __init__(self, target_class): if target_class.is_abstract(): raise LeApiQueryError("Trying to create an insert query on an \ abstract LeObject : %s" % target_class) super().__init__(target_class) ## @brief Implements an insert query operation, with only one insertion # @param datas : datas to be inserted def _query(self, datas): datas = self._target_class.prepare_datas(datas, True, False) id_inserted = self._rw_datasource.insert(self._target_class, datas) return id_inserted """ ## @brief Implements an insert query operation, with multiple insertions # @param datas : list of **datas to be inserted def _query(self, datas): nb_inserted = self._datasource.insert_multi( self._target_class,datas_list) if nb_inserted < 0: raise LeApiQueryError("Multiple insertions error") return nb_inserted """ ## @brief Execute the insert query def execute(self, datas): return super().execute(datas=datas) ##@brief A query to update datas for a given object # #@todo Change behavior, Huge optimization problem when updating using filters #and not instance. We have to run a GET and then 1 update by fecthed object... class LeUpdateQuery(LeFilteredQuery): _hook_prefix = 'leapi_update_' _data_check_args = {'complete': False, 'allow_internal': False} ##@brief Instanciate an update query # #If a class and not an instance is given, no query_filters are expected #and the update will be fast and simple. Else we have to run a get query #before updating (to fetch datas, update them and then, construct them #and check their consistency) #@param target LeObject clas or instance #@param query_filters list|None #@todo change strategy with instance update. We have to accept datas for #the execute method def __init__(self, target, query_filters=None): ##@brief This attr is set only if the target argument is an #instance of a LeObject subclass self.__leobject_instance_datas = None target_class = target if not inspect.isclass(target): if query_filters is not None: msg = "No query_filters accepted when an instance is given as \ target to LeUpdateQuery constructor" raise AttributeError(msg) target_class = target.__class__ if target_class.initialized: self.__leobject_instance_datas = target.datas(True) else: query_filters = [(target._uid[0], '=', target.uid())] super().__init__(target_class, query_filters) ##@brief Implements an update query #@param datas dict : datas to update #@returns the number of updated items #@todo change stategy for instance update. Datas should be allowed #for execute method (and query) def _query(self, datas): uid_name = self._target_class._uid[0] if self.__leobject_instance_datas is not None: #Instance update #Building query_filter filters = [( uid_name, '=', str(self.__leobject_instance_datas[uid_name]))] res = self._rw_datasource.update( self._target_class, filters, [], self.__leobject_instance_datas) else: #Update by filters, we have to fetch datas before updating res = self._ro_datasource.select( self._target_class, self._target_class.fieldnames(True), self._query_filter[0], self._query_filter[1]) #Checking and constructing datas upd_datas = dict() for res_data in res: res_data.update(datas) res_datas = self._target_class.prepare_datas( res_data, True, True) filters = [(uid_name, '=', res_data[uid_name])] res = self._rw_datasource.update( self._target_class, filters, [], res_datas) return res ## @brief Execute the update query def execute(self, datas=None): if self.__leobject_instance_datas is not None and datas is not None: raise LeApiQueryError("No datas expected when running an update \ query on an instance") if self.__leobject_instance_datas is None and datas is None: raise LeApiQueryError("Datas are mandatory when running an update \ query on a class with filters") return super().execute(datas=datas) ##@brief A query to delete an object class LeDeleteQuery(LeFilteredQuery): _hook_prefix = 'leapi_delete_' def __init__(self, target_class, query_filter): super().__init__(target_class, query_filter) ## @brief Execute the delete query # @param datas def execute(self, datas=None): return super().execute() ##@brief Implements delete query operations # @param datas #@returns the number of deleted items def _query(self, datas=None): filters, rel_filters = self._query_filter nb_deleted = self._rw_datasource.delete( self._target_class, filters, rel_filters) return nb_deleted class LeGetQuery(LeFilteredQuery): _hook_prefix = 'leapi_get_' ##@brief Instanciate a new get query #@param target_class LeObject : class of object the query is about #@param query_filters dict : {OP, list of query filters} # or tuple (FIELD, OPERATOR, VALUE) ) #@param kwargs dict : other query-related arguments and options # - field_list list|None : list of string representing fields see @ref leobject_filters # - order list : A list of field names or tuple (FIELDNAME,[ASC | DESC]) # - group list : A list of field names or tuple (FIELDNAME,[ASC | DESC]) # - limit int : The maximum number of returned results # - offset int : offset def __init__(self, target_class, query_filters, **kwargs): super().__init__(target_class, query_filters) ##@brief The fields to get self._field_list = None ##@brief An equivalent to the SQL ORDER BY self._order = None ##@brief An equivalent to the SQL GROUP BY self._group = None ##@brief An equivalent to the SQL LIMIT x self._limit = None ##@brief An equivalent to the SQL LIMIT x, OFFSET self._offset = 0 # Checking kwargs and assigning default values if there is some for argname in kwargs: if argname not in ( 'field_list', 'order', 'group', 'limit', 'offset'): raise TypeError("Unexpected argument '%s'" % argname) if 'field_list' not in kwargs: self.set_field_list(target_class.fieldnames(include_ro = True)) else: self.set_field_list(kwargs['field_list']) if 'order' in kwargs: #check kwargs['order'] self._order = kwargs['order'] if 'group' in kwargs: #check kwargs['group'] self._group = kwargs['group'] if 'limit' in kwargs and kwargs['limit'] is not None: try: self._limit = int(kwargs['limit']) if self._limit <= 0: raise ValueError() except ValueError: msg = "limit argument expected to be an interger > 0" raise ValueError(msg) if 'offset' in kwargs: try: self._offset = int(kwargs['offset']) if self._offset < 0: raise ValueError() except ValueError: msg = "offset argument expected to be an integer >= 0" raise ValueError(msg) ##@brief Set the field list # @param field_list list | None : If None use all fields # @return None # @throw LeApiQueryError if unknown field given def set_field_list(self, field_list): err_l = dict() if field_list is not None: for fieldname in field_list: ret = self._check_field(self._target_class, fieldname) if isinstance(ret, Exception): msg = "No field named '%s' in %s" msg %= (fieldname, self._target_class.__name__) expt = NameError(msg) err_l[fieldname] = expt if len(err_l) > 0: msg = "Error while setting field_list in a get query" raise LeApiQueryErrors(msg = msg, exceptions = err_l) self._field_list = list(set(field_list)) ##@brief Execute the get query def execute(self, datas=None): return super().execute() ##@brief Implements select query operations # @returns a list containing the item(s) def _query(self, datas=None): # select datas corresponding to query_filter fl = list(self._field_list) if self._field_list is not None else None l_datas=self._ro_datasource.select( target = self._target_class, field_list = fl, filters = self._query_filter[0], relational_filters = self._query_filter[1], order = self._order, group = self._group, limit = self._limit, offset = self._offset) return l_datas ##@return a dict with query infos def dump_infos(self): ret = super().dump_infos() ret.update({ 'field_list' : self._field_list, 'order' : self._order, 'group' : self._group, 'limit' : self._limit, 'offset': self._offset, }) return ret def __repr__(self): res = " 0: for n,subq in enumerate(self.subqueries): res += "\n\tSubquerie %d : %s" res %= (n, subq) res += ">" return res