#-*- coding: utf-8 -*- import re import copy import inspect import warnings from lodel.context import LodelContext LodelContext.expose_modules(globals(), { 'lodel.leapi.exceptions': ['LeApiError', 'LeApiErrors', 'LeApiDataCheckError', 'LeApiDataCheckErrors', 'LeApiQueryError', 'LeApiQueryErrors'], 'lodel.plugin.hooks': ['LodelHook'], 'lodel.logger': ['logger']}) # @todo check data when running query class LeQuery(object): ## @brief Hookname prefix _hook_prefix = None ## @brief arguments for the LeObject.check_data_value() _data_check_args = {'complete': False, 'allow_internal': False} ## @brief Abstract constructor # @param target_class LeObject : class of object the query is about def __init__(self, target_class): from .leobject import LeObject if self._hook_prefix is None: raise NotImplementedError("Abstract class") if not inspect.isclass(target_class) or \ not issubclass(target_class, LeObject): raise TypeError( "target class has to be a child class of LeObject but %s given" % target_class) self._target_class = target_class self._ro_datasource = target_class._ro_datasource self._rw_datasource = target_class._rw_datasource ## @brief Executes a query and returns the result #@param **data #@return the query result #@see LeQuery._query() #@todo check that the check_datas_value is not duplicated/useless def execute(self, data): if data is not None: self._target_class.check_datas_value( data, **self._data_check_args) self._target_class.prepare_datas(data) # not yet implemented if self._hook_prefix is None: raise NotImplementedError("Abstract method") LodelHook.call_hook(self._hook_prefix + 'pre', self._target_class, data) ret = self._query(data=data) ret = LodelHook.call_hook(self._hook_prefix + 'post', self._target_class, ret) return ret ## @brief Child classes implement this method to execute the query #@param **data #@return query result def _query(self, **data): raise NotImplementedError("Asbtract method") # @return a dict with query infos def dump_infos(self): return {'target_class': self._target_class} def __repr__(self): ret = "<{classname} target={target_class}>" return ret.format( classname=self.__class__.__name__, target_class=self._target_class) ## @brief Abstract class handling query with filters class LeFilteredQuery(LeQuery): ## @brief The available operators used in query definitions _query_operators = [ ' = ', ' <= ', ' >= ', ' != ', ' < ', ' > ', ' in ', ' not in ', ' like ', ' not like '] ## @brief Regular expression to process filters _query_re = None ## @brief Abtract constructor for queries with filter #@param target_class LeObject : class of object the query is about #@param query_filters list : with a tuple (only one filter) or a list of # tuple or a dict: {OP,list(filters)} with OP = 'OR' or 'AND for tuple # (FIELD,OPERATOR,VALUE) def __init__(self, target_class, query_filters=None): super().__init__(target_class) ## @brief The query filter tuple(std_filter, relational_filters) self._query_filter = None ## @brief Stores potential subqueries (used when a query implies # more than one datasource. # # Subqueries are tuple(target_class_ref_field, LeGetQuery) self.subqueries = None query_filters = [] if query_filters is None else query_filters self.set_query_filter(query_filters) ## @brief Abstract FilteredQuery execution method # # This method takes care to execute subqueries before calling super execute def execute(self, data=None): # copy originals filters orig_filters = copy.copy(self._query_filter) std_filters, rel_filters = self._query_filter for rfield, subq in self.subqueries: subq_res = subq.execute() std_filters.append( (rfield, ' in ', subq_res)) self._query_filter = (std_filters, rel_filters) try: filters, rel_filters = self._query_filter res = super().execute(data) except Exception as e: # restoring filters even if an exception is raised self.__query_filter = orig_filters raise e # reraise # restoring filters self._query_filter = orig_filters return res ## @brief Add filter(s) to the query # # This method is also able to slice query if different datasources are # implied in the request # #@param query_filter list|tuple|str : A single filter or a list of filters #@see LeFilteredQuery._prepare_filters() #@warning Does not support multiple UID def set_query_filter(self, query_filter): if isinstance(query_filter, str): query_filter = [query_filter] # Query filter preparation filters_orig, rel_filters = self._prepare_filters(query_filter) # Here we know that each relational filter concerns only one datasource # thank's to _prepare_relational_fields # Multiple datasources detection self_ds_name = self._target_class._datasource_name result_rel_filters = list() # The filters that will remain in the query other_ds_filters = dict() for rfilter in rel_filters: (rfield, ref_dict), op, value = rfilter # rfield : the field in self._target_class tmp_rel_filter = dict() # designed to store rel_field of same DS # First step : simplification # Trying to delete relational filters done on referenced class uid for tclass, tfield in copy.copy(ref_dict).items(): # tclass : referenced target class # tfield : referenced field from target class # # !!!WARNING!!! # The line below breaks multi UID support # if tfield == tclass.uid_fieldname()[0]: # This relational filter can be simplified as # ref_field, op, value # Note : we will have to dedup filters_orig filters_orig.append((rfield, op, value)) del(ref_dict[tclass]) if len(ref_dict) == 0: continue # Determines what to do with the other relational filters according # to the referenced class datasource # Remember : all classes in a relational filter have the same # datasource tclass = list(ref_dict.keys())[0] cur_ds = tclass._datasource_name if cur_ds == self_ds_name: # Same datasource, the filter stays in this query result_rel_filters.append(((rfield, ref_dict), op, value)) else: # Different datasources, we will have to create a subquery if cur_ds not in other_ds_filters: other_ds_filters[cur_ds] = list() other_ds_filters[cur_ds].append( ((rfield, ref_dict), op, value)) # deduplication of std filters filters_cp = set() if not isinstance(filters_orig, set): for i, cfilt in enumerate(filters_orig): a, b, c = cfilt if isinstance(c, list): # list are not hashable newc = tuple(c) else: newc = c old_len = len(filters_cp) filters_cp |= set((a, b, newc)) if len(filters_cp) == old_len: del(filters_orig[i]) # Sets _query_filter attribute of self query self._query_filter = (filters_orig, result_rel_filters) # Sub queries creation subq = list() for ds, rfilters in other_ds_filters.items(): for rfilter in rfilters: (rfield, ref_dict), op, value = rfilter for tclass, tfield in ref_dict.items(): query = LeGetQuery( target_class=tclass, query_filters=[(tfield, op, value)], field_list=[tfield]) subq.append((rfield, query)) self.subqueries = subq # @return informations def dump_infos(self): ret = super().dump_infos() ret['query_filter'] = self._query_filter ret['subqueries'] = self.subqueries return ret def __repr__(self): res = "<{classname} target={target_class} query_filter={query_filter}" res = res.format( classname=self.__class__.__name__, query_filter=self._query_filter, target_class=self._target_class) if len(self.subqueries) > 0: for n, subq in enumerate(self.subqueries): res += "\n\tSubquerie %d : %s" res %= (n, subq) res += '>' return res ## @brief Prepares filters for datasource # # A filter can be a string or a tuple with len = 3. # # This method divide filters in two categories : # #@par Simple filters # # Those filters concern fields that represent object values (a title, # the content, etc.) They are composed of three elements : FIELDNAME OP # VALUE . Where : #- FIELDNAME is the name of the field #- OP is one of the authorized comparison operands (see #@ref LeFilteredQuery.query_operators ) #- VALUE is... a value # #@par Relational filters # # Those filters concern on reference fields (see the corresponding # abstract datahandler @ref lodel.leapi.datahandlers.base_classes.Reference) # The filter as quite the same composition than simple filters : # FIELDNAME[.REF_FIELD] OP VALUE . Where : #- FIELDNAME is the name of the reference field #- REF_FIELD is an optionnal addon to the base field. It indicates on which # field of the referenced object the comparison as to be done. If no # REF_FIELD is indicated the comparison will be done on identifier. # #@param cls #@param filters_l list : This list of str or tuple (or both) #@return a tuple(FILTERS, RELATIONNAL_FILTERS #@todo move this doc in another place (a dedicated page ?) #@warning Does not support multiple UID for an EmClass def _prepare_filters(self, filters_l): filters = list() res_filters = list() rel_filters = list() err_l = dict() # Splitting in tuple if necessary for i, fil in enumerate(filters_l): if len(fil) == 3 and not isinstance(fil, str): filters.append(tuple(fil)) else: try: filters.append(self.split_filter(fil)) except ValueError as e: err_l["filter %d" % i] = e for field, operator, value in filters: err_key = "%s %s %s" % (field, operator, value) # to push in err_l # Splitting field name to be able to detect a relational field field_spl = field.split('.') if len(field_spl) == 2: field, ref_field = field_spl elif len(field_spl) == 1: ref_field = None else: err_l[field] = NameError("'%s' is not a valid relational \ field name" % field) continue # Checking field against target_class ret = self._check_field(self._target_class, field) if isinstance(ret, Exception): err_l[field] = ret continue field_datahandler = self._target_class.field(field) if isinstance(field_datahandler, Exception): err_l[field] = field_datahandler continue if ref_field is not None and not field_datahandler.is_reference(): # inconsistency err_l[field] = NameError("The field '%s' in %s is not \ a relational field, but %s.%s was present in the filter" % (field, self._target_class.__name__, field, ref_field)) if field_datahandler.is_reference(): # Relationnal field if ref_field is None: # ref_field default value # # !!! WARNING !!! # This piece of code does not supports multiple UID for an # emclass # ref_uid = [ lc._uid[0] for lc in field_datahandler.linked_classes] if len(set(ref_uid)) == 1: ref_field = ref_uid[0] else: if len(ref_uid) > 1: msg = "The referenced classes are identified by \ fields with different names. Unable to determine which field to use for the \ reference" else: msg = "Unknow error when trying to determine which \ field to use for the relational filter" err_l[err_key] = RuntimeError(msg) continue # Prepares relational field ret = self._prepare_relational_fields(field, ref_field) if isinstance(ret, Exception): err_l[err_key] = ret continue else: rel_filters.append((ret, operator, value)) else: value_orig = value value, error = field_datahandler.check_data_value(value) if isinstance(error, Exception): value = value_orig res_filters.append((field, operator, value)) if len(err_l) > 0: raise LeApiDataCheckErrors( "Error while preparing filters : ", err_l) return (res_filters, rel_filters) ## @brief Checks and splits a query filter # @note The query_filter format is "FIELD OPERATOR VALUE" # @param query_filter str : A query_filter string # @param cls # @return a tuple (FIELD, OPERATOR, VALUE) @classmethod def split_filter(cls, query_filter): if cls._query_re is None: cls.__compile_query_re() matches = cls._query_re.match(query_filter) if not matches: msg = "The query_filter '%s' seems to be invalid" raise ValueError(msg % query_filter) result = ( matches.group('field'), re.sub(r'\s', ' ', matches.group('operator'), count=0), matches.group('value').strip()) result = [r.strip() for r in result] for r in result: if len(r) == 0: msg = "The query_filter '%s' seems to be invalid" raise ValueError(msg % query_filter) return result ## @brief Compiles the regex for query_filter processing # @note Sets _LeObject._query_re @classmethod def __compile_query_re(cls): op_re_piece = '(?P(%s)' op_re_piece %= cls._query_operators[0].replace(' ', '\s') for operator in cls._query_operators[1:]: op_re_piece += '|(%s)' % operator.replace(' ', '\s') op_re_piece += ')' re_full = '^\s*(?P([a-z_][a-z0-9\-_]*\.)?[a-z_][a-z0-9\-_]*)\s*' re_full += op_re_piece + '\s*(?P.*)\s*$' cls._query_re = re.compile(re_full, flags=re.IGNORECASE) pass @classmethod def _check_field(cls, target_class, fieldname): try: target_class.field(fieldname) except NameError as e: msg = "No field named '%s' in %s'" msg %= (fieldname, target_class.__name__) return NameError(msg) ## @brief Prepares a relational filter # # Relational filters are composed of a tuple like the simple filters # but the first element of this tuple is also a tuple : # #((FIELDNAME, {REF_CLASS: REF_FIELD}), OP, VALUE) # Where : #- FIELDNAME is the field name is the target class #- the second element is a dict with : # - REF_CLASS as key. It's a LeObject child class # - REF_FIELD as value. The name of the referenced field in the REF_CLASS # # Visibly the REF_FIELD value of the dict will vary only when # no REF_FIELD is explicitly given in the filter string notation # and REF_CLASS classes have different uids # #@par String notation examples #
contributeur IN (1,2,3,5)
will be transformed into : #
(
    #       (
    #           contributeur,
    #           {
    #               auteur: 'lodel_id',
    #               traducteur: 'lodel_id'
    #          }
    #       ),
    #       ' IN ',
    #       [ 1,2,3,5 ])
#@todo move the documentation to another place # #@param fieldname str : The relational field name #@param ref_field str|None : The referenced field name (if None uses # uniq identifiers as referenced field #@return a well formed relational filter tuple or an Exception instance def _prepare_relational_fields(self, fieldname, ref_field=None): datahandler = self._target_class.field(fieldname) # now we are going to fetch the referenced class to see if the # reference field is valid ref_classes = datahandler.linked_classes ref_dict = dict() if ref_field is None: for ref_class in ref_classes: ref_dict[ref_class] = ref_class.uid_fieldname else: r_ds = None for ref_class in ref_classes: if r_ds is None: r_ds = ref_class._datasource_name elif ref_class._datasource_name != r_ds: return RuntimeError("All referenced classes don't have the\ same datasource. Query not possible") if ref_field in ref_class.fieldnames(True): ref_dict[ref_class] = ref_field else: msg = "Warning the class %s is not considered in \ the relational filter %s" msg %= (ref_class.__name__, ref_field) logger.debug(msg) if len(ref_dict) == 0: return NameError("No field named '%s' in referenced objects [%s]" % (ref_field, ','.join([rc.__name__ for rc in ref_classes]))) return (fieldname, ref_dict) ## @brief A query to insert a new object class LeInsertQuery(LeQuery): _hook_prefix = 'leapi_insert_' _data_check_args = {'complete': True, 'allow_internal': False} def __init__(self, target_class): if target_class.is_abstract(): raise LeApiQueryError("Trying to create an insert query on an \ abstract LeObject : %s" % target_class) super().__init__(target_class) #  @brief Implements an insert query operation, with only one insertion # @param data : data to be inserted def _query(self, data): data = self._target_class.prepare_datas(data, True, False) id_inserted = self._rw_datasource.insert(self._target_class, data) return id_inserted """ ## @brief Implements an insert query operation, with multiple insertions # @param data : list of **data to be inserted def _query(self, data): nb_inserted = self._datasource.insert_multi( self._target_class,data_list) if nb_inserted < 0: raise LeApiQueryError("Multiple insertions error") return nb_inserted """ #  @brief Executes the insert query def execute(self, data): return super().execute(data=data) ## @brief A query to update data for a given object # #@todo Change behavior, Huge optimization problem when updating using filters # and not instance. We have to run a GET and then one update by fetched object... class LeUpdateQuery(LeFilteredQuery): _hook_prefix = 'leapi_update_' _data_check_args = {'complete': False, 'allow_internal': False} ## @brief Instanciates an update query # # If a class and not an instance is given, no query_filters are expected # and the update will be fast and simple. Else we have to run a get query # before updating (to fetch data, update them and then, construct them # and check their consistency) #@param target LeObject clas or instance #@param query_filters list|None #@todo change strategy with instance update. We have to accept data for # the execute method def __init__(self, target, query_filters=None): ## @brief This attr is set only if the target argument is an # instance of a LeObject subclass self.__leobject_instance_datas = None target_class = target if not inspect.isclass(target): if query_filters is not None: msg = "No query_filters accepted when an instance is given as \ target to LeUpdateQuery constructor" raise AttributeError(msg) target_class = target.__class__ if target_class.initialized: self.__leobject_instance_datas = target.datas(True) else: query_filters = [(target._uid[0], '=', target.uid())] super().__init__(target_class, query_filters) ## @brief Implements an update query #@param data dict : data to be updated #@return the number of updated items #@todo change stategy for instance update. Data should be allowed # for execute method (and query) def _query(self, data): uid_name = self._target_class._uid[0] if self.__leobject_instance_datas is not None: # Instance update # Building query_filter filters = [( uid_name, '=', str(self.__leobject_instance_datas[uid_name]))] res = self._rw_datasource.update( self._target_class, filters, [], self.__leobject_instance_datas) else: # Update by filters, we have to fetch data before updating res = self._ro_datasource.select( self._target_class, self._target_class.fieldnames(True), self._query_filter[0], self._query_filter[1]) # Checking and constructing data upd_data = dict() for res_data in res: res_data.update(data) res_data = self._target_class.prepare_datas( res_data, True, True) filters = [(uid_name, '=', res_data[uid_name])] res = self._rw_datasource.update( self._target_class, filters, [], res_data) return res #  @brief Execute the update query def execute(self, data=None): if self.__leobject_instance_datas is not None and data is not None: raise LeApiQueryError("No data expected when running an update \ query on an instance") if self.__leobject_instance_datas is None and data is None: raise LeApiQueryError("Data are mandatory when running an update \ query on a class with filters") return super().execute(data=data) ## @brief A query to delete an object class LeDeleteQuery(LeFilteredQuery): _hook_prefix = 'leapi_delete_' def __init__(self, target_class, query_filter): super().__init__(target_class, query_filter) #  @brief Executes the delete query # @param data def execute(self, data=None): return super().execute() ## @brief Implements delete query operations # @param data #@return the number of deleted items def _query(self, data=None): filters, rel_filters = self._query_filter nb_deleted = self._rw_datasource.delete( self._target_class, filters, rel_filters) return nb_deleted class LeGetQuery(LeFilteredQuery): _hook_prefix = 'leapi_get_' ## @brief Instanciates a new get query #@param target_class LeObject : class of object the query is about #@param query_filters dict : {OP, list of query filters} # or tuple (FIELD, OPERATOR, VALUE) ) #@param kwargs dict : other query-related arguments and options # - field_list list|None : list of string representing fields see @ref leobject_filters # - order list : A list of field names or tuple (FIELDNAME,[ASC | DESC]) # - group list : A list of field names or tuple (FIELDNAME,[ASC | DESC]) # - limit int : The maximum number of returned results # - offset int : offset def __init__(self, target_class, query_filters, **kwargs): super().__init__(target_class, query_filters) ## @brief The fields to get self._field_list = None ## @brief An equivalent to the SQL ORDER BY self._order = None ## @brief An equivalent to the SQL GROUP BY self._group = None ## @brief An equivalent to the SQL LIMIT x self._limit = None ## @brief An equivalent to the SQL LIMIT x, OFFSET self._offset = 0 # Checking kwargs and assigning default values if there is some for argname in kwargs: if argname not in ( 'field_list', 'order', 'group', 'limit', 'offset'): raise TypeError("Unexpected argument '%s'" % argname) if 'field_list' not in kwargs: self.set_field_list(target_class.fieldnames(include_ro=True)) else: self.set_field_list(kwargs['field_list']) if 'order' in kwargs: # check kwargs['order'] self._order = kwargs['order'] if 'group' in kwargs: # check kwargs['group'] self._group = kwargs['group'] if 'limit' in kwargs and kwargs['limit'] is not None: try: self._limit = int(kwargs['limit']) if self._limit <= 0: raise ValueError() except ValueError: msg = "limit argument expected to be an interger > 0" raise ValueError(msg) if 'offset' in kwargs: try: self._offset = int(kwargs['offset']) if self._offset < 0: raise ValueError() except ValueError: msg = "offset argument expected to be an integer >= 0" raise ValueError(msg) ## @brief Set the field list # @param field_list list | None : If None use all fields # @return None # @throw LeApiQueryError if unknown field given def set_field_list(self, field_list): err_l = dict() if field_list is not None: for fieldname in field_list: ret = self._check_field(self._target_class, fieldname) if isinstance(ret, Exception): msg = "No field named '%s' in %s" msg %= (fieldname, self._target_class.__name__) expt = NameError(msg) err_l[fieldname] = expt if len(err_l) > 0: msg = "Error while setting field_list in a get query" raise LeApiQueryErrors(msg=msg, exceptions=err_l) self._field_list = list(set(field_list)) ## @brief Executes the get query def execute(self, data=None): return super().execute() ## @brief Implements select query operations # @return a list containing the item(s) def _query(self, data=None): # select data corresponding to query_filter fl = list(self._field_list) if self._field_list is not None else None l_data = self._ro_datasource.select( target=self._target_class, field_list=fl, filters=self._query_filter[0], relational_filters=self._query_filter[1], order=self._order, group=self._group, limit=self._limit, offset=self._offset) return l_data ## @brief Returns a dict with query infos # @return a dict def dump_infos(self): ret = super().dump_infos() ret.update({'field_list': self._field_list, 'order': self._order, 'group': self._group, 'limit': self._limit, 'offset': self._offset, }) return ret ## @brief Returns a string representation of the query # @return a string def __repr__(self): res = " 0: for n, subq in enumerate(self.subqueries): res += "\n\tSubquerie %d : %s" res %= (n, subq) res += ">" return res