Move of the MongoDb Datasource in a plugin

lodel/datasource/mongodb/utils.py

@@ -39,7 +39,7 @@ MONGODB_SORT_OPERATORS_MAP = {
 }
 
 
-MANDATORY_CONNECTION_ARGS = ('host', 'port', 'login', 'password', 'dbname')
+MANDATORY_CONNECTION_ARGS = ('host', 'port', 'username', 'password', 'db_name')
 
 
 class MongoDbConnectionError(Exception):
@@ -71,7 +71,7 @@ def mongodbconnect(connection_name):
 # @return dict
 # @todo Use the settings module to store the connections parameters
 def get_connection_args(connnection_name='default'):
-    return {'host': 'localhost', 'port': 28015, 'login': 'lodel_admin', 'password': 'lapwd', 'dbname': 'lodel'}
+    return {'host': 'localhost', 'port': 28015, 'username': 'lodel_admin', 'password': 'lapwd', 'db_name': 'lodel'}
 
 
 ## @brief Checks the settings given a connection name

plugins/mongodb_datasource/__init__.py

@@ -0,0 +1,16 @@
+from lodel.settings.validator import SettingValidator
+
+
+__loader__ = "main.py"
+__confspec__ = "confspec.py"
+__author__ = "Lodel2 dev team"
+__fullname__ = "MongoDB plugin"
+
+
+## @brief Activates the plugin
+#
+# @note It is possible there to add some specific actions (like checks, etc ...) for the plugin
+#
+# @return bool|str : True if all the checks are OK, an error message if not
+def _activate():
+    return True

plugins/mongodb_datasource/confspec.py

@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+
+from lodel.settings.validator import SettingValidator
+
+CONFSPEC = {
+    'lodel2.datasource.mongodb_datasource.*':{
+        'host': ('localhost', SettingValidator('host')),
+        'port': (None, SettingValidator('string')),
+        'db_name':('lodel', SettingValidator('string')),
+        'username': (None, SettingValidator('string')),
+        'password': (None, SettingValidator('string'))
+    }
+}

plugins/mongodb_datasource/main.py

@@ -0,0 +1,124 @@
+# -*- coding: utf-8 -*-
+
+import bson
+from bson.son import SON
+from collections import OrderedDict
+import pymongo
+from pymongo.errors import BulkWriteError
+import urllib
+
+from .utils import mongodbconnect, object_collection_name, parse_query_filters, parse_query_order, MONGODB_SORT_OPERATORS_MAP
+
+class MongoDbDataSourceError(Exception):
+    pass
+
+
+class MongoDbDatasource(object):
+
+    ## @brief instanciates a database object given a connection name
+    # @param connection_name str
+    def __init__(self, connection_name):
+        self.database = mongodbconnect(connection_name)
+
+    ## @brief returns a selection of documents from the datasource
+    # @param target Emclass
+    # @param field_list list
+    # @param filters list : List of filters
+    # @param rel_filters list : List of relational filters
+    # @param order list : List of column to order. ex: order = [('title', 'ASC'),]
+    # @param group list : List of tupple representing the column used as "group by" fields. ex: group = [('title', 'ASC'),]
+    # @param limit int : Number of records to be returned
+    # @param offset int: used with limit to choose the start record
+    # @param instanciate bool : If true, the records are returned as instances, else they are returned as dict
+    # @return list
+    # @todo Implement the relations
+    def select(self, target, field_list, filters, rel_filters=None, order=None, group=None, limit=None, offset=0, instanciate=True):
+        collection_name = object_collection_name(target.__class__)
+        collection = self.database[collection_name]
+        query_filters = parse_query_filters(filters)
+        query_result_ordering = parse_query_order(order) if order is not None else None
+        results_field_list = None if len(field_list) == 0 else field_list
+        limit = limit if limit is not None else 0
+
+        if group is None:
+            cursor = collection.find(filter=query_filters, projection=results_field_list, skip=offset, limit=limit, sort=query_result_ordering)
+        else:
+            pipeline = list()
+            unwinding_list = list()
+            grouping_dict = OrderedDict()
+            sorting_list = list()
+            for group_param in group:
+                field_name = group_param[0]
+                field_sort_option = group_param[1]
+                sort_option = MONGODB_SORT_OPERATORS_MAP[field_sort_option]
+                unwinding_list.append({'$unwind': '$%s' % field_name})
+                grouping_dict[field_name] = '$%s' % field_name
+                sorting_list.append((field_name, sort_option))
+
+            sorting_list.extends(query_result_ordering)
+
+            pipeline.append({'$match': query_filters})
+            if results_field_list is not None:
+                pipeline.append({'$project': SON([{field_name: 1} for field_name in field_list])})
+            pipeline.extend(unwinding_list)
+            pipeline.append({'$group': grouping_dict})
+            pipeline.extend({'$sort': SON(sorting_list)})
+            if offset > 0:
+                pipeline.append({'$skip': offset})
+            if limit is not None:
+                pipeline.append({'$limit': limit})
+
+        results = list()
+        for document in cursor:
+            results.append(document)
+
+        return results
+
+    ## @brief Deletes one record defined by its uid
+    # @param target Emclass : class of the record to delete
+    # @param uid dict|list : a dictionary of fields and values composing the unique identifier of the record or a list of several dictionaries
+    # @return int : number of deleted records
+    # @TODO Implement the error management
+    def delete(self, target, uid):
+        if isinstance(uid, dict):
+            uid = [uid]
+        collection_name = object_collection_name(target.__class__)
+        collection = self.database[collection_name]
+        result = collection.delete_many(uid)
+        return result.deleted_count
+
+    ## @brief updates one or a list of records
+    # @param target Emclass : class of the object to insert
+    # @param uids list : list of uids to update
+    # @param datas dict : datas to update (new values)
+    # @return int : Number of updated records
+    # @todo check if the values need to be parsed
+    def update(self, target, uids, **datas):
+        if not isinstance(uids, list):
+            uids = [uids]
+        collection_name = object_collection_name(target.__class__)
+        collection = self.database[collection_name]
+        results = collection.update_many({'uid': {'$in': uids}}, datas)
+        return results.modified_count()
+
+    ## @brief Inserts a record in a given collection
+    # @param target Emclass : class of the object to insert
+    # @param datas dict : datas to insert
+    # @return bool
+    # @TODO Implement the error management
+    def insert(self, target, **datas):
+        collection_name = object_collection_name(target.__class__)
+        collection = self.database[collection_name]
+        result = collection.insert_one(datas)
+        return len(result.inserted_id)
+
+    ## @brief Inserts a list of records in a given collection
+    # @param target Emclass : class of the objects inserted
+    # @param datas_list
+    # @return list : list of the inserted records' ids
+    # @TODO Implement the error management
+    def insert_multi(self, target, datas_list):
+        collection_name = object_collection_name(target.__class__)
+        collection = self.database[collection_name]
+        result = collection.insert_many(datas_list)
+        return len(result.inserted_ids)

plugins/mongodb_datasource/utils.py

@@ -0,0 +1,147 @@
+# -*- coding: utf-8 -*-
+
+import pymongo
+from pymongo import MongoClient
+
+from lodel.settings.settings import Settings as settings
+
+common_collections = {
+    'object': 'objects',
+    'relation': 'relation'
+}
+
+collection_prefix = {
+    'relation': 'rel_',
+    'object': 'class_'
+}
+
+LODEL_OPERATORS_MAP = {
+    '=': {'name': '$eq', 'value_type': None},
+    '<=': {'name': '$lte', 'value_type': None},
+    '>=': {'name': '$gte', 'value_type': None},
+    '!=': {'name': '$ne', 'value_type': None},
+    '<': {'name': '$lt', 'value_type': None},
+    '>': {'name': '$gt', 'value_type': None},
+    'in': {'name': '$in', 'value_type': list},
+    'not in': {'name': '$nin', 'value_type': list},
+    'OR': {'name': '$or', 'value_type': list},
+    'AND': {'name': '$and', 'value_type': list}
+}
+
+LODEL_SORT_OPERATORS_MAP = {
+    'ASC': pymongo.ASCENDING,
+    'DESC': pymongo.DESCENDING
+}
+
+MONGODB_SORT_OPERATORS_MAP = {
+    'ASC': 1,
+    'DESC': -1
+}
+
+MANDATORY_CONNECTION_ARGS = ('host', 'port', 'login', 'password', 'dbname')
+
+
+class MongoDbConnectionError(Exception):
+    pass
+
+
+## @brief gets the settings given a connection name
+# @param connection_name str
+# @return dict
+# @todo Use the settings module to store the connections parameters
+def get_connection_args(connnection_name='default'):
+    return {'host': 'localhost', 'port': 28015, 'login': 'lodel_admin', 'password': 'lapwd', 'dbname': 'lodel'}
+
+
+## @brief Creates a connection to a MongoDb Database
+# @param connection_name str
+# @return MongoClient
+def mongodbconnect(connection_name):
+    login, password, host, port, dbname = get_connection_args(connection_name)
+    connection_string = 'mongodb://%s:%s@%s:%s' % (login, password, host, port)
+    connection = MongoClient(connection_string)
+    database = connection[dbname]
+    return database
+
+
+## @brief Returns a collection name given a EmClass
+# @param class_object EmClass
+# @return str
+def object_collection_name(class_object):
+    if not class_object.pure_abstract:
+        class_parent = class_object.parents[0].uid
+        collection_name = ("%s%s" % (collection_prefix['object'], class_parent)).lower()
+    else:
+        collection_name = ("%s%s" % (collection_prefix['object'], class_object.name)).lower()
+
+    return collection_name
+
+
+## @brief Converts a Lodel query filter into a MongoDB filter
+# @param filter_params tuple: (FIELD, OPERATOR, VALUE) representing the query filter to convert
+# @return dict : {KEY:{OPERATOR:VALUE}}
+# @todo Add an error management for the operator mismatch
+def convert_filter(filter_params):
+    key, operator, value = filter_params
+
+    if operator == 'in' and not isinstance(value, list):
+            raise ValueError('A list should be used as value for an IN operator, %s given' % value.__class__)
+
+    if operator not in ('like', 'not like'):
+        converted_operator = LODEL_OPERATORS_MAP[operator]['name']
+        converted_filter = {key: {converted_operator: value}}
+    else:
+        is_starting_with = value.endswith('*')
+        is_ending_with = value.startswith('*')
+
+        if is_starting_with and not is is_ending_with:
+            regex_pattern = value.replace('*', '^')
+        elif is_ending_with and not is_starting_with:
+            regex_pattern = value.replace('*', '$')
+        elif is_starting_with and is_ending_with:
+            regex_pattern = '%s' % value
+        else:
+            regex_pattern = '^%s$' % value
+
+        regex_condition = {'$regex': regex_pattern, '$options': 'i'}
+        converted_filter = {key: regex_condition}
+        if operator.startswith('not'):
+            converted_filter = {key: {'$not': regex_condition}}
+
+    return converted_filter
+
+
+## @brief converts the query filters into MongoDB filters
+# @param query_filters list : list of query_filters as tuples or dicts
+# @param as_list bool : defines if the output will be a list (default: False)
+# @return dict|list
+def parse_query_filters(query_filters, as_list = False):
+    parsed_filters = dict() if not as_list else list()
+    for query_filter in query_filters:
+        if isinstance(query_filters, tuple):
+            if as_list:
+                parsed_filters.append(convert_filter(query_filter))
+            else:
+                parsed_filters.update(convert_filter(query_filter))
+        elif isinstance(query_filter, dict):
+            query_item = list(query_filter.items())[0]
+            key = LODEL_OPERATORS_MAP[query_item[0]]
+            if as_list:
+                parsed_filters.append({key: parse_query_filters(query_item[1], as_list=True)})
+            else:
+                parsed_filters.update({key: parse_query_filters(query_item[1], as_list=True)})
+        else:
+            # TODO add an exception management here in case the filter is neither a tuple nor a dict
+            pass
+    return parsed_filters
+
+
+## @brief Returns a list of orting options
+# @param query_filters_order list
+# @return list
+def parse_query_order(query_filters_order):
+    ordering = list()
+    for query_filter_order in query_filters_order:
+        field, direction = query_filter_order
+        ordering.append((field, LODEL_SORT_OPERATORS_MAP[direction]))
+    return ordering