Resolving conflict with merge

Makefile.am

@@ -27,7 +27,7 @@ doc_graphviz:
 
 do_subst = sed -e 's,\[@\]INSTALLMODEL_DIR\[@\],$(install_model_dir),g' 
 
-runtest: runtest.sh
+runtest: ./runtest.sh
 	$(do_subst) < $(srcdir)/runtest.sh > runtest
 	chmod +x runtest
 

lodel/context.py

@@ -123,6 +123,7 @@ site_id set to None when we are in MULTISITE beahavior")
                 self.__class__._current = self.__class__._contexts = self
                 self.__pkg_name = 'lodel'
                 self.__package = lodel
+                self.__instance_path = os.getcwd()
                 return
         else:
             #Multisite instanciation
@@ -145,6 +146,7 @@ instance path")
                 """
                 warnings.warn("It can be a really BAD idea to create a \
 a context without a path......")
+                self.__instance_path = None
             else:
                 self.__instance_path = os.path.realpath(instance_path)
             #Importing the site package to trigger its creation

lodel/leapi/datahandlers/base_classes.py

@@ -410,70 +410,6 @@ class MultipleRef(Reference):
             raise FieldValidationError("MultipleRef have for invalid values [%s]  :" % (",".join(error_list)))
         return new_val
 
-    ##@brief Construct a multiple ref data
-    def construct_data(self, emcomponent, fname, datas, cur_value):
-        cur_value = super().construct_data(emcomponent, fname, datas, cur_value)
-        if cur_value is not None:
-            if self.back_reference is not None:
-                br_class = self.back_reference[0]
-                for br_id in cur_value:
-                    query_filters = list()
-                    query_filters.append((br_class.uid_fieldname()[0], '=', br_id))
-                    br_obj = br_class.get(query_filters)
-                    if len(br_obj) != 0:
-                        br_list = br_obj[0].data(self.back_reference[1])
-                        if br_list is None:
-                            br_list = list()
-                        if br_id not in br_list:
-                            br_list.append(br_id)
-        return cur_value
-
-    ## @brief Checks the backreference, updates it if it is not complete
-    # @param emcomponent EmComponent : An EmComponent child class instance
-    # @param fname : the field name
-    # @param datas dict : dict storing fields values
-    # @note Not done in case of delete
-    def make_consistency(self, emcomponent, fname, datas, type_query):
-        dh = emcomponent.field(fname)
-        logger.info('Warning : multiple uid capabilities are broken here')
-        uid = datas[emcomponent.uid_fieldname()[0]]
-        if self.back_reference is not None:
-            target_class = self.back_reference[0]
-            target_field = self.back_reference[1]
-            target_uidfield = target_class.uid_fieldname()[0]
-            l_value = datas[fname]
-
-            if l_value is not None:
-                for value in l_value:
-                    query_filters = list()
-                    query_filters.append((target_uidfield, '=', value))
-                    obj = target_class.get(query_filters)
-                    if len(obj) == 0:
-                        logger.warning('Object referenced does not exist')
-                        return False
-                    l_uids_ref = obj[0].data(target_field)
-                    if l_uids_ref is None:
-                        l_uids_ref = list()
-                    if uid not in l_uids_ref:
-                        l_uids_ref.append(uid)
-                        obj[0].set_data(target_field, l_uids_ref)
-                        obj[0].update()
-
-            if type_query == 'update':
-                query_filters = list()
-                query_filters.append((uid, ' in ', target_field))
-                objects = target_class.get(query_filters)
-                if l_value is None:
-                    l_value = list()
-                if len(objects) != len(l_value):
-                    for obj in objects:
-                        l_uids_ref = obj.data(target_field)
-                        if obj.data(target_uidfield) not in l_value:
-                            l_uids_ref.remove(uid)
-                            obj.set_data(target_field, l_uids_ref)
-                            obj.update()
-
-
 
 ## @brief Class designed to handle datas access will fieldtypes are constructing datas
 #@ingroup lodel2_datahandlers

lodel/plugins/multisite/main.py

@@ -75,16 +75,23 @@ class LodelWSGIHandler(wsgiref.simple_server.WSGIRequestHandler):
         self.request.close()
         super().close()
 
-##@brief WSGIServer implementing ForkingTCPServer.
-#
-#Same features than wsgiref.simple_server.WSGIServer but process each requests
-#in a child process
-class ForkingWSGIServer(
-        wsgiref.simple_server.WSGIServer, socketserver.ForkingTCPServer):
-    
+   
+class CustomForkingTCPServer(socketserver.ForkingTCPServer):
     ##@brief static property indicating the max number of childs allowed
     max_children = 40
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if self.__class__.active_children is None:
+            self.__class__.active_childer = set()
+    
+    ##@brief Implements max_children limitations
+    def process_request(self, request, client_address):
+        while self.active_children is not None and \
+                len(self.active_children) > self.__class__.max_children:
+            self.collect_children()
+        super().process_request(request, client_address)
+        
     ##@brief Custom reimplementation of shutdown method in order to ensure
     #that we close all listening sockets
     #
@@ -107,6 +114,15 @@ class ForkingWSGIServer(
                     self.active_children.discard(pid)
         self.server_close()
 
+##@brief WSGIServer implementing ForkingTCPServer.
+#
+#Same features than wsgiref.simple_server.WSGIServer but process each requests
+#in a child process
+class ForkingWSGIServer(
+        wsgiref.simple_server.WSGIServer, CustomForkingTCPServer):
+    pass
+ 
+
 ##@brief utility function to extract site id from an url
 def site_id_from_url(url):
     res = ''

lodel/plugins/webui/interface/controllers/admin.py

@@ -45,9 +45,16 @@ def admin_update(request):
             raise HttpException(400)
         target_leo = dyncode.Object.name2class(datas['classname'])
         leo = target_leo.get_from_uid(datas['lodel_id'])
-        for in_value in datas:
-            if datas[in_value] == '':
-                datas[in_value] = None
+
+        #for in_value in datas:
+        #    if datas[in_value] == '':
+        #        datas[in_value] = None
+
+        if leo is None:
+            raise HttpException(404,
+                custom = 'No %s with id %s' % (
+                    target_leo.__name__, datas['lodel_id']))
+
         try:
             leo.update(
                 { f:datas[f] for f in datas if f not in ('classname', 'lodel_id')})

plugins

@@ -0,0 +1 @@
+lodel/plugins/

progs/mass_deploy.sh

@@ -37,6 +37,7 @@ test -f $conffile || badconf
 . $conffile
 test -z "$MONGODB_ADMIN_USER" && badconf
 test -z "$MONGODB_ADMIN_PASSWORD" && badconf
+$MONGODB_HOST=${MONGODB_HOST:="127.0.0.1"}
 
 
 if [ -z "$fixed_name" ]
@@ -65,17 +66,17 @@ fi
 
 if [ -f "/etc/mongodb.conf" ]
 then
-	if cat /etc/mongodb.conf  |grep -E '^ *auth *= *true *$' >/dev/null
-	then
-		echo "OK, auth enabled"
-	else
-		echo "WARNING : auth seems disabled on mongod !" >&2
-	fi
+        if cat /etc/mongodb.conf  |grep -E '^ *auth *= *true *$' >/dev/null
+        then
+                echo "OK, auth enabled"
+        else
+                echo "WARNING : auth seems disabled on mongod !" >&2
+        fi
 else
-	echo "/etc/mongodb.conf not found. Unable to check if auth is on"
+        echo "/etc/mongodb.conf not found. Unable to check if auth is on"
 fi
 
-echo "exit" | mongo $MONGODB_HOST --quiet -u "$MONGODB_ADMIN_USER" -p "$MONGODB_ADMIN_PASSWORD" admin &>/dev/null || mongodb_connfail
+echo "exit" | mongo $MONGODB_HOST --quiet -u "$MONGODB_ADMIN_USER" -p "$MONGODB_ADMIN_PASSWORD" --authenticationDatabase admin &>/dev/null || mongodb_connfail
 
 if [ "$1" == 'purgedb' ]
 then
@@ -88,9 +89,9 @@ then
 	read rep
 	if [ "$rep" = "Y" ]
 	then
-		for dbname in $(echo "show dbs" | mongo -u admin -p pass admin  |grep "^$MONGODB_DB_PREFIX"|cut -f1)
+		for dbname in $(echo "show dbs" | mongo $MONGODB_HOST --quiet -u "$MONGODB_ADMIN_USER" -p "$MONGODB_ADMIN_PASSWORD" --authenticationDatabase admin |grep "^$MONGODB_DB_PREFIX"|cut -f1)
 		do 
-			echo -e "use $dname\ndb.dropDatabase()\nexit\n" | mongo -u $MONGODB_ADMIN_USER -p $MONGODB_ADMIN_PASSWORD --quiet --authenticationDatabase admin $dbname && echo "$dbname succesfully deleted" || echo "$dbname deletion fails" >&2
+			echo -e "use $dname\ndb.dropDatabase()\nexit\n" | mongo $MONGODB_HOST -u "$MONGODB_ADMIN_USER" -p "$MONGODB_ADMIN_PASSWORD" --quiet --authenticationDatabase admin && echo "$dbname succesfully deleted" || echo "$dbname deletion fails" >&2
 		done
 		echo "Done."
 	else
@@ -139,13 +140,13 @@ do
 	dbname="${MONGODB_DB_PREFIX}_$iname"
 	dbuser="lodel2_$iname"
 	dbpass=$($rnd_pass_cmd)
-	mongo $MONGODB_HOST -u "$MONGODB_ADMIN_USER" -p "$MONGODB_ADMIN_PASSWORD" admin <<EOF
+	mongo $MONGODB_HOST -u "$MONGODB_ADMIN_USER" -p "$MONGODB_ADMIN_PASSWORD" --authenticationDatabase admin <<EOF
 use $dbname
 db.addUser({user:"$dbuser", pwd:"$dbpass", roles:["readWrite", "dbAdmin"]})
 exit
 EOF
 	#Append created db to instance conf
-	slim -n $iname -s --datasource_connectors mongodb --host localhost --user $dbuser --password $dbpass --db_name $dbname || slim_fails "configuring the instance's datasource"
+	slim -n $iname -s --datasource_connectors mongodb --host $MONGODB_HOST --user $dbuser --password $dbpass --db_name $dbname || slim_fails "configuring the instance's datasource"
 
 done
 

scripts/cbl.sh

@@ -2,7 +2,7 @@
 #
 # CBL : Curl Benchmark on Lodel
 #
-# Usage : $0 [HOSTNAME] [INSTANCE_LIST_FILE]
+# Usage : $0 [HOSTNAME] [INSTANCE_LIST_FILE] [N_CREATE] [N_EDIT] [N_DELETE]
 #
 # Instance_list_file is expected to be a file containing instances name (1 per
 # line). Uses by default /tmp/lodel2_instance_list.txt
@@ -81,20 +81,11 @@ n_delete=${n_delete:=10}
 
 for i in $(seq $#)
 do
-	echo $1 |grep -E "^-?-h" &>/dev/null && usage
+	echo $1 |grep -E "^-?-h" &>/dev/null
 	shift
 done
 
 
-logdir="/tmp/lodel2_cbl_logs"
-if [ -d "$logdir" ]
-then
-	echo "WARNING : $logdir allready exists. It's a better idea to delete it before running this script again"
-	echo "waiting 3s"
-	sleep 3
-fi
-mkdir -p $logdir
-
 #curl_options='--silent -o /dev/null -s -w %{http_code}:%{time_connect}:%{time_starttransfer}:%{time_total}\n'
 curl_options='--silent -o /dev/null -s -w %{url_effective};%{http_code};%{time_connect};%{time_starttransfer};%{time_total}\n'
 curl_debug_opt='-v -w %{url_effective};%{http_code};%{time_connect};%{time_starttransfer};%{time_total}\n'
@@ -128,9 +119,14 @@ mass_creation() {
 	instance_name=$1
 	iteration_count=$2
 	base_uri=$(_base_uri $1)
-	logfile="$logdir/mass_creation_${instance_name}.log"
 	cls_list_file=$(fetch_all_classes $1)
 
+	if [ -z "$(cat $cls_list_file)" ]
+	then
+		echo "Failed to fetch class list for instance $1. Abording..." >&2
+		exit
+	fi
+
 	if [ "$iteration_count" -le "0" ]
 	then
 		return
@@ -139,10 +135,9 @@ mass_creation() {
 	for i in $(seq $iteration_count)
 	do
 		cls=$(shuf -n1 $cls_list_file)
-		$curcurl -d "$(curl_opt_create_$cls)" "${base_uri}$(uri_create $cls)" | tee -a $logfile
+		echo "${base_uri}$(uri_create $cls) POST $(curl_opt_create_$cls)"
 	done
-
-	rm -v $cls_list_file
+	rm -v $cls_list_file >&2
 }
 
 mass_link_edit() {
@@ -152,7 +147,6 @@ mass_link_edit() {
 	instance_name=$1
 	iteration_count=$2
 	base_uri=$(_base_uri $1)
-	logfile="$logdir/mass_link_edit_${instance_name}.log"
 	cls_list_file=$(fetch_all_classes $1)
 
 	for cls in $(cat $cls_list_file)
@@ -171,17 +165,79 @@ mass_link_edit() {
 					ltext_count=$(shuf -i1-5 -n1)
 					alias_param=$(head -n $(expr $alias_count \* $i) $person_ids| tail -n$alias_count|tr -s "\n" "," | sed 's/,$//')
 					txt_param=$(head -n $(expr $ltext_count \* $i) $text_ids | tail -n$ltext_count|tr -s "\n" "," | sed 's/,$//')
-					$curcurl -d "$(curl_opt_create_$cls $alias_param $txt_param)&uid=$cur_id" "$base_uri/admin/update?classname=$cls&lodel_id=$cur_id" | tee -a $logfile
+					echo "$base_uri/admin/update?classname=$cls&lodel_id=$cur_id POST $(curl_opt_create_$cls $alias_param $txt_param)&uid=$cur_id"
 				done
 				rm -v $text_ids $person_ids $section_ids $subsection_ids
 				;;
-
+            
+            Collection)
+                collections_ids=$(fetch_all_ids $1 Collection)
+                publication_ids=$(fetch_all_ids $1 Publication)
+                for i in $(seq $iteration_count)
+                do
+                    cur_id=$(shuf -n1 $collections_ids)
+                    publications_count=$(shuf -i1-5 -n1)
+                    publication_param=$(head -n $(expr $publications_count \* $i) $publication_ids| tail -n$publications_count|tr -s "\n" ",")
+		    echo "$base_uri/admin/update?classname=$cls&lodel_id=$cur_id POST $(curl_opt_create_$cls $publication_param)&uid=$cur_id"
+                done
+                rm -v $collections_ids $publication_ids
+                ;;
+
+            Publication)
+                publication_ids=$(fetch_all_ids $1 Publication)
+                collection_ids=$(fetch_all_ids $1 Collection)
+                for i in $(seq $iteration_count)
+                do
+                    cur_id=$(shuf -n1 $publication_ids)
+                    collections_count=$(shuf -i1-5 -n1)
+                    collection_param=$(head -n $(expr $collections_count \* $i) $collection_ids| tail -n$collections_count|tr -s "\n" ",")
+		    echo "$base_uri/admin.update?classname=$cls&lodel_id=$cur_id POST $(curl_opt_create_$cls $collection_param)&uid=$cur_id"
+                done
+                rm -v $publication_ids $collection_ids >&2
+                ;;
+			
+			Section)
+                section_ids=$(fetch_all_ids $1 Section)
+                child_ids=$(fetch_all_ids $1 Subsection)
+                person_ids=$(fetch_all_ids $1 Person)
+                for i in $(seq $iteration_count)
+                do
+                    cur_id=$(shuf -n1 $section_ids)
+                    child_count=$(shuf -i1-5 -n1)
+                    person_count=$(shuf -i1-5 -n1)
+                    child_param=$(head -n $(expr $child_count \* $i) $child_ids| tail -n$child_count|tr -s "\n" ",")
+                    person_param=$(head -n $(expr $person_count \* $i) $person_ids| tail -n$person_count|tr -s "\n" ",")
+		    echo "$base_uri/admin/update?classname=$cls&lodel_id=$cur_id POST $(curl_opt_create_$cls $child_param $person_param)&uid=$cur_id"
+                done
+                rm -v $section_ids $child_ids $person_ids >&2
+                ;;
+                
+            Subsection)
+                subsection_ids=$(fetch_all_ids $1 Subsection)
+                section_ids=$(fetch_all_ids $1 Section)
+                persons_ids=$(fetch_all_ids $1 Person)
+                parent_ids=$($cmktemp)
+                cat $section_ids $subsection_ids | shuf > $parent_ids
+                child_ids=$subsection_ids
+                for i in $(seq $iteration_count)
+                do
+                    cur_id=$(shuf -n1 $subsection_ids)
+                    child_count=$(shuf -i1-5 -n1)
+                    parent_count=$(shuf -i1-5 -n1)
+                    person_count=$(shuf -i1-5 -n1)
+                    child_param=$(head -n $(expr $child_count \* $i) $child_ids| tail -n$child_count|tr -s "\n" ",")
+                    parent_param=$(head -n $(expr $parent_count \* $i) $parent_ids| tail -n$parent_count|tr -s "\n" ",")
+                    person_param=$(head -n $(expr $person_count \* $i) $persons_ids| tail -n$person_count|tr -s "\n" ",")
+		    echo "$base_uri/admin/update?classname=$cls&lodel_id=$cur_id POST $(curl_opt_create_$cls $child_param $person_param $parent_param)&uid=$cur_id"
+                done
+                rm -v $subsection_ids $parent_ids $section_ids $persons_ids >&2
+                ;;
 			*)
 				;;
 			
 		esac
 	done
-	rm -v $cls_list_file
+	rm -v $cls_list_file >&2
 }
 
 mass_deletion() {
@@ -191,7 +247,6 @@ mass_deletion() {
 	instance_name=$1
 	iteration_count=$2
 	base_uri=$(_base_uri $1)
-	logfile="$logdir/mass_deletion_${instance_name}.log"
 	cls_list_file=$(fetch_all_classes $1)
 	
 	for cls in $(cat $cls_list_file)
@@ -207,7 +262,7 @@ mass_deletion() {
 		for i in $(seq $max_iter)
 		do
 			id=$(tail -n $i $id_list_file | head -n1)
-			$curcurl "${base_uri}/admin/delete?classname=$cls&lodel_id=$id" | tee -a $logfile
+			echo "${base_uri}/admin/delete?classname=$cls&lodel_id=$id"
 		done
 		rm -v $id_list_file
 	done
@@ -219,6 +274,13 @@ fetch_all_classes() {
 	#$1 is intance name
 	cls_list_file=$($cmktemp)
 	$curl_raw "$(_base_uri $1)/list_classes" | grep -v Abstract |sed -nE 's/^ *<li> +<a href="show_class([^"]+)".*$/\1/p'|cut -d"=" -f2 > $cls_list_file
+	if [ -z "$(cat $cls_list_file)" ]
+	then
+		echo "Unable to fetch class list for $1" >&2
+		echo "Request was : $curl_raw '$(_base_uri $1)/list_classes'" >&2
+		rm $cls_list_file
+		exit 1
+	fi
 	echo $cls_list_file
 }
 
@@ -291,10 +353,30 @@ run_bg_with_param() {
 	rm -v $pidlist
 }
 
-run_bg_with_param "mass_creation" $instance_list $n_create
-run_bg_with_param "mass_link_edit" $instance_list $n_edit
-run_bg_with_param "mass_deletion" $instance_list $n_delete
+get_queries_with_params() {
+	#$1 is the function name to run
+	#$2 is the instance_list filename
+	#other parameters are given to the function
+	fun=$1
+	instance_list=$2
+	shift;shift
+	counter=0
+	for iname in $(cat $instance_list| sort)
+	do
+		echo "Running $fun $iname $@" >&2
+		beg=$(date "+%s")
+		$fun $iname $@
+		tsecs=$(expr $(date "+%s") - $beg)
+		left=$(expr $(cat $instance_list |wc -l) - $counter)
+		counter=$(expr $counter + 1)
+		tleft=$(expr $left \* $tsecs)
+		percent_done=$(echo "2k ${counter}.0 100.0 * $(cat $instance_list |wc -l).0 2k/ f" | dc)
+		echo -e "Done in ${tsecs}s\t$fun ${percent_done}% done ~$tleft secs" >&2
+
+	done | shuf
+}
 
-echo ""
-echo "Logs can be found in $logdir"
+get_queries_with_params "mass_creation" $instance_list $n_create
+get_queries_with_params "mass_link_edit" $instance_list $n_edit
+get_queries_with_params "mass_deletion" $instance_list $n_delete