Pool handling enhancement + bugfix in worker sighandling

The responder_loop handle better child creation & deletion. A conf struct
member indicates the number of seconds the pool have to be idle before deleting
a worker.
In a same way, the pool has to be busy for at least 1s before a new
process can be added (the -f flag indicate if the counter has to be reseted
when a new process is created, allowing spawn burst).
The spawn() function call sigaction 2 times for SIGINT & SIGTERM in order
to catch both.

src/responder.c

@@ -37,8 +37,10 @@ int responder_loop()
 	struct timespec timeout;
 	/**@brief watchdog timeout */
 	struct timespec pool_timeout;
-	short idle;
+	time_t idle_start, busy_start;
+	short idle, busy;
 	struct sigaction act;
+	char *statusstr;
 
 	act.sa_handler = pool_sighandler;
 	sigemptyset(&act.sa_mask);
@@ -57,7 +59,7 @@ int responder_loop()
 	sop.sem_flg = 0;
 	timeout.tv_sec = 0;
 	timeout.tv_nsec = 100000000;
-	idle = 0;
+	idle = busy = 0;
 
 	pyfcgi_logger_set_ident("Workpool");
 
@@ -126,53 +128,32 @@ int responder_loop()
 				       ret);
 				continue;
 			}
-			idle=0;
-			sop.sem_op = -1;
-			ret = semtimedop(semid, &sop, 1, &timeout);
-			sop.sem_op = 0;
-			if(ret < 0)
-			{
-				err = errno;
-				if(err != EAGAIN) //can fail if wrokers timeout
-				{
-					pyfcgi_log(LOG_ALERT,
-					       "Unable to dec sem after child exit : %s",
-					       strerror(err));
-					clean_exit(err);
-				}
-			}
-			if(status)
+			if(WIFSIGNALED(status))
 			{
-				if(WIFSIGNALED(status))
-				{
-					if(WTERMSIG(status) == 11)
-					{
-						pyfcgi_log(LOG_ALERT,
-							"Worker[%d] segfault !",
-							n);
-					}
-					else
-					{
-						pyfcgi_log(LOG_ALERT,
-							"Worker[%d] terminated by signal %d",
-							n, WTERMSIG(status));
-					}
-				}
-				if(WEXITSTATUS(status) & PYFCGI_FATAL)
+				if(WTERMSIG(status) == 11)
 				{
 					pyfcgi_log(LOG_ALERT,
-						"Worker[%d] exited with status FATAL",
+						"Worker[%d] segfault !",
 						n);
-					//TODO : restart ?
 				}
 				else
 				{
-					pyfcgi_log(LOG_WARNING,
-						"Worker[%d] exited with status %d",
-						n, WEXITSTATUS(status));
+					pyfcgi_log(LOG_ALERT,
+						"Worker[%d] terminated by signal %s(%d)",
+						n, strsignal(WTERMSIG(status)),
+						WTERMSIG(status));
 				}
 			}
-			else
+			if(WEXITSTATUS(status))
+			{
+				statusstr = status2str(WEXITSTATUS(status));
+				pyfcgi_log((WEXITSTATUS(status)&PYFCGI_FATAL)?
+						LOG_ALERT:LOG_WARNING,
+					"Worker[%d] exited with status %s",
+					n, statusstr);
+				free(statusstr);
+			}
+			if(!status)
 			{
 				pyfcgi_log(LOG_INFO,
 				       "Worker[%d] PID %d exited normally",
@@ -180,73 +161,88 @@ int responder_loop()
 			}
 
 			// respawn on same slot
-			pyfcgi_log(LOG_INFO, "respawn #%d", n);
+			pyfcgi_log(LOG_DEBUG, "respawning worker #%d", n);
 			wrk_pids[n] = spawn(n);
 
-			
-		}
-		// Stopping & deleting useless childs
-		if(wanted_n < n_wrk)
-		{	// need to shift the list and dec n_wrk
-			pyfcgi_log(LOG_DEBUG, "GC Workers");
-			n_wrk--;
-			kill(wrk_pids[n_wrk], SIGTERM);
-			nanosleep(&timeout, NULL);
-			kill(wrk_pids[n_wrk], SIGKILL);
-			nanosleep(&timeout, NULL);
-			if( (ret = waitpid(wrk_pids[n_wrk], &status, WNOHANG)) < 0 )
-			{
-				pyfcgi_log(LOG_ERR, "Unable to kill child %d (PID %d)",
-					n_wrk, wrk_pids[n_wrk]);
-			}
-			else
-			{
-				pyfcgi_log(LOG_INFO, "worker[%d](%d) killed",
-					n_wrk, wrk_pids[n_wrk]);
-			}
 			continue;
 		}
 
+		// Check if the pool is idle or busy
 		ret = semtimedop(semid, &sop, 1, &timeout);
-//pyfcgi_log( LOG_DEBUG, "semtimeop ret=%d want %d have %d", ret, wanted_n, n_wrk);
 		if(ret < 0)
 		{
 			err = errno;
-			if(err == EAGAIN)
+			if(err != EAGAIN)
 			{
-//pyfcgi_log(LOG_DEBUG, "IDLE want %d have %d\t min=%d", wanted_n, n_wrk, min_wrk);
-				// workers idle
-				if(!idle)
-				{
-					idle = 1;
-				}
-				else if(wanted_n > PyFCGI_conf.min_wrk
-					&& n_wrk - wanted_n < 2)
-				{
-					wanted_n--;
-				}
-				continue;
+				pyfcgi_log(LOG_ERR, "Unable to read semaphore : %s",
+				       strerror(err));
+				exit(PYFCGI_FATAL);
+			}
+			// workers idle
+			busy = 0;
+			if(!idle)
+			{
+				idle = 1;
+				idle_start = time(NULL);	
+			}
+			else if((time(NULL) - idle_start) > PyFCGI_conf.worker_gc_timeout &&
+				wanted_n > PyFCGI_conf.min_wrk
+				&& n_wrk - wanted_n < 2)
+			{
+				wanted_n--;
+				idle = 0;
 			}
-			pyfcgi_log(LOG_ERR, "Unable to read semaphore : %s",
-			       strerror(err));
 		}
-		if(!ret)
+		else if(!ret)
 		{
-			if(n_wrk < PyFCGI_conf.max_wrk)
+			idle = 0;
+			if(!busy)
+			{
+				busy = 1;
+				busy_start = time(NULL);
+			}
+			else if(time(NULL) - busy_start > 0 &&
+				wanted_n <= PyFCGI_conf.max_wrk)
 			{
-				idle=0;
 				pyfcgi_log( LOG_DEBUG,
 					"All workers busy, spawning a new one");
 				n = n_wrk;
 				n_wrk++;
 				wanted_n = n_wrk;
 				wrk_pids[n] = spawn(n);
+				if(!PyFCGI_conf.worker_fast_spawn)
+				{
+					busy_start = time(NULL);
+				}
+			}
+		}
+
+		// Stopping & deleting useless childs
+		if(wanted_n < n_wrk && idle)
+		{	// need to shift the list and dec n_wrk
+			busy = 0;
+			n_wrk--;
+			kill(wrk_pids[n_wrk], SIGTERM);
+			nanosleep(&timeout, NULL);
+			if( (ret = waitpid(wrk_pids[n_wrk], &status, WNOHANG)) < 0 )
+			{
+				pyfcgi_log(LOG_ERR, "Pool idle since %ds but unable to kill child %d (PID %d)",
+					PyFCGI_conf.worker_gc_timeout,
+					n_wrk, wrk_pids[n_wrk]);
+				kill(wrk_pids[n_wrk], SIGKILL);
 			}
 			else
 			{
-				nanosleep(&timeout, NULL);
+				pyfcgi_log(LOG_INFO, "Pool idle since %ds : worker[%d](%d) killed",
+					PyFCGI_conf.worker_gc_timeout,
+					n_wrk, wrk_pids[n_wrk]);
 			}
+			idle = 0;
+			continue;
 		}
+
+
+		nanosleep(&timeout, NULL);
 	}
 	
 	pyfcgi_wd_arm();
@@ -267,17 +263,12 @@ int responder_loop()
 pid_t spawn(int wrk_id)
 {
 	pid_t res;
-	struct timespec timeout;
 	struct timespec wd_timeout;
 	struct sigaction act;
 	char ident[128];
 
-	timeout.tv_sec = 0;
-	timeout.tv_nsec = 100000000;
-
 	act.sa_handler = worker_sighandler;
 	sigemptyset(&act.sa_mask);
-	sigaddset(&act.sa_mask, SIGTERM);
 	act.sa_flags = 0;
 	act.sa_restorer = NULL;
 
@@ -292,6 +283,8 @@ pid_t spawn(int wrk_id)
 	else if(!res)
 	{
 		// Child process
+		PyFCGI_conf.context.ppid = PyFCGI_conf.context.pid;
+		PyFCGI_conf.context.pid = getpid();
 		snprintf(ident, 128, "Worker%2d", wrk_id);
 		pyfcgi_logger_set_ident(ident);
 		// Set handler for SIGINT & SIGTERM
@@ -300,6 +293,11 @@ pid_t spawn(int wrk_id)
 			perror("Sigaction error for pool process");
 			exit(PYFCGI_FATAL);
 		}
+		if(sigaction(SIGTERM, &act, NULL))
+		{
+			perror("Sigaction2 error for pool process");
+			exit(PYFCGI_FATAL);
+		}
 		// Set watchdog
 		if(PyFCGI_conf.worker_timeout)
 		{
@@ -320,7 +318,7 @@ pid_t spawn(int wrk_id)
 	// Sleep to avoid spawning like hell thinking all workers are
 	// busy. Let some time to this one to go up...
 	// TODO: find a better way to avoid spawning to max_wrk
-	nanosleep(&timeout, NULL);
+	//nanosleep(&timeout, NULL);
 	pyfcgi_log(	LOG_INFO,
 		"Worker #%d spawned with PID %d", wrk_id, res);
 	return res;