Add timeout & watchdog for pool & workers

configure.ac

@@ -51,10 +51,11 @@ AC_C_INLINE
 
 # Checks for libraries.
 AC_CHECK_LIB([fcgi], [FCGI_Accept])
+AC_CHECK_LIB([rt], [timer_create])
 PKG_CHECK_MODULES([CHECK], [check >= 0.9.4])
 
 # Checks for header files.
-AC_CHECK_HEADERS([fcntl.h limits.h stdlib.h string.h syslog.h unistd.h sem.h])
+AC_CHECK_HEADERS([fcntl.h limits.h stdlib.h string.h syslog.h unistd.h sem.h time.h])
 
 # Checks for typedefs, structures, and compiler characteristics.
 AC_TYPE_PID_T
@@ -64,7 +65,7 @@ AC_TYPE_SIZE_T
 AC_FUNC_FORK
 AC_FUNC_MALLOC
 AC_FUNC_REALLOC
-AC_CHECK_FUNCS([bzero getcwd gettimeofday memmove strdup strerror strndup semop clone get_current_dir_name])
+AC_CHECK_FUNCS([bzero getcwd gettimeofday memmove strdup strerror strndup semop clone get_current_dir_name timer_settime])
 
 AM_INIT_AUTOMAKE
 AC_CONFIG_FILES([Makefile

include/conf.h

@@ -44,6 +44,7 @@
 
 /**@defgroup ret_status Function & process return status
  */
+#define PYFCGI_TIMEOUT 8
 #define PYFCGI_ERR 16
 /**@ingroup ret_status */
 #define PYFCGI_WORKER_FAIL 32
@@ -118,20 +119,35 @@ typedef struct pyfcgi_context_s pyfcgi_context_t;
 
 /**@brief Stores contextual informations about current process */
 struct pyfcgi_context_s {
+
+	/**@brief Stores current process PID */
 	pid_t pid;
+	/**@brief Stores parent process PID */
 	pid_t ppid;
 
+	/**@brief Stores the watchdog timer */
+	timer_t wd_timer;
+	/**@brief Security timer sending a sigkill */
+	timer_t wd_timerkill;
+	/**@brief Watchdog delay */
+	struct timespec wd_delay;
+	/**@brief Watchdog sig restorer */
+	struct sigaction wd_oldsig;
+	/**@brief watchdog flag */
+	short wd;
 
 	/**@brief array of worker pids (pool handler context) */
 	pid_t **wrk_pids;
 	/**@brief workers count */
 	unsigned int n_wrk;
 
+	/**@brief semid for worker<->pool semaphore */
+	int semid;
+
 	/**@brief Stores python_path (not dupped by python) */
 	wchar_t python_path[PATH_MAX];
 	/**@brief Stores venv_path for python home (not dupped by python) */
 	wchar_t venv_path[PATH_MAX];
-
 	/**@brief Stores a part of the environ (containing wsgi.* keys) */
 	PyObject *wsgi_dict;
 };
@@ -165,6 +181,12 @@ struct pyfcgi_conf_s
 	 * @ingroup conf_glob */
 	int max_reqs;
 
+	/**@brief Worker timeout in seconds (if 0 no timeout)*/
+	time_t worker_timeout;
+
+	/**@brief Pool timeout in seconds (if 0 no timeout)*/
+	time_t pool_timeout;
+
 	/**@brief 0 is silent 1 means logs to stderr */
 	short verbosity;
 
@@ -197,4 +219,18 @@ int check_entrypoint_import();
 
 int parse_optlog(const char*);
 
+/**@brief Initialize the watchdog
+ * @param void (*wd_sig_cleaner)(int) pointer on a signal handler (or NULL to
+ * use @ref pyfcgi_default_sighandler
+ * @param const struct timespec* delay a pointer on the watchdog timeout
+ * @return 0 if no error else -1
+ */
+int pyfcgi_wd_init(void (*wd_sig_cleaner)(int), const struct timespec *delay);
+int pyfcgi_wd_arm();
+int pyfcgi_wd_pause();
+int pyfcgi_wd_stop();
+
+/**@brief Watchdog default signal handler */
+void pyfcgi_wd_default_sighandler(int signum);
+
 #endif

include/pyworker.h

@@ -113,10 +113,9 @@ extern PyObject* response_headers;
  * @ref response_headers globals.
  * @param char* python_entrypoint a path to a python entrypoint
  * @param int worker uid
- * @param int semid for FCGI access
  * @return 0 if exit avec max requests
  */
-int work333(int, int);
+int work333(int);
 
 /**@brief the function that initialize the alternate python worker
  * @ingroup worker_process
@@ -125,10 +124,9 @@ int work333(int, int);
  * FCGI. This function clones for each request, running worker_piper()
  * @param char* python_entrypoint a path to a python entrypoint
  * @param int worker uid
- * @param int semid for FCGI access
  * @return 0 if exit avec max requests
  */
-int work(int, int);
+int work(int);
 
 /**@brief function for a subprocess designed to read stdin & stdout from
  * python & forward them to syslog or FCGI
@@ -168,6 +166,7 @@ void worker_log_pipes(int, int, PyObject*[2]);
 int ctl_get_rep_sz(int, size_t*);
 
 void worker_sighandler(int);
+void worker_sigalrmhandler(int signum);
 
 #endif
 

include/responder.h

@@ -58,8 +58,6 @@
 #include "pyworker.h"
 
 
-extern int pyfcgi_semid;
-
 void init_context();
 
 /**@brief The responder loop
@@ -81,10 +79,9 @@ int responder_loop();
  * @ingroup work_master_proc
  * Spawn a new worker process and prepare ENV & request forwarding
  * @param int worker uid
- * @param int semid for FCGI access
  * @return child PID
  */
-pid_t spawn(int, int);
+pid_t spawn(int);
 
 /**@brief Generate a new semaphore from given key
  * @ingroup work_master_proc
@@ -101,4 +98,7 @@ void clean_exit(int);
 /**@brief Handle signals and forward it to workers */
 void pool_sighandler(int signum);
 
+/**@brief Handle watchdog alarm signal */
+void pool_wd_sighandler(int signum);
+
 #endif

src/conf.c

@@ -46,6 +46,8 @@ void default_conf()
 	PyFCGI_conf.max_reqs = 1000;
 	PyFCGI_conf.pep333 = 1;
 	PyFCGI_conf.verbosity = 0;
+	PyFCGI_conf.pool_timeout = 5;
+	PyFCGI_conf.worker_timeout = 3;
 }
 
 int parse_args(int argc, char *argv[])
@@ -202,3 +204,167 @@ int parse_optlog(const char* logspec)
 	return 0;
 }
 
+int pyfcgi_wd_init(void (*wd_sig_cleaner)(int), const struct timespec *delay)
+{
+	struct sigaction act;
+	struct sigevent sev;
+	pyfcgi_context_t *context;
+	int err;
+
+	context = &(PyFCGI_conf.context);
+
+	PyFCGI_conf.context.wd_delay.tv_sec = delay->tv_sec;
+	PyFCGI_conf.context.wd_delay.tv_nsec = delay->tv_nsec;
+	pyfcgi_log(LOG_DEBUG, "Set watchdog with %ds timeout", PyFCGI_conf.context.wd_delay.tv_sec);
+
+	// Creating new timer with default sigevent (SIGEV_SIGNAL with SIGALRM)
+	if(timer_create(CLOCK_REALTIME, NULL, &(context->wd_timer)) < 0)
+	{
+		err = errno;
+		pyfcgi_log(LOG_ALERT, "Unable to create watchdog timer : %s",
+			strerror(err));
+		errno = err;
+		return -1;
+	}
+	// Creating a new timer sending SIGKILL after 1.5 delay
+	sev.sigev_notify = SIGEV_SIGNAL;
+	sev.sigev_signo = SIGKILL;
+	if(timer_create(CLOCK_REALTIME, &sev, &(context->wd_timerkill)) < 0)
+	{
+		err = errno;
+		pyfcgi_log(LOG_ALERT, "Unable to create kill watchdog timer : %s",
+			strerror(err));
+		errno = err;
+		return -1;
+	}
+
+	// Registering SIGALRM signal handler
+	act.sa_handler = wd_sig_cleaner?wd_sig_cleaner:pyfcgi_wd_default_sighandler;
+	sigemptyset(&act.sa_mask);
+	act.sa_flags = 0;
+	act.sa_restorer = NULL;
+	if(sigaction(SIGALRM, &act, &(context->wd_oldsig)) < 0)
+	{
+		err = errno;
+		pyfcgi_log(LOG_ALERT, "Unable to set the signal handler for watchdog timer : %s",
+			strerror(err));
+		errno = err;
+		return -1;
+	}
+	context->wd = 1;
+	return 0;
+}
+
+int pyfcgi_wd_arm()
+{
+	pyfcgi_context_t *context;
+	struct itimerspec timeout;
+	int err, res;
+
+	context = &(PyFCGI_conf.context);
+
+	if(!context->wd) { return 0; }
+
+	timeout.it_value = context->wd_delay;
+	timeout.it_interval.tv_sec = 0;
+	timeout.it_interval.tv_nsec = 0;
+
+	context = &(PyFCGI_conf.context);
+	res = 0;
+
+	if(timer_settime(context->wd_timer, 0, &timeout, NULL) < 0)
+	{
+		err = errno;
+		pyfcgi_log(LOG_ALERT, "Unable to arm watchdog : %s",
+			strerror(err));
+		res = -1;
+	}
+
+	timeout.it_value.tv_sec = (time_t)(timeout.it_value.tv_sec * 1.5);
+	timeout.it_value.tv_nsec = (long)(timeout.it_value.tv_nsec * 1.5);
+	timeout.it_interval = timeout.it_value;
+
+	if(timer_settime(context->wd_timerkill, 0, &timeout, NULL) < 0)
+	{
+		err = errno;
+		pyfcgi_log(LOG_ALERT, "Unable to pause killer watchdog : %s",
+			strerror(err));
+		res = -1;
+	}
+	return res;
+}
+
+int pyfcgi_wd_pause()
+{
+	pyfcgi_context_t *context;
+	struct itimerspec zero;
+	int err, res;
+
+	context = &(PyFCGI_conf.context);
+	if(!context->wd) { return 0; }
+
+	memset(&zero, 0, sizeof(struct itimerspec));
+
+	res = 0;
+
+	if(timer_settime(context->wd_timer, TIMER_ABSTIME, &zero, NULL) < 0)
+	{
+		err = errno;
+		pyfcgi_log(LOG_ALERT, "Unable to pause watchdog : %s",
+			strerror(err));
+		res = -1;
+	}
+	if(timer_settime(context->wd_timerkill, TIMER_ABSTIME, &zero, NULL) < 0)
+	{
+		err = errno;
+		pyfcgi_log(LOG_ALERT, "Unable to pause killer watchdog : %s",
+			strerror(err));
+		res = -1;
+	}
+	return res;
+}
+
+int pyfcgi_wd_stop()
+{	
+	int err, res;
+	pyfcgi_context_t *context;
+
+	context = &(PyFCGI_conf.context);
+	if(!context->wd) { return 0; }
+	res = 0;
+
+	if(timer_delete(context->wd_timer) < 0)
+	{
+		err = errno;
+		pyfcgi_log(LOG_WARNING, "Unable to delete watchdog timer : %s",
+			strerror(err));
+		res = -1;
+	}
+	if(timer_delete(context->wd_timerkill) < 0)
+	{
+		err = errno;
+		pyfcgi_log(LOG_WARNING, "Unable to delete killer watchdog timer : %s",
+			strerror(err));
+		res = -1;
+	}
+	if(sigaction(SIGALRM, &(context->wd_oldsig), NULL) < 0)
+	{
+		err = errno;
+		pyfcgi_log(LOG_WARNING, "Unable to restore signal handler : %s",
+			strerror(err));
+		res = -1;
+	}
+	context->wd = 0;
+pyfcgi_log(LOG_DEBUG, "Watchdog stopped");
+	return res;
+}
+
+void pyfcgi_wd_default_sighandler(int signum)
+{
+	pyfcgi_log(LOG_ALERT, "Timeout after %d.%09ds",
+		PyFCGI_conf.context.wd_delay.tv_sec,
+		PyFCGI_conf.context.wd_delay.tv_nsec);
+	pyfcgi_wd_stop();
+	exit(PYFCGI_TIMEOUT);
+}
+

src/main.c

@@ -56,7 +56,12 @@ pid_t pool_handler_pid;
 void sighandler(int signum)
 {
 	int ret;
-	if(signum == SIGINT)
+	if(signum == SIGALRM)
+	{
+		pyfcgi_log(LOG_WARNING, "Master process received SIGALRM !");
+		return;
+	}
+	else if(signum == SIGINT)
 	{
 		pyfcgi_log(LOG_INFO,
 			"Master process received ctrl+c, exiting...");
@@ -77,6 +82,13 @@ void sighandler(int signum)
 	exit(0);
 }
 
+void debug_sighandler(int signum)
+{
+	pyfcgi_log(LOG_WARNING, "Master process received signal %s(%d)",
+		strsignal(signum), signum);
+	return;
+}
+
 int main(int argc, char **argv)
 {
 	int child_ret;
@@ -94,6 +106,15 @@ int main(int argc, char **argv)
 		perror("Sigaction error");
 		exit(4);
 	}
+	sigfillset(&act.sa_mask);
+	sigdelset(&act.sa_mask, SIGINT);
+	sigdelset(&act.sa_mask, SIGTERM);
+	act.sa_handler = debug_sighandler;
+	if(sigaction(SIGALRM, &act, NULL))
+	{
+		perror("Sigaction2 error");
+		exit(4);
+	}
 
 
 	pool_handler_pid = 0;

src/pyworker.c

@@ -20,12 +20,10 @@
 #include "pyworker.h"
 
 
-/**@brief Indicate that a worker is idle
- * @param int semid */
-static inline void worker_set_idle(int);
-/**@brief Indicate that a worker is busy
- * @param int semid */
-static inline void worker_set_busy(int);
+/**@brief Indicate that a worker is idle */
+static inline void worker_set_idle();
+/**@brief Indicate that a worker is busy */
+static inline void worker_set_busy();
 
 /**@brief Process results from a pep333 worker
  * @param FCGX_Stream* out stream from libFCGI
@@ -36,7 +34,7 @@ static inline int work333_send_result(FCGX_Stream*, PyObject* ret);
 
 static int worker_piper_sigrcv = 0;
 
-int work333(int wrk_id, int semid)
+int work333(int wrk_id)
 {
 	PyObject *entry_fun, *pyflush[2], *py_osmod, *entry_ret, *environ,
 		*start_response, *args;
@@ -44,14 +42,10 @@ int work333(int wrk_id, int semid)
 	char **envp;
 	int count, pipe_out[2], pipe_err[2];
 	int max_reqs;
-	char ident[128];
 	struct timeval start, stop;
 
 	max_reqs = PyFCGI_conf.max_reqs;
 
-	snprintf(ident, 128, "Worker[%d]", wrk_id);
-	pyfcgi_logger_set_ident(ident);
-
 	pyfcgi_log(LOG_INFO, "Worker started with PEP333 App");
 
 	pyinit();
@@ -67,7 +61,7 @@ int work333(int wrk_id, int semid)
 	pyfcgi_log(LOG_INFO, "Waiting request with %s.%s()",
 		PyFCGI_conf.py_entrymod, PyFCGI_conf.py_entryfun);
 
-	worker_set_idle(semid); //before failing on import
+	worker_set_idle(); //before failing on import
 
 	if(!entry_fun) //but exit if import failed
 	{
@@ -83,8 +77,9 @@ int work333(int wrk_id, int semid)
 	while ((!count || count != max_reqs) &&
 		FCGX_Accept(&in_stream, &out_stream, &err_stream, &envp) >= 0)
 	{
+		pyfcgi_wd_arm();
 		gettimeofday(&start, NULL);
-		worker_set_busy(semid);
+		worker_set_busy();
 		count++;
 		environ = update_pyenv(py_osmod, envp);
 
@@ -126,7 +121,7 @@ int work333(int wrk_id, int semid)
 		FCGX_FClose(in_stream);
 		FCGX_FClose(err_stream);
 		FCGI_Finish();
-		worker_set_idle(semid);
+		worker_set_idle();
 		gettimeofday(&stop, NULL);
 		stop.tv_sec = stop.tv_sec - start.tv_sec;
 		stop.tv_usec = stop.tv_usec - start.tv_usec;
@@ -135,6 +130,7 @@ int work333(int wrk_id, int semid)
 			stop.tv_usec += 1000000;
 			stop.tv_sec -= 1;
 		}
+		pyfcgi_wd_pause();
 		pyfcgi_log(LOG_DEBUG, "Worker[%d] request %d END [OK] %lu bytes in %ld.%06lds",
 			wrk_id, count, libpyfcgi.rep_sz, stop.tv_sec, stop.tv_usec);
 	}
@@ -146,7 +142,7 @@ static inline int work333_send_result(FCGX_Stream *out, PyObject* ret)
 	return 0;
 }
 
-int work(int wrk_id, int semid)
+int work(int wrk_id)
 {
 	PyObject *entry_fun, *pystdout_flush, *pystderr_flush, *py_osmod,
 		*environ;
@@ -162,10 +158,6 @@ int work(int wrk_id, int semid)
 	piper_args_t piper_args;
 	char *piper_stack;
 
-	char ident[128];
-	snprintf(ident, 128, "Worker[%d]", wrk_id);
-	pyfcgi_logger_set_ident(ident);
-
 	max_reqs = PyFCGI_conf.max_reqs;
 	piper_args.wrk_id = wrk_id;
 	piper_args.act = &act;
@@ -215,7 +207,7 @@ int work(int wrk_id, int semid)
 		PyFCGI_conf.py_entrymod, PyFCGI_conf.py_entryfun);
 
 
-	worker_set_idle(semid); //before failing on import
+	worker_set_idle(); //before failing on import
 
 	if(!entry_fun) //but exit if import failed
 	{
@@ -237,8 +229,9 @@ int work(int wrk_id, int semid)
 	count = 0;
 	while ((!count || count != max_reqs) && FCGX_Accept(&in_stream, &out_stream, &err_stream, &envp) >= 0)
 	{
+		pyfcgi_wd_arm();
 		gettimeofday(&start, NULL);
-		worker_set_busy(semid);
+		worker_set_busy();
 
 		count++;
 		piper_args.out = out_stream;
@@ -304,7 +297,7 @@ int work(int wrk_id, int semid)
 			rep_sz = 0;
 		}
 		//Increase sem showing the worker is idle
-		worker_set_idle(semid);
+		worker_set_idle();
 		gettimeofday(&stop, NULL);
 		stop.tv_sec = stop.tv_sec - start.tv_sec;
 		stop.tv_usec = stop.tv_usec - start.tv_usec;
@@ -313,6 +306,7 @@ int work(int wrk_id, int semid)
 			stop.tv_usec += 1000000;
 			stop.tv_sec -= 1;
 		}
+		pyfcgi_wd_pause();
 		pyfcgi_log(LOG_DEBUG, "Worker[%d] request %d END [OK] %lu bytes in %ld.%06lds",
 			wrk_id, count, rep_sz, stop.tv_sec, stop.tv_usec);
 	}
@@ -573,7 +567,7 @@ int ctl_get_rep_sz(int ctl_pipe, size_t* rep_sz)
 	return 0;
 }
 
-static void worker_set_idle(int semid)
+static void worker_set_idle()
 {
 	int err;
 	struct sembuf sop;
@@ -582,7 +576,7 @@ static void worker_set_idle(int semid)
 	sop.sem_flg = 0;
 	sop.sem_op = 1;
 
-	if(semop(semid, &sop, 1) < 0)
+	if(semop(PyFCGI_conf.context.semid, &sop, 1) < 0)
 	{
 		err = errno;
 		pyfcgi_log(LOG_ERR, "error incrementing the semaphore : %s",
@@ -590,7 +584,7 @@ static void worker_set_idle(int semid)
 	}
 }
 
-static void worker_set_busy(int semid)
+static void worker_set_busy()
 {
 	int err;
 	struct sembuf sop;
@@ -599,7 +593,7 @@ static void worker_set_busy(int semid)
 	sop.sem_flg = 0;
 	sop.sem_op = -1;
 
-	if(semop(semid, &sop, 1) < 0)
+	if(semop(PyFCGI_conf.context.semid, &sop, 1) < 0)
 	{
 		err = errno;
 		pyfcgi_log(LOG_ERR, "error incrementing the semaphore : %s",
@@ -612,3 +606,9 @@ void worker_sighandler(int signum)
 	pyfcgi_log(LOG_INFO, "%s signal received, exiting...", strsignal(signum));
 	exit(0);
 }
+
+void worker_sigalrmhandler(int signum)
+{
+	pyfcgi_log(LOG_WARNING, "Timeout, exiting...");
+	exit(PYFCGI_TIMEOUT);
+}

src/responder.c

@@ -17,11 +17,12 @@
  * along with PyFCGI.  If not, see <http://www.gnu.org/licenses/>.
  */
 #include "responder.h"
-int pyfcgi_semid = 0;
 
 void init_context()
 {
-	pyfcgi_semid = 0;
+	PyFCGI_conf.context.semid = 0;
+	PyFCGI_conf.context.pid = getpid();
+	PyFCGI_conf.context.ppid = getppid();
 }
 
 int responder_loop()
@@ -32,7 +33,10 @@ int responder_loop()
 	int status;
 	pid_t ret;
 	struct sembuf sop;
+	/**@brief poll timeout */
 	struct timespec timeout;
+	/**@brief watchdog timeout */
+	struct timespec pool_timeout;
 	short idle;
 	struct sigaction act;
 
@@ -56,10 +60,20 @@ int responder_loop()
 	idle = 0;
 
 	pyfcgi_logger_set_ident("pool");
+
+	if(PyFCGI_conf.pool_timeout)
+	{
+		pool_timeout.tv_nsec = 0;
+		pool_timeout.tv_sec = PyFCGI_conf.pool_timeout;
+		pyfcgi_wd_init(pool_wd_sighandler, &pool_timeout);
+	}
+
 	pyfcgi_log(LOG_INFO, "Preparing workers");
 
 	init_context();
 
+	pyfcgi_wd_arm();
+
 	PyFCGI_conf.context.wrk_pids = &wrk_pids;
 	PyFCGI_conf.context.n_wrk = 0;
 	wrk_pids = malloc(sizeof(int) * PyFCGI_conf.max_wrk);
@@ -73,14 +87,14 @@ int responder_loop()
 	}
 	bzero(wrk_pids, sizeof(int) * PyFCGI_conf.max_wrk);
 
-	semid = new_semaphore();
+	PyFCGI_conf.context.semid = semid = new_semaphore();
 	
 	wanted_n = PyFCGI_conf.min_wrk;
 	n_wrk = 0;
 	// prespawning minimum worker count
 	for(n_wrk=0; n_wrk < wanted_n; n_wrk++)
 	{
-		wrk_pids[n_wrk] = spawn(n_wrk, semid);
+		wrk_pids[n_wrk] = spawn(n_wrk);
 		PyFCGI_conf.context.n_wrk = n_wrk;
 	}
 	//Wait at least for a process to be ready
@@ -90,6 +104,7 @@ int responder_loop()
 	// spawn new one if needed, etc.
 	while(1)
 	{
+		pyfcgi_wd_arm();
 		PyFCGI_conf.context.n_wrk = n_wrk;
 		if( (ret = waitpid(0, &status, WNOHANG)) )
 		{
@@ -97,10 +112,6 @@ int responder_loop()
 			{
 				//TODO : error
 			}
-			if(!ret)
-			{
-				continue;
-			}
 			for(n=0; n<n_wrk; n++)
 			{
 				if(wrk_pids[n] == ret)
@@ -117,15 +128,18 @@ int responder_loop()
 			}
 			idle=0;
 			sop.sem_op = -1;
-			ret = semop(semid, &sop, 1);
+			ret = semtimedop(semid, &sop, 1, &timeout);
 			sop.sem_op = 0;
 			if(ret < 0)
 			{
 				err = errno;
-				pyfcgi_log(LOG_ALERT,
-				       "Unable to dec sem after child exit : %s",
-				       strerror(err));
-				clean_exit(err);
+				if(err != EAGAIN) //can fail if wrokers timeout
+				{
+					pyfcgi_log(LOG_ALERT,
+					       "Unable to dec sem after child exit : %s",
+					       strerror(err));
+					clean_exit(err);
+				}
 			}
 			if(status)
 			{
@@ -164,22 +178,35 @@ int responder_loop()
 				       "Worker[%d] PID %d exited normally",
 				       n, wrk_pids[n]);
 			}
-			// child stopped, looking for it
-			if(wanted_n < n_wrk)
-			{	// need to shift the list and dec n_wrk
-				pyfcgi_log(LOG_DEBUG, "GC Workers");
-pyfcgi_log( LOG_DEBUG, "GC want %d have %d", wanted_n, n_wrk);
-				memmove(wrk_pids+n, wrk_pids+n+1,
-				        sizeof(pid_t) * (n_wrk - n));
-				n_wrk--;
+
+			// respawn on same slot
+			pyfcgi_log(LOG_INFO, "respawn #%d", n);
+			wrk_pids[n] = spawn(n);
+
+			
+		}
+		// Stopping & deleting useless childs
+		if(wanted_n < n_wrk)
+		{	// need to shift the list and dec n_wrk
+			pyfcgi_log(LOG_DEBUG, "GC Workers");
+			n_wrk--;
+			kill(wrk_pids[n_wrk], SIGTERM);
+			nanosleep(&timeout, NULL);
+			kill(wrk_pids[n_wrk], SIGKILL);
+			nanosleep(&timeout, NULL);
+			if( (ret = waitpid(wrk_pids[n_wrk], &status, WNOHANG)) < 0 )
+			{
+				pyfcgi_log(LOG_ERR, "Unable to kill child %d (PID %d)",
+					n_wrk, wrk_pids[n_wrk]);
 			}
 			else
-			{	// respawn on same slot
-				pyfcgi_log(LOG_INFO, "respawn #%d", n);
-				wrk_pids[n] = spawn(n, semid);
-				continue;
+			{
+				pyfcgi_log(LOG_INFO, "worker[%d](%d) killed",
+					n_wrk, wrk_pids[n_wrk]);
 			}
+			continue;
 		}
+
 		ret = semtimedop(semid, &sop, 1, &timeout);
 //pyfcgi_log( LOG_DEBUG, "semtimeop ret=%d want %d have %d", ret, wanted_n, n_wrk);
 		if(ret < 0)
@@ -203,18 +230,26 @@ pyfcgi_log( LOG_DEBUG, "GC want %d have %d", wanted_n, n_wrk);
 			pyfcgi_log(LOG_ERR, "Unable to read semaphore : %s",
 			       strerror(err));
 		}
-		if(!ret && n_wrk < PyFCGI_conf.max_wrk)
+		if(!ret)
 		{
-			idle=0;
-			pyfcgi_log( LOG_DEBUG,
-				"All workers busy, spawning a new one");
-			n = n_wrk;
-			n_wrk++;
-			wanted_n = n_wrk;
-			wrk_pids[n] = spawn(n, semid);
+			if(n_wrk < PyFCGI_conf.max_wrk)
+			{
+				idle=0;
+				pyfcgi_log( LOG_DEBUG,
+					"All workers busy, spawning a new one");
+				n = n_wrk;
+				n_wrk++;
+				wanted_n = n_wrk;
+				wrk_pids[n] = spawn(n);
+			}
+			else
+			{
+				nanosleep(&timeout, NULL);
+			}
 		}
 	}
 	
+	pyfcgi_wd_arm();
 	//Debug wait & exit
 	for(; n_wrk != 0; n_wrk--)
 	{
@@ -224,15 +259,18 @@ pyfcgi_log( LOG_DEBUG, "GC want %d have %d", wanted_n, n_wrk);
 		PyFCGI_conf.context.n_wrk = n_wrk;
 	}
 		//printf("Content-Type: text/html\r\n\r\nHello world !\n");
+	pyfcgi_wd_stop();
 	pyfcgi_log(LOG_INFO,"Child workers stoped, stopping responder");
 	exit(0);
 }
 
-pid_t spawn(int wrk_id, int semid)
+pid_t spawn(int wrk_id)
 {
 	pid_t res;
 	struct timespec timeout;
+	struct timespec wd_timeout;
 	struct sigaction act;
+	char ident[128];
 
 	timeout.tv_sec = 0;
 	timeout.tv_nsec = 100000000;
@@ -254,19 +292,29 @@ pid_t spawn(int wrk_id, int semid)
 	else if(!res)
 	{
 		// Child process
+		snprintf(ident, 128, "Worker[%d]", wrk_id);
+		pyfcgi_logger_set_ident(ident);
+		// Set handler for SIGINT & SIGTERM
 		if(sigaction(SIGINT, &act, NULL))
 		{
 			perror("Sigaction error for pool process");
 			exit(PYFCGI_FATAL);
 		}
+		// Set watchdog
+		if(PyFCGI_conf.worker_timeout)
+		{
+			wd_timeout.tv_nsec = 0;
+			wd_timeout.tv_sec = PyFCGI_conf.worker_timeout;
+			pyfcgi_wd_init(worker_sigalrmhandler, &wd_timeout);
+		}
 
 		if(PyFCGI_conf.pep333)
 		{
-			exit(work333(wrk_id, semid));
+			exit(work333(wrk_id));
 		}
 		else
 		{
-			exit(work(wrk_id, semid));
+			exit(work(wrk_id));
 		}
 	}
 	// Sleep to avoid spawning like hell thinking all workers are
@@ -292,20 +340,21 @@ int new_semaphore(key_t semkey)
 			strerror(err));
 		clean_exit(err);
 	}
-	if(pyfcgi_semid)
+	if(PyFCGI_conf.context.semid)
 	{
 		pyfcgi_log(	LOG_WARNING,
 			"The semid context was not zero when calling new_semaphore, attempt to closeing it.");
-		semctl(pyfcgi_semid, 0, IPC_RMID);
+		semctl(PyFCGI_conf.context.semid, 0, IPC_RMID);
 
 	}
-	pyfcgi_semid = semid;
+	PyFCGI_conf.context.semid = semid;
 	return semid;
 }
 
 void clean_exit(int status)
 {
-	if(pyfcgi_semid && semctl(pyfcgi_semid, 0, IPC_RMID) == -1)
+	if(PyFCGI_conf.context.semid &&
+		semctl(PyFCGI_conf.context.semid, 0, IPC_RMID) == -1)
 	{
 		/*silentely fails (when exiting from sighandler)
 		pyfcgi_log(	LOG_CRIT,
@@ -342,3 +391,23 @@ void pool_sighandler(int signum)
 	}
 	clean_exit(0);
 }
+
+void pool_wd_sighandler(int signum)
+{
+	unsigned int i;
+	pyfcgi_log(LOG_ALERT, "Worker pool timeout ! Attempt to kill all childs");
+	for(i=0; i<PyFCGI_conf.context.n_wrk; i++)
+	{
+		pyfcgi_log(LOG_ALERT, "Child[%d] PID %d", i, (*PyFCGI_conf.context.wrk_pids)[i]);
+		kill((*PyFCGI_conf.context.wrk_pids)[i], SIGALRM);
+	}
+	while(PyFCGI_conf.context.n_wrk)
+	{
+		kill((*PyFCGI_conf.context.wrk_pids)[PyFCGI_conf.context.n_wrk], SIGALRM);
+		PyFCGI_conf.context.n_wrk--;
+	}
+	pyfcgi_wd_stop();
+	kill(PyFCGI_conf.context.pid, SIGTERM);
+	clean_exit(PYFCGI_TIMEOUT);
+	exit(PYFCGI_TIMEOUT);
+}