Start implementing pep333 application support

Doxyfile

@@ -790,7 +790,7 @@ WARN_LOGFILE           =
 # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING
 # Note: If this tag is empty the current directory is searched.
 
-INPUT                  = src
+INPUT                  = src include
 
 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses

Makefile.am

@@ -1 +1 @@
-SUBDIRS = src tests
+SUBDIRS = src tests lib

autogen.sh

@@ -6,6 +6,7 @@ rm -fR configure aclocal.m4 autom4te.cache src/Makefile.in tests/Makefile.in Mak
 
 [ "$1" = "clean" ] && exit 0
 
+libtoolize
 aclocal
-automake -a -c
 autoconf
+automake -a -c

configure.ac

@@ -6,6 +6,7 @@ AC_INIT([PyFCGI], [0.0.1], [yann.weber@member.fsf.org])
 AC_CONFIG_SRCDIR([src/main.c])
 AC_CONFIG_HEADERS([include/config.h])
 
+
 AC_ARG_ENABLE([debug],
 	AS_HELP_STRING([--enable-debug], [Enable debug, disabling -Werror, etc.]))
 
@@ -21,6 +22,10 @@ if test x"${PYTHON_CONFIG}" == x"yes"; then
 	PYTHON_LDFLAGS=`python3-config --libs`
 	AC_SUBST([PYTHON_CFLAGS])
 	AC_SUBST([PYTHON_LDFLAGS])
+	PYTHON_SO_CFLAGS=`python3-config --cflags`
+	PYTHON_SO_LDFLAGS=`python3-config --cflags`
+	AC_SUBST([PYTHON_SO_CFLAGS])
+	AC_SUBST([PYTHON_SO_LDFLAGS])
 else
 	AC_MSG_ERROR([Unable to find python3-config])
 fi
@@ -31,6 +36,11 @@ else
 	AM_CFLAGS="-Wall -Werror -O2"
 fi
 
+AC_ENABLE_SHARED
+AC_DISABLE_STATIC
+AC_PROG_LIBTOOL
+LT_INIT
+
 AC_SUBST([AM_CFLAGS])
 
 AC_C_INLINE
@@ -55,5 +65,6 @@ AC_CHECK_FUNCS([bzero getcwd gettimeofday memmove strdup strerror strndup semop
 AM_INIT_AUTOMAKE
 AC_CONFIG_FILES([Makefile
                  src/Makefile
-		 tests/Makefile])
+		 tests/Makefile
+		 lib/Makefile])
 AC_OUTPUT

include/conf.h

@@ -44,14 +44,17 @@
 /**@ingroup ret_status */
 #define PYFCGI_FATAL 128
 
+#define EXIT_PYERR 42
+
 #define PYFCGI_NAME "spawn-fcgi [OPTIONS] -- pyfcgi"
 
-#define PYFCGI_SHORT_OPT "Ce:E:w:W:m:L:Svh"
+#define PYFCGI_SHORT_OPT "Ce:E:Aw:W:m:L:Svh"
 
 #define PYFCGI_LONG_OPT { \
 	{"config", required_argument, 0, 'C'},\
 	{"pymodule", required_argument, 0, 'e'},\
 	{"pyapp", required_argument, 0, 'E'},\
+	{"alt-io", no_argument, 0, 'A'},\
 	{"min-worker", required_argument, 0, 'w'},\
 	{"max-worker", required_argument, 0, 'W'},\
 	{"max-request", required_argument, 0, 'm'},\
@@ -67,6 +70,7 @@
 	{"Load options from configuration file", "CONFIG"},\
 	{"Search application function in given python module", "MODULE_NAME"},\
 	{"Python application entrypoint function name", "FUNC_NAME"},\
+	{"Use stdout to communicate with web server instead of entrypoint return as specified in PEP 333", NULL},\
 	{"Minimum worker in the pool", "INT"},\
 	{"Maximum worker in the pool", "INT"},\
 	{"Request count after wich the worker is restarted (if 0 never restart)", "INT"},\
@@ -121,6 +125,9 @@ struct pyfcgi_conf_s
 	/**@brief Entrypoint function name
 	 * @ingroup conf_glob */
 	char *py_entryfun;
+	/**@brief If 0 use stdout to communicate with webserver, else
+	 * PyFCGI will expect PEP333 compliant entrypoint */
+	short pep333;
 	/**@brief Minimum count worker in pool
 	 * @ingroup conf_glob */
 	int min_wrk;

include/logger.h

@@ -19,6 +19,7 @@
 #ifndef __LOGGER_H___
 #define __LOGGER_H___
 
+
 #include "config.h"
 #include <syslog.h>
 #include <stdlib.h>
@@ -38,6 +39,7 @@
  */
 /**@defgroup conf_logger Logging configuration
  * @ingroup conf_internal
+ * @ingroup logging
  */
 
 /**@defgroup log_facility Logger custom facilities */

include/pyutils.h

@@ -28,7 +28,10 @@
 #include <Python.h>
 
 #include "logger.h"
+#include "python_pyfcgi.h"
 
+extern PyObject* response_status;
+extern PyObject* response_headers;
 /**@brief Call Py_Initialize & update_python_path
  */
 void pyinit();
@@ -37,14 +40,55 @@ void pyinit();
  */
 void update_python_path();
 
-/**@brief Import & return the python entrypoint callable
- * from PyFCGI_conf.py_entrymod & PyFCGI_conf.py_entryfun
+/**@brief Fetch stdout & stderr python flush() function
+ * @param PyObject* pystdout_flush
+ * @param PyObject* pystderr_flush
+ */
+void fetch_pyflush(PyObject**, PyObject**);
+
+/**@brief Create two pipes for stdout & stderr
+ * @ingroup worker_process
+ * @params int[2] pystdout
+ * @params int[2] pystderr
  */
+void update_python_fd(int[2], int[2]);
+
+/**@brief Clear then update python sys.environ using current FCI environ
+ * @ingroup worker_process
+ * @note The environ has to be set without a call to os.putenv, the problem
+ * is that the os.environ is a special mapping calling putenv on setitem...
+ * For these reason the os.environ will be replaced by a new dict instance for
+ * each request...
+ * @param PyObject* os module
+ * @return Python env dict
+ */
+PyObject* update_pyenv(PyObject*, char**);
+
+/**@brief Import & return the python entrypoint callable
+ * from PyFCGI_conf.py_entrymod & PyFCGI_conf.py_entryfun */
 PyObject* import_entrypoint();
 
+/**@brief Return the start_response() python function for pep333 worker */
+PyObject* get_start_response();
+
+/**@brief Clean response_status & response_headers globals */
+inline void libpyfcgi_clean_response()
+{
+	if(response_status) { Py_DECREF(response_status); }
+	if(response_headers) { Py_DECREF(response_headers); }
+}
+
+
+/**@brief Logs an exception */
 void log_expt(int priority);
 
+/**@brief Set python version
+ * @param char[16] version buffer */
 void pyfcgi_python_version(char[16]);
 
+/**@brief Import os module and exit on error
+ * @return Imported module */
+PyObject* python_osmod();
+
 #endif
 

include/pyworker.h

@@ -81,7 +81,6 @@
 #include "logger.h"
 #include "pyutils.h"
 
-#define EXIT_PYERR 42
 #define WPIPER_SIG 30
 #define PIPER_STACK_SZ (1024 * 1024 * 4)
 
@@ -98,15 +97,20 @@ struct piper_args_s
 	struct sigaction *act;
 };
 
+extern PyObject* response_status;
+extern PyObject* response_headers;
+
 /**@todo TODO on all request (when updating env ?) update stdin so
  * python will be able to read the request content */
 
-/**@brief the function that initialize the python worker
+/**@brief the function that initialize the alternate python worker
  * @ingroup worker_process
+ *
+ * This function handles python worker using stdout to communicate with
+ * FCGI. This function clones for each request, running worker_piper()
  * @param char* python_entrypoint a path to a python entrypoint
  * @param int worker uid
  * @param int semid for FCGI access
- * @param int max request before worker restart
  * @return 0 if exit avec max requests
  */
 int work(int, int);
@@ -138,28 +142,5 @@ void worker_piper_sighandler(int);
  */
 int ctl_get_rep_sz(int, size_t*);
 
-/**@brief Fetch stdout & stderr python flush() function
- * @param PyObject* pystdout_flush
- * @param PyObject* pystderr_flush
- */
-void fetch_pyflush(PyObject**, PyObject**);
-
-/**@brief Create two pipes for stdout & stderr
- * @ingroup worker_process
- * @params int[2] pystdout
- * @params int[2] pystderr
- */
-void update_python_fd(int[2], int[2]);
-
-/**@brief Clear then update python sys.environ using current FCI environ
- * @ingroup worker_process
- * @note The environ has to be set without a call to os.putenv, the problem
- * is that the os.environ is a special mapping calling putenv on setitem...
- * For these reason the os.environ will be replaced by a new dict instance for
- * each request...
- * @param PyObject* os module
- */
-void update_pyenv(PyObject*, char**);
-
 #endif
 

lib/Makefile.am

@@ -0,0 +1,4 @@
+lib_LTLIBRARIES = libpyfcgi.la
+libpyfcgi_la_SOURCES = pyfcgi.c
+libpyfcgi_la_CFLAGS = $(PYTHON_SO_CFLAGS)
+libpyfcgi_la_LDFLAGS = $(PYTHON_SO_LDFLAGS)

lib/pyfcgi.c

@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) 2019 Weber Yann
+ * 
+ * This file is part of PyFCGI.
+ * 
+ * PyFCGI is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * any later version.
+ * 
+ * PyFCGI is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Affero General Public License
+ * along with PyFCGI.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "python_pyfcgi.h"
+
+PyMODINIT_FUNC
+PyInit_libpyfcgi(void)
+{
+	PyObject *module;
+	module = PyModule_Create(&pyfcgimodule);
+	if(module == NULL) { return NULL; }
+	return module;
+}
+
+PyObject* pyfcgi_start_response(PyObject* self, PyObject** argv, Py_ssize_t argc)
+{
+	char err[64];
+	if(argc != 2)
+	{
+		snprintf(err, 64,
+			"libpyfcgi.start_response() expected 2 arguments but %d given",
+			argc);
+		PyErr_SetString(PyExc_ValueError, err);
+		Py_RETURN_NONE;
+	}
+
+	libpyfcgi_clean_response();
+
+	response_status = argv[0];
+	response_headers = argv[1];
+	Py_RETURN_NONE;
+}

src/conf.c

@@ -44,6 +44,7 @@ void default_conf()
 	PyFCGI_conf.min_wrk = 1;
 	PyFCGI_conf.max_wrk = 5;
 	PyFCGI_conf.max_reqs = 1000;
+	PyFCGI_conf.pep333 = 1;
 }
 
 int parse_args(int argc, char *argv[])
@@ -71,6 +72,9 @@ int parse_args(int argc, char *argv[])
 			case 'E':
 				PyFCGI_conf.py_entryfun = strdup(optarg);
 				break;
+			case 'A':
+				PyFCGI_conf.pep333 = 0;
+				break;
 			case 'w':
 				PyFCGI_conf.min_wrk = atoi(optarg);
 				break;

src/pyutils.c

@@ -112,6 +112,240 @@ update_python_path_err:
 	exit(err);
 }
 
+static PyObject* _fetch_pyflush(const char *fdname)
+{
+	PyObject *pyfd, *pyflush;
+	pyfd = PySys_GetObject(fdname);
+	if(!pyfd)
+	{
+		pyfcgi_log(LOG_ALERT, "Unable to fetch sys.%s", fdname);
+		log_expt(LOG_ALERT);
+		Py_Exit(EXIT_PYERR);
+	}
+	pyflush = PyObject_GetAttrString(pyfd, "flush");
+	Py_DECREF(pyfd);
+	if(!pyflush)
+	{
+		pyfcgi_log(LOG_ALERT, "Unable to fetch sys.%s.flush", fdname);
+		log_expt(LOG_ALERT);
+		Py_Exit(EXIT_PYERR);
+	}
+	if(!PyCallable_Check(pyflush))
+	{
+		pyfcgi_log(LOG_ALERT, "sys.%s.flush is not callable !",
+		       fdname);
+		Py_Exit(EXIT_PYERR);
+	}
+
+	return pyflush;
+}
+
+void fetch_pyflush(PyObject** pystdout_flush, PyObject** pystderr_flush)
+{
+	*pystdout_flush = _fetch_pyflush("stdout");
+	*pystderr_flush = _fetch_pyflush("stderr");
+}
+
+void update_python_fd(int pipe_out[2], int pipe_err[2])
+{
+	int pri, err;
+	char *err_fmt;
+	PyObject *os_mod, *pyfdopen, *args, *new_fd;
+
+	pri = LOG_ALERT;
+
+	if(pipe2(pipe_out, O_DIRECT) == -1)
+	{
+		err = errno;
+		err_fmt = "Unable to create pipe for python stdout : %s";
+		goto update_python_fd_err;
+	}
+	if(pipe2(pipe_err, O_DIRECT) == -1)
+	{
+		err = errno;
+		err_fmt = "Unable to create pipe for python stderr : %s";
+		goto update_python_fd_err_pipeout;
+	}
+
+	os_mod = PyImport_ImportModule("os");
+	if(!os_mod)
+	{
+		if(PyErr_Occurred())
+		{
+			log_expt(LOG_ALERT);
+		}
+		else
+		{
+			pyfcgi_log(	LOG_ALERT,
+				"Unable to import python os module, got NULL.");
+		}
+		err_fmt = NULL;
+		goto update_python_fd_err_pipes;
+	}
+	pyfdopen = PyObject_GetAttrString(os_mod, "fdopen");
+	Py_DECREF(os_mod);
+	if(!pyfdopen)
+	{
+		if(PyErr_Occurred())
+		{
+			log_expt(LOG_ALERT);
+		}
+		else
+		{
+			pyfcgi_log(	LOG_ALERT,
+				"Unable to fetch os.fdopen() , got NULL.");
+		}
+		err_fmt = NULL;
+		goto update_python_fd_err_pipes;
+	}
+
+	args = Py_BuildValue("is", pipe_out[1], "w");
+	if(!args)
+	{
+		pyfcgi_log(	LOG_ERR, "Error building values with '%d', '%s' for stdout",
+			pipe_out[1], "w");
+		log_expt(LOG_ALERT);
+		err_fmt = NULL;
+		goto update_python_fd_err_fdopen;
+	}
+	new_fd = PyObject_CallObject(pyfdopen, args);
+	if(!new_fd || PyErr_Occurred())
+	{
+		pyfcgi_log(	LOG_ERR, "Error calling fdopen(%d, '%s')",
+			pipe_out[1], "w");
+		log_expt(LOG_ALERT);
+		err_fmt = NULL;
+		goto update_python_fd_err_args;
+	}
+	Py_DECREF(args);
+	if(PySys_SetObject("stdout", new_fd))
+	{
+		pyfcgi_log(LOG_ERR, "Unable to set sys.stdout");
+		log_expt(LOG_ALERT);
+		goto update_python_fd_err_newfd;
+	}
+	Py_DECREF(new_fd);
+
+	args = Py_BuildValue("is", pipe_err[1], "w");
+	if(!args)
+	{
+		pyfcgi_log(	LOG_ERR, "Error building values with '%d', '%s' for stderr",
+			pipe_out[1], "w");
+		log_expt(LOG_ALERT);
+		err_fmt = NULL;
+		goto update_python_fd_err_fdopen;
+	}
+	new_fd = PyObject_CallObject(pyfdopen, args);
+	if(!new_fd || PyErr_Occurred())
+	{
+		pyfcgi_log(	LOG_ERR, "Error calling fdopen(%d, '%s')",
+			pipe_out[1], "w");
+		log_expt(LOG_ALERT);
+		err_fmt = NULL;
+		goto update_python_fd_err_args;
+	}
+	Py_DECREF(args);
+	if(PySys_SetObject("stderr", new_fd))
+	{
+		pyfcgi_log(LOG_ERR, "Unable to set sys.stderr");
+		log_expt(LOG_ALERT);
+		goto update_python_fd_err_newfd;
+	}
+	Py_DECREF(new_fd);
+
+	return;
+
+update_python_fd_err_newfd:
+	Py_DECREF(new_fd);
+update_python_fd_err_args:
+	Py_DECREF(args);
+update_python_fd_err_fdopen:
+	Py_DECREF(fdopen);
+update_python_fd_err_pipes:
+	close(pipe_err[0]);
+	close(pipe_err[1]);
+update_python_fd_err_pipeout:
+	close(pipe_out[0]);
+	close(pipe_out[1]);
+update_python_fd_err:
+	if(err_fmt)
+	{
+		pyfcgi_log(pri, err_fmt, strerror(err));
+	}
+	exit(1);
+}
+
+
+PyObject* update_pyenv(PyObject *py_osmod, char **environ)
+{
+	PyObject *pyenv, *pykey, *pyval;
+	char *key, *value, **cur;
+
+	cur = environ;
+
+	pyenv = PyObject_GetAttrString(py_osmod, "environ");
+	if(!pyenv)
+	{
+		pyfcgi_log(LOG_WARNING, "Unable to get os.environ");
+		log_expt(LOG_ALERT);
+	}
+	else
+	{
+		Py_DECREF(pyenv);
+	}
+	pyenv = PyDict_New();
+
+
+
+	while(*cur)
+	{
+		key = value = *cur;
+		while(*value && *value != '=')
+		{
+			value++;
+		}
+		if(!*value)
+		{
+			pyfcgi_log(LOG_WARNING, "Droping environment value without value : '%s'",
+			       key);
+			cur++;
+			continue;
+		}
+		value++;
+		*(value-1) = '\0'; // dirty modification of **environ
+//pyfcgi_log(LOG_DEBUG, "PySetEnv '%s'='%s'", key, value);
+		pykey = PyUnicode_DecodeLocale(key, "surrogateescape");
+		if(!pykey)
+		{
+			*(value-1) = '='; // **environ restore
+			pyfcgi_log(LOG_ALERT, "Unable to parse environ key string '%s'",
+			       key);
+			log_expt(LOG_ALERT);
+			Py_Exit(EXIT_PYERR);
+		}
+		*(value-1) = '='; // **environ restore
+		pyval = PyUnicode_DecodeFSDefault(value);
+		if(!pyval)
+		{
+			pyfcgi_log(LOG_ALERT, "Unable to parse environ val string '%s'",
+			       value);
+			log_expt(LOG_ALERT);
+			Py_Exit(EXIT_PYERR);
+		}
+		if(PyDict_SetItem(pyenv, pykey, pyval) == -1)
+		{
+			pyfcgi_log(LOG_ERR, "Unable to set environ '%s'='%s'",
+			       key, value);
+			log_expt(LOG_ERR);
+		}
+		Py_DECREF(pyval);
+		Py_DECREF(pykey);
+		cur++;
+	}
+	PyObject_SetAttrString(py_osmod, "environ", pyenv);
+	return pyenv;
+}
+
 PyObject* import_entrypoint()
 {
 	PyObject *entry_fname, *entry_module, *entry_fun;
@@ -152,6 +386,29 @@ PyObject* import_entrypoint()
 	return entry_fun;
 }
 
+static PyObject* libpyfcgi_self = NULL;
+PyObject* get_start_response()
+{
+	PyObject *module;
+	response_status = NULL;
+	response_headers = NULL;
+	if(!libpyfcgi_self)
+	{
+		module = PyModule_Create(&pyfcgimodule);
+		if(module == NULL)
+		{
+			pyfcgi_log(LOG_ERR, "Unable to create libpyfcgi python module");
+			return NULL;
+		}
+		libpyfcgi_self = module;
+	}
+	else
+	{
+		module = libpyfcgi_self;
+	}
+	return PyObject_GetAttrString(libpyfcgi_self, "start_response");
+}
+
 void log_expt(int priority)
 {
 	if(!PyErr_Occurred())
@@ -200,3 +457,15 @@ void pyfcgi_python_version(char version[16])
 		PY_MICRO_VERSION);
 }
 
+PyObject* python_osmod()
+{
+	PyObject *ret;
+	ret = PyImport_ImportModule("os");
+	if(!ret)
+	{
+		pyfcgi_log(LOG_ALERT, "Unable to import os module");
+		log_expt(LOG_ALERT);
+		Py_Exit(EXIT_PYERR);
+	}
+	return ret;
+}

src/pyworker.c

@@ -19,17 +19,96 @@
 
 #include "pyworker.h"
 
+
+/**@brief Indicate that a worker is idle
+ * @param int semid */
+static inline void worker_set_idle(int);
+/**@brief Indicate that a worker is busy
+ * @param int semid */
+static inline void worker_set_busy(int);
+
 static int worker_piper_sigrcv = 0;
 
+int work333(int wrk_id, int semid)
+{
+	PyObject *entry_fun, *pyflush[2], *py_osmod, *entry_ret, *environ,
+		*start_response, *args;
+	FCGX_Stream *in_stream, *out_stream, *err_stream;
+	char **envp;
+	int count, pipe_out[2], pipe_err[2], err;
+	int max_reqs;
+	char ident[128];
+	struct timeval start, stop;
+
+	max_reqs = PyFCGI_conf.max_reqs;
+
+	snprintf(ident, 128, "Worker[%d]", wrk_id);
+	pyfcgi_logger_set_ident(ident);
+
+	pyfcgi_log(LOG_INFO, "Worker started with PEP333 App");
+
+	update_python_path(); // add cwd to python path
+	Py_Initialize();
+	update_python_fd(pipe_out, pipe_err);
+	fetch_pyflush(&(pyflush[0]), &(pyflush[1]));
+	pyfcgi_log(LOG_INFO, "Python started");
+
+	//importing os
+	py_osmod = python_osmod();
+	// loading module
+	entry_fun = import_entrypoint();
+
+	pyfcgi_log(LOG_INFO, "Waiting request with %s.%s()",
+		PyFCGI_conf.py_entrymod, PyFCGI_conf.py_entryfun);
+
+	worker_set_idle(semid); //before failing on import
+
+	if(!entry_fun) //but exit if import failed
+	{
+		pyfcgi_log(LOG_ALERT, "Unable to import entrypoint");
+		exit(PYFCGI_FATAL);
+
+	}
+
+	start_response = get_start_response();
+
+	// requests accepting loop
+	count = 0;
+	while ((!count || count != max_reqs) &&
+		FCGX_Accept(&in_stream, &out_stream, &err_stream, &envp) >= 0)
+	{
+		gettimeofday(&start, NULL);
+		worker_set_busy(semid);
+		count++;
+		environ = update_pyenv(py_osmod, envp);
+
+		libpyfcgi_clean_response();
+		args = Py_BuildValue("OO", environ, start_response);
+		entry_ret = PyObject_CallObject(entry_fun, args);
+		if(PyErr_Occurred())
+		{
+			log_expt(LOG_ERR);
+		}
+		// able to process returned value
+
+		// clean stuffs
+		Py_DECREF(args);
+		Py_DECREF(environ);
+		// flush & logs pystdout & pystderr
+		PyObject_CallObject(pyflush[0], NULL);
+		PyObject_CallObject(pyflush[1], NULL);
+	}
+	return 0;
+}
+
 int work(int wrk_id, int semid)
 {
-	PyObject *entry_fun, *pystdout_flush, *pystderr_flush,
-	         *py_osmod;
+	PyObject *entry_fun, *pystdout_flush, *pystderr_flush, *py_osmod,
+		*environ;
 	FCGX_Stream *in_stream, *out_stream, *err_stream;
 	char **envp;
 	int count, pipe_out[2], pipe_err[2], pipe_ctl[2], err, piper_status;
 	int max_reqs;
-	struct sembuf sop;
 	struct sigaction act;
 	struct timeval start, stop;
 	sigset_t emptyset;
@@ -41,8 +120,6 @@ int work(int wrk_id, int semid)
 	max_reqs = PyFCGI_conf.max_reqs;
 	piper_args.wrk_id = wrk_id;
 	piper_args.act = &act;
-	sop.sem_num = 0;
-	sop.sem_flg = 0;
 
 	// preparing sigaction for piper
 	if(sigemptyset(&emptyset))
@@ -81,14 +158,7 @@ int work(int wrk_id, int semid)
 		"Worker[%d] Python started", wrk_id);
 	
 	//importing os
-	py_osmod = PyImport_ImportModule("os");
-	if(!py_osmod)
-	{
-		pyfcgi_log(LOG_ALERT, "Unable to import os module");
-		log_expt(LOG_ALERT);
-		Py_Exit(EXIT_PYERR);
-	}
-
+	py_osmod = python_osmod();
 	// loading module
 	entry_fun = import_entrypoint();
 
@@ -97,14 +167,7 @@ int work(int wrk_id, int semid)
 		PyFCGI_conf.py_entrymod, PyFCGI_conf.py_entryfun);
 
 
-	sop.sem_op = 1; //indicate worker as ready & idle
-	if(semop(semid, &sop, 1) < 0)
-	{
-		err = errno;
-		pyfcgi_log(LOG_ERR,
-		       "Worker[%d] error incrementing the semaphore : %s",
-		       wrk_id, strerror(err));
-	}
+	worker_set_idle(semid); //before failing on import
 
 	if(!entry_fun) //but exit if import failed
 	{
@@ -127,20 +190,14 @@ int work(int wrk_id, int semid)
 	while ((!count || count != max_reqs) && FCGX_Accept(&in_stream, &out_stream, &err_stream, &envp) >= 0)
 	{
 		gettimeofday(&start, NULL);
-		sop.sem_op = -1; // decrementing sem to show worker busy
-		if(semop(semid, &sop, 1) < 0)
-		{
-			err = errno;
-			pyfcgi_log(LOG_ERR,
-			       "Worker[%d] error decrementing the semaphore : %s",
-			       wrk_id, strerror(err));
-		}
+		worker_set_busy(semid);
 
 		count++;
 		piper_args.out = out_stream;
 		//piper_args.req_id = count;
 		worker_piper_sigrcv = 0;
-
+//		if(!PyFCGI_conf.pep333)
+		/**@todo avoid clone on each request... (using named pipes ?) */
 		pid_t pid = clone(worker_piper, piper_stack + PIPER_STACK_SZ - 1,
 		                  CLONE_CHILD_CLEARTID | CLONE_CHILD_SETTID | \
 				  SIGCHLD | \
@@ -154,7 +211,8 @@ int work(int wrk_id, int semid)
 			       wrk_id, count, strerror(err));
 			exit(err);
 		}
-		update_pyenv(py_osmod, envp);
+		environ = update_pyenv(py_osmod, envp);
+		Py_DECREF(environ);
 		//close(pipe_ctl[1]);
 		PyObject_CallObject(entry_fun, NULL);
 		if(PyErr_Occurred())
@@ -176,15 +234,7 @@ int work(int wrk_id, int semid)
 		//close(pipe_ctl[0]);
 
 		kill(pid, WPIPER_SIG); //indicate child python call ended
-/*
-gettimeofday(&stop, NULL);
-pyfcgi_log(LOG_DEBUG, "W%dR%dtimer KIL %ld.%06ld us", wrk_id, count, stop.tv_sec - start.tv_sec, stop.tv_usec - start.tv_usec);
-*/
 		waitpid(pid, &piper_status, 0);
-/*
-gettimeofday(&stop, NULL);
-pyfcgi_log(LOG_DEBUG, "W%dR%dtimer W8  %ld.%06ld us", wrk_id, count, stop.tv_sec - start.tv_sec, stop.tv_usec - start.tv_usec);
-*/
 		if(WIFSIGNALED(piper_status))
 		{
 			pyfcgi_log(LOG_ERR,
@@ -206,14 +256,7 @@ pyfcgi_log(LOG_DEBUG, "W%dR%dtimer W8  %ld.%06ld us", wrk_id, count, stop.tv_sec
 			rep_sz = 0;
 		}
 		//Increase sem showing the worker is idle
-		sop.sem_op = 1;
-		if(semop(semid, &sop, 1) < 0)
-		{
-			err = errno;
-			pyfcgi_log(LOG_ERR,
-			       "Worker[%d] error incrementing the semaphore : %s",
-			       wrk_id, strerror(err));
-		}
+		worker_set_idle(semid);
 		gettimeofday(&stop, NULL);
 		stop.tv_sec = stop.tv_sec - start.tv_sec;
 		stop.tv_usec = stop.tv_usec - start.tv_usec;
@@ -434,237 +477,37 @@ int ctl_get_rep_sz(int ctl_pipe, size_t* rep_sz)
 	return 0;
 }
 
-static PyObject* _fetch_pyflush(const char *fdname)
-{
-	PyObject *pyfd, *pyflush;
-	pyfd = PySys_GetObject(fdname);
-	if(!pyfd)
-	{
-		pyfcgi_log(LOG_ALERT, "Unable to fetch sys.%s", fdname);
-		log_expt(LOG_ALERT);
-		Py_Exit(EXIT_PYERR);
-	}
-	pyflush = PyObject_GetAttrString(pyfd, "flush");
-	Py_DECREF(pyfd);
-	if(!pyflush)
-	{
-		pyfcgi_log(LOG_ALERT, "Unable to fetch sys.%s.flush", fdname);
-		log_expt(LOG_ALERT);
-		Py_Exit(EXIT_PYERR);
-	}
-	if(!PyCallable_Check(pyflush))
-	{
-		pyfcgi_log(LOG_ALERT, "sys.%s.flush is not callable !",
-		       fdname);
-		Py_Exit(EXIT_PYERR);
-	}
-
-	return pyflush;
-}
-
-void fetch_pyflush(PyObject** pystdout_flush, PyObject** pystderr_flush)
-{
-	*pystdout_flush = _fetch_pyflush("stdout");
-	*pystderr_flush = _fetch_pyflush("stderr");
-}
-
-void update_python_fd(int pipe_out[2], int pipe_err[2])
+static void worker_set_idle(int semid)
 {
-	int pri, err;
-	char *err_fmt;
-	PyObject *os_mod, *pyfdopen, *args, *new_fd;
+	int err;
+	struct sembuf sop;
 
-	pri = LOG_ALERT;
+	sop.sem_num = 0;
+	sop.sem_flg = 0;
+	sop.sem_op = 1;
 
-	if(pipe2(pipe_out, O_DIRECT) == -1)
-	{
-		err = errno;
-		err_fmt = "Unable to create pipe for python stdout : %s";
-		goto update_python_fd_err;
-	}
-	if(pipe2(pipe_err, O_DIRECT) == -1)
+	if(semop(semid, &sop, 1) < 0)
 	{
 		err = errno;
-		err_fmt = "Unable to create pipe for python stderr : %s";
-		goto update_python_fd_err_pipeout;
-	}
-
-	os_mod = PyImport_ImportModule("os");
-	if(!os_mod)
-	{
-		if(PyErr_Occurred())
-		{
-			log_expt(LOG_ALERT);
-		}
-		else
-		{
-			pyfcgi_log(	LOG_ALERT,
-				"Unable to import python os module, got NULL.");
-		}
-		err_fmt = NULL;
-		goto update_python_fd_err_pipes;
-	}
-	pyfdopen = PyObject_GetAttrString(os_mod, "fdopen");
-	Py_DECREF(os_mod);
-	if(!pyfdopen)
-	{
-		if(PyErr_Occurred())
-		{
-			log_expt(LOG_ALERT);
-		}
-		else
-		{
-			pyfcgi_log(	LOG_ALERT,
-				"Unable to fetch os.fdopen() , got NULL.");
-		}
-		err_fmt = NULL;
-		goto update_python_fd_err_pipes;
-	}
-
-	args = Py_BuildValue("is", pipe_out[1], "w");
-	if(!args)
-	{
-		pyfcgi_log(	LOG_ERR, "Error building values with '%d', '%s' for stdout",
-			pipe_out[1], "w");
-		log_expt(LOG_ALERT);
-		err_fmt = NULL;
-		goto update_python_fd_err_fdopen;
-	}
-	new_fd = PyObject_CallObject(pyfdopen, args);
-	if(!new_fd || PyErr_Occurred())
-	{
-		pyfcgi_log(	LOG_ERR, "Error calling fdopen(%d, '%s')",
-			pipe_out[1], "w");
-		log_expt(LOG_ALERT);
-		err_fmt = NULL;
-		goto update_python_fd_err_args;
-	}
-	Py_DECREF(args);
-	if(PySys_SetObject("stdout", new_fd))
-	{
-		pyfcgi_log(LOG_ERR, "Unable to set sys.stdout");
-		log_expt(LOG_ALERT);
-		goto update_python_fd_err_newfd;
-	}
-	Py_DECREF(new_fd);
-
-	args = Py_BuildValue("is", pipe_err[1], "w");
-	if(!args)
-	{
-		pyfcgi_log(	LOG_ERR, "Error building values with '%d', '%s' for stderr",
-			pipe_out[1], "w");
-		log_expt(LOG_ALERT);
-		err_fmt = NULL;
-		goto update_python_fd_err_fdopen;
-	}
-	new_fd = PyObject_CallObject(pyfdopen, args);
-	if(!new_fd || PyErr_Occurred())
-	{
-		pyfcgi_log(	LOG_ERR, "Error calling fdopen(%d, '%s')",
-			pipe_out[1], "w");
-		log_expt(LOG_ALERT);
-		err_fmt = NULL;
-		goto update_python_fd_err_args;
-	}
-	Py_DECREF(args);
-	if(PySys_SetObject("stderr", new_fd))
-	{
-		pyfcgi_log(LOG_ERR, "Unable to set sys.stderr");
-		log_expt(LOG_ALERT);
-		goto update_python_fd_err_newfd;
-	}
-	Py_DECREF(new_fd);
-
-	return;
-
-update_python_fd_err_newfd:
-	Py_DECREF(new_fd);
-update_python_fd_err_args:
-	Py_DECREF(args);
-update_python_fd_err_fdopen:
-	Py_DECREF(fdopen);
-update_python_fd_err_pipes:
-	close(pipe_err[0]);
-	close(pipe_err[1]);
-update_python_fd_err_pipeout:
-	close(pipe_out[0]);
-	close(pipe_out[1]);
-update_python_fd_err:
-	if(err_fmt)
-	{
-		pyfcgi_log(pri, err_fmt, strerror(err));
+		pyfcgi_log(LOG_ERR, "error incrementing the semaphore : %s",
+			strerror(err));
 	}
-	exit(1);
 }
 
-
-void update_pyenv(PyObject *py_osmod, char **environ)
+static void worker_set_busy(int semid)
 {
-	PyObject *pyenv, *pykey, *pyval;
-	char *key, *value, **cur;
-
-	cur = environ;
-
-	pyenv = PyObject_GetAttrString(py_osmod, "environ");
-	if(!pyenv)
-	{
-		pyfcgi_log(LOG_WARNING, "Unable to get os.environ");
-		log_expt(LOG_ALERT);
-	}
-	else
-	{
-		Py_DECREF(pyenv);
-	}
-	pyenv = PyDict_New();
-
+	int err;
+	struct sembuf sop;
 
+	sop.sem_num = 0;
+	sop.sem_flg = 0;
+	sop.sem_op = -1;
 
-	while(*cur)
+	if(semop(semid, &sop, 1) < 0)
 	{
-		key = value = *cur;
-		while(*value && *value != '=')
-		{
-			value++;
-		}
-		if(!*value)
-		{
-			pyfcgi_log(LOG_WARNING, "Droping environment value without value : '%s'",
-			       key);
-			cur++;
-			continue;
-		}
-		value++;
-		*(value-1) = '\0'; // dirty modification of **environ
-//pyfcgi_log(LOG_DEBUG, "PySetEnv '%s'='%s'", key, value);
-		pykey = PyUnicode_DecodeLocale(key, "surrogateescape");
-		if(!pykey)
-		{
-			*(value-1) = '='; // **environ restore
-			pyfcgi_log(LOG_ALERT, "Unable to parse environ key string '%s'",
-			       key);
-			log_expt(LOG_ALERT);
-			Py_Exit(EXIT_PYERR);
-		}
-		*(value-1) = '='; // **environ restore
-		pyval = PyUnicode_DecodeFSDefault(value);
-		if(!pyval)
-		{
-			pyfcgi_log(LOG_ALERT, "Unable to parse environ val string '%s'",
-			       value);
-			log_expt(LOG_ALERT);
-			Py_Exit(EXIT_PYERR);
-		}
-		if(PyDict_SetItem(pyenv, pykey, pyval) == -1)
-		{
-			pyfcgi_log(LOG_ERR, "Unable to set environ '%s'='%s'",
-			       key, value);
-			log_expt(LOG_ERR);
-		}
-		Py_DECREF(pyval);
-		Py_DECREF(pykey);
-		cur++;
+		err = errno;
+		pyfcgi_log(LOG_ERR, "error incrementing the semaphore : %s",
+			strerror(err));
 	}
-	PyObject_SetAttrString(py_osmod, "environ", pyenv);
-
 }
 

tests/Makefile.am

@@ -1,8 +1,8 @@
 TESTS = check_logger
 check_PROGRAMS = check_logger
 check_logger_SOURCES = check_logger.c $(top_builddir)/include/logger.h
-check_logger_CFLAGS = @CHECK_CFLAGS@
-check_logger_LDADD = $(top_builddir)/src/libpyfcgi.a @CHECK_LIBS@
+check_logger_CFLAGS = @CHECK_CFLAGS@ $(PYTHON_CFLAGS)
+check_logger_LDADD = $(top_builddir)/src/libpyfcgi.a @CHECK_LIBS@ $(PYTHON_LDFLAGS)
 
 clean-local:
 	-rm -rf /tmp/tmp_PyFCGI_checks*