a huge list of threading fixes, we are near to ROCK SOLID threading mode

plugins/python/gil.c

@@ -5,11 +5,11 @@ extern struct uwsgi_python up;
 
 void gil_real_get() {
 	PyEval_AcquireLock();
-	PyThreadState_Swap((PyThreadState *) pthread_getspecific(up.upt_save_key));
+	PyThreadState_Swap((PyThreadState *) pthread_getspecific(up.upt_gil_key));
 }
 
 void gil_real_release() {
-	pthread_setspecific(up.upt_save_key, (void *) PyThreadState_Swap(NULL));
+	pthread_setspecific(up.upt_gil_key, (void *) PyThreadState_Swap(NULL));
 	PyEval_ReleaseLock();
 }
 

plugins/python/pyloader.c

@@ -158,7 +158,6 @@ int init_uwsgi_app(int loader, void *arg1, struct wsgi_request *wsgi_req, PyThre
 	}
 #endif
 
-
 	// check function args
 	// by defaut it is a WSGI app
 	wi->argc = 2;
@@ -247,22 +246,22 @@ int init_uwsgi_app(int loader, void *arg1, struct wsgi_request *wsgi_req, PyThre
 #endif
 	}
 
-	if (interpreter == NULL && id) {
+	if (uwsgi.threads > 1 && id) {
 		// if we have multiple threads we need to initialize a PyThreadState for each one
-		if (uwsgi.threads > 1) {
-			for(i=0;i<uwsgi.threads;i++) {
-				uwsgi.workers[uwsgi.mywid].cores[i]->ts[id] = PyThreadState_New( ((PyThreadState *)wi->interpreter)->interp);
-				if (!uwsgi.workers[uwsgi.mywid].cores[i]->ts[id]) {
-					uwsgi_log("unable to allocate new PyThreadState structure for app %s", mountpoint);
-					goto doh;
-				}
+		for(i=0;i<uwsgi.threads;i++) {
+			uwsgi.core[i]->ts[id] = PyThreadState_New( ((PyThreadState *)wi->interpreter)->interp);
+			if (!uwsgi.core[i]->ts[id]) {
+				uwsgi_log("unable to allocate new PyThreadState structure for app %s", mountpoint);
+				goto doh;
 			}
-			PyThreadState_Swap((PyThreadState *) pthread_getspecific(up.upt_save_key));
-		}
-		else {
-			PyThreadState_Swap(up.main_thread);
 		}
+		PyThreadState_Swap((PyThreadState *) pthread_getspecific(up.upt_save_key) );
 	}
+	else if (interpreter == NULL && id) {
+		PyThreadState_Swap(up.main_thread);
+	}
+
+	uwsgi_log("done\n");
 
 	if (wi->argc == 1) {
 		uwsgi_log( "Web3 application %d (SCRIPT_NAME=%.*s) ready on interpreter %p", id, wi->mountpoint_len, wi->mountpoint, wi->interpreter);

plugins/python/python_plugin.c

@@ -124,6 +124,9 @@ int uwsgi_python_init() {
 	up.gil_get = gil_fake_get;
 	up.gil_release = gil_fake_release;
 
+	up.swap_ts = simple_swap_ts;
+	up.reset_ts = simple_reset_ts;
+
 	return 1;
 
 }
@@ -671,12 +674,19 @@ int uwsgi_python_mount_app(char *mountpoint, char *app) {
 				uwsgi_error("pthread_key_create()");
 				exit(1);
 			}
+			if (pthread_key_create(&up.upt_gil_key, NULL)) {
+				uwsgi_error("pthread_key_create()");
+				exit(1);
+			}
 			pthread_setspecific(up.upt_save_key, (void *) PyThreadState_Get());
+			pthread_setspecific(up.upt_gil_key, (void *) PyThreadState_Get());
 			pthread_mutex_init(&up.lock_pyloaders, NULL);
 			pthread_atfork(uwsgi_python_pthread_prepare, uwsgi_python_pthread_parent, uwsgi_python_pthread_child);
 			up.gil_get = gil_real_get;
 			up.gil_release = gil_real_release;
 
+			up.swap_ts = threaded_swap_ts;
+			up.reset_ts = threaded_reset_ts;
 			uwsgi_log("threads support enabled\n");
 		}
 
@@ -686,6 +696,7 @@ int uwsgi_python_mount_app(char *mountpoint, char *app) {
 			PyThreadState *pts;
 			pts = PyThreadState_New(up.main_thread->interp);
 			pthread_setspecific(up.upt_save_key, (void *) pts);
+			pthread_setspecific(up.upt_gil_key, (void *) pts);
 
 		}
 

plugins/python/uwsgi_pymodule.c

@@ -410,6 +410,16 @@ PyObject *py_uwsgi_signal(PyObject * self, PyObject * args) {
 	
 }
 
+PyObject *py_uwsgi_log_this(PyObject * self, PyObject * args) {
+
+        struct wsgi_request *wsgi_req = current_wsgi_req();
+
+	wsgi_req->log_this = 1;
+
+	Py_INCREF(Py_None);
+        return Py_None;
+}
+
 PyObject *py_uwsgi_recv_frame(PyObject * self, PyObject * args) {
 
 	struct wsgi_request *wsgi_req = current_wsgi_req();
@@ -763,19 +773,12 @@ PyObject *py_uwsgi_warning(PyObject * self, PyObject * args) {
 
 PyObject *py_uwsgi_log(PyObject * self, PyObject * args) {
 	char *logline;
-	time_t tt;
 
 	if (!PyArg_ParseTuple(args, "s:log", &logline)) {
 		return NULL;
 	}
 
-	tt = time(NULL);
-	if (logline[strlen(logline)] != '\n') {
-		uwsgi_log( UWSGI_LOGBASE " %.*s] %s\n", 24, ctime(&tt), logline);
-	}
-	else {
-		uwsgi_log( UWSGI_LOGBASE " %.*s] %s", 24, ctime(&tt), logline);
-	}
+	uwsgi_log( "%s\n", logline);
 
 	Py_INCREF(Py_True);
 	return Py_True;
@@ -1837,7 +1840,9 @@ PyObject *py_uwsgi_cl(PyObject * self, PyObject * args) {
 
 		struct wsgi_request *wsgi_req = current_wsgi_req();
 
+#ifdef UWSGI_DEBUG
 		uwsgi_log( "disconnecting worker %d (pid :%d) from session...\n", uwsgi.mywid, uwsgi.mypid);
+#endif
 
 		fclose(wsgi_req->async_post);
 		wsgi_req->fd_closed = 1;
@@ -2111,6 +2116,7 @@ static PyMethodDef uwsgi_advanced_methods[] = {
 		{"request_id", py_uwsgi_request_id, METH_VARARGS, ""},
 		{"worker_id", py_uwsgi_worker_id, METH_VARARGS, ""},
 		{"log", py_uwsgi_log, METH_VARARGS, ""},
+		{"log_this_request", py_uwsgi_log_this, METH_VARARGS, ""},
 		{"disconnect", py_uwsgi_disconnect, METH_VARARGS, ""},
 		{"grunt", py_uwsgi_grunt, METH_VARARGS, ""},
 		{"load_plugin", py_uwsgi_load_plugin, METH_VARARGS, ""},

plugins/python/uwsgi_python.h

@@ -108,8 +108,12 @@ struct uwsgi_python {
 	int current_recursion_depth;
 	struct _frame* current_frame;
 
+	void (*swap_ts)(struct wsgi_request *, struct uwsgi_app *);
+	void (*reset_ts)(struct wsgi_request *, struct uwsgi_app *);
+
 #ifdef UWSGI_THREADING
 	pthread_key_t upt_save_key;
+	pthread_key_t upt_gil_key;
 	pthread_mutex_t lock_pyloaders;
 	void (*gil_get) (void);
 	void (*gil_release) (void);
@@ -209,3 +213,8 @@ void init_uwsgi_module_sharedarea(PyObject *);
 void init_uwsgi_module_cache(PyObject *);
 
 PyObject *uwsgi_pyimport_by_filename(char *, char *);
+
+void threaded_swap_ts(struct wsgi_request *, struct uwsgi_app *);
+void simple_swap_ts(struct wsgi_request *, struct uwsgi_app *);
+void threaded_reset_ts(struct wsgi_request *, struct uwsgi_app *);
+void simple_reset_ts(struct wsgi_request *, struct uwsgi_app *);

plugins/python/wsgi_handlers.c

@@ -191,29 +191,15 @@ int uwsgi_request_wsgi(struct wsgi_request *wsgi_req) {
 
 	wi = &uwsgi.apps[wsgi_req->app_id];
 
-	if (uwsgi.single_interpreter == 0 && wi->interpreter != up.main_thread) {
-		if (!wi->interpreter) {
-			internal_server_error(wsgi_req->poll.fd, "wsgi application's %d interpreter not found");
-			goto clear2;
-		}
-
-		// set the interpreter
-		UWSGI_GET_GIL
-			if (uwsgi.threads > 1) {
-				PyThreadState_Swap(uwsgi.workers[uwsgi.mywid].cores[wsgi_req->async_id]->ts[wsgi_req->app_id]);
-			}
-			else {
-				PyThreadState_Swap(wi->interpreter);
-			}
-		UWSGI_RELEASE_GIL
-			if (wi->chdir) {
+	up.swap_ts(wsgi_req, wi);
+	
+	if (wi->chdir) {
 #ifdef UWSGI_DEBUG
-				uwsgi_debug("chdir to %s\n", wi->chdir);
+		uwsgi_debug("chdir to %s\n", wi->chdir);
 #endif
-				if (chdir(wi->chdir)) {
-					uwsgi_error("chdir()");
-				}
-			}
+		if (chdir(wi->chdir)) {
+			uwsgi_error("chdir()");
+		}
 	}
 
 
@@ -241,8 +227,8 @@ int uwsgi_request_wsgi(struct wsgi_request *wsgi_req) {
 
 	UWSGI_GET_GIL
 
-		// no fear of race conditions for this counter as it is already protected by the GIL
-		wi->requests++;
+	// no fear of race conditions for this counter as it is already protected by the GIL
+	wi->requests++;
 
 	Py_INCREF((PyObject *)wsgi_req->async_environ);
 
@@ -291,10 +277,11 @@ int uwsgi_request_wsgi(struct wsgi_request *wsgi_req) {
 
 	wsgi_req->async_result = wi->request_subhandler(wsgi_req, wi);
 
+	UWSGI_RELEASE_GIL
+
 	if (wsgi_req->async_result) {
 
 
-		UWSGI_RELEASE_GIL
 		while (wi->response_subhandler(wsgi_req) != UWSGI_OK) {
 			wsgi_req->switches++;
 #ifdef UWSGI_ASYNC
@@ -332,7 +319,9 @@ int uwsgi_request_wsgi(struct wsgi_request *wsgi_req) {
 			goto clear;
 		}
 		// print the error
+		UWSGI_GET_GIL
 		PyErr_Print();
+		UWSGI_RELEASE_GIL
 		// ...resume the original stderr, in case of error we are damaged forever !!!
 		if (dup2(tmp_stderr, 2) < 0) {
 			uwsgi_error("dup2()");
@@ -342,19 +331,7 @@ int uwsgi_request_wsgi(struct wsgi_request *wsgi_req) {
 
 clear:
 
-	UWSGI_GET_GIL
-
-		if (uwsgi.single_interpreter == 0 && wi->interpreter != up.main_thread) {
-			// restoring main interpreter
-			if (uwsgi.threads > 1) {
-				PyThreadState_Swap((PyThreadState *) pthread_getspecific(up.upt_save_key));
-			}
-			else {
-				PyThreadState_Swap(up.main_thread);
-			}
-		}
-
-	UWSGI_RELEASE_GIL
+	up.reset_ts(wsgi_req, wi);
 
 clear2:
 
@@ -365,7 +342,7 @@ clear2:
 void uwsgi_after_request_wsgi(struct wsgi_request *wsgi_req) {
 
 
-	if (uwsgi.shared->options[UWSGI_OPTION_LOGGING]) {
+	if (uwsgi.shared->options[UWSGI_OPTION_LOGGING] || wsgi_req->log_this) {
 		log_request(wsgi_req);
 	}
 	else {
@@ -416,3 +393,38 @@ PyObject *py_uwsgi_sendfile(PyObject * self, PyObject * args) {
 	return (PyObject *) wsgi_req->sendfile_obj;
 }
 #endif
+
+void threaded_swap_ts(struct wsgi_request *wsgi_req, struct uwsgi_app *wi) {
+
+	if (uwsgi.single_interpreter == 0 && wi->interpreter != up.main_thread) {
+		UWSGI_GET_GIL
+                PyThreadState_Swap(uwsgi.core[wsgi_req->async_id]->ts[wsgi_req->app_id]);
+		UWSGI_RELEASE_GIL
+	}
+
+}
+
+void threaded_reset_ts(struct wsgi_request *wsgi_req, struct uwsgi_app *wi) {
+	if (uwsgi.single_interpreter == 0 && wi->interpreter != up.main_thread) {
+		UWSGI_GET_GIL
+        	PyThreadState_Swap((PyThreadState *) pthread_getspecific(up.upt_save_key));
+        	UWSGI_RELEASE_GIL
+	}
+}
+
+
+void simple_reset_ts(struct wsgi_request *wsgi_req, struct uwsgi_app *wi) {
+	if (uwsgi.single_interpreter == 0 && wi->interpreter != up.main_thread) {
+        	// restoring main interpreter
+                PyThreadState_Swap(up.main_thread);
+	}
+}
+
+
+void simple_swap_ts(struct wsgi_request *wsgi_req, struct uwsgi_app *wi) {
+
+	if (uwsgi.single_interpreter == 0 && wi->interpreter != up.main_thread) {
+                // set the interpreter
+                PyThreadState_Swap(wi->interpreter);
+	}
+}