various stabilization fixes

This commit is contained in:
roberto@mrspurr
2010-10-05 05:48:20 +02:00
parent 17e2dbf85b
commit ca43ad0a91
11 changed files with 205 additions and 136 deletions
+20
View File
@@ -0,0 +1,20 @@
#include "uwsgi.h"
extern struct uwsgi_server uwsgi;
void gil_real_get() {
PyEval_AcquireLock();
PyThreadState_Swap((PyThreadState *) pthread_getspecific(uwsgi.ut_save_key));
}
void gil_real_release() {
pthread_setspecific(uwsgi.ut_save_key, (void *) PyThreadState_Swap(NULL));
PyEval_ReleaseLock();
}
struct wsgi_request* threaded_current_wsgi_req() { return pthread_getspecific(uwsgi.ut_key); }
struct wsgi_request* simple_current_wsgi_req() { return uwsgi.wsgi_req ; }
void gil_fake_get() {}
void gil_fake_release() {}
+87 -5
View File
@@ -5,19 +5,36 @@ extern struct uwsgi_server uwsgi;
void *simple_loop(void *arg1) {
long core_id = (long) arg1;
PyThreadState *pts;
struct wsgi_request *wsgi_req = uwsgi.wsgi_requests[core_id];
pthread_setspecific(uwsgi.ut_key, (void *) wsgi_req);
#ifdef UWSGI_THREADING
PyThreadState *pts;
if (core_id > 0) {
pts = PyThreadState_New(uwsgi.main_thread->interp);
pthread_setspecific(uwsgi.ut_save_key, (void *) pts);
if (uwsgi.threads > 1) {
pthread_setspecific(uwsgi.ut_key, (void *) wsgi_req);
if (core_id > 0) {
pts = PyThreadState_New(uwsgi.main_thread->interp);
pthread_setspecific(uwsgi.ut_save_key, (void *) pts);
}
}
#endif
while (uwsgi.workers[uwsgi.mywid].manage_next_request) {
#ifndef __linux__
if (uwsgi.no_orphans && uwsgi.master_process) {
// am i a son of init ?
if (getppid() == 1) {
uwsgi_log("UAAAAAAH my parent died :( i will follow him...\n");
exit(1);
}
}
#endif
UWSGI_CLEAR_STATUS;
wsgi_req_setup(wsgi_req, core_id);
@@ -35,3 +52,68 @@ void *simple_loop(void *arg1) {
pthread_exit(NULL);
}
void complex_loop() {
int current_async_timeout = 0;
int i;
while (uwsgi.workers[uwsgi.mywid].manage_next_request) {
current_async_timeout = async_get_timeout() ;
uwsgi.async_nevents = async_wait(uwsgi.async_queue, uwsgi.async_events, uwsgi.async, uwsgi.async_running, current_async_timeout);
async_expire_timeouts();
if (uwsgi.async_nevents < 0) {
continue;
}
for(i=0; i<uwsgi.async_nevents;i++) {
if ( (int) uwsgi.async_events[i].ASYNC_FD == uwsgi.sockets[0].fd) {
uwsgi.wsgi_req = find_first_available_wsgi_req();
if (uwsgi.wsgi_req == NULL) {
// async system is full !!!
goto cycle;
}
wsgi_req_setup(uwsgi.wsgi_req, ( (uint8_t *)uwsgi.wsgi_req - (uint8_t *)uwsgi.wsgi_requests)/sizeof(struct wsgi_request) );
if (wsgi_req_accept(uwsgi.wsgi_req)) {
continue;
}
if (wsgi_req_recv(uwsgi.wsgi_req)) {
continue;
}
if (uwsgi.wsgi_req->async_status == UWSGI_OK) {
goto reqclear;
}
}
else {
uwsgi.wsgi_req = find_wsgi_req_by_fd(uwsgi.async_events[i].ASYNC_FD, uwsgi.async_events[i].ASYNC_EV);
if (uwsgi.wsgi_req) {
uwsgi.wsgi_req->async_status = UWSGI_AGAIN ;
uwsgi.wsgi_req->async_waiting_fd = -1 ;
uwsgi.wsgi_req->async_waiting_fd_monitored = 0 ;
}
async_del(uwsgi.async_queue, uwsgi.async_events[i].ASYNC_FD, uwsgi.async_events[i].ASYNC_EV);
}
}
cycle:
uwsgi.wsgi_req = async_loop();
if (uwsgi.wsgi_req == NULL)
continue ;
uwsgi.wsgi_req->async_status = UWSGI_OK ;
reqclear:
uwsgi_close_request(uwsgi.wsgi_req);
}
}
+5 -5
View File
@@ -53,7 +53,7 @@ void master_loop(char **argv, char **environ) {
struct timeval check_interval = {.tv_sec = 1,.tv_usec = 0 };
// release the GIL
uwsgi_release_gil();
UWSGI_RELEASE_GIL
/* route signals to workers... */
signal(SIGHUP, (void *) &grace_them_all);
@@ -100,12 +100,12 @@ void master_loop(char **argv, char **environ) {
#endif
#ifdef UWSGI_UDP
uwsgi_get_gil();
UWSGI_GET_GIL
udp_callable = PyDict_GetItemString(uwsgi.embedded_dict, "udp_callable");
if (udp_callable) {
udp_callable_args = PyTuple_New(3);
}
uwsgi_release_gil();
UWSGI_RELEASE_GIL
#endif
for (;;) {
if (ready_to_die >= uwsgi.numproc && uwsgi.to_hell) {
@@ -232,7 +232,7 @@ void master_loop(char **argv, char **environ) {
#endif
else {
if (udp_callable && udp_callable_args) {
uwsgi_get_gil();
UWSGI_GET_GIL
PyTuple_SetItem(udp_callable_args, 0, PyString_FromString(udp_client_addr));
PyTuple_SetItem(udp_callable_args, 1, PyInt_FromLong(ntohs(udp_client.sin_port)));
PyTuple_SetItem(udp_callable_args, 2, PyString_FromStringAndSize(uwsgi.wsgi_req->buffer, rlen));
@@ -243,7 +243,7 @@ void master_loop(char **argv, char **environ) {
if (PyErr_Occurred())
PyErr_Print();
uwsgi_release_gil();
UWSGI_RELEASE_GIL
}
else {
// a simple udp logger
+2 -2
View File
@@ -36,7 +36,7 @@ ssize_t uwsgi_sendfile(struct wsgi_request *wsgi_req) {
struct stat stat_buf;
ssize_t sst = 0;
uwsgi_release_gil();
UWSGI_RELEASE_GIL
if (!wsgi_req->sendfile_fd_size) {
@@ -57,7 +57,7 @@ ssize_t uwsgi_sendfile(struct wsgi_request *wsgi_req) {
}
end:
uwsgi_get_gil();
UWSGI_GET_GIL
return sst;
}
+2 -2
View File
@@ -460,11 +460,11 @@ inline struct wsgi_request *current_wsgi_req() {
void sanitize_args() {
if (uwsgi.async > 0) {
if (uwsgi.async > 1) {
uwsgi.cores = uwsgi.async;
}
if (uwsgi.threads > 0) {
if (uwsgi.threads > 1) {
uwsgi.has_threads = 1;
uwsgi.cores = uwsgi.threads;
}
+44 -96
View File
@@ -344,10 +344,7 @@ void what_i_am_doing() {
PyMethodDef uwsgi_spit_method[] = { {"uwsgi_spit", py_uwsgi_spit, METH_VARARGS, ""} };
PyMethodDef uwsgi_write_method[] = { {"uwsgi_write", py_uwsgi_write, METH_VARARGS, ""} };
// process manager is now (20090725) available on Unbit
pid_t masterpid;
pid_t diedpid;
int waitpid_status;
struct timeval last_respawn;
@@ -400,11 +397,6 @@ int main(int argc, char *argv[], char *envp[]) {
int uwsgi_will_starts = 0;
#ifdef UWSGI_ASYNC
int current_async_timeout = 0;
#endif
pid_t pid;
FILE *pidfile;
@@ -849,7 +841,7 @@ int main(int argc, char *argv[], char *envp[]) {
uwsgi.wsgi_req = uwsgi.wsgi_requests[0] ;
if (uwsgi.cores > 1) {
uwsgi_log("allocated %llu bytes (%llu KB) for %d request's buffer.\n", (uint64_t) (sizeof(struct wsgi_request) * uwsgi.cores),
uwsgi_log("allocated %llu bytes (%llu KB) for %d cores per worker.\n", (uint64_t) (sizeof(struct wsgi_request) * uwsgi.cores),
(uint64_t)( (sizeof(struct wsgi_request) * uwsgi.cores ) / 1024),
uwsgi.cores);
}
@@ -949,6 +941,9 @@ int main(int argc, char *argv[], char *envp[]) {
uwsgi.main_thread = PyThreadState_Get();
uwsgi.gil_get = gil_fake_get;
uwsgi.gil_release = gil_fake_release;
uwsgi.current_wsgi_req = simple_current_wsgi_req;
#ifdef UWSGI_NAGIOS
@@ -973,6 +968,9 @@ int main(int argc, char *argv[], char *envp[]) {
exit(1);
}
pthread_setspecific(uwsgi.ut_save_key, (void *) PyThreadState_Get());
uwsgi.gil_get = gil_real_get;
uwsgi.gil_release = gil_real_release;
uwsgi.current_wsgi_req = threaded_current_wsgi_req;
}
#endif
@@ -1234,6 +1232,34 @@ int main(int argc, char *argv[], char *envp[]) {
uwsgi_log( "done.\n");
uwsgi_log("*** Operational MODE: ");
if (uwsgi.threads > 1) {
uwsgi_log("threaded");
}
#ifdef UWSGI_UGREEN
else if (uwsgi.ugreen) {
uwsgi_log("uGreen");
}
#endif
#ifdef UWSGI_STACKLESS
else if (uwsgi.stackless) {
uwsgi_log("stackless");
}
#endif
#ifdef UWSGI_ASYNC
else if (uwsgi.async > 1) {
uwsgi_log("async");
}
#endif
else if (uwsgi.numproc > 1) {
uwsgi_log("preforking");
}
else {
uwsgi_log("single process");
}
uwsgi_log(" ***\n");
#ifdef UWSGI_EMBED_PLUGINS
embed_plugins();
#endif
@@ -1361,10 +1387,10 @@ int main(int argc, char *argv[], char *envp[]) {
if (!uwsgi.master_process) {
if (uwsgi.numproc == 1) {
uwsgi_log( "spawned uWSGI worker 1 (and the only) (pid: %d)\n", masterpid);
uwsgi_log( "spawned uWSGI worker 1 (and the only) (pid: %d, cores: %d)\n", masterpid, uwsgi.cores);
}
else {
uwsgi_log( "spawned uWSGI worker 1 (pid: %d)\n", masterpid);
uwsgi_log( "spawned uWSGI worker 1 (pid: %d, cores: %d)\n", masterpid, uwsgi.cores);
}
uwsgi.workers[1].pid = masterpid;
uwsgi.workers[1].id = 1;
@@ -1399,7 +1425,7 @@ int main(int argc, char *argv[], char *envp[]) {
exit(1);
}
else {
uwsgi_log( "spawned uWSGI worker %d (pid: %d)\n", i, pid);
uwsgi_log( "spawned uWSGI worker %d (pid: %d, cores: %d)\n", i, pid, uwsgi.cores);
gettimeofday(&last_respawn, NULL);
uwsgi.respawn_delta = last_respawn.tv_sec;
}
@@ -1506,7 +1532,7 @@ int main(int argc, char *argv[], char *envp[]) {
#endif
// release the GIL
uwsgi_release_gil();
UWSGI_RELEASE_GIL
#ifdef UWSGI_ASYNC
@@ -1554,96 +1580,18 @@ int main(int argc, char *argv[], char *envp[]) {
exit(1);
}
for(i=1;i<uwsgi.threads;i++) {
int j = i;
long j = i;
a_thread = malloc(sizeof(pthread_t));
pthread_create(a_thread, &pa, simple_loop, (void *) j);
}
}
while (uwsgi.workers[uwsgi.mywid].manage_next_request) {
#ifndef __linux__
if (uwsgi.no_orphans && uwsgi.master_process) {
// am i a son of init ?
if (getppid() == 1) {
uwsgi_log("UAAAAAAH my parent died :( i will follow him...\n");
exit(1);
}
}
#endif
// clear all status bits
UWSGI_CLEAR_STATUS;
#ifdef UWSGI_ASYNC
if (uwsgi.async > 1) {
current_async_timeout = async_get_timeout() ;
uwsgi.async_nevents = async_wait(uwsgi.async_queue, uwsgi.async_events, uwsgi.async, uwsgi.async_running, current_async_timeout);
async_expire_timeouts();
if (uwsgi.async_nevents < 0) {
continue;
}
for(i=0; i<uwsgi.async_nevents;i++) {
if ( (int) uwsgi.async_events[i].ASYNC_FD == uwsgi.sockets[0].fd) {
uwsgi.wsgi_req = find_first_available_wsgi_req();
if (uwsgi.wsgi_req == NULL) {
// async system is full !!!
goto cycle;
}
wsgi_req_setup(uwsgi.wsgi_req, ( (uint8_t *)uwsgi.wsgi_req - (uint8_t *)uwsgi.wsgi_requests)/sizeof(struct wsgi_request) );
if (wsgi_req_accept(uwsgi.wsgi_req)) {
continue;
}
if (wsgi_req_recv(uwsgi.wsgi_req)) {
continue;
}
if (uwsgi.wsgi_req->async_status == UWSGI_OK) {
goto reqclear;
}
}
else {
uwsgi.wsgi_req = find_wsgi_req_by_fd(uwsgi.async_events[i].ASYNC_FD, uwsgi.async_events[i].ASYNC_EV);
if (uwsgi.wsgi_req) {
uwsgi.wsgi_req->async_status = UWSGI_AGAIN ;
uwsgi.wsgi_req->async_waiting_fd = -1 ;
uwsgi.wsgi_req->async_waiting_fd_monitored = 0 ;
}
async_del(uwsgi.async_queue, uwsgi.async_events[i].ASYNC_FD, uwsgi.async_events[i].ASYNC_EV);
}
}
cycle:
uwsgi.wsgi_req = async_loop();
if (uwsgi.wsgi_req == NULL)
continue ;
uwsgi.wsgi_req->async_status = UWSGI_OK ;
if (uwsgi.async < 2) {
long y = 0;
simple_loop((void *) y);
}
else {
#endif
int y = 0;
simple_loop((void *) y);
#ifdef UWSGI_ASYNC
}
reqclear:
#endif
uwsgi_close_request(uwsgi.wsgi_req);
complex_loop();
}
if (uwsgi.workers[uwsgi.mywid].manage_next_request == 0) {
+24 -5
View File
@@ -14,11 +14,11 @@
#define MAX_PYTHONPATH 64
#ifdef UWSGI_THREADING
#define uwsgi_get_gil() if (uwsgi.has_threads) { PyEval_AcquireLock(); PyThreadState_Swap((PyThreadState *) pthread_getspecific(uwsgi.ut_save_key)); }
#define uwsgi_release_gil() if (uwsgi.has_threads) { pthread_setspecific(uwsgi.ut_save_key, (void *) PyThreadState_Swap(NULL)); PyEval_ReleaseLock();}
#define UWSGI_GET_GIL (*uwsgi.gil_get)();
#define UWSGI_RELEASE_GIL (*uwsgi.gil_release)();
#else
#define uwsgi_get_gil()
#define uwsgi_release_gil()
#define UWSGI_GET_GIL
#define UWSGI_RELEASE_GIL
#endif
#include <stdio.h>
@@ -797,6 +797,14 @@ struct uwsgi_server {
int threads;
pthread_key_t ut_key;
pthread_key_t ut_save_key;
#ifdef UWSGI_THREADING
void (*gil_get) (void);
void (*gil_release) (void);
#endif
struct wsgi_request* (*current_wsgi_req)(void);
};
struct uwsgi_cluster_node {
@@ -1117,7 +1125,7 @@ struct wsgi_request *current_wsgi_req(void);
inline struct wsgi_request *current_wsgi_req(void);
#endif
#else
#define current_wsgi_req() pthread_getspecific(uwsgi.ut_key)
#define current_wsgi_req() (*uwsgi.current_wsgi_req)()
#endif
void sanitize_args(void);
@@ -1262,3 +1270,14 @@ char *get_uwsgi_pymodule(char *);
PyObject *get_uwsgi_pydict(char *);
void *simple_loop(void *);
void complex_loop(void);
void gil_real_get(void);
void gil_real_release(void);
void gil_fake_get(void);
void gil_fake_release(void);
struct wsgi_request* threaded_current_wsgi_req(void);
struct wsgi_request* simple_current_wsgi_req(void);
+8 -8
View File
@@ -61,7 +61,7 @@ int uwsgi_request_eval(struct wsgi_request *wsgi_req) {
PyObject *code, *py_dict;
uwsgi_get_gil();
UWSGI_GET_GIL
PyObject *m = PyImport_AddModule("__main__");
if (m == NULL) {
PyErr_Print();
@@ -75,18 +75,18 @@ int uwsgi_request_eval(struct wsgi_request *wsgi_req) {
code = Py_CompileString(wsgi_req->buffer, "uWSGI", Py_file_input);
if (code == NULL) {
PyErr_Print();
uwsgi_release_gil();
UWSGI_RELEASE_GIL
return -1;
}
PyEval_EvalCode((PyCodeObject *)code, py_dict, py_dict );
Py_DECREF(code);
if (PyErr_Occurred()) {
PyErr_Print();
uwsgi_release_gil();
UWSGI_RELEASE_GIL
return -1;
}
uwsgi_release_gil();
UWSGI_RELEASE_GIL
return UWSGI_OK;
}
@@ -96,7 +96,7 @@ int uwsgi_request_fastfunc(struct wsgi_request *wsgi_req) {
PyObject *ffunc;
int ret = UWSGI_OK ;
uwsgi_get_gil();
UWSGI_GET_GIL
// CHECK HERE
ffunc = PyList_GetItem(uwsgi.fastfuncslist, wsgi_req->uh.modifier2);
@@ -105,7 +105,7 @@ int uwsgi_request_fastfunc(struct wsgi_request *wsgi_req) {
ret = uwsgi_python_call(wsgi_req, ffunc, NULL);
}
uwsgi_release_gil();
UWSGI_RELEASE_GIL
return ret;
}
@@ -113,7 +113,7 @@ int uwsgi_request_fastfunc(struct wsgi_request *wsgi_req) {
int uwsgi_request_marshal(struct wsgi_request *wsgi_req) {
PyObject *func_result;
uwsgi_get_gil();
UWSGI_GET_GIL
PyObject *umm = PyDict_GetItemString(uwsgi.embedded_dict,
"message_manager_marshal");
@@ -159,6 +159,6 @@ int uwsgi_request_marshal(struct wsgi_request *wsgi_req) {
}
PyErr_Clear();
uwsgi_release_gil();
UWSGI_RELEASE_GIL
return 0;
}
+1 -1
View File
@@ -89,7 +89,7 @@ gcc_major = int(gcc_version.split('.')[0])
gcc_minor = int(gcc_version.split('.')[1])
gcc_list = ['utils', 'pyutils', 'pyloader', 'protocol', 'socket', 'logging', 'master', 'wsgi_handlers', 'wsgi_subhandler', 'wsgi_headers', 'uwsgi_handlers', 'plugins', 'loop', 'uwsgi']
gcc_list = ['utils', 'pyutils', 'pyloader', 'protocol', 'socket', 'logging', 'master', 'wsgi_handlers', 'wsgi_subhandler', 'wsgi_headers', 'uwsgi_handlers', 'plugins', 'loop', 'gil', 'uwsgi']
cflags = ['-O2', '-Wall', '-Werror', '-D_LARGEFILE_SOURCE', '-D_FILE_OFFSET_BITS=64'] + os.environ.get("CFLAGS", "").split()
+10 -10
View File
@@ -13,9 +13,9 @@ PyObject *py_uwsgi_write(PyObject * self, PyObject * args) {
if (PyString_Check(data)) {
content = PyString_AsString(data);
len = PyString_Size(data);
uwsgi_release_gil();
UWSGI_RELEASE_GIL
wsgi_req->response_size = write(wsgi_req->poll.fd, content, len);
uwsgi_get_gil();
UWSGI_GET_GIL
}
Py_INCREF(Py_None);
@@ -142,9 +142,9 @@ int uwsgi_request_wsgi(struct wsgi_request *wsgi_req) {
#endif
) {
// a bit of magic: 1-1 = 0 / 0-1 = -1
uwsgi_get_gil();
UWSGI_GET_GIL
wsgi_req->app_id = init_uwsgi_app(LOADER_DYN, (void *) wsgi_req, wsgi_req, uwsgi.single_interpreter-1);
uwsgi_release_gil();
UWSGI_RELEASE_GIL
}
}
}
@@ -176,9 +176,9 @@ int uwsgi_request_wsgi(struct wsgi_request *wsgi_req) {
}
// set the interpreter
uwsgi_get_gil();
UWSGI_GET_GIL
PyThreadState_Swap(wi->interpreter);
uwsgi_release_gil();
UWSGI_RELEASE_GIL
if (wi->chdir) {
#ifdef UWSGI_DEBUG
uwsgi_debug("chdir to %s\n", wi->chdir);
@@ -215,7 +215,7 @@ int uwsgi_request_wsgi(struct wsgi_request *wsgi_req) {
wsgi_req->async_args = wi->wsgi_args;
#endif
uwsgi_get_gil();
UWSGI_GET_GIL
Py_INCREF((PyObject *)wsgi_req->async_environ);
@@ -255,7 +255,7 @@ int uwsgi_request_wsgi(struct wsgi_request *wsgi_req) {
if (uwsgi.post_buffering > 0 && wsgi_req->post_cl > (size_t) uwsgi.post_buffering) {
uwsgi_release_gil();
UWSGI_RELEASE_GIL
wsgi_req->async_post = tmpfile();
if (!wsgi_req->async_post) {
uwsgi_error("tmpfile()");
@@ -284,7 +284,7 @@ int uwsgi_request_wsgi(struct wsgi_request *wsgi_req) {
post_remains -= post_chunk;
}
rewind(wsgi_req->async_post);
uwsgi_get_gil();
UWSGI_GET_GIL
}
else {
wsgi_req->async_post = fdopen(wsgi_req->poll.fd, "r");
@@ -351,7 +351,7 @@ clear:
PyThreadState_Swap(uwsgi.main_thread);
}
uwsgi_release_gil();
UWSGI_RELEASE_GIL
clear2:
+2 -2
View File
@@ -141,9 +141,9 @@ PyObject *py_uwsgi_spit(PyObject * self, PyObject * args) {
wsgi_req->hvec[j].iov_base = nl;
wsgi_req->hvec[j].iov_len = NL_SIZE;
uwsgi_release_gil();
UWSGI_RELEASE_GIL
wsgi_req->headers_size = writev(wsgi_req->poll.fd, wsgi_req->hvec, j + 1);
uwsgi_get_gil();
UWSGI_GET_GIL
if (wsgi_req->headers_size < 0) {
uwsgi_error("writev()");
}