diff --git a/master.c b/master.c index 6989fa3d..5851ced3 100644 --- a/master.c +++ b/master.c @@ -110,6 +110,30 @@ void *logger_thread_loop(void *noarg) { return NULL; } +void *cache_sweeper_loop(void *noarg) { + + int i; + // block all signals + sigset_t smask; + sigfillset(&smask); + pthread_sigmask(SIG_BLOCK, &smask, NULL); + + // remove expired cache items TODO use rb_tree timeouts + for(;;) { + sleep(1); + for (i = 0; i < (int) uwsgi.cache_max_items; i++) { + uwsgi_wlock(uwsgi.cache_lock); + if (uwsgi.cache_items[i].expires) { + if (uwsgi.cache_items[i].expires < (uint64_t) uwsgi.current_time) { + uwsgi_cache_del(uwsgi.cache_items[i].key, uwsgi.cache_items[i].keysize); + } + } + uwsgi_rwunlock(uwsgi.cache_lock); + } + }; + +} + void uwsgi_subscribe(char *subscription, uint8_t cmd) { int subfile_size; @@ -279,6 +303,7 @@ int master_loop(char **argv, char **environ) { uint64_t last_request_count = 0; pthread_t logger_thread; + pthread_t cache_sweeper; #ifdef UWSGI_UDP struct sockaddr_in udp_client; @@ -374,6 +399,16 @@ int master_loop(char **argv, char **environ) { } } + if (uwsgi.cache_max_items > 0 && !uwsgi.cache_no_expire) { + if (pthread_create(&cache_sweeper, NULL, cache_sweeper_loop, NULL)) { + uwsgi_error("pthread_create()"); + uwsgi_log("unable to run the cache sweeper !!!\n"); + } + else { + uwsgi_log("cache sweeper thread enabled\n"); + } + } + uwsgi.wsgi_req->buffer = uwsgi.async_buf[0]; @@ -1189,19 +1224,6 @@ int master_loop(char **argv, char **environ) { } } - // remove expired cache items TODO use rb_tree timeouts - if (uwsgi.cache_max_items > 0) { - for (i = 0; i < (int) uwsgi.cache_max_items; i++) { - uwsgi_wlock(uwsgi.cache_lock); - if (uwsgi.cache_items[i].expires) { - if (uwsgi.cache_items[i].expires < (uint64_t) uwsgi.current_time) { - uwsgi_cache_del(uwsgi.cache_items[i].key, uwsgi.cache_items[i].keysize); - } - } - uwsgi_rwunlock(uwsgi.cache_lock); - } - } - check_interval = uwsgi.shared->options[UWSGI_OPTION_MASTER_INTERVAL]; if (!check_interval) check_interval = 1; diff --git a/plugins/python/uwsgi_pymodule.c b/plugins/python/uwsgi_pymodule.c index 8ca132c0..be20e5fd 100644 --- a/plugins/python/uwsgi_pymodule.c +++ b/plugins/python/uwsgi_pymodule.c @@ -3363,29 +3363,29 @@ PyObject *py_uwsgi_queue_pull(PyObject * self, PyObject * args) { char *message; uint64_t size; PyObject *res; + char *storage; if (uwsgi.queue_size) { UWSGI_RELEASE_GIL uwsgi_wlock(uwsgi.queue_lock); + message = uwsgi_queue_pull(&size); - UWSGI_GET_GIL - if (message && size > 0) { - res = PyString_FromStringAndSize(NULL, size); -#ifdef PYTHREE - char *storage = PyBytes_AsString(res); -#else - char *storage = PyString_AS_STRING(res); -#endif - UWSGI_RELEASE_GIL - memcpy(storage, message, size); - } - else { + + if (!message || size == 0) { + uwsgi_rwunlock(uwsgi.queue_lock); + UWSGI_GET_GIL Py_INCREF(Py_None); - res = Py_None; - UWSGI_RELEASE_GIL + return Py_None; } + + storage = uwsgi_malloc(size); + memcpy(storage, message, size); + uwsgi_rwunlock(uwsgi.queue_lock); UWSGI_GET_GIL + + res = PyString_FromStringAndSize(storage, size); + free(storage); return res; } @@ -3399,29 +3399,29 @@ PyObject *py_uwsgi_queue_pop(PyObject * self, PyObject * args) { char *message; uint64_t size; PyObject *res; + char *storage; if (uwsgi.queue_size) { + UWSGI_RELEASE_GIL uwsgi_wlock(uwsgi.queue_lock); + message = uwsgi_queue_pop(&size); - UWSGI_GET_GIL - if (message && size > 0) { - res = PyString_FromStringAndSize(NULL, size); -#ifdef PYTHREE - char *storage = PyBytes_AsString(res); -#else - char *storage = PyString_AS_STRING(res); -#endif - UWSGI_RELEASE_GIL - memcpy(storage, message, size); - } - else { + if (!message || size == 0) { + uwsgi_rwunlock(uwsgi.queue_lock); + UWSGI_GET_GIL Py_INCREF(Py_None); - res = Py_None; - UWSGI_RELEASE_GIL - } + return Py_None; + } + + storage = uwsgi_malloc(size); + memcpy(storage, message, size); + uwsgi_rwunlock(uwsgi.queue_lock); UWSGI_GET_GIL + + res = PyString_FromStringAndSize(storage, size); + free(storage); return res; } @@ -3437,6 +3437,7 @@ PyObject *py_uwsgi_queue_get(PyObject * self, PyObject * args) { uint64_t size = 0; char *message; PyObject *res; + char *storage; if (!PyArg_ParseTuple(args, "l:queue_get", &index)) { return NULL; @@ -3445,25 +3446,23 @@ PyObject *py_uwsgi_queue_get(PyObject * self, PyObject * args) { if (uwsgi.queue_size) { UWSGI_RELEASE_GIL uwsgi_rlock(uwsgi.queue_lock); + message = uwsgi_queue_get(index, &size); - UWSGI_GET_GIL - if (message && size > 0) { - res = PyString_FromStringAndSize(NULL, size); -#ifdef PYTHREE - char *storage = PyBytes_AsString(res); -#else - char *storage = PyString_AS_STRING(res); -#endif - UWSGI_RELEASE_GIL - memcpy(storage, message, size); - } - else { + if (!message || size == 0) { + uwsgi_rwunlock(uwsgi.queue_lock); + UWSGI_GET_GIL Py_INCREF(Py_None); - res = Py_None; - UWSGI_RELEASE_GIL - } + return Py_None; + } + + storage = uwsgi_malloc(size); + memcpy(storage, message, size); + uwsgi_rwunlock(uwsgi.queue_lock); UWSGI_GET_GIL + + res = PyString_FromStringAndSize(storage, size); + free(storage); return res; } @@ -3473,11 +3472,12 @@ PyObject *py_uwsgi_queue_get(PyObject * self, PyObject * args) { PyObject *py_uwsgi_queue_last(PyObject * self, PyObject * args) { - long num = 0; + long i, num = 0; uint64_t size = 0; char *message; PyObject *res = NULL; uint64_t base; + char *storage; if (!PyArg_ParseTuple(args, "|l:queue_last", &num)) { return NULL; @@ -3501,50 +3501,41 @@ PyObject *py_uwsgi_queue_last(PyObject * self, PyObject * args) { if (num == 0) { message = uwsgi_queue_get(base, &size); - UWSGI_GET_GIL - if (message && size > 0) { - res = PyString_FromStringAndSize(NULL, size); -#ifdef PYTHREE - char *storage = PyBytes_AsString(res); -#else - char *storage = PyString_AS_STRING(res); -#endif - UWSGI_RELEASE_GIL - memcpy(storage, message, size); - } - else { + if (!message || size == 0) { + uwsgi_rwunlock(uwsgi.queue_lock); + UWSGI_GET_GIL Py_INCREF(Py_None); - res = Py_None; - UWSGI_RELEASE_GIL - } + return Py_None; + } + + storage = uwsgi_malloc(size); + memcpy(storage, message, size); + uwsgi_rwunlock(uwsgi.queue_lock); UWSGI_GET_GIL + + res = PyString_FromStringAndSize(storage, size); + free(storage); return res; } if (num > (long)uwsgi.queue_size) num = uwsgi.queue_size; + char **queue_items = uwsgi_malloc(sizeof(char *) * num); + uint64_t *queue_items_size = uwsgi_malloc(sizeof(uint64_t) * num); + long item_pos = 0; while(num) { message = uwsgi_queue_get(base, &size); - UWSGI_GET_GIL - if (message && size) { - PyObject *zero = PyString_FromStringAndSize(NULL, size); - PyList_Append(res, zero); - Py_DECREF(zero); -#ifdef PYTHREE - char *storage = PyBytes_AsString(res); -#else - char *storage = PyString_AS_STRING(res); -#endif - UWSGI_RELEASE_GIL - memcpy(storage, message, size); - } - else { - UWSGI_RELEASE_GIL - uwsgi_rwunlock(uwsgi.queue_lock); - UWSGI_GET_GIL - return res; + if (!message || size == 0) { + queue_items[item_pos] = NULL; + queue_items_size[item_pos] = 0; } + else { + queue_items[item_pos] = uwsgi_malloc(size); + memcpy(queue_items[item_pos], message, size); + queue_items_size[item_pos] = size; + } + item_pos++; if (base > 0) { base--; } @@ -3553,9 +3544,24 @@ PyObject *py_uwsgi_queue_last(PyObject * self, PyObject * args) { } num--; } - UWSGI_RELEASE_GIL + uwsgi_rwunlock(uwsgi.queue_lock); UWSGI_GET_GIL + + for(i=0;i 30000) { @@ -3622,6 +3621,8 @@ PyObject *py_uwsgi_cache_get(PyObject * self, PyObject * args) { memcpy(storage, value, valsize); uwsgi_rwunlock(uwsgi.cache_lock); UWSGI_GET_GIL + ret = PyString_FromStringAndSize(storage, valsize); + free(storage); return ret; } diff --git a/uwsgi.c b/uwsgi.c index 9b88a592..f3a75013 100644 --- a/uwsgi.c +++ b/uwsgi.c @@ -158,6 +158,7 @@ static struct uwsgi_option uwsgi_base_options[] = { {"cache-store-sync", required_argument, 0, "set frequency of sync for persistent cache", uwsgi_opt_set_int, &uwsgi.cache_store_sync,0}, {"cache-server", required_argument, 0, "enable the threaded cache server", uwsgi_opt_set_str, &uwsgi.cache_server, 0}, {"cache-server-threads", required_argument, 0, "set the number of threads for the cache server", uwsgi_opt_set_int, &uwsgi.cache_server_threads,0}, + {"cache-no-expire", required_argument, 0, "disable auto sweep of expired items", uwsgi_opt_true, &uwsgi.cache_no_expire,0}, {"queue", required_argument, 0, "enable shared queue", uwsgi_opt_set_int, &uwsgi.queue_size, 0}, {"queue-blocksize", required_argument, 0, "set queue blocksize", uwsgi_opt_set_int, &uwsgi.queue_store_sync, 0}, diff --git a/uwsgi.h b/uwsgi.h index e51367da..ad13c631 100644 --- a/uwsgi.h +++ b/uwsgi.h @@ -1572,6 +1572,7 @@ struct uwsgi_server { char *cache_store; size_t cache_filesize; int cache_store_sync; + int cache_no_expire; char *cache_server; int cache_server_threads;