clearlinux/uwsgi
2
0
Fork 0
mirror of https://github.com/clearlinux/uwsgi.git synced 2026-09-04 20:51:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

improved the caching subsystem to allows dynamic key

Browse Source
This commit is contained in:
Unbit
2013-01-23 20:16:49 +01:00
parent 2447f819cb
commit b90779aecf
5 changed files with 75 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/cache.c
+36 -33
View File
@@ -1,6 +1,7 @@
#include "uwsgi.h"
#include <uwsgi.h>
extern struct uwsgi_server uwsgi;
#define cache_item(x) (struct uwsgi_cache_item *) (((char *)uc->items) + ((sizeof(struct uwsgi_cache_item)+uc->keysize) * x))
// block bitmap manager
/*
@@ -119,11 +120,11 @@ void uwsgi_cache_init(struct uwsgi_cache *uc) {
uint64_t i;
for (i = 0; i < uc->max_items; i++) {
// here we only need to clear the item header
memset(&uc->items[i], 0, sizeof(struct uwsgi_cache_item));
memset(cache_item(i), 0, sizeof(struct uwsgi_cache_item));
}
}
uc->data = uc->items + ((sizeof(struct uwsgi_cache_item)+uc->keysize) * uc->max_items);
uc->data = ((char *)uc->items) + ((sizeof(struct uwsgi_cache_item)+uc->keysize) * uc->max_items);
uc->lock = uwsgi_rwlock_init("cache");
@@ -209,20 +210,18 @@ static inline uint64_t uwsgi_cache_get_index(struct uwsgi_cache *uc, char *key,
uint32_t hash = uc->hash->func(key, keylen);
int hash_key = hash % uc->hashsize;
uint32_t hash_key = hash % uc->hashsize;
uint64_t slot = uc->hashtable[hash_key];
struct uwsgi_cache_item *uci;
struct uwsgi_cache_item *uci = cache_item(slot);
uint64_t rounds = 0;
//uwsgi_log("found slot %d for key %d\n", slot, hash_key);
uci = &uc->items[slot];
// first round
if (uci->hash != hash)
if (uci->hash % uc->hashsize != hash_key)
return 0;
if (uci->hash != hash)
goto cycle;
if (uci->keysize != keylen)
goto cycle;
if (memcmp(uci->key, key, keylen))
@@ -233,7 +232,7 @@ static inline uint64_t uwsgi_cache_get_index(struct uwsgi_cache *uc, char *key,
cycle:
while (uci->next) {
slot = uci->next;
uci = &uc->items[slot];
uci = cache_item(slot);
rounds++;
if (rounds > uc->max_items) {
uwsgi_log("ALARM !!! cache-loop (and potential deadlock) detected slot = %lu prev = %lu next = %lu\n", slot, uci->prev, uci->next);
@@ -248,7 +247,7 @@ cycle:
}
}
if (uci->hash != hash)
return 0;
continue;
if (uci->keysize != keylen)
continue;
if (!memcmp(uci->key, key, keylen))
@@ -268,10 +267,11 @@ char *uwsgi_cache_get2(struct uwsgi_cache *uc, char *key, uint16_t keylen, uint6
uint64_t index = uwsgi_cache_get_index(uc, key, keylen);
if (index) {
if (uc->items[index].flags & UWSGI_CACHE_FLAG_UNGETTABLE)
struct uwsgi_cache_item *uci = cache_item(index);
if (uci->flags & UWSGI_CACHE_FLAG_UNGETTABLE)
return NULL;
*valsize = uc->items[index].valsize;
uc->items[index].hits++;
*valsize = uci->valsize;
uci->hits++;
uc->hits++;
return uc->data + (index * uc->blocksize);
}
@@ -290,7 +290,7 @@ int uwsgi_cache_del2(struct uwsgi_cache *uc, char *key, uint16_t keylen, uint64_
index = uwsgi_cache_get_index(uc, key, keylen);
if (index) {
uci = &uc->items[index];
uci = cache_item(index);
uci->keysize = 0;
cache_unmark_blocks(uc, index, uci->valsize);
uci->valsize = 0;
@@ -302,7 +302,8 @@ int uwsgi_cache_del2(struct uwsgi_cache *uc, char *key, uint16_t keylen, uint64_
ret = 0;
// relink collisioned entry
if (uci->prev) {
uc->items[uci->prev].next = uci->next;
struct uwsgi_cache_item *ucii = cache_item(uci->prev);
ucii->next = uci->next;
}
else {
// set next as the new entry point (could be 0)
@@ -310,7 +311,8 @@ int uwsgi_cache_del2(struct uwsgi_cache *uc, char *key, uint16_t keylen, uint64_
}
if (uci->next) {
uc->items[uci->next].prev = uci->prev;
struct uwsgi_cache_item *ucii = cache_item(uci->next);
ucii->prev = uci->prev;
}
if (!uci->prev && !uci->next) {
@@ -340,10 +342,11 @@ void uwsgi_cache_fix(struct uwsgi_cache *uc) {
for (i = 0; i < uc->max_items; i++) {
// valid record ?
if (uc->items[i].keysize) {
if (!uc->items[i].prev) {
struct uwsgi_cache_item *uci = cache_item(i);
if (uci->keysize) {
if (!uci->prev) {
// put value in hash_table
uc->hashtable[uc->items[i].hash % uc->hashsize] = i;
uc->hashtable[uci->hash % uc->hashsize] = i;
restored++;
}
}
@@ -392,7 +395,7 @@ int uwsgi_cache_set2(struct uwsgi_cache *uc, char *key, uint16_t keylen, char *v
uc->first_available_block++;
}
}
uci = &uc->items[index];
uci = cache_item(index);
if (expires && !(flags & UWSGI_CACHE_FLAG_ABSEXPIRE))
expires += uwsgi_now();
uci->expires = expires;
@@ -400,30 +403,29 @@ int uwsgi_cache_set2(struct uwsgi_cache *uc, char *key, uint16_t keylen, char *v
uci->hits = 0;
uci->flags = flags;
memcpy(uci->key, key, keylen);
memcpy(uc->data + (index * uc->blocksize), val, vallen);
memcpy(((char *) uc->data) + (index * uc->blocksize), val, vallen);
// set this as late as possibile (to reduce races risk)
uci->valsize = vallen;
uci->keysize = keylen;
ret = 0;
// now put the value in the 16bit hashtable
int slot = uci->hash % uc->hashsize;
// now put the value in the hashtable
uint32_t slot = uci->hash % uc->hashsize;
// reset values
uci->prev = 0;
uci->next = 0;
if (uc->hashtable[slot] == 0) {
last_index = uc->hashtable[slot];
if (last_index == 0) {
uc->hashtable[slot] = index;
}
else {
//uwsgi_log("HASH COLLISION !!!!\n");
// append to first available next
last_index = uc->hashtable[slot];
ucii = &uc->items[last_index];
ucii = cache_item(last_index);
while (ucii->next) {
last_index = ucii->next;
ucii = &uc->items[last_index];
ucii = cache_item(last_index);
}
ucii->next = index;
uci->prev = last_index;
@@ -432,7 +434,7 @@ int uwsgi_cache_set2(struct uwsgi_cache *uc, char *key, uint16_t keylen, char *v
uc->n_items++ ;
}
else if (flags & UWSGI_CACHE_FLAG_UPDATE) {
uci = &uc->items[index];
uci = cache_item(index);
if (expires && !(flags & UWSGI_CACHE_FLAG_ABSEXPIRE)) {
expires += uwsgi_now();
uci->expires = expires;
@@ -762,8 +764,9 @@ void *cache_sweeper_loop(void *ucache) {
// skip the first slot
for (i = 1; i < uc->max_items; i++) {
uwsgi_wlock(uc->lock);
if (uc->items[i].expires) {
if (uc->items[i].expires < (uint64_t) uwsgi.current_time) {
struct uwsgi_cache_item *uci = cache_item(i);
if (uci->expires) {
if (uci->expires < (uint64_t) uwsgi.current_time) {
uwsgi_cache_del2(uc, NULL, 0, i, UWSGI_CACHE_FLAG_LOCAL);
freed_items++;
}
core/uwsgi.c
+5 -2
View File
@@ -176,8 +176,8 @@ static struct uwsgi_option uwsgi_base_options[] = {
{"ftok", required_argument, 0, "set the ipcsem key via ftok() for avoiding duplicates", uwsgi_opt_set_str, &uwsgi.ftok, 0},
{"sharedarea", required_argument, 'A', "create a raw shared memory area of specified pages", uwsgi_opt_set_int, &uwsgi.sharedareasize, 0},
{"cache", required_argument, 0, "create a shared cache containing given elements", uwsgi_opt_set_int, &uwsgi.cache_max_items, 0},
{"cache-blocksize", required_argument, 0, "set cache blocksize", uwsgi_opt_set_int, &uwsgi.cache_blocksize, 0},
{"cache", required_argument, 0, "create a shared cache containing given elements", uwsgi_opt_set_64bit, &uwsgi.cache_max_items, 0},
{"cache-blocksize", required_argument, 0, "set cache blocksize", uwsgi_opt_set_64bit, &uwsgi.cache_blocksize, 0},
{"cache-store", required_argument, 0, "enable persistent cache to disk", uwsgi_opt_set_str, &uwsgi.cache_store, UWSGI_OPT_MASTER},
{"cache-store-sync", required_argument, 0, "set frequency of sync for persistent cache", uwsgi_opt_set_int, &uwsgi.cache_store_sync, 0},
{"cache-server", required_argument, 0, "enable the threaded cache server", uwsgi_opt_set_str, &uwsgi.cache_server, 0},
@@ -2287,6 +2287,9 @@ int uwsgi_start(void *v_argv) {
uwsgi_init_queue();
}
// register embedded hash algorithms
uwsgi_hash_algo_register_all();
// setup default cache
if (uwsgi.cache_max_items > 0) {
uwsgi_cache_create(NULL);
plugins/cache/cache.c
Vendored
+4 -4
View File
@@ -109,12 +109,12 @@ int uwsgi_cache_request(struct wsgi_request *wsgi_req) {
if (uwsgi_response_write_body_do(wsgi_req, (char *)&wsgi_req->uh, 4)) return -1;
uwsgi_response_write_body_do(wsgi_req, cache_dump->buf, cache_dump->pos);
uwsgi_buffer_destroy(cache_dump);
uwsgi_wlock(uwsgi.cache_lock);
int ret = uwsgi_write_nb(wsgi_req->poll.fd, (char *)uwsgi.cache_items, uwsgi.cache_filesize, uwsgi.shared->options[UWSGI_OPTION_SOCKET_TIMEOUT]);
uwsgi_wlock(uwsgi.caches->lock);
int ret = uwsgi_write_nb(wsgi_req->poll.fd, (char *)uwsgi.caches->items, uwsgi.caches->filesize, uwsgi.shared->options[UWSGI_OPTION_SOCKET_TIMEOUT]);
if (!ret) {
wsgi_req->response_size += uwsgi.cache_filesize;
wsgi_req->response_size += uwsgi.caches->filesize;
}
uwsgi_rwunlock(uwsgi.cache_lock);
uwsgi_rwunlock(uwsgi.caches->lock);
break;
}
plugins/python/uwsgi_pymodule.c
+27 -27
View File
@@ -3491,11 +3491,11 @@ PyObject *py_uwsgi_cache_clear(PyObject * self, PyObject * args) {
uint64_t i;
// skip the first slot
for (i = 1; i < uwsgi.cache_max_items; i++) {
for (i = 1; i < uwsgi.caches->max_items; i++) {
UWSGI_RELEASE_GIL
uwsgi_wlock(uwsgi.cache_lock);
uwsgi_wlock(uwsgi.caches->lock);
uwsgi_cache_del(NULL, 0, i, 0);
uwsgi_rwunlock(uwsgi.cache_lock);
uwsgi_rwunlock(uwsgi.caches->lock);
UWSGI_GET_GIL
}
@@ -3519,16 +3519,16 @@ PyObject *py_uwsgi_cache_del(PyObject * self, PyObject * args) {
uwsgi_simple_send_string(remote, 111, 2, key, keylen, uwsgi.shared->options[UWSGI_OPTION_SOCKET_TIMEOUT]);
UWSGI_GET_GIL
}
else if (uwsgi.cache_max_items) {
else if (uwsgi.caches) {
UWSGI_RELEASE_GIL
uwsgi_wlock(uwsgi.cache_lock);
uwsgi_wlock(uwsgi.caches->lock);
if (uwsgi_cache_del(key, keylen, 0, 0)) {
uwsgi_rwunlock(uwsgi.cache_lock);
uwsgi_rwunlock(uwsgi.caches->lock);
UWSGI_GET_GIL
Py_INCREF(Py_None);
return Py_None;
}
uwsgi_rwunlock(uwsgi.cache_lock);
uwsgi_rwunlock(uwsgi.caches->lock);
UWSGI_GET_GIL
}
@@ -3552,8 +3552,8 @@ PyObject *py_uwsgi_cache_set(PyObject * self, PyObject * args) {
return NULL;
}
if ((uint64_t)vallen > uwsgi.cache_blocksize) {
return PyErr_Format(PyExc_ValueError, "uWSGI cache items size must be < %llu, requested %llu bytes", (unsigned long long)uwsgi.cache_blocksize, (unsigned long long) vallen);
if ((uint64_t)vallen > uwsgi.caches->blocksize) {
return PyErr_Format(PyExc_ValueError, "uWSGI cache items size must be < %llu, requested %llu bytes", (unsigned long long)uwsgi.caches->blocksize, (unsigned long long) vallen);
}
if (remote && strlen(remote) > 0) {
@@ -3561,16 +3561,16 @@ PyObject *py_uwsgi_cache_set(PyObject * self, PyObject * args) {
uwsgi_simple_send_string2(remote, 111, 1, key, keylen, value, vallen, uwsgi.shared->options[UWSGI_OPTION_SOCKET_TIMEOUT]);
UWSGI_GET_GIL
}
else if (uwsgi.cache_max_items) {
else if (uwsgi.caches) {
UWSGI_RELEASE_GIL
uwsgi_wlock(uwsgi.cache_lock);
uwsgi_wlock(uwsgi.caches->lock);
if (uwsgi_cache_set(key, keylen, value, vallen, expires, 0)) {
uwsgi_rwunlock(uwsgi.cache_lock);
uwsgi_rwunlock(uwsgi.caches->lock);
UWSGI_GET_GIL
Py_INCREF(Py_None);
return Py_None;
}
uwsgi_rwunlock(uwsgi.cache_lock);
uwsgi_rwunlock(uwsgi.caches->lock);
UWSGI_GET_GIL
}
@@ -3593,8 +3593,8 @@ PyObject *py_uwsgi_cache_update(PyObject * self, PyObject * args) {
return NULL;
}
if ((uint64_t)vallen > uwsgi.cache_blocksize) {
return PyErr_Format(PyExc_ValueError, "uWSGI cache items size must be < %llu, requested %llu bytes", (unsigned long long)uwsgi.cache_blocksize, (unsigned long long) vallen);
if ((uint64_t)vallen > uwsgi.caches->blocksize) {
return PyErr_Format(PyExc_ValueError, "uWSGI cache items size must be < %llu, requested %llu bytes", (unsigned long long)uwsgi.caches->blocksize, (unsigned long long) vallen);
}
if (remote && strlen(remote) > 0) {
@@ -3602,16 +3602,16 @@ PyObject *py_uwsgi_cache_update(PyObject * self, PyObject * args) {
uwsgi_simple_send_string2(remote, 111, 1, key, keylen, value, vallen, uwsgi.shared->options[UWSGI_OPTION_SOCKET_TIMEOUT]);
UWSGI_GET_GIL
}
else if (uwsgi.cache_max_items) {
else if (uwsgi.caches) {
UWSGI_RELEASE_GIL
uwsgi_wlock(uwsgi.cache_lock);
uwsgi_wlock(uwsgi.caches->lock);
if (uwsgi_cache_set(key, keylen, value, vallen, expires, UWSGI_CACHE_FLAG_UPDATE)) {
uwsgi_rwunlock(uwsgi.cache_lock);
uwsgi_rwunlock(uwsgi.caches->lock);
UWSGI_GET_GIL
Py_INCREF(Py_None);
return Py_None;
}
uwsgi_rwunlock(uwsgi.cache_lock);
uwsgi_rwunlock(uwsgi.caches->lock);
UWSGI_GET_GIL
}
@@ -3645,16 +3645,16 @@ PyObject *py_uwsgi_cache_exists(PyObject * self, PyObject * args) {
return Py_True;
}
}
else if (uwsgi.cache_max_items) {
else if (uwsgi.caches) {
UWSGI_RELEASE_GIL
uwsgi_rlock(uwsgi.cache_lock);
uwsgi_rlock(uwsgi.caches->lock);
if (uwsgi_cache_exists(key, keylen)) {
uwsgi_rwunlock(uwsgi.cache_lock);
uwsgi_rwunlock(uwsgi.caches->lock);
UWSGI_GET_GIL
Py_INCREF(Py_True);
return Py_True;
}
uwsgi_rwunlock(uwsgi.cache_lock);
uwsgi_rwunlock(uwsgi.caches->lock);
UWSGI_GET_GIL
}
@@ -3982,15 +3982,15 @@ PyObject *py_uwsgi_cache_get(PyObject * self, PyObject * args) {
valsize = valsize16;
}
}
else if (uwsgi.cache_max_items) {
else if (uwsgi.caches) {
#ifdef UWSGI_DEBUG
gettimeofday(&tv, NULL);
#endif
UWSGI_RELEASE_GIL
uwsgi_rlock(uwsgi.cache_lock);
uwsgi_rlock(uwsgi.caches->lock);
value = uwsgi_cache_get(key, keylen, &valsize);
if (!value) {
uwsgi_rwunlock(uwsgi.cache_lock);
uwsgi_rwunlock(uwsgi.caches->lock);
UWSGI_GET_GIL
Py_INCREF(Py_None);
return Py_None;
@@ -4003,7 +4003,7 @@ PyObject *py_uwsgi_cache_get(PyObject * self, PyObject * args) {
}
#endif
memcpy(storage, value, valsize);
uwsgi_rwunlock(uwsgi.cache_lock);
uwsgi_rwunlock(uwsgi.caches->lock);
UWSGI_GET_GIL
ret = PyString_FromStringAndSize(storage, valsize);
free(storage);
uwsgi.h
+3 -2
View File
@@ -598,6 +598,7 @@ struct uwsgi_hash_algo {
struct uwsgi_hash_algo *uwsgi_hash_algo_get(char *);
void uwsgi_hash_algo_register(char *, uint32_t (*)(char *, uint64_t));
void uwsgi_hash_algo_register_all(void);
// maintain alignment here !!!
struct uwsgi_cache_item {
@@ -1704,8 +1705,8 @@ struct uwsgi_server {
int cache_expire_freq;
int cache_report_freed_items;
int cache_no_expire;
int cache_max_items;
int cache_blocksize;
uint64_t cache_max_items;
uint64_t cache_blocksize;
char *cache_store;
int cache_store_sync;