From 01ab3e28958f30d438ee7fa9288b87b24d83c02d Mon Sep 17 00:00:00 2001 From: "roberto@freebsd64" Date: Tue, 28 Feb 2012 09:05:37 +0100 Subject: [PATCH] broken commit for new locking api --- lock.c | 117 ++++++++++++++++++++++++---------------- master.c | 2 + plugins/erlang/erlang.c | 3 +- uwsgi.c | 5 ++ uwsgi.h | 46 +++++++++++----- uwsgiconfig.py | 12 ++--- 6 files changed, 118 insertions(+), 67 deletions(-) diff --git a/lock.c b/lock.c index b0b20394..bcf22578 100644 --- a/lock.c +++ b/lock.c @@ -9,7 +9,12 @@ static struct uwsgi_lock_item *uwsgi_register_lock(char *id, int rw) { uwsgi.registered_locks = uwsgi_malloc_shared(sizeof(struct uwsgi_lock_item)); uwsgi.registered_locks->id = id; uwsgi.registered_locks->pid = 0; - uwsgi.registered_locks->lock_ptr = uwsgi_mmap_shared_lock();; + if (rw) { + uwsgi.registered_locks->lock_ptr = uwsgi_malloc_shared(uwsgi.rwlock_size); + } + else { + uwsgi.registered_locks->lock_ptr = uwsgi_malloc_shared(uwsgi.lock_size); + } uwsgi.registered_locks->rw = rw; uwsgi.registered_locks->next = NULL; return uwsgi.registered_locks; @@ -18,7 +23,12 @@ static struct uwsgi_lock_item *uwsgi_register_lock(char *id, int rw) { while(uli) { if (!uli->next) { uli->next = uwsgi_malloc_shared(sizeof(struct uwsgi_lock_item)); - uli->next->lock_ptr = uwsgi_mmap_shared_lock();; + if (rw) { + uwsgi_malloc_shared(uwsgi.rwlock_size); + } + else { + uli->next->lock_ptr = uwsgi_malloc_shared(uwsgi.lock_size); + } uli->next->id = id; uli->next->pid = 0; uli->next->rw = rw; @@ -33,7 +43,6 @@ static struct uwsgi_lock_item *uwsgi_register_lock(char *id, int rw) { } - #ifdef UWSGI_LOCK_USE_MUTEX #define UWSGI_LOCK_SIZE sizeof(pthread_mutexattr_t) + sizeof(pthread_mutex_t) @@ -159,9 +168,9 @@ struct uwsgi_lock_item *uwsgi_rwlock_init(char *id) { -#endif +#elif defined(UWSGI_LOCK_USE_UMTX) -#ifdef UWSGI_LOCK_USE_UMTX +/* Warning: FreeBSD is still not ready for process-shared UMTX */ #include #include @@ -181,18 +190,18 @@ struct uwsgi_lock_item *uwsgi_lock_init(char *id) { } void uwsgi_lock(struct uwsgi_lock_item *uli) { - umtx_lock((struct umtx*) uli->lock_ptr, 1); + umtx_lock((struct umtx*) uli->lock_ptr, (u_long) getpid() ); uli->pid = uwsgi.mypid; } void uwsgi_unlock(struct uwsgi_lock_item *uli) { - umtx_unlock((struct umtx*) uli->lock_ptr, 1); + umtx_unlock((struct umtx*) uli->lock_ptr, (u_long) getpid() ); uli->pid = 0; } pid_t uwsgi_lock_check(struct uwsgi_lock_item *uli) { - if (umtx_trylock((struct umtx*) uli->lock_ptr, 1)) { - umtx_unlock((struct umtx*) uli->lock_ptr, 1); + if (umtx_trylock((struct umtx*) uli->lock_ptr, (u_long) getpid() )) { + umtx_unlock((struct umtx*) uli->lock_ptr, (u_long) getpid() ); return 0; } return uli->pid; @@ -200,10 +209,7 @@ pid_t uwsgi_lock_check(struct uwsgi_lock_item *uli) { pid_t uwsgi_rwlock_check(struct uwsgi_lock_item *uli) { return uwsgi_lock_check(uli); } -#endif - - -#ifdef UWSGI_LOCK_USE_OSX_SPINLOCK +#elif defined(UWSGI_LOCK_USE_OSX_SPINLOCK) #define UWSGI_LOCK_SIZE sizeof(OSSpinLock) #define UWSGI_RWLOCK_SIZE sizeof(OSSpinLock) @@ -249,17 +255,29 @@ pid_t uwsgi_rwlock_check(struct uwsgi_lock_item *uli) { return uwsgi_lock_check( void uwsgi_rwunlock(struct uwsgi_lock_item *uli) { uwsgi_unlock(uli); } +#else +#define uwsgi_lock_fast_init uwsgi_lock_flock_init +#define uwsgi_lock_fast_check uwsgi_lock_flock_check +#define uwsgi_lock_fast uwsgi_lock_flock +#define uwsgi_unlock_fast uwsgi_unlock_flock -#endif +#define uwsgi_rwlock_fast_init uwsgi_rwlock_flock_init +#define uwsgi_rwlock_fast_check uwsgi_rwlock_flock_check - -#ifdef UWSGI_LOCK_USE_FLOCK +#define uwsgi_rlock_fast uwsgi_rlock_flock +#define uwsgi_wlock_fast uwsgi_wlock_flock +#define uwsgi_rwunlock_fast uwsgi_rwunlock_flock #define UWSGI_LOCK_SIZE 8 #define UWSGI_RWLOCK_SIZE 8 -struct uwsgi_lock_item *uwsgi_lock_init(char *id) { +#define UWSGI_LOCK_ENGINE_NAME "flock" + +#endif + + +struct uwsgi_lock_item *uwsgi_lock_flock_init(char *id) { struct uwsgi_lock_item *uli = uwsgi_register_lock(id, 0); FILE *tf = tmpfile(); @@ -275,25 +293,25 @@ struct uwsgi_lock_item *uwsgi_lock_init(char *id) { return uli; } -void uwsgi_lock(struct uwsgi_lock_item *uli) { +void uwsgi_lock_flock(struct uwsgi_lock_item *uli) { int fd; memcpy(&fd, uli->lock_ptr, sizeof(int)); if (flock(fd, LOCK_EX)) { uwsgi_error("flock()"); } } -void uwsgi_unlock(struct uwsgi_lock_item *uli) { +void uwsgi_unlock_flock(struct uwsgi_lock_item *uli) { int fd; memcpy(&fd, uli->lock_ptr, sizeof(int)); if (flock(fd, LOCK_UN)) { uwsgi_error("flock()"); } } -struct uwsgi_lock_item *uwsgi_rwlock_init(char *id) { return uwsgi_lock_init(id);} -void uwsgi_rlock(struct uwsgi_lock_item *uli) { uwsgi_lock(uli);} -void uwsgi_wlock(struct uwsgi_lock_item *uli) { uwsgi_lock(uli);} -void uwsgi_rwunlock(struct uwsgi_lock_item *uli) { uwsgi_unlock(uli); } +struct uwsgi_lock_item *uwsgi_rwlock_flock_init(char *id) { return uwsgi_lock_flock_init(id);} +void uwsgi_rlock_flock(struct uwsgi_lock_item *uli) { uwsgi_lock_flock(uli);} +void uwsgi_wlock_flock(struct uwsgi_lock_item *uli) { uwsgi_lock_flock(uli);} +void uwsgi_rwunlock_flock(struct uwsgi_lock_item *uli) { uwsgi_unlock_flock(uli); } -pid_t uwsgi_lock_check(struct uwsgi_lock_item *uli) { +pid_t uwsgi_lock_flock_check(struct uwsgi_lock_item *uli) { int fd; memcpy(&fd, uli->lock_ptr, sizeof(int)); if (flock(fd, LOCK_EX|LOCK_NB) < 0) { @@ -309,31 +327,38 @@ pid_t uwsgi_lock_check(struct uwsgi_lock_item *uli) { } -pid_t uwsgi_rwlock_check(struct uwsgi_lock_item *uli) { return uwsgi_lock_check(uli); } - -#endif +pid_t uwsgi_rwlock_flock_check(struct uwsgi_lock_item *uli) { return uwsgi_lock_flock_check(uli); } -void *uwsgi_mmap_shared_lock() { - void *addr = NULL; - addr = mmap(NULL, UWSGI_LOCK_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANON, -1, 0); +void uwsgi_setup_locking() { - if (addr == NULL) { - uwsgi_error("mmap()"); - exit(1); + // use the fastest avaikable locking + if (uwsgi.lock_engine) { + if (!strcmp(uwsgi.lock_engine, "flock")) { + uwsgi_log("lock engine: flock\n"); + goto fallback; + } + else if (!strcmp(uwsgi.lock_engine, "ipcsem")) { + uwsgi_log("lock engine: ipc semaphores\n"); + return; + } + else { + uwsgi_log("lock engine: %s\n", UWSGI_LOCK_ENGINE_NAME); + goto fallback; + } } - - return addr; + uwsgi_log("lock engine: %s\n", UWSGI_LOCK_ENGINE_NAME); +fallback: + uwsgi.lock_ops.lock_init = uwsgi_lock_fast_init; + uwsgi.lock_ops.lock_check = uwsgi_lock_fast_check; + uwsgi.lock_ops.lock = uwsgi_lock_fast; + uwsgi.lock_ops.unlock = uwsgi_unlock_fast; + uwsgi.lock_ops.rwlock_init = uwsgi_rwlock_fast_init; + uwsgi.lock_ops.rwlock_check = uwsgi_rwlock_fast_check; + uwsgi.lock_ops.rlock = uwsgi_rlock_fast; + uwsgi.lock_ops.wlock = uwsgi_wlock_fast; + uwsgi.lock_ops.rwunlock = uwsgi_rwunlock_fast; + uwsgi.lock_size = UWSGI_LOCK_SIZE; + uwsgi.rwlock_size = UWSGI_RWLOCK_SIZE; } -void *uwsgi_mmap_shared_rwlock() { - void *addr = NULL; - addr = mmap(NULL, UWSGI_RWLOCK_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANON, -1, 0); - - if (addr == NULL) { - uwsgi_error("mmap()"); - exit(1); - } - - return addr; -} diff --git a/master.c b/master.c index 174d637b..c326ea02 100644 --- a/master.c +++ b/master.c @@ -1357,6 +1357,7 @@ int master_loop(char **argv, char **environ) { // check for deadlocks first struct uwsgi_lock_item *uli = uwsgi.registered_locks; while(uli) { + if (!uli->can_deadlock) goto nextlock; pid_t locked_pid = 0; if (uli->rw) { locked_pid = uwsgi_rwlock_check(uli); @@ -1373,6 +1374,7 @@ int master_loop(char **argv, char **environ) { uwsgi_unlock(uli); } } +nextlock: uli = uli->next; } } diff --git a/plugins/erlang/erlang.c b/plugins/erlang/erlang.c index e880fe13..d5d76d0a 100644 --- a/plugins/erlang/erlang.c +++ b/plugins/erlang/erlang.c @@ -363,8 +363,7 @@ int erlang_init() { char *nodename; struct in_addr addr; - uerl.lock = uwsgi_mmap_shared_lock(); - uwsgi_lock_init(uerl.lock); + uerl.lock = uwsgi_lock_init("erlang"); if (uerl.name) { diff --git a/uwsgi.c b/uwsgi.c index fe8ed118..e548d7f8 100644 --- a/uwsgi.c +++ b/uwsgi.c @@ -142,6 +142,7 @@ static struct uwsgi_option uwsgi_base_options[] = { {"socket-timeout", required_argument, 'z', "set internal sockets timeout", uwsgi_opt_set_dyn, (void *) UWSGI_OPTION_SOCKET_TIMEOUT, 0}, {"no-fd-passing", no_argument, 0, "disable file descriptor passing", uwsgi_opt_true, &uwsgi.no_fd_passing, 0}, {"locks", required_argument, 0, "create the specified number of shared locks", uwsgi_opt_set_int, &uwsgi.locks, 0}, + {"lock-engine", required_argument, 0, "set the lock engine", uwsgi_opt_set_str, &uwsgi.lock_engine, 0}, {"sharedarea", required_argument, 'A', "create a raw shared memory area of specified pages", uwsgi_opt_set_int, &uwsgi.sharedareasize, 0}, {"cache", required_argument, 0, "create a shared cache containing given elements", uwsgi_opt_set_int, &uwsgi.cache_max_items, 0}, @@ -1503,6 +1504,7 @@ int main(int argc, char *argv[], char *envp[]) { // ok, the options dictionary is available, lets manage it uwsgi_configure(); + if (uwsgi.daemonize) { if (uwsgi.has_emperor) { logto(uwsgi.daemonize); @@ -1953,6 +1955,9 @@ int uwsgi_start(void *v_argv) { uwsgi_log("VirtualHosting mode enabled.\n"); } + + // setup locking + uwsgi_setup_locking(); // event queue lock (mitigate same event on multiple queues) if (uwsgi.threads > 1) { diff --git a/uwsgi.h b/uwsgi.h index 5fd51fdf..e523cbff 100644 --- a/uwsgi.h +++ b/uwsgi.h @@ -297,10 +297,36 @@ struct uwsgi_lock_item { void *lock_ptr; int rw; pid_t pid; + int can_deadlock; struct uwsgi_lock_item *next; }; +struct uwsgi_lock_ops { + struct uwsgi_lock_item* (*lock_init)(char *); + pid_t (*lock_check)(struct uwsgi_lock_item *); + void (*lock)(struct uwsgi_lock_item *); + void (*unlock)(struct uwsgi_lock_item *); + + struct uwsgi_lock_item * (*rwlock_init)(char *); + pid_t (*rwlock_check)(struct uwsgi_lock_item *); + void (*rlock)(struct uwsgi_lock_item *); + void (*wlock)(struct uwsgi_lock_item *); + void (*rwunlock)(struct uwsgi_lock_item *); +}; + +#define uwsgi_lock_init(x) uwsgi.lock_ops.lock_init(x) +#define uwsgi_lock_check(x) uwsgi.lock_ops.lock_check(x) +#define uwsgi_lock(x) uwsgi.lock_ops.lock(x) +#define uwsgi_unlock(x) uwsgi.lock_ops.unlock(x) + +#define uwsgi_rwlock_init(x) uwsgi.lock_ops.rwlock_init(x) +#define uwsgi_rwlock_check(x) uwsgi.lock_ops.rwlock_check(x) +#define uwsgi_rlock(x) uwsgi.lock_ops.rlock(x) +#define uwsgi_wlock(x) uwsgi.lock_ops.wlock(x) +#define uwsgi_rwunlock(x) uwsgi.lock_ops.rwunlock(x) + + struct uwsgi_dyn_dict { char *key; @@ -1492,6 +1518,11 @@ struct uwsgi_server { struct sockaddr_in mc_cluster_addr; struct uwsgi_lock_item *registered_locks; + struct uwsgi_lock_ops lock_ops; + char *lock_engine; + char *lock_id; + size_t lock_size; + size_t rwlock_size; int check_cache; @@ -2048,16 +2079,6 @@ int uwsgi_cache_del(char *, uint16_t); char *uwsgi_cache_get(char *, uint16_t, uint64_t *); uint32_t uwsgi_cache_exists(char *, uint16_t); -struct uwsgi_lock_item *uwsgi_lock_init(char *); -pid_t uwsgi_lock_check(struct uwsgi_lock_item *); -void uwsgi_lock(struct uwsgi_lock_item *); -void uwsgi_unlock(struct uwsgi_lock_item *); - -struct uwsgi_lock_item *uwsgi_rwlock_init(char *); -pid_t uwsgi_rwlock_check(struct uwsgi_lock_item *); -void uwsgi_rlock(struct uwsgi_lock_item *); -void uwsgi_wlock(struct uwsgi_lock_item *); -void uwsgi_rwunlock(struct uwsgi_lock_item *); inline void *uwsgi_malloc(size_t); inline void *uwsgi_calloc(size_t); @@ -2082,9 +2103,6 @@ int event_queue_add_file_monitor(int, char *, int *); struct uwsgi_fmon *event_queue_ack_file_monitor(int, int); -void *uwsgi_mmap_shared_lock(void); -void *uwsgi_mmap_shared_rwlock(void); - int uwsgi_register_signal(uint8_t, char *, void *, uint8_t); int uwsgi_add_file_monitor(uint8_t, char *); int uwsgi_add_timer(uint8_t, int); @@ -2673,6 +2691,8 @@ void uwsgi_flush_logs(void); void uwsgi_register_cheaper_algo(char *, int(*) (void)); +void uwsgi_setup_locking(void); + void uwsgi_emulate_cow_for_apps(int); #ifdef UWSGI_AS_SHARED_LIBRARY diff --git a/uwsgiconfig.py b/uwsgiconfig.py index 64da3275..362737ec 100644 --- a/uwsgiconfig.py +++ b/uwsgiconfig.py @@ -414,19 +414,19 @@ class uConf(object): if locking_mode == 'auto': if uwsgi_os == 'Linux' or uwsgi_os == 'SunOS': locking_mode = 'pthread_mutex' - elif uwsgi_os == 'FreeBSD': - locking_mode = 'umtx' + # FreeBSD umtx is still not ready for process shared locking + #elif uwsgi_os == 'FreeBSD': + # locking_mode = 'umtx' elif uwsgi_os == 'Darwin': locking_mode = 'osx_spinlock' if locking_mode == 'pthread_mutex': self.cflags.append('-DUWSGI_LOCK_USE_MUTEX') - elif locking_mode == 'umtx': - self.cflags.append('-DUWSGI_LOCK_USE_UMTX') + # FreeBSD umtx is still not ready for process shared locking + #elif locking_mode == 'umtx': + # self.cflags.append('-DUWSGI_LOCK_USE_UMTX') elif locking_mode == 'osx_spinlock': self.cflags.append('-DUWSGI_LOCK_USE_OSX_SPINLOCK') - else: - self.cflags.append('-DUWSGI_LOCK_USE_FLOCK') # set event subsystem event_mode = self.get('event','auto')