diff --git a/core/init.c b/core/init.c index 3b4645f8..b6137651 100644 --- a/core/init.c +++ b/core/init.c @@ -130,6 +130,11 @@ void uwsgi_init_default() { uwsgi.shared->worker_log_pipe[0] = -1; uwsgi.shared->worker_log_pipe[1] = -1; + uwsgi.shared->worker_req_log_pipe[0] = -1; + uwsgi.shared->worker_req_log_pipe[1] = -1; + + uwsgi.req_log_fd = 2; + #ifdef UWSGI_SSL // 1 day of tolerance uwsgi.subscriptions_sign_check_tolerance = 3600 * 24; diff --git a/core/logging.c b/core/logging.c index ca504837..7d9a1f6d 100644 --- a/core/logging.c +++ b/core/logging.c @@ -144,6 +144,21 @@ void create_logpipe(void) { exit(1); } + if (uwsgi.req_log_master) { +#if defined(SOCK_SEQPACKET) && defined(__linux__) + if (socketpair(AF_UNIX, SOCK_SEQPACKET, 0, uwsgi.shared->worker_req_log_pipe)) { +#else + if (socketpair(AF_UNIX, SOCK_DGRAM, 0, uwsgi.shared->worker_req_log_pipe)) { +#endif + uwsgi_error("socketpair()\n"); + exit(1); + } + + uwsgi_socket_nb(uwsgi.shared->worker_req_log_pipe[0]); + uwsgi_socket_nb(uwsgi.shared->worker_req_log_pipe[1]); + uwsgi.req_log_fd = uwsgi.shared->worker_req_log_pipe[1]; + } + } #ifdef UWSGI_ZEROMQ @@ -294,62 +309,71 @@ void uwsgi_setup_log() { } +static struct uwsgi_logger *setup_choosen_logger(struct uwsgi_string_list *usl) { + char *id = NULL; + char *name = usl->value; + + char *space = strchr(name, ' '); + if (space) { + int is_id = 1; + int i; + for (i = 0; i < (space - name); i++) { + if (!isalnum(name[i])) { + is_id = 0; + break; + } + } + if (is_id) { + id = uwsgi_concat2n(name, space - name, "", 0); + name = space + 1; + } + } + + char *colon = strchr(name, ':'); + if (colon) { + *colon = 0; + } + + struct uwsgi_logger *choosen_logger = uwsgi_get_logger(name); + if (!choosen_logger) { + uwsgi_log("unable to find logger %s\n", name); + exit(1); + } + + // make a copy of the logger + struct uwsgi_logger *copy_of_choosen_logger = uwsgi_malloc(sizeof(struct uwsgi_logger)); + memcpy(copy_of_choosen_logger, choosen_logger, sizeof(struct uwsgi_logger)); + choosen_logger = copy_of_choosen_logger; + choosen_logger->id = id; + choosen_logger->next = NULL; + + if (colon) { + choosen_logger->arg = colon + 1; + // check for empty string + if (*choosen_logger->arg == 0) { + choosen_logger->arg = NULL; + } + *colon = ':'; + } + return choosen_logger; +} + void uwsgi_setup_log_master(void) { struct uwsgi_string_list *usl = uwsgi.requested_logger; while (usl) { - char *id = NULL; - char *name = usl->value; - - char *space = strchr(name, ' '); - if (space) { - int is_id = 1; - int i; - for (i = 0; i < (space - name); i++) { - if (!isalnum(name[i])) { - is_id = 0; - break; - } - } - if (is_id) { - id = uwsgi_concat2n(name, space - name, "", 0); - name = space + 1; - } - } - - char *colon = strchr(name, ':'); - if (colon) { - *colon = 0; - } - - struct uwsgi_logger *choosen_logger = uwsgi_get_logger(name); - if (!choosen_logger) { - uwsgi_log("unable to find logger %s\n", name); - exit(1); - } - - // make a copy of the logger - struct uwsgi_logger *copy_of_choosen_logger = uwsgi_malloc(sizeof(struct uwsgi_logger)); - memcpy(copy_of_choosen_logger, choosen_logger, sizeof(struct uwsgi_logger)); - choosen_logger = copy_of_choosen_logger; - choosen_logger->id = id; - choosen_logger->next = NULL; - - if (colon) { - choosen_logger->arg = colon + 1; - // check for empty string - if (*choosen_logger->arg == 0) { - choosen_logger->arg = NULL; - } - *colon = ':'; - } - + struct uwsgi_logger *choosen_logger = setup_choosen_logger(usl); uwsgi_append_logger(choosen_logger); - usl = usl->next; - } + usl = uwsgi.requested_req_logger; + while (usl) { + struct uwsgi_logger *choosen_logger = setup_choosen_logger(usl); + uwsgi_append_req_logger(choosen_logger); + usl = usl->next; + } + #ifdef UWSGI_PCRE // set logger by its id struct uwsgi_regexp_list *url = uwsgi.log_route; @@ -357,6 +381,11 @@ void uwsgi_setup_log_master(void) { url->custom_ptr = uwsgi_get_logger_from_id(url->custom_str); url = url->next; } + url = uwsgi.log_req_route; + while (url) { + url->custom_ptr = uwsgi_get_logger_from_id(url->custom_str); + url = url->next; + } #endif uwsgi.original_log_fd = dup(1); @@ -637,7 +666,7 @@ void uwsgi_logit_simple(struct wsgi_request *wsgi_req) { logvec[logvecpos].iov_len = rlen; // do not check for errors - rlen = writev(2, logvec, logvecpos + 1); + rlen = writev(uwsgi.req_log_fd, logvec, logvecpos + 1); } void get_memusage(uint64_t * rss, uint64_t * vsz) { @@ -786,6 +815,24 @@ void uwsgi_append_logger(struct uwsgi_logger *ul) { } } +void uwsgi_append_req_logger(struct uwsgi_logger *ul) { + + if (!uwsgi.choosen_req_logger) { + uwsgi.choosen_req_logger = ul; + return; + } + + struct uwsgi_logger *ucl = uwsgi.choosen_req_logger; + while (ucl) { + if (!ucl->next) { + ucl->next = ul; + return; + } + ucl = ucl->next; + } +} + + struct uwsgi_logger *uwsgi_get_logger(char *name) { struct uwsgi_logger *ul = uwsgi.loggers; @@ -856,7 +903,7 @@ void uwsgi_logit_lf(struct wsgi_request *wsgi_req) { } // do not check for errors - rlen = writev(2, uwsgi.logvectors[wsgi_req->async_id], uwsgi.logformat_vectors); + rlen = writev(uwsgi.req_log_fd, uwsgi.logvectors[wsgi_req->async_id], uwsgi.logformat_vectors); // free allocated memory logchunk = uwsgi.logchunks; @@ -1107,3 +1154,115 @@ void uwsgi_add_logchunk(int variable, int pos, char *ptr, size_t len) { } } } + +int uwsgi_master_log(void) { + + ssize_t rlen = read(uwsgi.shared->worker_log_pipe[0], uwsgi.log_master_buf, uwsgi.log_master_bufsize); + if (rlen > 0) { +#ifdef UWSGI_ALARM + uwsgi_alarm_log_check(uwsgi.log_master_buf, rlen); +#endif +#ifdef UWSGI_PCRE + struct uwsgi_regexp_list *url = uwsgi.log_drain_rules; + while (url) { + if (uwsgi_regexp_match(url->pattern, url->pattern_extra, uwsgi.log_master_buf, rlen) >= 0) { + return 0; + } + url = url->next; + } + if (uwsgi.log_filter_rules) { + int show = 0; + url = uwsgi.log_filter_rules; + while (url) { + if (uwsgi_regexp_match(url->pattern, url->pattern_extra, uwsgi.log_master_buf, rlen) >= 0) { + show = 1; + break; + } + url = url->next; + } + if (!show) + return 0; + } + + url = uwsgi.log_route; + int finish = 0; + while (url) { + if (uwsgi_regexp_match(url->pattern, url->pattern_extra, uwsgi.log_master_buf, rlen) >= 0) { + struct uwsgi_logger *ul_route = (struct uwsgi_logger *) url->custom_ptr; + if (ul_route) { + ul_route->func(ul_route, uwsgi.log_master_buf, rlen); + finish = 1; + } + } + url = url->next; + } + if (finish) + return 0; +#endif + + int raw_log = 1; + + struct uwsgi_logger *ul = uwsgi.choosen_logger; + while (ul) { + // check for named logger + if (ul->id) { + goto next; + } + ul->func(ul, uwsgi.log_master_buf, rlen); + raw_log = 0; +next: + ul = ul->next; + } + + if (raw_log) { + rlen = write(uwsgi.original_log_fd, uwsgi.log_master_buf, rlen); + } + return 0; + } + + return -1; +} + +int uwsgi_master_req_log(void) { + + ssize_t rlen = read(uwsgi.shared->worker_req_log_pipe[0], uwsgi.log_master_buf, uwsgi.log_master_bufsize); + if (rlen > 0) { +#ifdef UWSGI_PCRE + url = uwsgi.log_req_route; + int finish = 0; + while (url) { + if (uwsgi_regexp_match(url->pattern, url->pattern_extra, uwsgi.log_master_buf, rlen) >= 0) { + struct uwsgi_logger *ul_route = (struct uwsgi_logger *) url->custom_ptr; + if (ul_route) { + ul_route->func(ul_route, uwsgi.log_master_buf, rlen); + finish = 1; + } + } + url = url->next; + } + if (finish) + return 0; +#endif + + int raw_log = 1; + + struct uwsgi_logger *ul = uwsgi.choosen_req_logger; + while (ul) { + // check for named logger + if (ul->id) { + goto next; + } + ul->func(ul, uwsgi.log_master_buf, rlen); + raw_log = 0; +next: + ul = ul->next; + } + + if (raw_log) { + rlen = write(uwsgi.original_log_fd, uwsgi.log_master_buf, rlen); + } + return 0; + } + + return -1; +} diff --git a/core/master.c b/core/master.c index 717d31d6..0488fa06 100644 --- a/core/master.c +++ b/core/master.c @@ -255,91 +255,40 @@ void expire_rb_timeouts(struct rb_root *root) { } } -int uwsgi_master_log(void) { - - ssize_t rlen = read(uwsgi.shared->worker_log_pipe[0], uwsgi.log_master_buf, uwsgi.log_master_bufsize); - if (rlen > 0) { -#ifdef UWSGI_ALARM - uwsgi_alarm_log_check(uwsgi.log_master_buf, rlen); -#endif -#ifdef UWSGI_PCRE - struct uwsgi_regexp_list *url = uwsgi.log_drain_rules; - while (url) { - if (uwsgi_regexp_match(url->pattern, url->pattern_extra, uwsgi.log_master_buf, rlen) >= 0) { - return 0; - } - url = url->next; - } - if (uwsgi.log_filter_rules) { - int show = 0; - url = uwsgi.log_filter_rules; - while (url) { - if (uwsgi_regexp_match(url->pattern, url->pattern_extra, uwsgi.log_master_buf, rlen) >= 0) { - show = 1; - break; - } - url = url->next; - } - if (!show) - return 0; - } - - url = uwsgi.log_route; - int finish = 0; - while (url) { - if (uwsgi_regexp_match(url->pattern, url->pattern_extra, uwsgi.log_master_buf, rlen) >= 0) { - struct uwsgi_logger *ul_route = (struct uwsgi_logger *) url->custom_ptr; - if (ul_route) { - ul_route->func(ul_route, uwsgi.log_master_buf, rlen); - finish = 1; - } - } - url = url->next; - } - if (finish) - return 0; -#endif - - int raw_log = 1; - - struct uwsgi_logger *ul = uwsgi.choosen_logger; - while (ul) { - // check for named logger - if (ul->id) { - goto next; - } - ul->func(ul, uwsgi.log_master_buf, rlen); - raw_log = 0; -next: - ul = ul->next; - } - - if (raw_log) { - rlen = write(uwsgi.original_log_fd, uwsgi.log_master_buf, rlen); - } - return 0; - } - - return -1; -} void *logger_thread_loop(void *noarg) { - struct pollfd logpoll; + struct pollfd logpoll[2]; // block all signals sigset_t smask; sigfillset(&smask); pthread_sigmask(SIG_BLOCK, &smask, NULL); - logpoll.events = POLLIN; - logpoll.fd = uwsgi.shared->worker_log_pipe[0]; + logpoll[0].events = POLLIN; + logpoll[0].fd = uwsgi.shared->worker_log_pipe[0]; + + int logpolls = 1; + + if (uwsgi.req_log_master) { + logpoll[1].events = POLLIN; + logpoll[1].fd = uwsgi.shared->worker_req_log_pipe[0]; + } + for (;;) { - int ret = poll(&logpoll, 1, -1); - if (ret > 0 && logpoll.revents & POLLIN) { - pthread_mutex_lock(&uwsgi.threaded_logger_lock); - uwsgi_master_log(); - pthread_mutex_unlock(&uwsgi.threaded_logger_lock); + int ret = poll(logpoll, logpolls, -1); + if (ret > 0) { + if (logpoll[0].revents & POLLIN) { + pthread_mutex_lock(&uwsgi.threaded_logger_lock); + uwsgi_master_log(); + pthread_mutex_unlock(&uwsgi.threaded_logger_lock); + } + else if (logpolls > 1 && logpoll[1].revents & POLLIN) { + pthread_mutex_lock(&uwsgi.threaded_logger_lock); + uwsgi_master_req_log(); + pthread_mutex_unlock(&uwsgi.threaded_logger_lock); + } + } } @@ -524,12 +473,18 @@ int master_loop(char **argv, char **environ) { uwsgi_log("adding %d to master logging\n", uwsgi.shared->worker_log_pipe[0]); #endif event_queue_add_fd_read(uwsgi.master_queue, uwsgi.shared->worker_log_pipe[0]); + if (uwsgi.req_log_master) { + event_queue_add_fd_read(uwsgi.master_queue, uwsgi.shared->worker_req_log_pipe[0]); + } } else { if (pthread_create(&logger_thread, NULL, logger_thread_loop, NULL)) { uwsgi_error("pthread_create()"); uwsgi_log("falling back to non-threaded logger...\n"); event_queue_add_fd_read(uwsgi.master_queue, uwsgi.shared->worker_log_pipe[0]); + if (uwsgi.req_log_master) { + event_queue_add_fd_read(uwsgi.master_queue, uwsgi.shared->worker_req_log_pipe[0]); + } uwsgi.threaded_logger = 0; } } @@ -873,6 +828,10 @@ int master_loop(char **argv, char **environ) { uwsgi_master_log(); goto health_cycle; } + if (uwsgi.req_log_master && interesting_fd == uwsgi.shared->worker_req_log_pipe[0]) { + uwsgi_master_req_log(); + goto health_cycle; + } } if (uwsgi.stats && uwsgi.stats_fd > -1) { diff --git a/core/utils.c b/core/utils.c index fa05eb17..81715b3b 100644 --- a/core/utils.c +++ b/core/utils.c @@ -1522,6 +1522,11 @@ add: uwsgi.master_process = 1; uwsgi.log_master = 1; } + if (op->flags & UWSGI_OPT_REQ_LOG_MASTER) { + uwsgi.master_process = 1; + uwsgi.log_master = 1; + uwsgi.req_log_master = 1; + } // requires threads ? if (op->flags & UWSGI_OPT_THREADS) { uwsgi.has_threads = 1; diff --git a/core/uwsgi.c b/core/uwsgi.c index e3aa00eb..46ca5b80 100644 --- a/core/uwsgi.c +++ b/core/uwsgi.c @@ -399,6 +399,8 @@ static struct uwsgi_option uwsgi_base_options[] = { {"logfile-chmod", required_argument, 0, "chmod logfiles", uwsgi_opt_logfile_chmod, NULL, 0}, {"log-syslog", optional_argument, 0, "log to syslog", uwsgi_opt_set_logger, "syslog", UWSGI_OPT_MASTER | UWSGI_OPT_LOG_MASTER}, {"log-socket", required_argument, 0, "send logs to the specified socket", uwsgi_opt_set_logger, "socket", UWSGI_OPT_MASTER | UWSGI_OPT_LOG_MASTER}, + {"req-logger", required_argument, 0, "set/append a request logger", uwsgi_opt_set_req_logger, NULL, UWSGI_OPT_REQ_LOG_MASTER}, + {"logger-req", required_argument, 0, "set/append a request logger", uwsgi_opt_set_req_logger, NULL, UWSGI_OPT_REQ_LOG_MASTER}, {"logger", required_argument, 0, "set/append a logger", uwsgi_opt_set_logger, NULL, UWSGI_OPT_MASTER | UWSGI_OPT_LOG_MASTER}, {"logger-list", no_argument, 0, "list enabled loggers", uwsgi_opt_true, &uwsgi.loggers_list, 0}, {"loggers-list", no_argument, 0, "list enabled loggers", uwsgi_opt_true, &uwsgi.loggers_list, 0}, @@ -407,6 +409,7 @@ static struct uwsgi_option uwsgi_base_options[] = { {"log-drain", required_argument, 0, "drain (do not show) log lines matching the specified regexp", uwsgi_opt_add_regexp_list, &uwsgi.log_drain_rules, UWSGI_OPT_MASTER | UWSGI_OPT_LOG_MASTER}, {"log-filter", required_argument, 0, "show only log lines matching the specified regexp", uwsgi_opt_add_regexp_list, &uwsgi.log_filter_rules, UWSGI_OPT_MASTER | UWSGI_OPT_LOG_MASTER}, {"log-route", required_argument, 0, "log to the specified named logger if regexp applied on logline matches", uwsgi_opt_add_regexp_custom_list, &uwsgi.log_route, UWSGI_OPT_MASTER | UWSGI_OPT_LOG_MASTER}, + {"log-req-route", required_argument, 0, "log requests to the specified named logger if regexp applied on logline matches", uwsgi_opt_add_regexp_custom_list, &uwsgi.log_req_route, UWSGI_OPT_REQ_LOG_MASTER}, #endif #ifdef UWSGI_ALARM {"alarm", required_argument, 0, "create a new alarm, syntax: ", uwsgi_opt_add_string_list, &uwsgi.alarm_list, UWSGI_OPT_MASTER | UWSGI_OPT_LOG_MASTER}, @@ -509,7 +512,7 @@ static struct uwsgi_option uwsgi_base_options[] = { {"static-index", required_argument, 0, "search for specified file if a directory is requested", uwsgi_opt_add_string_list, &uwsgi.static_index, UWSGI_OPT_MIME}, {"static-safe", required_argument, 0, "skip security checks if the file is under the specified path", uwsgi_opt_add_string_list, &uwsgi.static_safe, UWSGI_OPT_MIME}, {"static-cache-paths", required_argument, 0, "put resolved paths in the uWSGI cache for the specified amount of seconds", uwsgi_opt_set_int, &uwsgi.static_cache_paths, UWSGI_OPT_MIME|UWSGI_OPT_MASTER}, - {"static-cache-name", required_argument, 0, "use the specified cache for static paths", uwsgi_opt_set_str, &uwsgi.static_cache_paths_name, UWSGI_OPT_MIME|UWSGI_OPT_MASTER}, + {"static-cache-paths-name", required_argument, 0, "use the specified cache for static paths", uwsgi_opt_set_str, &uwsgi.static_cache_paths_name, UWSGI_OPT_MIME|UWSGI_OPT_MASTER}, {"mimefile", required_argument, 0, "set mime types file path (default /etc/mime.types)", uwsgi_opt_add_string_list, &uwsgi.mime_file, UWSGI_OPT_MIME}, {"mime-file", required_argument, 0, "set mime types file path (default /etc/mime.types)", uwsgi_opt_add_string_list, &uwsgi.mime_file, UWSGI_OPT_MIME}, @@ -3367,6 +3370,19 @@ void uwsgi_opt_set_logger(char *opt, char *value, void *prefix) { } } +void uwsgi_opt_set_req_logger(char *opt, char *value, void *prefix) { + + if (!value) + value = ""; + + if (prefix) { + uwsgi_string_new_list(&uwsgi.requested_req_logger, uwsgi_concat3((char *) prefix, ":", value)); + } + else { + uwsgi_string_new_list(&uwsgi.requested_req_logger, uwsgi_str(value)); + } +} + void uwsgi_opt_set_str_spaced(char *opt, char *value, void *key) { char **ptr = (char **) key; *ptr = uwsgi_concat2((char *) value, " "); diff --git a/uwsgi.h b/uwsgi.h index 94e06206..afa6dfd9 100644 --- a/uwsgi.h +++ b/uwsgi.h @@ -57,6 +57,7 @@ extern "C" { #define UWSGI_OPT_POST_BUFFERING (1 << 11) #define UWSGI_OPT_CLUSTER (1 << 12) #define UWSGI_OPT_MIME (1 << 13) +#define UWSGI_OPT_REQ_LOG_MASTER (1 << 14) #define MAX_GENERIC_PLUGINS 64 #define MAX_RPC 64 @@ -659,6 +660,9 @@ struct uwsgi_cache { int thread_server_fd; + struct uwsgi_string_list *nodes; + struct uwsgi_string_list *sync_nodes; + struct uwsgi_lock_item *lock; struct uwsgi_cache *next; @@ -1632,6 +1636,7 @@ struct uwsgi_server { int restore_tc; // route all of the logs to the master process + int req_log_master; int log_master; char *log_master_buf; size_t log_master_bufsize; @@ -1642,6 +1647,7 @@ struct uwsgi_server { char *log_backupname; int original_log_fd; + int req_log_fd; // static file serving int file_serve_mode; @@ -1666,7 +1672,9 @@ struct uwsgi_server { struct uwsgi_logger *loggers; struct uwsgi_logger *choosen_logger; + struct uwsgi_logger *choosen_req_logger; struct uwsgi_string_list *requested_logger; + struct uwsgi_string_list *requested_req_logger; #ifdef UWSGI_PCRE int pcre_jit; @@ -2268,6 +2276,8 @@ struct uwsgi_shared { int rpc_count; int worker_log_pipe[2]; + // used for request logging + int worker_req_log_pipe[2]; #if defined(__linux__) || defined(__FreeBSD__) struct tcp_info ti; @@ -3204,6 +3214,7 @@ void uwsgi_build_cap(char *); void uwsgi_register_logger(char *, ssize_t (*func)(struct uwsgi_logger *, char *, size_t)); void uwsgi_append_logger(struct uwsgi_logger *); +void uwsgi_append_req_logger(struct uwsgi_logger *); struct uwsgi_logger *uwsgi_get_logger(char *); struct uwsgi_logger *uwsgi_get_logger_from_id(char *); @@ -3230,6 +3241,7 @@ void uwsgi_opt_print(char *, char *, void *); void uwsgi_opt_true(char *, char *, void *); void uwsgi_opt_set_str(char *, char *, void *); void uwsgi_opt_set_logger(char *, char *, void *); +void uwsgi_opt_set_req_logger(char *, char *, void *); void uwsgi_opt_set_str_spaced(char *, char *, void *); void uwsgi_opt_add_string_list(char *, char *, void *); void uwsgi_opt_add_addr_list(char *, char *, void *); @@ -3367,6 +3379,7 @@ int uwsgi_cheaper_algo_backlog(void); int uwsgi_cheaper_algo_backlog2(void); int uwsgi_master_log(void); +int uwsgi_master_req_log(void); void uwsgi_flush_logs(void); void uwsgi_register_cheaper_algo(char *, int(*) (void));