From ea44525c222857e46cf923590d7c73447f90f43b Mon Sep 17 00:00:00 2001 From: Roberto De Ioris Date: Fri, 15 Nov 2013 07:48:49 +0100 Subject: [PATCH 01/94] fixed zeromq support --- uwsgi.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/uwsgi.h b/uwsgi.h index d4b654ca..45e7201a 100644 --- a/uwsgi.h +++ b/uwsgi.h @@ -3265,6 +3265,9 @@ void uwsgi_proto_http_setup(struct uwsgi_socket *); void uwsgi_proto_https_setup(struct uwsgi_socket *); void uwsgi_proto_suwsgi_setup(struct uwsgi_socket *); #endif +#ifdef UWSGI_ZEROMQ +void uwsgi_proto_zmq_setup(struct uwsgi_socket *); +#endif void uwsgi_proto_fastcgi_setup(struct uwsgi_socket *); void uwsgi_proto_fastcgi_nph_setup(struct uwsgi_socket *); From 0abbabbe10040f562c49dcefd27c502356c5ef44 Mon Sep 17 00:00:00 2001 From: Unbit Date: Wed, 13 Nov 2013 19:51:22 +0100 Subject: [PATCH 02/94] added accumulator metrics collector --- core/metrics.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/core/metrics.c b/core/metrics.c index 47dc8ee0..b9e7d269 100644 --- a/core/metrics.c +++ b/core/metrics.c @@ -970,6 +970,18 @@ static int64_t uwsgi_metric_collector_sum(struct uwsgi_metric *um) { return total; } +static int64_t uwsgi_metric_collector_accumulator(struct uwsgi_metric *um) { + int64_t total = *um->value; + struct uwsgi_metric_child *umc = um->children; + while(umc) { + struct uwsgi_metric *c = umc->um; + total += *c->value; + umc = umc->next; + } + + return total; +} + static int64_t uwsgi_metric_collector_avg(struct uwsgi_metric *um) { int64_t total = 0; int64_t count = 0; @@ -1001,6 +1013,7 @@ void uwsgi_metrics_collectors_setup() { uwsgi_register_metric_collector("ptr", uwsgi_metric_collector_ptr); uwsgi_register_metric_collector("file", uwsgi_metric_collector_file); uwsgi_register_metric_collector("sum", uwsgi_metric_collector_sum); + uwsgi_register_metric_collector("accumulator", uwsgi_metric_collector_accumulator); uwsgi_register_metric_collector("avg", uwsgi_metric_collector_avg); uwsgi_register_metric_collector("func", uwsgi_metric_collector_func); } From d1383d774971d45e87660ed4c1ecf8f4544697b3 Mon Sep 17 00:00:00 2001 From: Unbit Date: Wed, 13 Nov 2013 19:53:47 +0100 Subject: [PATCH 03/94] added multiplier and adder collector --- core/metrics.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/core/metrics.c b/core/metrics.c index b9e7d269..5ccdf647 100644 --- a/core/metrics.c +++ b/core/metrics.c @@ -982,6 +982,30 @@ static int64_t uwsgi_metric_collector_accumulator(struct uwsgi_metric *um) { return total; } +static int64_t uwsgi_metric_collector_multiplier(struct uwsgi_metric *um) { + int64_t total = 0; + struct uwsgi_metric_child *umc = um->children; + while(umc) { + struct uwsgi_metric *c = umc->um; + total += *c->value; + umc = umc->next; + } + + return total * um->arg1n; +} + +static int64_t uwsgi_metric_collector_adder(struct uwsgi_metric *um) { + int64_t total = 0; + struct uwsgi_metric_child *umc = um->children; + while(umc) { + struct uwsgi_metric *c = umc->um; + total += *c->value; + umc = umc->next; + } + + return total + um->arg1n; +} + static int64_t uwsgi_metric_collector_avg(struct uwsgi_metric *um) { int64_t total = 0; int64_t count = 0; @@ -1014,6 +1038,8 @@ void uwsgi_metrics_collectors_setup() { uwsgi_register_metric_collector("file", uwsgi_metric_collector_file); uwsgi_register_metric_collector("sum", uwsgi_metric_collector_sum); uwsgi_register_metric_collector("accumulator", uwsgi_metric_collector_accumulator); + uwsgi_register_metric_collector("adder", uwsgi_metric_collector_adder); + uwsgi_register_metric_collector("multiplier", uwsgi_metric_collector_multiplier); uwsgi_register_metric_collector("avg", uwsgi_metric_collector_avg); uwsgi_register_metric_collector("func", uwsgi_metric_collector_func); } From 6a18d10c56cdc3720aa68f240075d7a1d67cb3a4 Mon Sep 17 00:00:00 2001 From: Unbit Date: Wed, 13 Nov 2013 21:38:56 +0100 Subject: [PATCH 04/94] fixed typo in total_tx metric --- core/metrics.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/metrics.c b/core/metrics.c index 5ccdf647..f83f8c7c 100644 --- a/core/metrics.c +++ b/core/metrics.c @@ -752,7 +752,7 @@ void uwsgi_setup_metrics() { uwsgi_metric_name("worker.%d.avg_response_time", i) ; uwsgi_metric_oid("3.%d.8", i); uwsgi_register_metric(buf, buf2, UWSGI_METRIC_GAUGE, "ptr", &uwsgi.workers[i].avg_response_time, 0, NULL); - uwsgi_metric_name("worker.%d.total_rx", i) ; uwsgi_metric_oid("3.%d.9", i); + uwsgi_metric_name("worker.%d.total_tx", i) ; uwsgi_metric_oid("3.%d.9", i); struct uwsgi_metric *tx = uwsgi_register_metric(buf, buf2, UWSGI_METRIC_COUNTER, "ptr", &uwsgi.workers[i].tx, 0, NULL); uwsgi_metric_add_child(total_tx, tx); From eeb203afd496117f602e877ca83aa1ed142e1448 Mon Sep 17 00:00:00 2001 From: Pavlo Kapyshin Date: Sat, 16 Nov 2013 17:41:26 +0200 Subject: [PATCH 05/94] Fix typos --- core/async.c | 10 +++++----- core/cache.c | 2 +- core/uwsgi.c | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/core/async.c b/core/async.c index 87af14d2..547e3a96 100644 --- a/core/async.c +++ b/core/async.c @@ -4,18 +4,18 @@ extern struct uwsgi_server uwsgi; /* - This is a general purpose async loop engine (it expects a coroutine based approach) + This is a general-purpose async loop engine (it expects a coroutine-based approach) You can see it as an hub holding the following structures: - 1) the runqueue, cores ready to be run are appended in this list + 1) the runqueue, cores ready to be run are appended to this list 2) the fd list, this is a list of monitored file descriptors, a core can wait for all the file descriptors it needs - 3) the timeout value, if set the current core will timeout aftert he specified number of seconds (unless an event cancel it) + 3) the timeout value, if set, the current core will timeout after the specified number of seconds (unless an event cancels it) - IMPORTANT: this is not a callback based engine !!! + IMPORTANT: this is not a callback-based engine !!! */ @@ -159,7 +159,7 @@ static void async_expire_timeouts(uint64_t now) { wsgi_req = (struct wsgi_request *) urbt->data; // timeout expired wsgi_req->async_timed_out = 1; - // reset teh request + // reset the request async_reset_request(wsgi_req); // push it in the runqueue runqueue_push(wsgi_req); diff --git a/core/cache.c b/core/cache.c index 186ad158..dbdb02d4 100644 --- a/core/cache.c +++ b/core/cache.c @@ -762,7 +762,7 @@ int uwsgi_cache_set2(struct uwsgi_cache *uc, char *key, uint16_t keylen, char *v } // mark used blocks; uint64_t needed_blocks = cache_mark_blocks(uc, uci->first_block, vallen); - // optimize teh scan + // optimize the scan if (uc->blocks_bitmap_pos + (needed_blocks+1) > uc->blocks) { uc->blocks_bitmap_pos = 0; } diff --git a/core/uwsgi.c b/core/uwsgi.c index 6aa8aae0..e5b1536e 100644 --- a/core/uwsgi.c +++ b/core/uwsgi.c @@ -241,7 +241,7 @@ static struct uwsgi_option uwsgi_base_options[] = { {"worker-reload-mercy", required_argument, 0, "set the maximum time (in seconds) a worker can take to reload/shutdown (default is 60)", uwsgi_opt_set_int, &uwsgi.worker_reload_mercy, 0}, {"exit-on-reload", no_argument, 0, "force exit even if a reload is requested", uwsgi_opt_true, &uwsgi.exit_on_reload, 0}, {"die-on-term", no_argument, 0, "exit instead of brutal reload on SIGTERM", uwsgi_opt_true, &uwsgi.die_on_term, 0}, - {"force-gateway", no_argument, 0, "force teh spawn of the first registered gateway without a master", uwsgi_opt_true, &uwsgi.force_gateway, 0}, + {"force-gateway", no_argument, 0, "force the spawn of the first registered gateway without a master", uwsgi_opt_true, &uwsgi.force_gateway, 0}, {"help", no_argument, 'h', "show this help", uwsgi_help, NULL, UWSGI_OPT_IMMEDIATE}, {"usage", no_argument, 'h', "show this help", uwsgi_help, NULL, UWSGI_OPT_IMMEDIATE}, @@ -886,7 +886,7 @@ static struct uwsgi_option uwsgi_base_options[] = { {"worker-exec", required_argument, 0, "run the specified command as worker", uwsgi_opt_set_str, &uwsgi.worker_exec, 0}, {"worker-exec2", required_argument, 0, "run the specified command as worker (after post_fork hook)", uwsgi_opt_set_str, &uwsgi.worker_exec2, 0}, {"attach-daemon", required_argument, 0, "attach a command/daemon to the master process (the command has to not go in background)", uwsgi_opt_add_daemon, NULL, UWSGI_OPT_MASTER}, - {"attach-control-daemon", required_argument, 0, "attach a command/daemon to the master process (the command has to not go in background), when teh daemon dies, the master dies too", uwsgi_opt_add_daemon, NULL, UWSGI_OPT_MASTER}, + {"attach-control-daemon", required_argument, 0, "attach a command/daemon to the master process (the command has to not go in background), when the daemon dies, the master dies too", uwsgi_opt_add_daemon, NULL, UWSGI_OPT_MASTER}, {"smart-attach-daemon", required_argument, 0, "attach a command/daemon to the master process managed by a pidfile (the command has to daemonize)", uwsgi_opt_add_daemon, NULL, UWSGI_OPT_MASTER}, {"smart-attach-daemon2", required_argument, 0, "attach a command/daemon to the master process managed by a pidfile (the command has to NOT daemonize)", uwsgi_opt_add_daemon, NULL, UWSGI_OPT_MASTER}, #ifdef UWSGI_SSL From 29c339a3c30a880afade6c50160116b39ac0dc7a Mon Sep 17 00:00:00 2001 From: Unbit Date: Sun, 17 Nov 2013 06:41:46 +0100 Subject: [PATCH 06/94] gemspec for 1.9.20 --- uwsgi.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uwsgi.gemspec b/uwsgi.gemspec index 1fbbde05..f841e7fd 100644 --- a/uwsgi.gemspec +++ b/uwsgi.gemspec @@ -2,7 +2,7 @@ Gem::Specification.new do |s| s.name = 'uwsgi' s.license = 'GPL-2' s.version = `python -c "import uwsgiconfig as uc; print uc.uwsgi_version"`.sub(/-dev-.*/,'') - s.date = '2013-11-09' + s.date = '2013-11-17' s.summary = "uWSGI" s.description = "The uWSGI server for Ruby/Rack" s.authors = ["Unbit"] From 95d20e9f4821e6e2ba1318141a6ca6e67dd29f6b Mon Sep 17 00:00:00 2001 From: Unbit Date: Sun, 17 Nov 2013 06:44:16 +0100 Subject: [PATCH 07/94] add rack to gemspec --- uwsgi.gemspec | 1 + 1 file changed, 1 insertion(+) diff --git a/uwsgi.gemspec b/uwsgi.gemspec index f841e7fd..0b607685 100644 --- a/uwsgi.gemspec +++ b/uwsgi.gemspec @@ -12,4 +12,5 @@ Gem::Specification.new do |s| s.require_paths = ['.'] s.executables << 'uwsgi' s.homepage = 'http://projects.unbit.it/uwsgi' + s.add_runtime_dependency 'rack' end From ec954b8aa774a1d5edda15bf37caf5490e303c51 Mon Sep 17 00:00:00 2001 From: Unbit Date: Sun, 17 Nov 2013 07:02:21 +0100 Subject: [PATCH 08/94] fixed v8 RPC --- plugins/v8/v8_uwsgi.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/v8/v8_uwsgi.cc b/plugins/v8/v8_uwsgi.cc index 91a8cfdc..14167f18 100644 --- a/plugins/v8/v8_uwsgi.cc +++ b/plugins/v8/v8_uwsgi.cc @@ -285,7 +285,7 @@ extern "C" uint64_t uwsgi_v8_rpc(void * func, uint8_t argc, char **argv, uint16_ if (!*robj) { return 0; } - uint16_t rlen = robj->Length(); + uint64_t rlen = (uint64_t) robj->Length(); if (rlen > 0) { *buffer = (char *)uwsgi_malloc(rlen); memcpy(*buffer, *r_value, rlen); From e606d1e5d11c094801bed84a1df45897bf99f845 Mon Sep 17 00:00:00 2001 From: Unbit Date: Mon, 18 Nov 2013 11:11:46 +0100 Subject: [PATCH 09/94] removal of embedded zeromq api --- core/logging.c | 43 ---- core/socket.c | 7 +- core/utils.c | 6 +- core/uwsgi.c | 22 -- plugins/emperor_zeromq/emperor_zeromq.c | 14 +- plugins/logzmq/plugin.c | 61 ++++++ plugins/logzmq/uwsgiplugin.py | 7 + proto/zeromq.c => plugins/mongrel2/mongrel2.c | 202 +++++++++--------- plugins/mongrel2/uwsgiplugin.py | 7 + uwsgi.h | 18 -- uwsgiconfig.py | 15 -- 11 files changed, 190 insertions(+), 212 deletions(-) create mode 100644 plugins/logzmq/plugin.c create mode 100644 plugins/logzmq/uwsgiplugin.py rename proto/zeromq.c => plugins/mongrel2/mongrel2.c (82%) create mode 100644 plugins/mongrel2/uwsgiplugin.py diff --git a/core/logging.c b/core/logging.c index cc284c66..c08a68c2 100644 --- a/core/logging.c +++ b/core/logging.c @@ -211,49 +211,6 @@ void create_logpipe(void) { } -#ifdef UWSGI_ZEROMQ -// the zeromq logger -ssize_t uwsgi_zeromq_logger(struct uwsgi_logger *ul, char *message, size_t len) { - - if (!ul->configured) { - - if (!ul->arg) { - uwsgi_log_safe("invalid zeromq syntax\n"); - exit(1); - } - - void *ctx = uwsgi_zeromq_init(); - - ul->data = zmq_socket(ctx, ZMQ_PUSH); - if (ul->data == NULL) { - uwsgi_error_safe("zmq_socket()"); - exit(1); - } - - if (zmq_connect(ul->data, ul->arg) < 0) { - uwsgi_error_safe("zmq_connect()"); - exit(1); - } - - ul->configured = 1; - } - - zmq_msg_t msg; - if (zmq_msg_init_size(&msg, len) == 0) { - memcpy(zmq_msg_data(&msg), message, len); -#if ZMQ_VERSION >= ZMQ_MAKE_VERSION(3,0,0) - zmq_sendmsg(ul->data, &msg, 0); -#else - zmq_send(ul->data, &msg, 0); -#endif - zmq_msg_close(&msg); - } - - return 0; -} -#endif - - // log to the specified file or udp address void logto(char *logfile) { diff --git a/core/socket.c b/core/socket.c index 8f35394d..bd0577fe 100644 --- a/core/socket.c +++ b/core/socket.c @@ -1931,17 +1931,18 @@ found: void uwsgi_protocols_register() { uwsgi_register_protocol("uwsgi", uwsgi_proto_uwsgi_setup); uwsgi_register_protocol("puwsgi", uwsgi_proto_puwsgi_setup); + uwsgi_register_protocol("http", uwsgi_proto_http_setup); + #ifdef UWSGI_SSL uwsgi_register_protocol("suwsgi", uwsgi_proto_suwsgi_setup); uwsgi_register_protocol("https", uwsgi_proto_https_setup); #endif uwsgi_register_protocol("fastcgi", uwsgi_proto_fastcgi_setup); uwsgi_register_protocol("fastcgi-nph", uwsgi_proto_fastcgi_nph_setup); + uwsgi_register_protocol("scgi", uwsgi_proto_scgi_setup); uwsgi_register_protocol("scgi-nph", uwsgi_proto_scgi_nph_setup); -#ifdef UWSGI_ZEROMQ - uwsgi_register_protocol("zmq", uwsgi_proto_zmq_setup); -#endif + uwsgi_register_protocol("raw", uwsgi_proto_raw_setup); } diff --git a/core/utils.c b/core/utils.c index 98f2cfc3..78233585 100644 --- a/core/utils.c +++ b/core/utils.c @@ -3922,9 +3922,9 @@ char *uwsgi_strip(char *src) { void uwsgi_uuid(char *buf) { #ifdef UWSGI_UUID - uuid_t uuid_zmq; - uuid_generate(uuid_zmq); - uuid_unparse(uuid_zmq, buf); + uuid_t uuid_value; + uuid_generate(uuid_value); + uuid_unparse(uuid_value, buf); #else int i,r[11]; if (!uwsgi_file_exists("/dev/urandom")) goto fallback; diff --git a/core/uwsgi.c b/core/uwsgi.c index e5b1536e..c7a1c1a0 100644 --- a/core/uwsgi.c +++ b/core/uwsgi.c @@ -449,12 +449,6 @@ static struct uwsgi_option uwsgi_base_options[] = { #ifdef UWSGI_JSON {"json", required_argument, 'j', "load config from json file", uwsgi_opt_load_json, NULL, UWSGI_OPT_IMMEDIATE}, {"js", required_argument, 'j', "load config from json file", uwsgi_opt_load_json, NULL, UWSGI_OPT_IMMEDIATE}, -#endif -#ifdef UWSGI_ZEROMQ - {"zeromq", required_argument, 0, "create a zeromq pub/sub pair", uwsgi_opt_add_lazy_socket, "zmq", 0}, - {"zmq", required_argument, 0, "create a zeromq pub/sub pair", uwsgi_opt_add_lazy_socket, "zmq", 0}, - {"zeromq-socket", required_argument, 0, "create a zeromq pub/sub pair", uwsgi_opt_add_lazy_socket, "zmq", 0}, - {"zmq-socket", required_argument, 0, "create a zeromq pub/sub pair", uwsgi_opt_add_lazy_socket, "zmq", 0}, #endif {"weight", required_argument, 0, "weight of the instance (used by clustering/lb/subscriptions)", uwsgi_opt_set_64bit, &uwsgi.weight, 0}, {"auto-weight", required_argument, 0, "set weight of the instance (used by clustering/lb/subscriptions) automatically", uwsgi_opt_true, &uwsgi.auto_weight, 0}, @@ -649,9 +643,6 @@ static struct uwsgi_option uwsgi_base_options[] = { {"alarm-list", no_argument, 0, "list enabled alarms", uwsgi_opt_true, &uwsgi.alarms_list, 0}, {"alarms-list", no_argument, 0, "list enabled alarms", uwsgi_opt_true, &uwsgi.alarms_list, 0}, {"alarm-msg-size", required_argument, 0, "set the max size of an alarm message (default 8192)", uwsgi_opt_set_64bit, &uwsgi.alarm_msg_size, 0}, -#ifdef UWSGI_ZEROMQ - {"log-zeromq", required_argument, 0, "send logs to a zeromq server", uwsgi_opt_set_logger, "zeromq", UWSGI_OPT_MASTER | UWSGI_OPT_LOG_MASTER}, -#endif {"log-master", no_argument, 0, "delegate logging to master process", uwsgi_opt_true, &uwsgi.log_master, UWSGI_OPT_MASTER}, {"log-master-bufsize", required_argument, 0, "set the buffer size for the master logger. bigger log messages will be truncated", uwsgi_opt_set_64bit, &uwsgi.log_master_bufsize, 0}, {"log-master-stream", no_argument, 0, "create the master logpipe as SOCK_STREAM", uwsgi_opt_true, &uwsgi.log_master_stream, 0}, @@ -1990,11 +1981,6 @@ int main(int argc, char *argv[], char *envp[]) { } uwsgi.hostname_len = strlen(uwsgi.hostname); -#ifdef UWSGI_ZEROMQ - uwsgi_register_logger("zeromq", uwsgi_zeromq_logger); - uwsgi_register_logger("zmq", uwsgi_zeromq_logger); -#endif - #ifdef UWSGI_ROUTING uwsgi_register_embedded_routers(); #endif @@ -3166,14 +3152,6 @@ void uwsgi_worker_run() { // some apps could be mounted only on specific workers uwsgi_init_worker_mount_apps(); - -#ifdef UWSGI_ZEROMQ - // setup zeromq context (if required) one per-worker - if (uwsgi.zeromq) { - uwsgi_zeromq_init_sockets(); - } -#endif - //postpone the queue initialization as kevent //do not pass kfd after fork() if (uwsgi.async > 1) { diff --git a/plugins/emperor_zeromq/emperor_zeromq.c b/plugins/emperor_zeromq/emperor_zeromq.c index 319db68b..eb88ecf7 100644 --- a/plugins/emperor_zeromq/emperor_zeromq.c +++ b/plugins/emperor_zeromq/emperor_zeromq.c @@ -52,9 +52,7 @@ s.send_multipart(['touch','foo.ini',"[uwsgi]\nsocket=:4142"]) */ -#ifdef UWSGI_ZEROMQ - -#include "../../uwsgi.h" +#include #include extern struct uwsgi_server uwsgi; @@ -215,7 +213,11 @@ static void uwsgi_imperial_monitor_zeromq_event(struct uwsgi_emperor_scanner *ue // initialize the zmq PULL socket static void uwsgi_imperial_monitor_zeromq_init(struct uwsgi_emperor_scanner *ues) { - void *context = uwsgi_zeromq_init(); + void *context = zmq_init(1); + if (!context) { + uwsgi_error("uwsgi_imperial_monitor_zeromq_init()/zmq_init()"); + exit(1); + } ues->data = zmq_socket(context, ZMQ_PULL); if (!ues->data) { @@ -256,7 +258,3 @@ struct uwsgi_plugin emperor_zeromq_plugin = { .name = "emperor_zeromq", .on_load = emperor_zeromq_init, }; - -#else -#error the uWSGI ZeroMQ imperial monitor requires uWSGI zeromq support -#endif diff --git a/plugins/logzmq/plugin.c b/plugins/logzmq/plugin.c new file mode 100644 index 00000000..502d607b --- /dev/null +++ b/plugins/logzmq/plugin.c @@ -0,0 +1,61 @@ +#include +#include + +extern struct uwsgi_server uwsgi; + +static struct uwsgi_option uwsgi_zmq_logger_options[] = { + {"log-zeromq", required_argument, 0, "send logs to a zeromq server", uwsgi_opt_set_logger, "zeromq", UWSGI_OPT_MASTER | UWSGI_OPT_LOG_MASTER}, + {NULL, 0, 0, NULL, NULL, NULL, 0}, +}; + +// the zeromq logger +static ssize_t uwsgi_zeromq_logger(struct uwsgi_logger *ul, char *message, size_t len) { + + if (!ul->configured) { + + if (!ul->arg) { + uwsgi_log_safe("invalid zeromq syntax\n"); + exit(1); + } + + void *ctx = zmq_init(1); + if (!ctx) exit(1); + + ul->data = zmq_socket(ctx, ZMQ_PUSH); + if (ul->data == NULL) { + uwsgi_error_safe("zmq_socket()"); + exit(1); + } + + if (zmq_connect(ul->data, ul->arg) < 0) { + uwsgi_error_safe("zmq_connect()"); + exit(1); + } + + ul->configured = 1; + } + + zmq_msg_t msg; + if (zmq_msg_init_size(&msg, len) == 0) { + memcpy(zmq_msg_data(&msg), message, len); +#if ZMQ_VERSION >= ZMQ_MAKE_VERSION(3,0,0) + zmq_sendmsg(ul->data, &msg, 0); +#else + zmq_send(ul->data, &msg, 0); +#endif + zmq_msg_close(&msg); + } + + return 0; +} + +static void uwsgi_zmq_logger_register() { + uwsgi_register_logger("zmq", uwsgi_zeromq_logger); + uwsgi_register_logger("zeromq", uwsgi_zeromq_logger); +} + +struct uwsgi_plugin logzmq_plugin = { + .name = "logzmq", + .options = uwsgi_zmq_logger_options, + .on_load = uwsgi_zmq_logger_register, +}; diff --git a/plugins/logzmq/uwsgiplugin.py b/plugins/logzmq/uwsgiplugin.py new file mode 100644 index 00000000..0799e325 --- /dev/null +++ b/plugins/logzmq/uwsgiplugin.py @@ -0,0 +1,7 @@ + +NAME='logzmq' +CFLAGS = [] +LDFLAGS = [] +LIBS = ['-lzmq'] + +GCC_LIST = ['plugin'] diff --git a/proto/zeromq.c b/plugins/mongrel2/mongrel2.c similarity index 82% rename from proto/zeromq.c rename to plugins/mongrel2/mongrel2.c index b666ce4f..90f6f55b 100644 --- a/proto/zeromq.c +++ b/plugins/mongrel2/mongrel2.c @@ -1,39 +1,21 @@ /* - generic ZeroMQ functions + Mongrel2 protocol parser + Mongrel2 protocol parser */ + #include +#include extern struct uwsgi_server uwsgi; -void *uwsgi_zeromq_init() { - if (!uwsgi.zmq_context) { - uwsgi.zmq_context = zmq_init(1); - if (uwsgi.zmq_context == NULL) { - uwsgi_error("zmq_init()"); - exit(1); - } - } - return uwsgi.zmq_context; -} - -void uwsgi_zeromq_init_sockets() { - - uwsgi_zeromq_init(); - - struct uwsgi_socket *uwsgi_sock = uwsgi.sockets; - while (uwsgi_sock) { - if (!uwsgi_sock->proto_name || strcmp(uwsgi_sock->proto_name, "zmq")) { - goto zmq_next; - } - uwsgi_proto_zeromq_setup(uwsgi_sock); -zmq_next: - uwsgi_sock = uwsgi_sock->next; - - } - -} +static struct uwsgi_option mongrel2_options[] = { + {"zeromq", required_argument, 0, "create a mongrel2/zeromq pub/sub pair", uwsgi_opt_add_lazy_socket, "mongrel2", 0}, + {"zmq", required_argument, 0, "create a mongrel2/zeromq pub/sub pair", uwsgi_opt_add_lazy_socket, "mongrel2", 0}, + {"zeromq-socket", required_argument, 0, "create a mongrel2/zeromq pub/sub pair", uwsgi_opt_add_lazy_socket, "mongrel2", 0}, + {"zmq-socket", required_argument, 0, "create a mongrel2/zeromq pub/sub pair", uwsgi_opt_add_lazy_socket, "mongrel2", 0}, + {"mongrel2", required_argument, 0, "create a mongrel2/zeromq pub/sub pair", uwsgi_opt_add_lazy_socket, "mongrel2", 0}, +}; #ifdef UWSGI_JSON #include @@ -290,7 +272,7 @@ int uwsgi_proto_zeromq_parser(struct wsgi_request *wsgi_req) { void uwsgi_proto_zeromq_thread_fixup(struct uwsgi_socket *uwsgi_sock, int async_id) { - void *tmp_zmq_pull = zmq_socket(uwsgi.zmq_context, ZMQ_PULL); + void *tmp_zmq_pull = zmq_socket(uwsgi_sock->ctx, ZMQ_PULL); if (tmp_zmq_pull == NULL) { uwsgi_error("zmq_socket()"); exit(1); @@ -315,7 +297,7 @@ void uwsgi_proto_zeromq_thread_fixup(struct uwsgi_socket *uwsgi_sock, int async_ #endif } -// fake function, the bosy is i na file or completely in memory +// fake function, the body is in a file or completely in memory ssize_t uwsgi_proto_zeromq_read_body(struct wsgi_request *wsgi_req, char *buf, size_t len) { return 0; } @@ -565,38 +547,6 @@ int uwsgi_proto_zeromq_sendfile(struct wsgi_request *wsgi_req, int fd, size_t po void uwsgi_proto_zeromq_setup(struct uwsgi_socket *uwsgi_sock) { - char *responder = strchr(uwsgi_sock->name, ','); - if (!responder) { - uwsgi_log("invalid zeromq address\n"); - exit(1); - } - uwsgi_sock->receiver = uwsgi_concat2n(uwsgi_sock->name, responder - uwsgi_sock->name, "", 0); - responder++; - - uwsgi_sock->pub = zmq_socket(uwsgi.zmq_context, ZMQ_PUB); - if (uwsgi_sock->pub == NULL) { - uwsgi_error("zmq_socket()"); - exit(1); - } - - - // generate uuid - uuid_t uuid_zmq; - uuid_generate(uuid_zmq); - uuid_unparse(uuid_zmq, uwsgi_sock->uuid); - - if (zmq_setsockopt(uwsgi_sock->pub, ZMQ_IDENTITY, uwsgi_sock->uuid, 36) < 0) { - uwsgi_error("zmq_setsockopt()"); - exit(1); - } - - if (zmq_connect(uwsgi_sock->pub, responder) < 0) { - uwsgi_error("zmq_connect()"); - exit(1); - } - - uwsgi_log("zeromq UUID for responder %s on worker %d: %.*s\n", responder, uwsgi.mywid, 36, uwsgi_sock->uuid); - uwsgi_sock->proto = uwsgi_proto_zeromq_parser; uwsgi_sock->proto_accept = uwsgi_proto_zeromq_accept; uwsgi_sock->proto_prepare_headers = uwsgi_proto_base_prepare_headers; @@ -614,55 +564,107 @@ void uwsgi_proto_zeromq_setup(struct uwsgi_socket *uwsgi_sock) { uwsgi_sock->retry = uwsgi_malloc(sizeof(int) * uwsgi.threads); uwsgi_sock->retry[0] = 1; - // inform loop engine about edge trigger status - uwsgi.is_et = 1; + uwsgi_sock->fd = -1; +} + +static void uwsgi_proto_mongrel2_setup(struct uwsgi_socket *uwsgi_sock) { +}; + +static void mongrel2_register_proto() { + uwsgi_register_protocol("mongrel2", uwsgi_proto_mongrel2_setup); +} + +static void mongrel2_connect() { + struct uwsgi_socket *uwsgi_sock = uwsgi.sockets; + while(uwsgi_sock) { + uwsgi_sock->ctx = zmq_init(1); + if (!uwsgi_sock->ctx) { + uwsgi_error("mongrel2_connect()/zmq_init()"); + exit(1); + } + char *responder = strchr(uwsgi_sock->name, ','); + if (!responder) { + uwsgi_log("invalid zeromq address\n"); + exit(1); + } + uwsgi_sock->receiver = uwsgi_concat2n(uwsgi_sock->name, responder - uwsgi_sock->name, "", 0); + responder++; + + uwsgi_sock->pub = zmq_socket(uwsgi_sock->ctx, ZMQ_PUB); + if (uwsgi_sock->pub == NULL) { + uwsgi_error("mongrel2_connect()/zmq_socket()"); + exit(1); + } + + // generate uuid + uuid_t uuid_zmq; + uuid_generate(uuid_zmq); + uuid_unparse(uuid_zmq, uwsgi_sock->uuid); + + if (zmq_setsockopt(uwsgi_sock->pub, ZMQ_IDENTITY, uwsgi_sock->uuid, 36) < 0) { + uwsgi_error("mongrel2_connect()/zmq_setsockopt()"); + exit(1); + } + + if (zmq_connect(uwsgi_sock->pub, responder) < 0) { + uwsgi_error("mongrel2_connect()/zmq_connect()"); + exit(1); + } + + uwsgi_log("zeromq UUID for responder %s on worker %d: %.*s\n", responder, uwsgi.mywid, 36, uwsgi_sock->uuid); + + // inform loop engine about edge trigger status + uwsgi.is_et = 1; - // initialize a lock for multithread usage - if (uwsgi.threads > 1) { - pthread_mutex_init(&uwsgi_sock->lock, NULL); - } + // initialize a lock for multithread usage + if (uwsgi.threads > 1) { + pthread_mutex_init(&uwsgi_sock->lock, NULL); + } - // one pull per-thread - if (pthread_key_create(&uwsgi_sock->key, NULL)) { - uwsgi_error("pthread_key_create()"); - exit(1); - } + // one pull per-thread + if (pthread_key_create(&uwsgi_sock->key, NULL)) { + uwsgi_error("mongrel2_connect()/pthread_key_create()"); + exit(1); + } - void *tmp_zmq_pull = zmq_socket(uwsgi.zmq_context, ZMQ_PULL); - if (tmp_zmq_pull == NULL) { - uwsgi_error("zmq_socket()"); - exit(1); - } - if (zmq_connect(tmp_zmq_pull, uwsgi_sock->receiver) < 0) { - uwsgi_error("zmq_connect()"); - exit(1); - } + void *tmp_zmq_pull = zmq_socket(uwsgi_sock->ctx, ZMQ_PULL); + if (tmp_zmq_pull == NULL) { + uwsgi_error("mongrel2_connect()/zmq_socket()"); + exit(1); + } + if (zmq_connect(tmp_zmq_pull, uwsgi_sock->receiver) < 0) { + uwsgi_error("mongrel2_connect()/zmq_connect()"); + exit(1); + } - pthread_setspecific(uwsgi_sock->key, tmp_zmq_pull); + pthread_setspecific(uwsgi_sock->key, tmp_zmq_pull); #ifdef ZMQ_FD - size_t zmq_socket_len = sizeof(int); - if (zmq_getsockopt(pthread_getspecific(uwsgi_sock->key), ZMQ_FD, &uwsgi_sock->fd, &zmq_socket_len) < 0) { - uwsgi_error("zmq_getsockopt()"); - exit(1); - } - if (uwsgi.threads > 1) { - uwsgi_sock->fd_threads = uwsgi_malloc(sizeof(int) * uwsgi.threads); - uwsgi_sock->fd_threads[0] = uwsgi_sock->fd; - } -#else - uwsgi_sock->fd = -1; + size_t zmq_socket_len = sizeof(int); + if (zmq_getsockopt(pthread_getspecific(uwsgi_sock->key), ZMQ_FD, &uwsgi_sock->fd, &zmq_socket_len) < 0) { + uwsgi_error("mongrel2_connect()/zmq_getsockopt()"); + exit(1); + } + if (uwsgi.threads > 1) { + uwsgi_sock->fd_threads = uwsgi_malloc(sizeof(int) * uwsgi.threads); + uwsgi_sock->fd_threads[0] = uwsgi_sock->fd; + } #endif - uwsgi_sock->bound = 1; + uwsgi_sock->bound = 1; #if ZMQ_VERSION >= ZMQ_MAKE_VERSION(3,0,0) - uwsgi_sock->recv_flag = ZMQ_DONTWAIT; + uwsgi_sock->recv_flag = ZMQ_DONTWAIT; #else - uwsgi_sock->recv_flag = ZMQ_NOBLOCK; + uwsgi_sock->recv_flag = ZMQ_NOBLOCK; #endif + uwsgi_sock = uwsgi_sock->next; + } } -void uwsgi_proto_zmq_setup(struct uwsgi_socket *uwsgi_sock) { - uwsgi.zeromq = 1; -} +struct uwsgi_plugin mongrel2_plugin = { + .name = "mongrel2", + .options = mongrel2_options, + .post_fork = mongrel2_connect, + .on_load = mongrel2_register_proto, +}; diff --git a/plugins/mongrel2/uwsgiplugin.py b/plugins/mongrel2/uwsgiplugin.py new file mode 100644 index 00000000..723885cf --- /dev/null +++ b/plugins/mongrel2/uwsgiplugin.py @@ -0,0 +1,7 @@ + +NAME='mongrel2' +CFLAGS = [] +LDFLAGS = [] +LIBS = ['-lzmq'] + +GCC_LIST = ['mongrel2'] diff --git a/uwsgi.h b/uwsgi.h index 45e7201a..4e20aa46 100644 --- a/uwsgi.h +++ b/uwsgi.h @@ -337,10 +337,6 @@ extern int pivot_root(const char *new_root, const char *put_old); #undef __EXTENSIONS__ #endif -#ifdef UWSGI_ZEROMQ -#include -#endif - #define UWSGI_CACHE_FLAG_UNGETTABLE 0x01 #define UWSGI_CACHE_FLAG_UPDATE 1 << 1 #define UWSGI_CACHE_FLAG_LOCAL 1 << 2 @@ -2410,10 +2406,6 @@ struct uwsgi_server { int signal_socket; int my_signal_socket; -#ifdef UWSGI_ZEROMQ - int zeromq; - void *zmq_context; -#endif struct uwsgi_protocol *protocols; struct uwsgi_socket *sockets; struct uwsgi_socket *shared_sockets; @@ -3249,13 +3241,6 @@ void uwsgi_proto_ssl_close(struct wsgi_request *); uint16_t proto_base_add_uwsgi_header(struct wsgi_request *, char *, uint16_t, char *, uint16_t); uint16_t proto_base_add_uwsgi_var(struct wsgi_request *, char *, uint16_t, char *, uint16_t); -#ifdef UWSGI_ZEROMQ -void uwsgi_proto_zeromq_setup(struct uwsgi_socket *); -ssize_t uwsgi_zeromq_logger(struct uwsgi_logger *, char *, size_t len); -void *uwsgi_zeromq_init(void); -void uwsgi_zeromq_init_sockets(void); -#endif - // protocols void uwsgi_proto_uwsgi_setup(struct uwsgi_socket *); void uwsgi_proto_puwsgi_setup(struct uwsgi_socket *); @@ -3265,9 +3250,6 @@ void uwsgi_proto_http_setup(struct uwsgi_socket *); void uwsgi_proto_https_setup(struct uwsgi_socket *); void uwsgi_proto_suwsgi_setup(struct uwsgi_socket *); #endif -#ifdef UWSGI_ZEROMQ -void uwsgi_proto_zmq_setup(struct uwsgi_socket *); -#endif void uwsgi_proto_fastcgi_setup(struct uwsgi_socket *); void uwsgi_proto_fastcgi_nph_setup(struct uwsgi_socket *); diff --git a/uwsgiconfig.py b/uwsgiconfig.py index a18e10e7..46e9f982 100644 --- a/uwsgiconfig.py +++ b/uwsgiconfig.py @@ -71,7 +71,6 @@ report = { 'yaml': False, 'json': False, 'ssl': False, - 'zeromq': False, 'xml': False, 'debug': False, 'plugin_dir': False, @@ -1127,20 +1126,6 @@ class uConf(object): self.gcc_list.append('core/legion') report['ssl'] = True - - if has_uuid and self.get('zeromq'): - if self.get('zeromq') == 'auto': - if self.has_include('zmq.h'): - self.cflags.append("-DUWSGI_ZEROMQ") - self.gcc_list.append('proto/zeromq') - self.libs.append('-lzmq') - report['zeromq'] = True - else: - self.cflags.append("-DUWSGI_ZEROMQ") - self.gcc_list.append('proto/zeromq') - self.libs.append('-lzmq') - report['zeromq'] = True - if self.get('xml'): if self.get('xml') == 'auto': xmlconf = spcall('xml2-config --libs') From 5806c333f55275379931b0173077a92e25fbc28b Mon Sep 17 00:00:00 2001 From: Unbit Date: Mon, 18 Nov 2013 16:21:36 +0100 Subject: [PATCH 10/94] do not spit errors on non x86 timerfd_create --- core/event.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/event.c b/core/event.c index 3cd63119..a9a99c0d 100644 --- a/core/event.c +++ b/core/event.c @@ -1241,6 +1241,8 @@ static int timerfd_settime(int __ufd, int __flags, __const struct itimerspec *__ return syscall(286, __ufd, __flags, __utmr, __otmr); #elif defined(__i386__) return syscall(325, __ufd, __flags, __utmr, __otmr); +#else + return -1; #endif } #endif From 8b006c0874c448de9218f783a0ee3b387f00d88b Mon Sep 17 00:00:00 2001 From: Unbit Date: Mon, 18 Nov 2013 16:34:45 +0100 Subject: [PATCH 11/94] support timerfd_create/timerfd_settime on __arm__ --- core/event.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/event.c b/core/event.c index a9a99c0d..cc98cdf1 100644 --- a/core/event.c +++ b/core/event.c @@ -1231,6 +1231,8 @@ static int timerfd_create(clockid_t __clock_id, int __flags) { return syscall(283, __clock_id, __flags); #elif defined(__i386__) return syscall(322, __clock_id, __flags); +#elif defined(__arm__) + return syscall(350, __clock_id, __flags); #else return -1; #endif @@ -1241,6 +1243,8 @@ static int timerfd_settime(int __ufd, int __flags, __const struct itimerspec *__ return syscall(286, __ufd, __flags, __utmr, __otmr); #elif defined(__i386__) return syscall(325, __ufd, __flags, __utmr, __otmr); +#elif defined(__arm__) + return syscall(353, __ufd, __flags, __utmr, __otmr); #else return -1; #endif From 826f49c010467967f9bd4bfa21ed17cb380dba57 Mon Sep 17 00:00:00 2001 From: Pavlo Kapyshin Date: Tue, 19 Nov 2013 11:22:07 +0200 Subject: [PATCH 12/94] Fix few typos --- core/daemons.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/core/daemons.c b/core/daemons.c index aa84c289..35fced08 100644 --- a/core/daemons.c +++ b/core/daemons.c @@ -6,7 +6,7 @@ extern struct uwsgi_server uwsgi; External uwsgi daemons - There are 3 kind of daemons (read: external applications) that can be managed + There are 3 kinds of daemons (read: external applications) that can be managed by uWSGI. 1) dumb daemons (attached with --attach-daemon) @@ -14,12 +14,12 @@ extern struct uwsgi_server uwsgi; and the process is respawned 2) smart daemons with daemonization you specify a pidfile and a command - - on startup - if the pidfile does not exists or contains a not-available pid (checked with kill(pid, 0)) - the daemon is respawned - - on master ckeck - if the pidfile does not exist or if it point to a non-existent pid - the daemon is respawned + - on startup - if the pidfile does not exist or contains a non-available pid (checked with kill(pid, 0)) + the daemon is respawned + - on master check - if the pidfile does not exist or if it points to a non-existent pid + the daemon is respawned 3) smart daemons without daemonization - same as 2, but the daemonization and pidfile creation is managed by uWSGI + same as 2, but the daemonization and pidfile creation are managed by uWSGI status: @@ -232,7 +232,7 @@ void uwsgi_detach_daemons() { if (ud->pid > 0 && !ud->pidfile) { #endif uwsgi_log("[uwsgi-daemons] stopping daemon (pid: %d): %s\n", (int) ud->pid, ud->command); - // try to gracefully stop daemon, kill it if it won't die + // try to stop daemon gracefully, kill it if it won't die // if mercy is not set then wait up to 3 seconds time_t timeout = uwsgi_now() + (uwsgi.reload_mercy ? uwsgi.reload_mercy : 3); int waitpid_status; From 36143381e17b48adb247b8cab25f5b549d5e3b77 Mon Sep 17 00:00:00 2001 From: Unbit Date: Wed, 20 Nov 2013 05:28:32 +0100 Subject: [PATCH 13/94] tiny uwsgi.h refactoring --- plugins/mongrel2/mongrel2.c | 4 +-- uwsgi.h | 57 ++++++++++++++----------------------- 2 files changed, 22 insertions(+), 39 deletions(-) diff --git a/plugins/mongrel2/mongrel2.c b/plugins/mongrel2/mongrel2.c index 90f6f55b..bdb4dbb6 100644 --- a/plugins/mongrel2/mongrel2.c +++ b/plugins/mongrel2/mongrel2.c @@ -597,9 +597,7 @@ static void mongrel2_connect() { } // generate uuid - uuid_t uuid_zmq; - uuid_generate(uuid_zmq); - uuid_unparse(uuid_zmq, uwsgi_sock->uuid); + uwsgi_uuid(uwsgi_sock->uuid); if (zmq_setsockopt(uwsgi_sock->pub, ZMQ_IDENTITY, uwsgi_sock->uuid, 36) < 0) { uwsgi_error("mongrel2_connect()/zmq_setsockopt()"); diff --git a/uwsgi.h b/uwsgi.h index 4e20aa46..244c6c0b 100644 --- a/uwsgi.h +++ b/uwsgi.h @@ -444,13 +444,14 @@ struct uwsgi_dyn_dict { uint64_t hits; int status; + struct uwsgi_dyn_dict *prev; + struct uwsgi_dyn_dict *next; + #ifdef UWSGI_PCRE pcre *pattern; pcre_extra *pattern_extra; #endif - struct uwsgi_dyn_dict *prev; - struct uwsgi_dyn_dict *next; }; struct uwsgi_hook { @@ -579,12 +580,13 @@ struct uwsgi_daemon { // frequency of pidfile checks (default 10 secs) int freq; + int control; + struct uwsgi_daemon *next; + #ifdef UWSGI_SSL char *legion; #endif - int control; - struct uwsgi_daemon *next; }; struct uwsgi_logger { @@ -933,11 +935,8 @@ struct uwsgi_socket { // this is a special map for having socket->thread mapping int *fd_threads; -#ifdef UWSGI_UUID + // generally used by zeromq handlers char uuid[37]; -#endif - - // currently used by zeromq handlers void *pub; void *pull; pthread_key_t key; @@ -954,12 +953,13 @@ struct uwsgi_socket { int shared; int from_shared; + // used for avoiding vacuum mess + ino_t inode; + #ifdef UWSGI_SSL SSL_CTX *ssl_ctx; #endif - // used for avoiding vacuum mess - ino_t inode; }; struct uwsgi_protocol { @@ -1530,15 +1530,6 @@ struct wsgi_request { #ifdef UWSGI_SSL SSL *ssl; #endif - - struct msghdr msg; - union { - struct cmsghdr cmsg; - // should be enough... - char control[64]; - } msg_control; - - }; @@ -2636,9 +2627,6 @@ struct uwsgi_cron { uint8_t sig; char *command; -#ifdef UWSGI_SSL - char *legion; -#endif void (*func)(struct uwsgi_cron *, time_t); time_t started_at; @@ -2652,6 +2640,10 @@ struct uwsgi_cron { pid_t pid; struct uwsgi_cron *next; + +#ifdef UWSGI_SSL + char *legion; +#endif }; struct uwsgi_shared { @@ -3193,16 +3185,9 @@ int uwsgi_str4_num(char *); #ifdef __linux__ #if !defined(__ia64__) - void linux_namespace_start(void *); - void linux_namespace_jail(void); +void linux_namespace_start(void *); +void linux_namespace_jail(void); #endif - int uwsgi_netlink_veth(char *, char *); - int uwsgi_netlink_veth_attach(char *, pid_t); - int uwsgi_netlink_ifup(char *); - int uwsgi_netlink_ip(char *, char *, int); - int uwsgi_netlink_gw(char *, char *); - int uwsgi_netlink_rt(char *, char *, int, char *); - int uwsgi_netlink_del(char *); #endif @@ -3425,16 +3410,16 @@ struct uwsgi_subscribe_slot { uint64_t hits; + struct uwsgi_subscribe_node *nodes; + + struct uwsgi_subscribe_slot *prev; + struct uwsgi_subscribe_slot *next; + #ifdef UWSGI_SSL EVP_PKEY *sign_public_key; EVP_MD_CTX *sign_ctx; #endif - - struct uwsgi_subscribe_node *nodes; - - struct uwsgi_subscribe_slot *prev; - struct uwsgi_subscribe_slot *next; }; void mule_send_msg(int, char *, size_t); From 9825f95ed55a529887a4d9a772fc9d1536f1ba33 Mon Sep 17 00:00:00 2001 From: Roberto De Ioris Date: Wed, 20 Nov 2013 18:47:55 +0100 Subject: [PATCH 14/94] improved chain reloading --- core/master_checks.c | 2 ++ core/master_utils.c | 4 ++++ core/uwsgi.c | 2 ++ uwsgi.h | 2 ++ 4 files changed, 10 insertions(+) diff --git a/core/master_checks.c b/core/master_checks.c index 6f62b139..2774ebfb 100644 --- a/core/master_checks.c +++ b/core/master_checks.c @@ -48,6 +48,8 @@ void uwsgi_master_check_chain() { } uwsgi_block_signal(SIGHUP); for(i=1;i<=uwsgi.numproc;i++) { + // do not curse a worker until the old one is ready + if (uwsgi.workers[i].ready == 0) break; if (uwsgi.workers[i].pid > 0 && uwsgi.workers[i].cheaped == 0 && uwsgi.workers[i].cursed_at == 0 && i == uwsgi.status.chain_reloading) { uwsgi_curse(i, SIGHUP); break; diff --git a/core/master_utils.c b/core/master_utils.c index 9ffd499d..894ed252 100644 --- a/core/master_utils.c +++ b/core/master_utils.c @@ -592,6 +592,8 @@ void uwsgi_fixup_fds(int wid, int muleid, struct uwsgi_gateway *ug) { int uwsgi_respawn_worker(int wid) { int respawns = uwsgi.workers[wid].respawn_count; + // the workers is not ready (obviously) + uwsgi.workers[wid].ready = 0; // we count the respawns before errors... uwsgi.workers[wid].respawn_count++; // ... same for update time @@ -990,6 +992,8 @@ struct uwsgi_stats *uwsgi_master_generate_stats() { goto end; if (uwsgi_stats_keylong_comma(us, "pid", (unsigned long long) uwsgi.workers[i + 1].pid)) goto end; + if (uwsgi_stats_keylong_comma(us, "ready", (unsigned long long) uwsgi.workers[i + 1].ready)) + goto end; if (uwsgi_stats_keylong_comma(us, "requests", (unsigned long long) uwsgi.workers[i + 1].requests)) goto end; if (uwsgi_stats_keylong_comma(us, "delta_requests", (unsigned long long) uwsgi.workers[i + 1].delta_requests)) diff --git a/core/uwsgi.c b/core/uwsgi.c index c7a1c1a0..582250d1 100644 --- a/core/uwsgi.c +++ b/core/uwsgi.c @@ -3249,6 +3249,8 @@ void uwsgi_ignition() { } } + // mark the worker as "ready" (this is a mark used by chain reloading) + uwsgi.workers[uwsgi.mywid].ready = 1; if (uwsgi.loop) { void (*u_loop) (void) = uwsgi_get_loop(uwsgi.loop); diff --git a/uwsgi.h b/uwsgi.h index 244c6c0b..77d75418 100644 --- a/uwsgi.h +++ b/uwsgi.h @@ -2786,6 +2786,8 @@ struct uwsgi_worker { struct uwsgi_core *cores; + int ready; + char name[0xff]; }; From c01cf1f6bea19a77c845ee5aeb74e1fcd29fc2ed Mon Sep 17 00:00:00 2001 From: Roberto De Ioris Date: Thu, 21 Nov 2013 06:39:23 +0100 Subject: [PATCH 15/94] added accepting and ready state to the Emperor --- core/emperor.c | 54 +++++++++++++++++++++++++++++++++++++------- core/master_checks.c | 2 +- core/master_utils.c | 6 ++--- core/notify.c | 2 +- core/utils.c | 2 +- core/uwsgi.c | 12 ++++++++-- uwsgi.h | 8 ++++++- 7 files changed, 69 insertions(+), 17 deletions(-) diff --git a/core/emperor.c b/core/emperor.c index 2a64a00d..51769e87 100644 --- a/core/emperor.c +++ b/core/emperor.c @@ -59,12 +59,12 @@ void uwsgi_emperor_blacklist_add(char *id) { uebi->throttle_level += (uwsgi.emperor_throttle * 1000); } else { - uwsgi_log("[emperor] maximum throttle level for vassal %s reached !!!\n", id); + uwsgi_log_verbose("[emperor] maximum throttle level for vassal %s reached !!!\n", id); uebi->throttle_level = uebi->throttle_level / 2; } uebi->attempt++; if (uebi->attempt == 2) { - uwsgi_log("[emperor] unloyal bad behaving vassal found: %s throttling it...\n", id); + uwsgi_log_verbose("[emperor] unloyal bad behaving vassal found: %s throttling it...\n", id); } return; } @@ -603,7 +603,7 @@ void emperor_del(struct uwsgi_instance *c_ui) { uwsgi_log("[emperor] %s stop-hook returned %d\n", c_ui->name, stop_hook_ret); } - uwsgi_log("[emperor] removed uwsgi instance %s\n", c_ui->name); + uwsgi_log_verbose("[emperor] removed uwsgi instance %s\n", c_ui->name); // put the instance in the blacklist (or update its throttling value) if (!c_ui->loyal) { uwsgi_emperor_blacklist_add(c_ui->name); @@ -632,7 +632,7 @@ void emperor_stop(struct uwsgi_instance *c_ui) { c_ui->status = 1; c_ui->cursed_at = uwsgi_now(); - uwsgi_log("[emperor] stop the uwsgi instance %s\n", c_ui->name); + uwsgi_log_verbose("[emperor] stop the uwsgi instance %s\n", c_ui->name); } void emperor_curse(struct uwsgi_instance *c_ui) { @@ -642,7 +642,7 @@ void emperor_curse(struct uwsgi_instance *c_ui) { c_ui->status = 1; c_ui->cursed_at = uwsgi_now(); - uwsgi_log("[emperor] curse the uwsgi instance %s (pid: %d)\n", c_ui->name, (int) c_ui->pid); + uwsgi_log_verbose("[emperor] curse the uwsgi instance %s (pid: %d)\n", c_ui->name, (int) c_ui->pid); } @@ -675,8 +675,12 @@ void emperor_respawn(struct uwsgi_instance *c_ui, time_t mod) { c_ui->respawns++; c_ui->last_mod = mod; c_ui->last_run = uwsgi_now(); + // reset readyness + c_ui->ready = 0; + // reset accepting + c_ui->accepting = 0; - uwsgi_log("[emperor] reload the uwsgi instance %s\n", c_ui->name); + uwsgi_log_verbose("[emperor] reload the uwsgi instance %s\n", c_ui->name); } void emperor_add(struct uwsgi_emperor_scanner *ues, char *name, time_t born, char *config, uint32_t config_size, uid_t uid, gid_t gid, char *socket_name) { @@ -763,6 +767,9 @@ void emperor_add(struct uwsgi_emperor_scanner *ues, char *name, time_t born, cha n_ui->uid = uid; n_ui->gid = gid; n_ui->last_mod = born; + // start non-ready + n_ui->last_ready = 0; + n_ui->ready = 0; // start without loyalty n_ui->last_loyal = 0; n_ui->loyal = 0; @@ -1552,7 +1559,7 @@ void emperor_loop() { if (byte == 17) { ui_current->loyal = 1; ui_current->last_loyal = uwsgi_now(); - uwsgi_log("[emperor] vassal %s is now loyal\n", ui_current->name); + uwsgi_log_verbose("[emperor] vassal %s is now loyal\n", ui_current->name); // remove it from the blacklist uwsgi_emperor_blacklist_remove(ui_current->name); // TODO post-start hook @@ -1565,12 +1572,22 @@ void emperor_loop() { emperor_stop(ui_current); } else if (byte == 30 && uwsgi.emperor_broodlord > 0 && uwsgi.emperor_broodlord_count < uwsgi.emperor_broodlord) { - uwsgi_log("[emperor] going in broodlord mode: launching zergs for %s\n", ui_current->name); + uwsgi_log_verbose("[emperor] going in broodlord mode: launching zergs for %s\n", ui_current->name); char *zerg_name = uwsgi_concat3(ui_current->name, ":", "zerg"); // here we discard socket name as broodlord/zerg cannot be on demand emperor_add(ui_current->scanner, zerg_name, uwsgi_now(), NULL, 0, ui_current->uid, ui_current->gid, NULL); free(zerg_name); } + else if (byte == 5) { + ui_current->accepting = 1; + ui_current->last_accepting = uwsgi_now(); + uwsgi_log_verbose("[emperor] vassal %s is ready to accept requests\n", ui_current->name); + } + else if (byte == 1) { + ui_current->ready = 1; + ui_current->last_ready = uwsgi_now(); + uwsgi_log_verbose("[emperor] vassal %s has been spawned\n", ui_current->name); + } } } else { @@ -1773,8 +1790,16 @@ void emperor_send_stats(int fd) { goto end0; if (uwsgi_stats_keylong_comma(us, "loyal", (unsigned long long) c_ui->loyal)) goto end0; + if (uwsgi_stats_keylong_comma(us, "ready", (unsigned long long) c_ui->ready)) + goto end0; + if (uwsgi_stats_keylong_comma(us, "accepting", (unsigned long long) c_ui->accepting)) + goto end0; if (uwsgi_stats_keylong_comma(us, "last_loyal", (unsigned long long) c_ui->last_loyal)) goto end0; + if (uwsgi_stats_keylong_comma(us, "last_ready", (unsigned long long) c_ui->last_ready)) + goto end0; + if (uwsgi_stats_keylong_comma(us, "last_accepting", (unsigned long long) c_ui->last_accepting)) + goto end0; if (uwsgi_stats_keylong_comma(us, "first_run", (unsigned long long) c_ui->first_run)) goto end0; if (uwsgi_stats_keylong_comma(us, "last_run", (unsigned long long) c_ui->last_run)) @@ -2105,3 +2130,16 @@ void uwsgi_master_manage_emperor_proxy() { close(ep_client); } + +static void emperor_notify_ready() { + if (!uwsgi.has_emperor) return; + char byte = 1; + if (write(uwsgi.emperor_fd, &byte, 1) != 1) { + uwsgi_error("emperor_notify_ready()/write()"); + } +} + +void uwsgi_setup_emperor() { + if (!uwsgi.has_emperor) return; + uwsgi.notify_ready = emperor_notify_ready; +} diff --git a/core/master_checks.c b/core/master_checks.c index 2774ebfb..6b58522e 100644 --- a/core/master_checks.c +++ b/core/master_checks.c @@ -49,7 +49,7 @@ void uwsgi_master_check_chain() { uwsgi_block_signal(SIGHUP); for(i=1;i<=uwsgi.numproc;i++) { // do not curse a worker until the old one is ready - if (uwsgi.workers[i].ready == 0) break; + if (uwsgi.workers[i].accepting == 0) break; if (uwsgi.workers[i].pid > 0 && uwsgi.workers[i].cheaped == 0 && uwsgi.workers[i].cursed_at == 0 && i == uwsgi.status.chain_reloading) { uwsgi_curse(i, SIGHUP); break; diff --git a/core/master_utils.c b/core/master_utils.c index 894ed252..3125bc27 100644 --- a/core/master_utils.c +++ b/core/master_utils.c @@ -592,8 +592,8 @@ void uwsgi_fixup_fds(int wid, int muleid, struct uwsgi_gateway *ug) { int uwsgi_respawn_worker(int wid) { int respawns = uwsgi.workers[wid].respawn_count; - // the workers is not ready (obviously) - uwsgi.workers[wid].ready = 0; + // the workers is not accepting (obviously) + uwsgi.workers[wid].accepting = 0; // we count the respawns before errors... uwsgi.workers[wid].respawn_count++; // ... same for update time @@ -992,7 +992,7 @@ struct uwsgi_stats *uwsgi_master_generate_stats() { goto end; if (uwsgi_stats_keylong_comma(us, "pid", (unsigned long long) uwsgi.workers[i + 1].pid)) goto end; - if (uwsgi_stats_keylong_comma(us, "ready", (unsigned long long) uwsgi.workers[i + 1].ready)) + if (uwsgi_stats_keylong_comma(us, "accepting", (unsigned long long) uwsgi.workers[i + 1].accepting)) goto end; if (uwsgi_stats_keylong_comma(us, "requests", (unsigned long long) uwsgi.workers[i + 1].requests)) goto end; diff --git a/core/notify.c b/core/notify.c index 4a550aa4..4e47e735 100644 --- a/core/notify.c +++ b/core/notify.c @@ -1,4 +1,4 @@ -#include "uwsgi.h" +#include extern struct uwsgi_server uwsgi; diff --git a/core/utils.c b/core/utils.c index 78233585..38e918af 100644 --- a/core/utils.c +++ b/core/utils.c @@ -1162,7 +1162,7 @@ void uwsgi_close_request(struct wsgi_request *wsgi_req) { } - // ready to accept request, if i am a vassal signal Emperor about my loyalty + // after the first request, if i am a vassal, signal Emperor about my loyalty if (uwsgi.has_emperor && !uwsgi.loyal) { uwsgi_log("announcing my loyalty to the Emperor...\n"); char byte = 17; diff --git a/core/uwsgi.c b/core/uwsgi.c index 582250d1..ce7f3f27 100644 --- a/core/uwsgi.c +++ b/core/uwsgi.c @@ -2601,6 +2601,7 @@ int uwsgi_start(void *v_argv) { uwsgi_setup_systemd(); uwsgi_setup_upstart(); uwsgi_setup_zerg(); + uwsgi_setup_emperor(); } @@ -3249,8 +3250,15 @@ void uwsgi_ignition() { } } - // mark the worker as "ready" (this is a mark used by chain reloading) - uwsgi.workers[uwsgi.mywid].ready = 1; + // mark the worker as "accepting" (this is a mark used by chain reloading) + uwsgi.workers[uwsgi.mywid].accepting = 1; + // ready to accept request, if i am a vassal signal Emperor about it + if (uwsgi.has_emperor && uwsgi.mywid == 1) { + char byte = 5; + if (write(uwsgi.emperor_fd, &byte, 1) != 1) { + uwsgi_error("write()"); + } + } if (uwsgi.loop) { void (*u_loop) (void) = uwsgi_get_loop(uwsgi.loop); diff --git a/uwsgi.h b/uwsgi.h index 77d75418..eb4da133 100644 --- a/uwsgi.h +++ b/uwsgi.h @@ -2786,7 +2786,7 @@ struct uwsgi_worker { struct uwsgi_core *cores; - int ready; + int accepting; char name[0xff]; }; @@ -3776,6 +3776,7 @@ void uwsgi_setup_systemd(); void uwsgi_setup_upstart(); void uwsgi_setup_zerg(); void uwsgi_setup_inherited_sockets(); +void uwsgi_setup_emperor(); #ifdef UWSGI_SSL void uwsgi_ssl_init(void); @@ -3846,6 +3847,8 @@ struct uwsgi_instance { time_t born; time_t last_mod; time_t last_loyal; + time_t last_accepting; + time_t last_ready; time_t last_run; time_t first_run; @@ -3865,6 +3868,9 @@ struct uwsgi_instance { int zerg; + int ready; + int accepting; + struct uwsgi_emperor_scanner *scanner; uid_t uid; From 25dc608fa38000e24ee76416de0cd467fb2d635b Mon Sep 17 00:00:00 2001 From: Unbit Date: Thu, 21 Nov 2013 14:10:41 +0100 Subject: [PATCH 16/94] prepare for embedded plugins builder --- core/plugins_builder.c | 16 ++++++++++++++++ core/uwsgi.c | 4 ++++ uwsgi.h | 3 +++ uwsgiconfig.py | 2 +- 4 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 core/plugins_builder.c diff --git a/core/plugins_builder.c b/core/plugins_builder.c new file mode 100644 index 00000000..a1646d22 --- /dev/null +++ b/core/plugins_builder.c @@ -0,0 +1,16 @@ +#include + +/* + + steps: + + mkdir(.uwsgi_plugin_builder) + generate .uwsgi_plugin_builder/uwsgi.h + setenv(CFLAGS=uwsgi_cflags) + pipe uwsgiconfig.py to python passing args + +*/ + +int uwsgi_build_plugin(char *directory) { + return 0; +} diff --git a/core/uwsgi.c b/core/uwsgi.c index ce7f3f27..f74e77f5 100644 --- a/core/uwsgi.c +++ b/core/uwsgi.c @@ -907,6 +907,7 @@ static struct uwsgi_option uwsgi_base_options[] = { {"exit", optional_argument, 0, "force exit() of the instance", uwsgi_opt_exit, NULL, UWSGI_OPT_IMMEDIATE}, {"cflags", no_argument, 0, "report uWSGI CFLAGS (useful for building external plugins)", uwsgi_opt_cflags, NULL, UWSGI_OPT_IMMEDIATE}, {"dot-h", no_argument, 0, "dump the uwsgi.h used for building the core (useful for building external plugins)", uwsgi_opt_dot_h, NULL, UWSGI_OPT_IMMEDIATE}, + {"build-plugin", no_argument, 0, "build a uWSGI plugin for the current binary", uwsgi_opt_build_plugin, NULL, UWSGI_OPT_IMMEDIATE}, {"version", no_argument, 0, "print uWSGI version", uwsgi_opt_print, UWSGI_VERSION, 0}, {0, 0, 0, 0, 0, 0, 0} }; @@ -4506,6 +4507,9 @@ void uwsgi_opt_dot_h(char *opt, char *filename, void *foobar) { exit(0); } +void uwsgi_opt_build_plugin(char *opt, char *directory, void *foobar) { + _exit(uwsgi_build_plugin(directory)); +} void uwsgi_opt_connect_and_read(char *opt, char *address, void *foobar) { diff --git a/uwsgi.h b/uwsgi.h index eb4da133..daaecc3b 100644 --- a/uwsgi.h +++ b/uwsgi.h @@ -3761,6 +3761,7 @@ char *uwsgi_substitute(char *, char *, char *); void uwsgi_opt_add_custom_option(char *, char *, void *); void uwsgi_opt_cflags(char *, char *, void *); +void uwsgi_opt_build_plugin(char *, char *, void *); void uwsgi_opt_dot_h(char *, char *, void *); void uwsgi_opt_connect_and_read(char *, char *, void *); void uwsgi_opt_extract(char *, char *, void *); @@ -4528,6 +4529,8 @@ struct uwsgi_protocol *uwsgi_register_protocol(char *, void (*)(struct uwsgi_soc void uwsgi_protocols_register(void); +int uwsgi_build_plugin(char *dir); + #ifdef __cplusplus } #endif diff --git a/uwsgiconfig.py b/uwsgiconfig.py index 46e9f982..0bef019b 100644 --- a/uwsgiconfig.py +++ b/uwsgiconfig.py @@ -546,7 +546,7 @@ class uConf(object): 'core/setup_utils', 'core/clock', 'core/init', 'core/buffer', 'core/reader', 'core/writer', 'core/alarm', 'core/cron', 'core/hooks', 'core/plugins', 'core/lock', 'core/cache', 'core/daemons', 'core/errors', 'core/hash', 'core/master_events', 'core/chunked', 'core/queue', 'core/event', 'core/signal', 'core/strings', 'core/progress', 'core/timebomb', 'core/ini', 'core/fsmon', 'core/mount', - 'core/metrics', + 'core/metrics', 'core/plugins_builder', 'core/rpc', 'core/gateway', 'core/loop', 'core/cookie', 'core/querystring', 'core/rb_timers', 'core/transformations', 'core/uwsgi'] # add protocols self.gcc_list.append('proto/base') From b83fecf4d33e7f01e89b10804aea694054c6dd04 Mon Sep 17 00:00:00 2001 From: Unbit Date: Thu, 21 Nov 2013 14:37:58 +0100 Subject: [PATCH 17/94] another step for embedded plugins builder --- core/plugins_builder.c | 57 +++++++++++++++++++++++++++++++++++++++++- core/uwsgi.c | 2 +- 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/core/plugins_builder.c b/core/plugins_builder.c index a1646d22..da708c9d 100644 --- a/core/plugins_builder.c +++ b/core/plugins_builder.c @@ -1,5 +1,7 @@ #include +#define UWSGI_BUILD_DIR ".uwsgi_plugins_builder" + /* steps: @@ -12,5 +14,58 @@ */ int uwsgi_build_plugin(char *directory) { - return 0; + uwsgi_log("building uWSGI plugin %s...\n", directory); + + if (!uwsgi_file_exists(UWSGI_BUILD_DIR)) { + if (mkdir(UWSGI_BUILD_DIR, S_IRWXU) < 0) { + uwsgi_error("uwsgi_build_plugin()/mkdir() " UWSGI_BUILD_DIR "/"); + return 1; + } + } + + char *dot_h = uwsgi_get_dot_h(); + if (!dot_h) { + uwsgi_log("unable to generate uwsgi.h"); + return 1; + } + + if (strlen(dot_h) == 0) { + free(dot_h); + uwsgi_log("invalid uwsgi.h"); + return 1; + } + + int dot_h_fd = open(UWSGI_BUILD_DIR "/uwsgi.h", O_WRONLY|O_CREAT|O_TRUNC, S_IRUSR|S_IWUSR); + if (dot_h_fd < 0) { + uwsgi_error_open(UWSGI_BUILD_DIR "/uwsgi.h"); + free(dot_h); + return 1; + } + + ssize_t dot_h_len = (ssize_t) strlen(dot_h); + if (write(dot_h_fd, dot_h, dot_h_len) != dot_h_len) { + uwsgi_error("uwsgi_build_plugin()/write()"); + close(dot_h_fd); + free(dot_h); + return 1; + } + + free(dot_h); + close(dot_h_fd); + + // now extract the python build script from the process + // and pipe it to python + + char *argv[4]; + + argv[0] = getenv("PYTHON"); + if (!argv[0]) argv[0] = "python"; + + argv[1] = "-"; + argv[2] = "--extra-plugin"; + argv[3] = directory; + + execvp(argv[0], argv); + // never here... + return 1; } diff --git a/core/uwsgi.c b/core/uwsgi.c index f74e77f5..7a94e2be 100644 --- a/core/uwsgi.c +++ b/core/uwsgi.c @@ -907,7 +907,7 @@ static struct uwsgi_option uwsgi_base_options[] = { {"exit", optional_argument, 0, "force exit() of the instance", uwsgi_opt_exit, NULL, UWSGI_OPT_IMMEDIATE}, {"cflags", no_argument, 0, "report uWSGI CFLAGS (useful for building external plugins)", uwsgi_opt_cflags, NULL, UWSGI_OPT_IMMEDIATE}, {"dot-h", no_argument, 0, "dump the uwsgi.h used for building the core (useful for building external plugins)", uwsgi_opt_dot_h, NULL, UWSGI_OPT_IMMEDIATE}, - {"build-plugin", no_argument, 0, "build a uWSGI plugin for the current binary", uwsgi_opt_build_plugin, NULL, UWSGI_OPT_IMMEDIATE}, + {"build-plugin", required_argument, 0, "build a uWSGI plugin for the current binary", uwsgi_opt_build_plugin, NULL, UWSGI_OPT_IMMEDIATE}, {"version", no_argument, 0, "print uWSGI version", uwsgi_opt_print, UWSGI_VERSION, 0}, {0, 0, 0, 0, 0, 0, 0} }; From 859c386b4020250a71454784b79453e33b2d8a15 Mon Sep 17 00:00:00 2001 From: Unbit Date: Thu, 21 Nov 2013 14:54:42 +0100 Subject: [PATCH 18/94] improved --extra-plugin --- uwsgiconfig.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/uwsgiconfig.py b/uwsgiconfig.py index 0bef019b..fd9e7af7 100644 --- a/uwsgiconfig.py +++ b/uwsgiconfig.py @@ -1431,13 +1431,10 @@ if __name__ == "__main__": print("*** uWSGI building and linking plugin %s ***" % options.plugin[0] ) build_plugin(options.plugin[0], uc, cflags, ldflags, libs, name) elif options.extra_plugin: - print("*** uWSGI building and linking plugin ***") - cflags = spcall("%s --cflags" % options.extra_plugin[0]).split() - try: - cflags.append('-I%s' % options.extra_plugin[1]) - except: - pass - build_plugin('.', None, cflags, [], [], None) + print("*** uWSGI building and linking plugin %s ***" % options.extra_plugin[0]) + cflags = os.environ['UWSGI_PLUGINS_BUILDER_CFLAGS'].split() + cflags.append('-I.uwsgi_plugins_builder/') + build_plugin(options.extra_plugin[0], None, cflags, [], [], None) elif options.clean: os.system("rm -f core/*.o") os.system("rm -f proto/*.o") From e4903ee0bfb640465628f33ea5a433934ef9be27 Mon Sep 17 00:00:00 2001 From: Unbit Date: Thu, 21 Nov 2013 15:29:30 +0100 Subject: [PATCH 19/94] first working prototype of embedded plugins builder --- core/plugins_builder.c | 82 +++++++++++++++++++++++++++++++----------- core/uwsgi.c | 39 +++++++++++++++++++- uwsgi.h | 4 ++- uwsgiconfig.py | 28 +++++++++++++-- 4 files changed, 129 insertions(+), 24 deletions(-) diff --git a/core/plugins_builder.c b/core/plugins_builder.c index da708c9d..b4341f6c 100644 --- a/core/plugins_builder.c +++ b/core/plugins_builder.c @@ -8,64 +8,106 @@ mkdir(.uwsgi_plugin_builder) generate .uwsgi_plugin_builder/uwsgi.h - setenv(CFLAGS=uwsgi_cflags) - pipe uwsgiconfig.py to python passing args + generate .uwsgi_plugin_builder/uwsgiconfig.py + setenv(UWSGI_PLUGINS_BUILDER_CFLAGS=uwsgi_cflags) + exec PYTHON .uwsgi_plugin_builder/uwsgiconfig.py --extra-plugin [name] */ -int uwsgi_build_plugin(char *directory) { +void uwsgi_build_plugin(char *directory) { uwsgi_log("building uWSGI plugin %s...\n", directory); if (!uwsgi_file_exists(UWSGI_BUILD_DIR)) { if (mkdir(UWSGI_BUILD_DIR, S_IRWXU) < 0) { uwsgi_error("uwsgi_build_plugin()/mkdir() " UWSGI_BUILD_DIR "/"); - return 1; + _exit(1); } } char *dot_h = uwsgi_get_dot_h(); if (!dot_h) { uwsgi_log("unable to generate uwsgi.h"); - return 1; + _exit(1); } if (strlen(dot_h) == 0) { free(dot_h); uwsgi_log("invalid uwsgi.h"); - return 1; + _exit(1); } int dot_h_fd = open(UWSGI_BUILD_DIR "/uwsgi.h", O_WRONLY|O_CREAT|O_TRUNC, S_IRUSR|S_IWUSR); if (dot_h_fd < 0) { uwsgi_error_open(UWSGI_BUILD_DIR "/uwsgi.h"); free(dot_h); - return 1; + _exit(1); } ssize_t dot_h_len = (ssize_t) strlen(dot_h); if (write(dot_h_fd, dot_h, dot_h_len) != dot_h_len) { uwsgi_error("uwsgi_build_plugin()/write()"); - close(dot_h_fd); - free(dot_h); - return 1; + _exit(1); + } + + char *config_py = uwsgi_get_config_py(); + if (!config_py) { + uwsgi_log("unable to generate uwsgiconfig.py"); + _exit(1); + } + + if (strlen(config_py) == 0) { + uwsgi_log("invalid uwsgiconfig.py"); + _exit(1); + } + + int config_py_fd = open(UWSGI_BUILD_DIR "/uwsgiconfig.py", O_WRONLY|O_CREAT|O_TRUNC, S_IRUSR|S_IWUSR); + if (config_py_fd < 0) { + uwsgi_error_open(UWSGI_BUILD_DIR "/uwsgiconfig.py"); + _exit(1); + } + + ssize_t config_py_len = (ssize_t) strlen(config_py); + if (write(config_py_fd, config_py, config_py_len) != config_py_len) { + uwsgi_error("uwsgi_build_plugin()/write()"); + _exit(1); + } + + char *cflags = uwsgi_get_cflags(); + if (!cflags) { + uwsgi_log("unable to find cflags\n"); + _exit(1); + } + if (strlen(cflags) == 0) { + uwsgi_log("invalid cflags\n"); + _exit(1); + } + + if (setenv("UWSGI_PLUGINS_BUILDER_CFLAGS", cflags, 1)) { + uwsgi_error("uwsgi_build_plugin()/setenv()"); + _exit(1); } - free(dot_h); - close(dot_h_fd); - - // now extract the python build script from the process - // and pipe it to python - - char *argv[4]; + // now run the python script + char *argv[6]; argv[0] = getenv("PYTHON"); if (!argv[0]) argv[0] = "python"; - argv[1] = "-"; + argv[1] = UWSGI_BUILD_DIR "/uwsgiconfig.py"; argv[2] = "--extra-plugin"; - argv[3] = directory; + char *space = strchr(directory, ' '); + if (space) { + *space = 0; + argv[3] = directory; + argv[4] = space+1; + argv[5] = NULL; + } + else { + argv[3] = directory; + argv[4] = NULL; + } execvp(argv[0], argv); // never here... - return 1; + _exit(1); } diff --git a/core/uwsgi.c b/core/uwsgi.c index 7a94e2be..a18c0b6b 100644 --- a/core/uwsgi.c +++ b/core/uwsgi.c @@ -907,6 +907,7 @@ static struct uwsgi_option uwsgi_base_options[] = { {"exit", optional_argument, 0, "force exit() of the instance", uwsgi_opt_exit, NULL, UWSGI_OPT_IMMEDIATE}, {"cflags", no_argument, 0, "report uWSGI CFLAGS (useful for building external plugins)", uwsgi_opt_cflags, NULL, UWSGI_OPT_IMMEDIATE}, {"dot-h", no_argument, 0, "dump the uwsgi.h used for building the core (useful for building external plugins)", uwsgi_opt_dot_h, NULL, UWSGI_OPT_IMMEDIATE}, + {"config-py", no_argument, 0, "dump the uwsgiconfig.py used for building the core (useful for building external plugins)", uwsgi_opt_config_py, NULL, UWSGI_OPT_IMMEDIATE}, {"build-plugin", required_argument, 0, "build a uWSGI plugin for the current binary", uwsgi_opt_build_plugin, NULL, UWSGI_OPT_IMMEDIATE}, {"version", no_argument, 0, "print uWSGI version", uwsgi_opt_print, UWSGI_VERSION, 0}, {0, 0, 0, 0, 0, 0, 0} @@ -4507,8 +4508,44 @@ void uwsgi_opt_dot_h(char *opt, char *filename, void *foobar) { exit(0); } +extern char *uwsgi_config_py; +char *uwsgi_get_config_py() { + char *src = uwsgi_config_py; + size_t len = strlen(src); + char *ptr = uwsgi_malloc(len / 2); + char *base = ptr; + size_t i; + unsigned int u; + for (i = 0; i < len; i += 2) { + sscanf(src + i, "%2x", &u); + *ptr++ = (char) u; + } +#ifdef UWSGI_ZLIB + struct uwsgi_buffer *ub = uwsgi_zlib_decompress(base, ptr-base); + if (!ub) { + free(base); + return ""; + } + // add final null byte + uwsgi_buffer_append(ub, "\0", 1); + free(base); + // base is the final blob + base = ub->buf; + ub->buf = NULL; + uwsgi_buffer_destroy(ub); +#endif + return base; +} + +void uwsgi_opt_config_py(char *opt, char *filename, void *foobar) { + fprintf(stdout, "%s\n", uwsgi_get_config_py()); + exit(0); +} + + void uwsgi_opt_build_plugin(char *opt, char *directory, void *foobar) { - _exit(uwsgi_build_plugin(directory)); + uwsgi_build_plugin(directory); + exit(1); } void uwsgi_opt_connect_and_read(char *opt, char *address, void *foobar) { diff --git a/uwsgi.h b/uwsgi.h index daaecc3b..7fd480a7 100644 --- a/uwsgi.h +++ b/uwsgi.h @@ -3763,10 +3763,12 @@ void uwsgi_opt_add_custom_option(char *, char *, void *); void uwsgi_opt_cflags(char *, char *, void *); void uwsgi_opt_build_plugin(char *, char *, void *); void uwsgi_opt_dot_h(char *, char *, void *); +void uwsgi_opt_config_py(char *, char *, void *); void uwsgi_opt_connect_and_read(char *, char *, void *); void uwsgi_opt_extract(char *, char *, void *); char *uwsgi_get_dot_h(); +char *uwsgi_get_config_py(); char *uwsgi_get_cflags(); struct uwsgi_string_list *uwsgi_string_list_has_item(struct uwsgi_string_list *, char *, size_t); @@ -4529,7 +4531,7 @@ struct uwsgi_protocol *uwsgi_register_protocol(char *, void (*)(struct uwsgi_soc void uwsgi_protocols_register(void); -int uwsgi_build_plugin(char *dir); +void uwsgi_build_plugin(char *dir); #ifdef __cplusplus } diff --git a/uwsgiconfig.py b/uwsgiconfig.py index fd9e7af7..c5375098 100644 --- a/uwsgiconfig.py +++ b/uwsgiconfig.py @@ -305,6 +305,24 @@ def build_uwsgi(uc, print_only=False, gcll=None): uwsgi_dot_h = uwsgi_dot_h_content.encode('hex') open('core/dot_h.c', 'w').write('char *uwsgi_dot_h = "%s";\n' % uwsgi_dot_h); gcc_list.append('core/dot_h') + + # embed uwsgiconfig.py in the server binary. It increases the binary size, but will be very useful + # if possibile, the blob is compressed + if sys.version_info[0] >= 3: + uwsgi_config_py_content = open('uwsgiconfig.py', 'rb').read() + else: + uwsgi_config_py_content = open('uwsgiconfig.py').read() + if report['zlib']: + import zlib + # maximum level of compression + uwsgi_config_py_content = zlib.compress(uwsgi_config_py_content, 9) + if sys.version_info[0] >= 3: + import binascii + uwsgi_config_py = binascii.b2a_hex(uwsgi_config_py_content).decode('ascii') + else: + uwsgi_config_py = uwsgi_config_py_content.encode('hex') + open('core/config_py.c', 'w').write('char *uwsgi_config_py = "%s";\n' % uwsgi_config_py); + gcc_list.append('core/config_py') cflags.append('-DUWSGI_CFLAGS=\\"%s\\"' % uwsgi_cflags) cflags.append('-DUWSGI_BUILD_DATE="\\"%s\\""' % time.strftime("%d %B %Y %H:%M:%S")) @@ -1363,7 +1381,7 @@ if __name__ == "__main__": parser.add_option("-f", "--cflags", action="callback", callback=vararg_callback, dest="cflags", help="same as --build but less verbose", metavar="PROFILE") parser.add_option("-u", "--unbit", action="store_true", dest="unbit", help="build unbit profile") parser.add_option("-p", "--plugin", action="callback", callback=vararg_callback, dest="plugin", help="build a plugin as shared library, optionally takes a build profile name", metavar="PLUGIN [PROFILE]") - parser.add_option("-x", "--extra-plugin", action="callback", callback=vararg_callback, dest="extra_plugin", help="build an external plugin as shared library, takes an optional include dir", metavar="PLUGIN [INCLUDE_DIR]") + parser.add_option("-x", "--extra-plugin", action="callback", callback=vararg_callback, dest="extra_plugin", help="build an external plugin as shared library, takes an optional include dir", metavar="PLUGIN [NAME]") parser.add_option("-c", "--clean", action="store_true", dest="clean", help="clean the build") parser.add_option("-e", "--check", action="store_true", dest="check", help="run cppcheck") parser.add_option("-v", "--verbose", action="store_true", dest="verbose", help="more verbose build") @@ -1434,7 +1452,12 @@ if __name__ == "__main__": print("*** uWSGI building and linking plugin %s ***" % options.extra_plugin[0]) cflags = os.environ['UWSGI_PLUGINS_BUILDER_CFLAGS'].split() cflags.append('-I.uwsgi_plugins_builder/') - build_plugin(options.extra_plugin[0], None, cflags, [], [], None) + name = None + try: + name = options.extra_plugin[1] + except: + pass + build_plugin(options.extra_plugin[0], None, cflags, [], [], name) elif options.clean: os.system("rm -f core/*.o") os.system("rm -f proto/*.o") @@ -1442,6 +1465,7 @@ if __name__ == "__main__": os.system("rm -f plugins/*/*.o") os.system("rm -f build/*.o") os.system("rm -f core/dot_h.c") + os.system("rm -f core/config_py.c") elif options.check: os.system("cppcheck --max-configs=1000 --enable=all -q core/ plugins/ proto/ lib/ apache2/") else: From 30e0af382655f23e88f29bd97e3121f0839d0d58 Mon Sep 17 00:00:00 2001 From: Roberto De Ioris Date: Fri, 22 Nov 2013 06:45:14 +0100 Subject: [PATCH 20/94] only GCC_LIST and NAME are required --- uwsgiconfig.py | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/uwsgiconfig.py b/uwsgiconfig.py index c5375098..fcd54bf0 100644 --- a/uwsgiconfig.py +++ b/uwsgiconfig.py @@ -377,7 +377,10 @@ def build_uwsgi(uc, print_only=False, gcll=None): f.close() p_cflags = cflags[:] - p_cflags += up['CFLAGS'] + try: + p_cflags += up['CFLAGS'] + except: + pass if uwsgi_os.startswith('CYGWIN'): try: @@ -448,7 +451,13 @@ def build_uwsgi(uc, print_only=False, gcll=None): except: pass - libs += up['LIBS'] + try: + libs += up['LIBS'] + except: + pass + + if not 'LDFLAGS' in up: + up['LDFLAGS'] = [] if uwsgi_os == 'Darwin': found_arch = False @@ -1215,9 +1224,20 @@ def build_plugin(path, uc, cflags, ldflags, libs, name = None): p_cflags = cflags[:] p_ldflags = ldflags[:] - p_cflags += up['CFLAGS'] - p_ldflags += up['LDFLAGS'] - p_libs = up['LIBS'] + try: + p_cflags += up['CFLAGS'] + except: + pass + + try: + p_ldflags += up['LDFLAGS'] + except: + pass + + try: + p_libs = up['LIBS'] + except: + p_libs = [] post_build = None From 06791376af5a5e55bd858ead0bdf20ede7808eb2 Mon Sep 17 00:00:00 2001 From: unbit Date: Fri, 22 Nov 2013 09:22:08 +0100 Subject: [PATCH 21/94] honour CFLAGS and LDFLAGS in embedded builder --- uwsgiconfig.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/uwsgiconfig.py b/uwsgiconfig.py index fcd54bf0..7e4efb5e 100644 --- a/uwsgiconfig.py +++ b/uwsgiconfig.py @@ -1470,14 +1470,15 @@ if __name__ == "__main__": build_plugin(options.plugin[0], uc, cflags, ldflags, libs, name) elif options.extra_plugin: print("*** uWSGI building and linking plugin %s ***" % options.extra_plugin[0]) - cflags = os.environ['UWSGI_PLUGINS_BUILDER_CFLAGS'].split() + cflags = os.environ['UWSGI_PLUGINS_BUILDER_CFLAGS'].split() + os.environ.get("CFLAGS", "").split() cflags.append('-I.uwsgi_plugins_builder/') + ldflags = os.environ.get("LDFLAGS", "").split() name = None try: name = options.extra_plugin[1] except: pass - build_plugin(options.extra_plugin[0], None, cflags, [], [], name) + build_plugin(options.extra_plugin[0], None, cflags, ldflags, None, name) elif options.clean: os.system("rm -f core/*.o") os.system("rm -f proto/*.o") From 3371dd2f9457e822b25c9c4747ba5074165e86be Mon Sep 17 00:00:00 2001 From: Unbit Date: Fri, 22 Nov 2013 18:42:30 +0100 Subject: [PATCH 22/94] added error-page management --- core/errors.c | 23 +++++++++++++++++++++++ core/uwsgi.c | 5 +++++ uwsgi.h | 4 ++++ 3 files changed, 32 insertions(+) diff --git a/core/errors.c b/core/errors.c index afd4c96c..1459ec7c 100644 --- a/core/errors.c +++ b/core/errors.c @@ -1,9 +1,30 @@ #include +extern struct uwsgi_server uwsgi; + +static int error_page(struct wsgi_request *wsgi_req, struct uwsgi_string_list *l) { + struct uwsgi_string_list *usl = NULL; + uwsgi_foreach(usl, l) { + struct stat st; + if (!stat(usl->value, &st)) { + int fd = open(usl->value, O_RDONLY); + if (fd >= 0) { + if (uwsgi_response_add_header(wsgi_req, "Content-Type", 12, "text/html", 9)) return 0; + if (uwsgi_response_add_content_length(wsgi_req, st.st_size)) return 0; + uwsgi_response_sendfile_do(wsgi_req, fd, 0, st.st_size); + return 0; + } + } + } + return -1; +} + // generate internal server error message void uwsgi_500(struct wsgi_request *wsgi_req) { if (uwsgi_response_prepare_headers(wsgi_req, "500 Internal Server Error", 25)) return; if (uwsgi_response_add_connection_close(wsgi_req)) return; + if (error_page(wsgi_req, uwsgi.error_page_500)) return; + if (uwsgi_response_add_header(wsgi_req, "Content-Type", 12, "text/plain", 10)) return; uwsgi_response_write_body_do(wsgi_req, "Internal Server Error", 21); } @@ -11,6 +32,7 @@ void uwsgi_500(struct wsgi_request *wsgi_req) { void uwsgi_404(struct wsgi_request *wsgi_req) { if (uwsgi_response_prepare_headers(wsgi_req, "404 Not Found", 13)) return; if (uwsgi_response_add_connection_close(wsgi_req)) return; + if (error_page(wsgi_req, uwsgi.error_page_404)) return; if (uwsgi_response_add_header(wsgi_req, "Content-Type", 12, "text/plain", 10)) return; uwsgi_response_write_body_do(wsgi_req, "Not Found", 9); } @@ -18,6 +40,7 @@ void uwsgi_404(struct wsgi_request *wsgi_req) { void uwsgi_403(struct wsgi_request *wsgi_req) { if (uwsgi_response_prepare_headers(wsgi_req, "403 Forbidden", 13)) return; if (uwsgi_response_add_connection_close(wsgi_req)) return; + if (error_page(wsgi_req, uwsgi.error_page_403)) return; if (uwsgi_response_add_content_type(wsgi_req, "text/plain", 10)) return; uwsgi_response_write_body_do(wsgi_req, "Forbidden", 9); } diff --git a/core/uwsgi.c b/core/uwsgi.c index a18c0b6b..67147503 100644 --- a/core/uwsgi.c +++ b/core/uwsgi.c @@ -761,6 +761,11 @@ static struct uwsgi_option uwsgi_base_options[] = { {"routers-list", no_argument, 0, "list enabled routers", uwsgi_opt_true, &uwsgi.router_list, 0}, #endif + + {"error-page-403", required_argument, 0, "add an error page (html) for managed 403 response", uwsgi_opt_add_string_list, &uwsgi.error_page_403, 0}, + {"error-page-404", required_argument, 0, "add an error page (html) for managed 404 response", uwsgi_opt_add_string_list, &uwsgi.error_page_404, 0}, + {"error-page-500", required_argument, 0, "add an error page (html) for managed 500 response", uwsgi_opt_add_string_list, &uwsgi.error_page_500, 0}, + {"websockets-ping-freq", required_argument, 0, "set the frequency (in seconds) of websockets automatic ping packets", uwsgi_opt_set_int, &uwsgi.websockets_ping_freq, 0}, {"websocket-ping-freq", required_argument, 0, "set the frequency (in seconds) of websockets automatic ping packets", uwsgi_opt_set_int, &uwsgi.websockets_ping_freq, 0}, diff --git a/uwsgi.h b/uwsgi.h index 7fd480a7..4f466f10 100644 --- a/uwsgi.h +++ b/uwsgi.h @@ -2354,6 +2354,10 @@ struct uwsgi_server { struct uwsgi_route_var *route_vars; #endif + struct uwsgi_string_list *error_page_403; + struct uwsgi_string_list *error_page_404; + struct uwsgi_string_list *error_page_500; + int single_interpreter; struct uwsgi_shared *shared; From 1154aa8ddcfe50258946f97dfecf09bedbedf44a Mon Sep 17 00:00:00 2001 From: Unbit Date: Fri, 22 Nov 2013 18:46:07 +0100 Subject: [PATCH 23/94] added accepting hook --- core/uwsgi.c | 4 ++++ uwsgi.h | 1 + 2 files changed, 5 insertions(+) diff --git a/core/uwsgi.c b/core/uwsgi.c index 67147503..f7da5d8a 100644 --- a/core/uwsgi.c +++ b/core/uwsgi.c @@ -373,6 +373,7 @@ static struct uwsgi_option uwsgi_base_options[] = { {"hook-as-user-atexit", required_argument, 0, "run the specified hook before app exit and reload", uwsgi_opt_add_string_list, &uwsgi.hook_as_user_atexit, 0}, {"hook-pre-app", required_argument, 0, "run the specified hook before app loading", uwsgi_opt_add_string_list, &uwsgi.hook_pre_app, 0}, {"hook-post-app", required_argument, 0, "run the specified hook after app loading", uwsgi_opt_add_string_list, &uwsgi.hook_post_app, 0}, + {"hook-accepting", required_argument, 0, "run the specified hook after the app enters the accepting phase", uwsgi_opt_add_string_list, &uwsgi.hook_accepting, 0}, {"hook-as-vassal", required_argument, 0, "run the specified command before exec()ing the vassal", uwsgi_opt_add_string_list, &uwsgi.hook_as_vassal, 0}, {"hook-as-emperor", required_argument, 0, "run the specified command in the emperor after the vassal has been started", uwsgi_opt_add_string_list, &uwsgi.hook_as_emperor, 0}, @@ -3267,6 +3268,9 @@ void uwsgi_ignition() { } } + // run accepting hooks + uwsgi_hooks_run(uwsgi.hook_accepting, "accepting", 1); + if (uwsgi.loop) { void (*u_loop) (void) = uwsgi_get_loop(uwsgi.loop); if (!u_loop) { diff --git a/uwsgi.h b/uwsgi.h index 4f466f10..cfe5fb1b 100644 --- a/uwsgi.h +++ b/uwsgi.h @@ -1990,6 +1990,7 @@ struct uwsgi_server { struct uwsgi_string_list *hook_as_user_atexit; struct uwsgi_string_list *hook_pre_app; struct uwsgi_string_list *hook_post_app; + struct uwsgi_string_list *hook_accepting; struct uwsgi_string_list *hook_as_vassal; struct uwsgi_string_list *hook_as_emperor; From 036df0e5ff0a873aafa5ff85cd5b0a053c9cd0a6 Mon Sep 17 00:00:00 2001 From: Unbit Date: Fri, 22 Nov 2013 18:52:02 +0100 Subject: [PATCH 24/94] fixed fd leak in error pages --- core/errors.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/errors.c b/core/errors.c index 1459ec7c..d1ff4e57 100644 --- a/core/errors.c +++ b/core/errors.c @@ -9,8 +9,8 @@ static int error_page(struct wsgi_request *wsgi_req, struct uwsgi_string_list *l if (!stat(usl->value, &st)) { int fd = open(usl->value, O_RDONLY); if (fd >= 0) { - if (uwsgi_response_add_header(wsgi_req, "Content-Type", 12, "text/html", 9)) return 0; - if (uwsgi_response_add_content_length(wsgi_req, st.st_size)) return 0; + if (uwsgi_response_add_header(wsgi_req, "Content-Type", 12, "text/html", 9)) { close(fd); return 0;} + if (uwsgi_response_add_content_length(wsgi_req, st.st_size)) { close(fd); return 0;} uwsgi_response_sendfile_do(wsgi_req, fd, 0, st.st_size); return 0; } From 9dde5f9d826e538fb999466ed15368847708ef43 Mon Sep 17 00:00:00 2001 From: Unbit Date: Fri, 22 Nov 2013 18:53:05 +0100 Subject: [PATCH 25/94] completed error pages --- core/errors.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/errors.c b/core/errors.c index d1ff4e57..8df221e7 100644 --- a/core/errors.c +++ b/core/errors.c @@ -12,11 +12,11 @@ static int error_page(struct wsgi_request *wsgi_req, struct uwsgi_string_list *l if (uwsgi_response_add_header(wsgi_req, "Content-Type", 12, "text/html", 9)) { close(fd); return 0;} if (uwsgi_response_add_content_length(wsgi_req, st.st_size)) { close(fd); return 0;} uwsgi_response_sendfile_do(wsgi_req, fd, 0, st.st_size); - return 0; + return -1; } } } - return -1; + return 0; } // generate internal server error message From eb33fa7b19293377aecdce22b32e562cc6c1dbad Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Fri, 22 Nov 2013 19:32:56 +0100 Subject: [PATCH 26/94] core/config: fix memory leak Reported by Coverity s CID #1128441 --- core/config.c | 1 + 1 file changed, 1 insertion(+) diff --git a/core/config.c b/core/config.c index 86873517..bded412b 100644 --- a/core/config.c +++ b/core/config.c @@ -795,6 +795,7 @@ char *uwsgi_manage_placeholder(char *key) { // reset state to concat state = concat; } + free(tmp_value); return current_value; } From de82b589520cb7cc4b41476d57b147cbed1e3be0 Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Fri, 22 Nov 2013 19:44:04 +0100 Subject: [PATCH 27/94] plugins/gccgo: fix memory leak Reported by Coverity as CID #1128440 --- plugins/gccgo/gccgo_plugin.c | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/gccgo/gccgo_plugin.c b/plugins/gccgo/gccgo_plugin.c index 482a8f28..7f99bcf2 100644 --- a/plugins/gccgo/gccgo_plugin.c +++ b/plugins/gccgo/gccgo_plugin.c @@ -203,6 +203,7 @@ static void uwsgi_gccgo_initialize() { n++; } runtime_args(argc, argv); + free(argv_list); } else { char *argv[2] = {0,0}; From d0b3c442c106f7a01d1d96286bc330abc41d58b0 Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Fri, 22 Nov 2013 20:13:54 +0100 Subject: [PATCH 28/94] core: Fix unintential integer overflow The expression would be evaluated as 32bit unsigned before promoting to 64bit unsigned so cast the strtoul return value to avoid integer overflows. Please note that the value passed here comes from the user. Reported by Coverity as CID #971244. --- core/uwsgi.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/uwsgi.c b/core/uwsgi.c index a18c0b6b..16713e18 100644 --- a/core/uwsgi.c +++ b/core/uwsgi.c @@ -3708,7 +3708,7 @@ void uwsgi_opt_set_16bit(char *opt, char *value, void *key) { void uwsgi_opt_set_megabytes(char *opt, char *value, void *key) { uint64_t *ptr = (uint64_t *) key; - *ptr = (strtoul(value, NULL, 10)) * 1024 * 1024; + *ptr = (uint64_t)strtoul(value, NULL, 10) * 1024 * 1024; } void uwsgi_opt_set_dyn(char *opt, char *value, void *key) { From 08525a91eefb42f77e41bc70d002a3de0e88a435 Mon Sep 17 00:00:00 2001 From: Unbit Date: Sat, 23 Nov 2013 04:59:12 +0100 Subject: [PATCH 29/94] reverted gccgo argv_list free --- core/config.c | 8 ++++---- plugins/gccgo/gccgo_plugin.c | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/core/config.c b/core/config.c index bded412b..dd638ebf 100644 --- a/core/config.c +++ b/core/config.c @@ -720,18 +720,18 @@ char *uwsgi_manage_placeholder(char *key) { } else if (!strcmp(p, "++")) { if (current_value) { - int64_t tmp_value = strtoll(current_value, NULL, 10); + int64_t tmp_num = strtoll(current_value, NULL, 10); free(current_value); - current_value = uwsgi_64bit2str(tmp_value+1); + current_value = uwsgi_64bit2str(tmp_num+1); } state = concat; continue; } else if (!strcmp(p, "--")) { if (current_value) { - int64_t tmp_value = strtoll(current_value, NULL, 10); + int64_t tmp_num = strtoll(current_value, NULL, 10); free(current_value); - current_value = uwsgi_64bit2str(tmp_value-1); + current_value = uwsgi_64bit2str(tmp_num-1); } state = concat; continue; diff --git a/plugins/gccgo/gccgo_plugin.c b/plugins/gccgo/gccgo_plugin.c index 7f99bcf2..482a8f28 100644 --- a/plugins/gccgo/gccgo_plugin.c +++ b/plugins/gccgo/gccgo_plugin.c @@ -203,7 +203,6 @@ static void uwsgi_gccgo_initialize() { n++; } runtime_args(argc, argv); - free(argv_list); } else { char *argv[2] = {0,0}; From 69c49f713e66669174b002b7bb67f2f9ef1103f3 Mon Sep 17 00:00:00 2001 From: Unbit Date: Sat, 23 Nov 2013 05:09:50 +0100 Subject: [PATCH 30/94] remove zeromq from build profiles --- buildconf/base.ini | 1 - buildconf/minimal.ini | 1 - buildconf/ppa.ini | 1 - buildconf/unbit.ini | 1 - 4 files changed, 4 deletions(-) diff --git a/buildconf/base.ini b/buildconf/base.ini index 31b26814..6dbe8e33 100644 --- a/buildconf/base.ini +++ b/buildconf/base.ini @@ -2,7 +2,6 @@ xml = auto yaml = true json = auto -zeromq = auto ssl = auto pcre = auto routing = auto diff --git a/buildconf/minimal.ini b/buildconf/minimal.ini index 99be3ac2..82afe1e1 100644 --- a/buildconf/minimal.ini +++ b/buildconf/minimal.ini @@ -2,7 +2,6 @@ xml = false yaml = false json = false -zeromq = false ssl = false pcre = false routing = false diff --git a/buildconf/ppa.ini b/buildconf/ppa.ini index 9ce8e737..06c21aff 100644 --- a/buildconf/ppa.ini +++ b/buildconf/ppa.ini @@ -2,7 +2,6 @@ xml = true yaml = true json = true -zeromq = true ssl = true pcre = true routing = true diff --git a/buildconf/unbit.ini b/buildconf/unbit.ini index f695891b..051cf701 100644 --- a/buildconf/unbit.ini +++ b/buildconf/unbit.ini @@ -2,7 +2,6 @@ xml = true yaml = true json = false -zeromq = false ssl = true pcre = true routing = true From 62aee259fadf08604eea3dbd61855f1eb9d99005 Mon Sep 17 00:00:00 2001 From: Unbit Date: Sat, 23 Nov 2013 05:11:02 +0100 Subject: [PATCH 31/94] prepare for 1.9.21 --- uwsgiconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uwsgiconfig.py b/uwsgiconfig.py index 7e4efb5e..288704a9 100644 --- a/uwsgiconfig.py +++ b/uwsgiconfig.py @@ -1,6 +1,6 @@ # uWSGI build system -uwsgi_version = '1.9.20' +uwsgi_version = '1.9.21' import os import re From 18dead4cd2a11176786f7e1465b2eac958fe3fa4 Mon Sep 17 00:00:00 2001 From: Unbit Date: Sat, 23 Nov 2013 07:31:20 +0100 Subject: [PATCH 32/94] added hook-accepting1 --- core/uwsgi.c | 7 ++++++- uwsgi.h | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/core/uwsgi.c b/core/uwsgi.c index 6212df28..20d71ae9 100644 --- a/core/uwsgi.c +++ b/core/uwsgi.c @@ -373,7 +373,8 @@ static struct uwsgi_option uwsgi_base_options[] = { {"hook-as-user-atexit", required_argument, 0, "run the specified hook before app exit and reload", uwsgi_opt_add_string_list, &uwsgi.hook_as_user_atexit, 0}, {"hook-pre-app", required_argument, 0, "run the specified hook before app loading", uwsgi_opt_add_string_list, &uwsgi.hook_pre_app, 0}, {"hook-post-app", required_argument, 0, "run the specified hook after app loading", uwsgi_opt_add_string_list, &uwsgi.hook_post_app, 0}, - {"hook-accepting", required_argument, 0, "run the specified hook after the app enters the accepting phase", uwsgi_opt_add_string_list, &uwsgi.hook_accepting, 0}, + {"hook-accepting", required_argument, 0, "run the specified hook after each worker enter the accepting phase", uwsgi_opt_add_string_list, &uwsgi.hook_accepting, 0}, + {"hook-accepting1", required_argument, 0, "run the specified hook after the first worker enters the accepting phase", uwsgi_opt_add_string_list, &uwsgi.hook_accepting1, 0}, {"hook-as-vassal", required_argument, 0, "run the specified command before exec()ing the vassal", uwsgi_opt_add_string_list, &uwsgi.hook_as_vassal, 0}, {"hook-as-emperor", required_argument, 0, "run the specified command in the emperor after the vassal has been started", uwsgi_opt_add_string_list, &uwsgi.hook_as_emperor, 0}, @@ -3271,6 +3272,10 @@ void uwsgi_ignition() { // run accepting hooks uwsgi_hooks_run(uwsgi.hook_accepting, "accepting", 1); + if (uwsgi.mywid == 1) { + uwsgi_hooks_run(uwsgi.hook_accepting1, "accepting1", 1); + } + if (uwsgi.loop) { void (*u_loop) (void) = uwsgi_get_loop(uwsgi.loop); if (!u_loop) { diff --git a/uwsgi.h b/uwsgi.h index cfe5fb1b..296bb85b 100644 --- a/uwsgi.h +++ b/uwsgi.h @@ -1991,6 +1991,7 @@ struct uwsgi_server { struct uwsgi_string_list *hook_pre_app; struct uwsgi_string_list *hook_post_app; struct uwsgi_string_list *hook_accepting; + struct uwsgi_string_list *hook_accepting1; struct uwsgi_string_list *hook_as_vassal; struct uwsgi_string_list *hook_as_emperor; From c5003561dda8f027682ebe765178604aedcfb7a6 Mon Sep 17 00:00:00 2001 From: Unbit Date: Sat, 23 Nov 2013 07:35:08 +0100 Subject: [PATCH 33/94] added write hook --- core/hooks.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/core/hooks.c b/core/hooks.c index 59b06f82..93af8c9b 100644 --- a/core/hooks.c +++ b/core/hooks.c @@ -78,6 +78,30 @@ static int uwsgi_hook_print(char *arg) { return 0; } +static int uwsgi_hook_write(char *arg) { + char *space = strchr(arg, ' '); + if (!space) { + uwsgi_log("invalid hook write syntax, must be: \n"); + return -1; + } + *space = 0; + int fd = open(arg, O_WRONLY); + if (fd < 0) { + uwsgi_error_open(arg); + *space = ' '; + return -1; + } + *space = ' '; + size_t l = strlen(space+1); + if (write(fd, space+1, l) != (ssize_t) l) { + uwsgi_error("uwsgi_hook_write()/write()"); + close(fd); + return -1; + } + close(fd); + return 0; +} + static int uwsgi_hook_callint(char *arg) { char *space = strchr(arg, ' '); if (space) { @@ -177,6 +201,8 @@ void uwsgi_register_base_hooks() { uwsgi_register_hook("cd", uwsgi_hook_chdir); uwsgi_register_hook("exec", uwsgi_hook_exec); + uwsgi_register_hook("write", uwsgi_hook_write); + uwsgi_register_hook("mount", uwsgi_mount_hook); uwsgi_register_hook("umount", uwsgi_umount_hook); From b6c969254da23c0090ba78c3d6a15caebb103fe7 Mon Sep 17 00:00:00 2001 From: Unbit Date: Sat, 23 Nov 2013 07:40:53 +0100 Subject: [PATCH 34/94] added accepting once variant --- core/uwsgi.c | 8 ++++++++ uwsgi.h | 2 ++ 2 files changed, 10 insertions(+) diff --git a/core/uwsgi.c b/core/uwsgi.c index 20d71ae9..bfd8015f 100644 --- a/core/uwsgi.c +++ b/core/uwsgi.c @@ -375,6 +375,8 @@ static struct uwsgi_option uwsgi_base_options[] = { {"hook-post-app", required_argument, 0, "run the specified hook after app loading", uwsgi_opt_add_string_list, &uwsgi.hook_post_app, 0}, {"hook-accepting", required_argument, 0, "run the specified hook after each worker enter the accepting phase", uwsgi_opt_add_string_list, &uwsgi.hook_accepting, 0}, {"hook-accepting1", required_argument, 0, "run the specified hook after the first worker enters the accepting phase", uwsgi_opt_add_string_list, &uwsgi.hook_accepting1, 0}, + {"hook-accepting-once", required_argument, 0, "run the specified hook after each worker enter the accepting phase (once per-instance)", uwsgi_opt_add_string_list, &uwsgi.hook_accepting_once, 0}, + {"hook-accepting1-once", required_argument, 0, "run the specified hook after the first worker enters the accepting phase (once per instance)", uwsgi_opt_add_string_list, &uwsgi.hook_accepting1_once, 0}, {"hook-as-vassal", required_argument, 0, "run the specified command before exec()ing the vassal", uwsgi_opt_add_string_list, &uwsgi.hook_as_vassal, 0}, {"hook-as-emperor", required_argument, 0, "run the specified command in the emperor after the vassal has been started", uwsgi_opt_add_string_list, &uwsgi.hook_as_emperor, 0}, @@ -3271,9 +3273,15 @@ void uwsgi_ignition() { // run accepting hooks uwsgi_hooks_run(uwsgi.hook_accepting, "accepting", 1); + if (uwsgi.workers[uwsgi.mywid].respawn_count == 1) { + uwsgi_hooks_run(uwsgi.hook_accepting1_once, "accepting-once", 1); + } if (uwsgi.mywid == 1) { uwsgi_hooks_run(uwsgi.hook_accepting1, "accepting1", 1); + if (uwsgi.workers[uwsgi.mywid].respawn_count == 1) { + uwsgi_hooks_run(uwsgi.hook_accepting1_once, "accepting1-once", 1); + } } if (uwsgi.loop) { diff --git a/uwsgi.h b/uwsgi.h index 296bb85b..ecf86f4a 100644 --- a/uwsgi.h +++ b/uwsgi.h @@ -1992,6 +1992,8 @@ struct uwsgi_server { struct uwsgi_string_list *hook_post_app; struct uwsgi_string_list *hook_accepting; struct uwsgi_string_list *hook_accepting1; + struct uwsgi_string_list *hook_accepting_once; + struct uwsgi_string_list *hook_accepting1_once; struct uwsgi_string_list *hook_as_vassal; struct uwsgi_string_list *hook_as_emperor; From f44a36900ab379ec73873bd70c605a2cf34f88e3 Mon Sep 17 00:00:00 2001 From: Unbit Date: Sat, 23 Nov 2013 07:48:50 +0100 Subject: [PATCH 35/94] added unlink command --- core/hooks.c | 9 +++++++++ core/uwsgi.c | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/core/hooks.c b/core/hooks.c index 93af8c9b..c2029265 100644 --- a/core/hooks.c +++ b/core/hooks.c @@ -78,6 +78,14 @@ static int uwsgi_hook_print(char *arg) { return 0; } +static int uwsgi_hook_unlink(char *arg) { + int ret = unlink(arg); + if (ret) { + uwsgi_error("uwsgi_hook_unlink()/unlink()"); + } + return ret; +} + static int uwsgi_hook_write(char *arg) { char *space = strchr(arg, ' '); if (!space) { @@ -202,6 +210,7 @@ void uwsgi_register_base_hooks() { uwsgi_register_hook("exec", uwsgi_hook_exec); uwsgi_register_hook("write", uwsgi_hook_write); + uwsgi_register_hook("unlink", uwsgi_hook_unlink); uwsgi_register_hook("mount", uwsgi_mount_hook); uwsgi_register_hook("umount", uwsgi_umount_hook); diff --git a/core/uwsgi.c b/core/uwsgi.c index bfd8015f..ee890913 100644 --- a/core/uwsgi.c +++ b/core/uwsgi.c @@ -3274,7 +3274,7 @@ void uwsgi_ignition() { // run accepting hooks uwsgi_hooks_run(uwsgi.hook_accepting, "accepting", 1); if (uwsgi.workers[uwsgi.mywid].respawn_count == 1) { - uwsgi_hooks_run(uwsgi.hook_accepting1_once, "accepting-once", 1); + uwsgi_hooks_run(uwsgi.hook_accepting_once, "accepting-once", 1); } if (uwsgi.mywid == 1) { From b4d9eb4f73654543683e7444a33ca048b287a535 Mon Sep 17 00:00:00 2001 From: Unbit Date: Sat, 23 Nov 2013 08:01:56 +0100 Subject: [PATCH 36/94] various fifo improvements --- core/fifo.c | 22 ++++++++++++---------- core/hooks.c | 38 ++++++++++++++++++++++++++++++++++---- 2 files changed, 46 insertions(+), 14 deletions(-) diff --git a/core/fifo.c b/core/fifo.c index ddfa27aa..aa2c39db 100644 --- a/core/fifo.c +++ b/core/fifo.c @@ -24,16 +24,18 @@ static char *uwsgi_fifo_by_slot() { return uwsgi.master_fifo->value; } -static void uwsgi_fifo_set_slot_zero() { uwsgi.master_fifo_slot = 0; } -static void uwsgi_fifo_set_slot_one() { uwsgi.master_fifo_slot = 1; } -static void uwsgi_fifo_set_slot_two() { uwsgi.master_fifo_slot = 2; } -static void uwsgi_fifo_set_slot_three() { uwsgi.master_fifo_slot = 3; } -static void uwsgi_fifo_set_slot_four() { uwsgi.master_fifo_slot = 4; } -static void uwsgi_fifo_set_slot_five() { uwsgi.master_fifo_slot = 5; } -static void uwsgi_fifo_set_slot_six() { uwsgi.master_fifo_slot = 6; } -static void uwsgi_fifo_set_slot_seven() { uwsgi.master_fifo_slot = 7; } -static void uwsgi_fifo_set_slot_eight() { uwsgi.master_fifo_slot = 8; } -static void uwsgi_fifo_set_slot_nine() { uwsgi.master_fifo_slot = 9; } +#define announce_fifo uwsgi_log_verbose("active master fifo is now %s\n", uwsgi_fifo_by_slot()) + +static void uwsgi_fifo_set_slot_zero() { uwsgi.master_fifo_slot = 0; announce_fifo; } +static void uwsgi_fifo_set_slot_one() { uwsgi.master_fifo_slot = 1; announce_fifo; } +static void uwsgi_fifo_set_slot_two() { uwsgi.master_fifo_slot = 2; announce_fifo; } +static void uwsgi_fifo_set_slot_three() { uwsgi.master_fifo_slot = 3; announce_fifo; } +static void uwsgi_fifo_set_slot_four() { uwsgi.master_fifo_slot = 4; announce_fifo; } +static void uwsgi_fifo_set_slot_five() { uwsgi.master_fifo_slot = 5; announce_fifo; } +static void uwsgi_fifo_set_slot_six() { uwsgi.master_fifo_slot = 6; announce_fifo; } +static void uwsgi_fifo_set_slot_seven() { uwsgi.master_fifo_slot = 7; announce_fifo; } +static void uwsgi_fifo_set_slot_eight() { uwsgi.master_fifo_slot = 8; announce_fifo; } +static void uwsgi_fifo_set_slot_nine() { uwsgi.master_fifo_slot = 9; announce_fifo; } /* diff --git a/core/hooks.c b/core/hooks.c index c2029265..49dde6d2 100644 --- a/core/hooks.c +++ b/core/hooks.c @@ -86,23 +86,27 @@ static int uwsgi_hook_unlink(char *arg) { return ret; } -static int uwsgi_hook_write(char *arg) { +static int uwsgi_hook_writefifo(char *arg) { char *space = strchr(arg, ' '); if (!space) { - uwsgi_log("invalid hook write syntax, must be: \n"); + uwsgi_log("invalid hook writefifo syntax, must be: \n"); return -1; } *space = 0; - int fd = open(arg, O_WRONLY); + int fd = open(arg, O_WRONLY|O_NONBLOCK); if (fd < 0) { uwsgi_error_open(arg); *space = ' '; + if (errno == ENODEV) return 0; +#ifdef ENXIO + if (errno == ENXIO) return 0; +#endif return -1; } *space = ' '; size_t l = strlen(space+1); if (write(fd, space+1, l) != (ssize_t) l) { - uwsgi_error("uwsgi_hook_write()/write()"); + uwsgi_error("uwsgi_hook_writefifo()/write()"); close(fd); return -1; } @@ -110,6 +114,31 @@ static int uwsgi_hook_write(char *arg) { return 0; } +static int uwsgi_hook_write(char *arg) { + char *space = strchr(arg, ' '); + if (!space) { + uwsgi_log("invalid hook write syntax, must be: \n"); + return -1; + } + *space = 0; + int fd = open(arg, O_WRONLY); + if (fd < 0) { + uwsgi_error_open(arg); + *space = ' '; + return -1; + } + *space = ' '; + size_t l = strlen(space+1); + if (write(fd, space+1, l) != (ssize_t) l) { + uwsgi_error("uwsgi_hook_write()/write()"); + close(fd); + return -1; + } + close(fd); + return 0; +} + + static int uwsgi_hook_callint(char *arg) { char *space = strchr(arg, ' '); if (space) { @@ -210,6 +239,7 @@ void uwsgi_register_base_hooks() { uwsgi_register_hook("exec", uwsgi_hook_exec); uwsgi_register_hook("write", uwsgi_hook_write); + uwsgi_register_hook("writefifo", uwsgi_hook_writefifo); uwsgi_register_hook("unlink", uwsgi_hook_unlink); uwsgi_register_hook("mount", uwsgi_mount_hook); From 347ef1a5bf746251a24c72fb2f0708b0efa491a1 Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Sat, 23 Nov 2013 16:05:56 +0100 Subject: [PATCH 37/94] plugins/router_hash: Avoid possible divide by zero Reported by Coverity as CID #1100804. --- plugins/router_hash/router_hash.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/router_hash/router_hash.c b/plugins/router_hash/router_hash.c index 812a4992..9092b93b 100644 --- a/plugins/router_hash/router_hash.c +++ b/plugins/router_hash/router_hash.c @@ -56,6 +56,8 @@ static int uwsgi_routing_func_hash(struct wsgi_request *wsgi_req, struct uwsgi_r // skip last semicolon if (urhc->items[ilen-1] == ';') items--; + if (items < 1) return UWSGI_ROUTE_BREAK; + uint32_t hashed_result = h % items; uint32_t found = 0; char *value = urhc->items; From b1f06bce32274cc614cec90555e1e4e415884fff Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Sat, 23 Nov 2013 16:18:09 +0100 Subject: [PATCH 38/94] plugins/rpc_plugin: check return value of uwsgi_response_prepare_headers Reported by Coverity as CID #1100798. --- plugins/rpc/rpc_plugin.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/plugins/rpc/rpc_plugin.c b/plugins/rpc/rpc_plugin.c index d8ecec8b..134d8864 100644 --- a/plugins/rpc/rpc_plugin.c +++ b/plugins/rpc/rpc_plugin.c @@ -112,7 +112,11 @@ static int uwsgi_rpc_xmlrpc(struct wsgi_request *wsgi_req, xmlDoc *doc, char **a xmlChar *xmlbuf; int xlen = 0; xmlDocDumpFormatMemory(rdoc, &xmlbuf, &xlen, 1); - uwsgi_response_prepare_headers(wsgi_req,"200 OK", 6); + if (uwsgi_response_prepare_headers(wsgi_req,"200 OK", 6)) { + xmlFreeDoc(rdoc); + xmlFree(xmlbuf); + return -1; + } uwsgi_response_add_content_length(wsgi_req, xlen); uwsgi_response_add_content_type(wsgi_req, "application/xml", 15); uwsgi_response_write_body_do(wsgi_req, (char *) xmlbuf, xlen); From 0ab1cfacc95abecc2d4894bdd4a1fdc5f0efe28c Mon Sep 17 00:00:00 2001 From: Unbit Date: Tue, 26 Nov 2013 07:13:07 +0100 Subject: [PATCH 39/94] added read errors counters and --log-ioerror --- core/logging.c | 3 +++ core/master_utils.c | 3 +++ core/metrics.c | 3 +++ core/reader.c | 22 ++++++++++++++++++++++ core/utils.c | 1 + core/uwsgi.c | 1 + uwsgi.h | 3 +++ 7 files changed, 36 insertions(+) diff --git a/core/logging.c b/core/logging.c index c08a68c2..8efdf8d7 100644 --- a/core/logging.c +++ b/core/logging.c @@ -575,6 +575,9 @@ void log_request(struct wsgi_request *wsgi_req) { if (uwsgi.shared->options[UWSGI_OPTION_LOG_SENDFILE] && wsgi_req->via == UWSGI_VIA_SENDFILE) { goto logit; } + if (uwsgi.shared->options[UWSGI_OPTION_LOG_IOERROR] && wsgi_req->read_errors > 0 && wsgi_req->write_errors > 0) { + goto logit; + } if (!log_it) return; diff --git a/core/master_utils.c b/core/master_utils.c index 3125bc27..b046d81d 100644 --- a/core/master_utils.c +++ b/core/master_utils.c @@ -1133,6 +1133,9 @@ struct uwsgi_stats *uwsgi_master_generate_stats() { if (uwsgi_stats_keylong_comma(us, "write_errors", (unsigned long long) uc->write_errors)) goto end; + if (uwsgi_stats_keylong_comma(us, "read_errors", (unsigned long long) uc->read_errors)) + goto end; + if (uwsgi_stats_keylong_comma(us, "in_request", (unsigned long long) uc->in_request)) goto end; diff --git a/core/metrics.c b/core/metrics.c index f83f8c7c..0c8741cd 100644 --- a/core/metrics.c +++ b/core/metrics.c @@ -784,6 +784,9 @@ void uwsgi_setup_metrics() { uwsgi_metric_name2("worker.%d.core.%d.exceptions", i, j) ; uwsgi_metric_oid2("3.%d.2.%d.7", i, j); uwsgi_register_metric(buf, buf2, UWSGI_METRIC_COUNTER, "ptr", &uwsgi.workers[i].cores[j].exceptions, 0, NULL); + uwsgi_metric_name2("worker.%d.core.%d.read_errors", i, j) ; uwsgi_metric_oid2("3.%d.2.%d.8", i, j); + uwsgi_register_metric(buf, buf2, UWSGI_METRIC_COUNTER, "ptr", &uwsgi.workers[i].cores[j].read_errors, 0, NULL); + } } diff --git a/core/reader.c b/core/reader.c index 1d402113..6cec6540 100644 --- a/core/reader.c +++ b/core/reader.c @@ -33,6 +33,7 @@ void uwsgi_request_body_seek(struct wsgi_request *wsgi_req, off_t pos) { if (pos < 0) { if (fseek(wsgi_req->post_file, pos, SEEK_CUR)) { uwsgi_error("uwsgi_request_body_seek()/fseek()"); + wsgi_req->read_errors++; } wsgi_req->post_pos = ftell(wsgi_req->post_file); return; @@ -40,6 +41,7 @@ void uwsgi_request_body_seek(struct wsgi_request *wsgi_req, off_t pos) { if (fseek(wsgi_req->post_file, pos, SEEK_SET)) { uwsgi_error("uwsgi_request_body_seek()/fseek()"); + wsgi_req->read_errors++; } wsgi_req->post_pos = ftell(wsgi_req->post_file); return; @@ -140,6 +142,7 @@ static int consume_body_for_readline(struct wsgi_request *wsgi_req) { } if (len == 0) { uwsgi_read_error(remains); + wsgi_req->read_errors++; return -1; } if (len < 0) { @@ -147,6 +150,7 @@ static int consume_body_for_readline(struct wsgi_request *wsgi_req) { goto wait; } uwsgi_read_error(remains); + wsgi_req->read_errors++; return -1; } wait: @@ -159,6 +163,7 @@ wait: return 0; } uwsgi_read_error(remains); + wsgi_req->read_errors++; return -1; } // 0 means timeout @@ -167,6 +172,7 @@ wait: return -1; } uwsgi_read_error(remains); + wsgi_req->read_errors++; return -1; } @@ -205,6 +211,7 @@ char *uwsgi_request_body_readline(struct wsgi_request *wsgi_req, ssize_t hint, s wsgi_req->post_readline_buf = malloc(amount); if (!wsgi_req->post_readline_buf) { uwsgi_error("uwsgi_request_body_readline()/malloc()"); + wsgi_req->read_errors++; *rlen = -1; return NULL; } @@ -217,6 +224,7 @@ char *uwsgi_request_body_readline(struct wsgi_request *wsgi_req, ssize_t hint, s if (wsgi_req->post_pos >= wsgi_req->post_cl) break; if (consume_body_for_readline(wsgi_req)) { + wsgi_req->read_errors++; *rlen = -1; return NULL; } @@ -274,6 +282,7 @@ char *uwsgi_request_body_read(struct wsgi_request *wsgi_req, ssize_t hint, ssize if (!tmp_buf) { uwsgi_error("uwsgi_request_body_read()/realloc()"); *rlen = -1; + wsgi_req->read_errors++; return NULL; } wsgi_req->post_read_buf = tmp_buf; @@ -321,6 +330,7 @@ char *uwsgi_request_body_read(struct wsgi_request *wsgi_req, ssize_t hint, ssize wsgi_req->post_read_buf = malloc(remains); if (!wsgi_req->post_read_buf) { uwsgi_error("uwsgi_request_body_read()/malloc()"); + wsgi_req->read_errors++; *rlen = -1; return NULL; } @@ -332,6 +342,7 @@ char *uwsgi_request_body_read(struct wsgi_request *wsgi_req, ssize_t hint, ssize char *tmp_buf = realloc(wsgi_req->post_read_buf, (remains+*rlen)); if (!tmp_buf) { uwsgi_error("uwsgi_request_body_read()/realloc()"); + wsgi_req->read_errors++; *rlen = -1; return NULL; } @@ -349,6 +360,7 @@ char *uwsgi_request_body_read(struct wsgi_request *wsgi_req, ssize_t hint, ssize if (fread(wsgi_req->post_read_buf + *rlen, remains, 1, wsgi_req->post_file) != 1) { *rlen = -1; uwsgi_error("uwsgi_request_body_read()/fread()"); + wsgi_req->read_errors++; return NULL; } *rlen += remains; @@ -378,6 +390,7 @@ char *uwsgi_request_body_read(struct wsgi_request *wsgi_req, ssize_t hint, ssize } *rlen = -1; uwsgi_read_error(remains); + wsgi_req->read_errors++; return NULL; } wait: @@ -398,6 +411,7 @@ wait: } else { uwsgi_read_error(remains); + wsgi_req->read_errors++; } return NULL; } @@ -409,6 +423,7 @@ wait: } *rlen = -1; uwsgi_read_error(remains); + wsgi_req->read_errors++; return NULL; } @@ -447,6 +462,7 @@ int uwsgi_postbuffer_do_in_mem(struct wsgi_request *wsgi_req) { goto wait; } uwsgi_read_error(remains); + wsgi_req->read_errors++; return -1; } @@ -462,6 +478,7 @@ wait: } if (ret < 0) { uwsgi_read_error(remains); + wsgi_req->read_errors++; return -1; } uwsgi_read_timeout(remains); @@ -483,6 +500,7 @@ int uwsgi_postbuffer_do_in_disk(struct wsgi_request *wsgi_req) { wsgi_req->post_file = uwsgi_tmpfile(); if (!wsgi_req->post_file) { uwsgi_error("uwsgi_postbuffer_do_in_disk()/uwsgi_tmpfile()"); + wsgi_req->read_errors++; return -1; } @@ -518,6 +536,7 @@ int uwsgi_postbuffer_do_in_disk(struct wsgi_request *wsgi_req) { goto wait; } uwsgi_read_error(remains); + wsgi_req->read_errors++; goto end; } @@ -531,11 +550,13 @@ wait: } else { uwsgi_read_error(remains); + wsgi_req->read_errors++; } goto end; } if (ret < 0) { uwsgi_read_error(remains); + wsgi_req->read_errors++; goto end; } uwsgi_read_timeout(remains); @@ -544,6 +565,7 @@ wait: write: if (fwrite(wsgi_req->post_buffering_buf, rlen, 1, wsgi_req->post_file) != 1) { uwsgi_error("uwsgi_postbuffer_do_in_disk()/fwrite()"); + wsgi_req->read_errors++; goto end; } diff --git a/core/utils.c b/core/utils.c index 38e918af..ea94ebf8 100644 --- a/core/utils.c +++ b/core/utils.c @@ -1038,6 +1038,7 @@ void uwsgi_close_request(struct wsgi_request *wsgi_req) { uwsgi.workers[uwsgi.mywid].requests++; uwsgi.workers[uwsgi.mywid].cores[wsgi_req->async_id].requests++; uwsgi.workers[uwsgi.mywid].cores[wsgi_req->async_id].write_errors += wsgi_req->write_errors; + uwsgi.workers[uwsgi.mywid].cores[wsgi_req->async_id].read_errors += wsgi_req->read_errors; // this is used for MAX_REQUESTS uwsgi.workers[uwsgi.mywid].delta_requests++; } diff --git a/core/uwsgi.c b/core/uwsgi.c index ee890913..a80de4e3 100644 --- a/core/uwsgi.c +++ b/core/uwsgi.c @@ -666,6 +666,7 @@ static struct uwsgi_option uwsgi_base_options[] = { {"log-5xx", no_argument, 0, "log requests with a 5xx response", uwsgi_opt_dyn_true, (void *) UWSGI_OPTION_LOG_5xx, 0}, {"log-big", required_argument, 0, "log requestes bigger than the specified size", uwsgi_opt_set_dyn, (void *) UWSGI_OPTION_LOG_BIG, 0}, {"log-sendfile", required_argument, 0, "log sendfile requests", uwsgi_opt_dyn_true, (void *) UWSGI_OPTION_LOG_SENDFILE, 0}, + {"log-ioerror", required_argument, 0, "log requests with io errors", uwsgi_opt_dyn_true, (void *) UWSGI_OPTION_LOG_IOERROR, 0}, {"log-micros", no_argument, 0, "report response time in microseconds instead of milliseconds", uwsgi_opt_true, &uwsgi.log_micros, 0}, {"log-x-forwarded-for", no_argument, 0, "use the ip from X-Forwarded-For header instead of REMOTE_ADDR", uwsgi_opt_true, &uwsgi.log_x_forwarded_for, 0}, {"master-as-root", no_argument, 0, "leave master process running as root", uwsgi_opt_true, &uwsgi.master_as_root, 0}, diff --git a/uwsgi.h b/uwsgi.h index ecf86f4a..99862804 100644 --- a/uwsgi.h +++ b/uwsgi.h @@ -846,6 +846,7 @@ struct uwsgi_opt { #define UWSGI_OPTION_MULE_HARAKIRI 18 #define UWSGI_OPTION_MAX_WORKER_LIFETIME 19 #define UWSGI_OPTION_MIN_WORKER_LIFETIME 20 +#define UWSGI_OPTION_LOG_IOERROR 21 #define UWSGI_SPOOLER_EXTERNAL 1 @@ -1436,6 +1437,7 @@ struct wsgi_request { int suspended; uint64_t write_errors; + uint64_t read_errors; int *ovector; size_t post_cl; @@ -2725,6 +2727,7 @@ struct uwsgi_core { uint64_t offloaded_requests; uint64_t write_errors; + uint64_t read_errors; uint64_t exceptions; pthread_t thread_id; From b3047667183465fe6eb7540a4df0c64e12d19ca9 Mon Sep 17 00:00:00 2001 From: Unbit Date: Tue, 26 Nov 2013 07:20:36 +0100 Subject: [PATCH 40/94] added werr, rerr and ioerr to logvars --- core/logging.c | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/core/logging.c b/core/logging.c index 8efdf8d7..b48ac14c 100644 --- a/core/logging.c +++ b/core/logging.c @@ -1174,6 +1174,21 @@ static ssize_t uwsgi_lf_headers(struct wsgi_request * wsgi_req, char **buf) { return strlen(*buf); } +static ssize_t uwsgi_lf_werr(struct wsgi_request * wsgi_req, char **buf) { + *buf = uwsgi_num2str((int) wsgi_req->write_errors); + return strlen(*buf); +} + +static ssize_t uwsgi_lf_rerr(struct wsgi_request * wsgi_req, char **buf) { + *buf = uwsgi_num2str((int) wsgi_req->read_errors); + return strlen(*buf); +} + +static ssize_t uwsgi_lf_ioerr(struct wsgi_request * wsgi_req, char **buf) { + *buf = uwsgi_num2str((int) (wsgi_req->write_errors + wsgi_req->read_errors)); + return strlen(*buf); +} + void uwsgi_add_logchunk(int variable, int pos, char *ptr, size_t len) { struct uwsgi_logchunk *logchunk = uwsgi.logchunks; @@ -1365,6 +1380,21 @@ void uwsgi_add_logchunk(int variable, int pos, char *ptr, size_t len) { logchunk->func = uwsgi_lf_headers; logchunk->free = 1; } + else if (!uwsgi_strncmp(ptr, len, "werr", 4)) { + logchunk->type = 3; + logchunk->func = uwsgi_lf_werr; + logchunk->free = 1; + } + else if (!uwsgi_strncmp(ptr, len, "rerr", 4)) { + logchunk->type = 3; + logchunk->func = uwsgi_lf_rerr; + logchunk->free = 1; + } + else if (!uwsgi_strncmp(ptr, len, "ioerr", 5)) { + logchunk->type = 3; + logchunk->func = uwsgi_lf_ioerr; + logchunk->free = 1; + } else if (!uwsgi_starts_with(ptr, len, "metric.", 7)) { logchunk->type = 4; logchunk->ptr = uwsgi_concat2n(ptr+7, len - 7, "", 0); From 31c73ce29801af8f14ba211d3ecb87213bf50382 Mon Sep 17 00:00:00 2001 From: Unbit Date: Tue, 26 Nov 2013 09:33:02 +0100 Subject: [PATCH 41/94] report request data in writers and readers --- core/reader.c | 22 +++++++++++----------- core/writer.c | 12 ++++++------ 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/core/reader.c b/core/reader.c index 6cec6540..70441cf7 100644 --- a/core/reader.c +++ b/core/reader.c @@ -32,7 +32,7 @@ void uwsgi_request_body_seek(struct wsgi_request *wsgi_req, off_t pos) { if (wsgi_req->post_file) { if (pos < 0) { if (fseek(wsgi_req->post_file, pos, SEEK_CUR)) { - uwsgi_error("uwsgi_request_body_seek()/fseek()"); + uwsgi_req_error("uwsgi_request_body_seek()/fseek()"); wsgi_req->read_errors++; } wsgi_req->post_pos = ftell(wsgi_req->post_file); @@ -40,7 +40,7 @@ void uwsgi_request_body_seek(struct wsgi_request *wsgi_req, off_t pos) { } if (fseek(wsgi_req->post_file, pos, SEEK_SET)) { - uwsgi_error("uwsgi_request_body_seek()/fseek()"); + uwsgi_req_error("uwsgi_request_body_seek()/fseek()"); wsgi_req->read_errors++; } wsgi_req->post_pos = ftell(wsgi_req->post_file); @@ -97,7 +97,7 @@ static int consume_body_for_readline(struct wsgi_request *wsgi_req) { if (wsgi_req->post_readline_size - wsgi_req->post_readline_watermark < remains) { char *tmp_buf = realloc(wsgi_req->post_readline_buf, wsgi_req->post_readline_size + remains); if (!tmp_buf) { - uwsgi_error("consume_body_for_readline()/realloc()"); + uwsgi_req_error("consume_body_for_readline()/realloc()"); return -1; } wsgi_req->post_readline_buf = tmp_buf; @@ -116,7 +116,7 @@ static int consume_body_for_readline(struct wsgi_request *wsgi_req) { if (wsgi_req->post_file) { size_t ret = fread(wsgi_req->post_readline_buf + wsgi_req->post_readline_watermark, remains, 1, wsgi_req->post_file); if (ret == 0) { - uwsgi_error("consume_body_for_readline()/fread()"); + uwsgi_req_error("consume_body_for_readline()/fread()"); return -1; } wsgi_req->post_pos += remains; @@ -210,7 +210,7 @@ char *uwsgi_request_body_readline(struct wsgi_request *wsgi_req, ssize_t hint, s size_t amount = UMIN(uwsgi.buffer_size, wsgi_req->post_cl); wsgi_req->post_readline_buf = malloc(amount); if (!wsgi_req->post_readline_buf) { - uwsgi_error("uwsgi_request_body_readline()/malloc()"); + uwsgi_req_error("uwsgi_request_body_readline()/malloc()"); wsgi_req->read_errors++; *rlen = -1; return NULL; @@ -280,7 +280,7 @@ char *uwsgi_request_body_read(struct wsgi_request *wsgi_req, ssize_t hint, ssize if (avail > wsgi_req->post_read_buf_size) { char *tmp_buf = realloc(wsgi_req->post_read_buf, avail); if (!tmp_buf) { - uwsgi_error("uwsgi_request_body_read()/realloc()"); + uwsgi_req_error("uwsgi_request_body_read()/realloc()"); *rlen = -1; wsgi_req->read_errors++; return NULL; @@ -329,7 +329,7 @@ char *uwsgi_request_body_read(struct wsgi_request *wsgi_req, ssize_t hint, ssize if (!wsgi_req->post_read_buf) { wsgi_req->post_read_buf = malloc(remains); if (!wsgi_req->post_read_buf) { - uwsgi_error("uwsgi_request_body_read()/malloc()"); + uwsgi_req_error("uwsgi_request_body_read()/malloc()"); wsgi_req->read_errors++; *rlen = -1; return NULL; @@ -341,7 +341,7 @@ char *uwsgi_request_body_read(struct wsgi_request *wsgi_req, ssize_t hint, ssize if ((remains+*rlen) > wsgi_req->post_read_buf_size) { char *tmp_buf = realloc(wsgi_req->post_read_buf, (remains+*rlen)); if (!tmp_buf) { - uwsgi_error("uwsgi_request_body_read()/realloc()"); + uwsgi_req_error("uwsgi_request_body_read()/realloc()"); wsgi_req->read_errors++; *rlen = -1; return NULL; @@ -359,7 +359,7 @@ char *uwsgi_request_body_read(struct wsgi_request *wsgi_req, ssize_t hint, ssize if (wsgi_req->post_file) { if (fread(wsgi_req->post_read_buf + *rlen, remains, 1, wsgi_req->post_file) != 1) { *rlen = -1; - uwsgi_error("uwsgi_request_body_read()/fread()"); + uwsgi_req_error("uwsgi_request_body_read()/fread()"); wsgi_req->read_errors++; return NULL; } @@ -499,7 +499,7 @@ int uwsgi_postbuffer_do_in_disk(struct wsgi_request *wsgi_req) { wsgi_req->post_file = uwsgi_tmpfile(); if (!wsgi_req->post_file) { - uwsgi_error("uwsgi_postbuffer_do_in_disk()/uwsgi_tmpfile()"); + uwsgi_req_error("uwsgi_postbuffer_do_in_disk()/uwsgi_tmpfile()"); wsgi_req->read_errors++; return -1; } @@ -564,7 +564,7 @@ wait: write: if (fwrite(wsgi_req->post_buffering_buf, rlen, 1, wsgi_req->post_file) != 1) { - uwsgi_error("uwsgi_postbuffer_do_in_disk()/fwrite()"); + uwsgi_req_error("uwsgi_postbuffer_do_in_disk()/fwrite()"); wsgi_req->read_errors++; goto end; } diff --git a/core/writer.c b/core/writer.c index 4bbd58d4..e7ee2475 100644 --- a/core/writer.c +++ b/core/writer.c @@ -213,7 +213,7 @@ int uwsgi_response_write_headers_do(struct wsgi_request *wsgi_req) { int ret = wsgi_req->socket->proto_write_headers(wsgi_req, wsgi_req->headers->buf, wsgi_req->headers->pos); if (ret < 0) { if (!uwsgi.ignore_write_errors) { - uwsgi_error("uwsgi_response_write_headers_do()"); + uwsgi_req_error("uwsgi_response_write_headers_do()"); } wsgi_req->write_errors++; return -1; @@ -293,7 +293,7 @@ sendbody: int ret = wsgi_req->socket->proto_write(wsgi_req, buf, len); if (ret < 0) { if (!uwsgi.ignore_write_errors) { - uwsgi_error("uwsgi_response_write_body_do()"); + uwsgi_req_error("uwsgi_response_write_body_do()"); } wsgi_req->write_errors++; return -1; @@ -349,7 +349,7 @@ sendfile: if (len == 0) { struct stat st; if (fstat(fd, &st)) { - uwsgi_error("uwsgi_response_sendfile_do()/fstat()"); + uwsgi_req_error("uwsgi_response_sendfile_do()/fstat()"); wsgi_req->write_errors++; if (can_close) close(fd); return -1; @@ -367,7 +367,7 @@ sendfile: if (!can_close) { int tmp_fd = dup(fd); if (tmp_fd < 0) { - uwsgi_error("uwsgi_response_sendfile_do()/dup()"); + uwsgi_req_error("uwsgi_response_sendfile_do()/dup()"); wsgi_req->write_errors++; return -1; } @@ -391,7 +391,7 @@ sendfile: int ret = wsgi_req->socket->proto_sendfile(wsgi_req, fd, pos, len); if (ret < 0) { if (!uwsgi.ignore_write_errors) { - uwsgi_error("uwsgi_response_sendfile_do()"); + uwsgi_req_error("uwsgi_response_sendfile_do()"); } wsgi_req->write_errors++; if (can_close) close(fd); @@ -457,7 +457,7 @@ int uwsgi_simple_write(struct wsgi_request *wsgi_req, char *buf, size_t len) { int ret = wsgi_req->socket->proto_write(wsgi_req, buf, len); if (ret < 0) { if (!uwsgi.ignore_write_errors) { - uwsgi_error("uwsgi_simple_write()"); + uwsgi_req_error("uwsgi_simple_write()"); } wsgi_req->write_errors++; return -1; From e76a40530b79d418938fe1f060a3110609874c40 Mon Sep 17 00:00:00 2001 From: Unbit Date: Tue, 26 Nov 2013 09:41:15 +0100 Subject: [PATCH 42/94] in request context uwsgi_req_error should always be used --- core/routing.c | 2 +- plugins/transformation_offload/offload.c | 4 ++-- plugins/transformation_tofile/tofile.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/routing.c b/core/routing.c index 2300f941..4513cf47 100644 --- a/core/routing.c +++ b/core/routing.c @@ -962,7 +962,7 @@ static int uwsgi_router_chdir_func(struct wsgi_request *wsgi_req, struct uwsgi_r struct uwsgi_buffer *ub = uwsgi_routing_translate(wsgi_req, ur, *subject, *subject_len, ur->data, ur->data_len); if (!ub) return UWSGI_ROUTE_BREAK; if (chdir(ub->buf)) { - uwsgi_error("uwsgi_router_chdir_func()/chdir()"); + uwsgi_req_error("uwsgi_router_chdir_func()/chdir()"); uwsgi_buffer_destroy(ub); return UWSGI_ROUTE_BREAK; } diff --git a/plugins/transformation_offload/offload.c b/plugins/transformation_offload/offload.c index 52b753d6..c599f56d 100644 --- a/plugins/transformation_offload/offload.c +++ b/plugins/transformation_offload/offload.c @@ -66,7 +66,7 @@ static int transform_offload(struct wsgi_request *wsgi_req, struct uwsgi_transfo if (ut->ub) { ssize_t wlen = write(ut->fd, ut->ub->buf, ut->ub->pos); if (wlen != (ssize_t) ut->ub->pos) { - uwsgi_error("transform_offload/write()"); + uwsgi_req_error("transform_offload/write()"); return -1; } } @@ -76,7 +76,7 @@ static int transform_offload(struct wsgi_request *wsgi_req, struct uwsgi_transfo if (ut->fd > -1) { ssize_t wlen = write(ut->fd, ut->chunk->buf, ut->chunk->pos); if (wlen != (ssize_t) ut->chunk->pos) { - uwsgi_error("transform_offload/write()"); + uwsgi_req_error("transform_offload/write()"); return -1; } ut->len += wlen; diff --git a/plugins/transformation_tofile/tofile.c b/plugins/transformation_tofile/tofile.c index 72b795ac..bb6671e9 100644 --- a/plugins/transformation_tofile/tofile.c +++ b/plugins/transformation_tofile/tofile.c @@ -43,7 +43,7 @@ static int transform_tofile(struct wsgi_request *wsgi_req, struct uwsgi_transfor while(remains) { ssize_t rlen = write(fd, ub->buf + (ub->pos - remains), remains); if (rlen <= 0) { - uwsgi_error("transform_tofile()/write()"); + uwsgi_req_error("transform_tofile()/write()"); unlink(uttc->filename->buf); break; } From c6550ece68e7a74158609ff4b5e0293acf40f673 Mon Sep 17 00:00:00 2001 From: Unbit Date: Wed, 27 Nov 2013 16:01:49 +0100 Subject: [PATCH 43/94] added as-mule hook --- core/mule.c | 1 + core/uwsgi.c | 6 ++++-- uwsgi.h | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/core/mule.c b/core/mule.c index 11fba3b7..d2aa8498 100644 --- a/core/mule.c +++ b/core/mule.c @@ -77,6 +77,7 @@ void uwsgi_mule(int id) { } } + uwsgi_hooks_run(uwsgi.hook_as_mule, "as-mule", 1); uwsgi_mule_run(); } diff --git a/core/uwsgi.c b/core/uwsgi.c index a80de4e3..23c57956 100644 --- a/core/uwsgi.c +++ b/core/uwsgi.c @@ -378,8 +378,10 @@ static struct uwsgi_option uwsgi_base_options[] = { {"hook-accepting-once", required_argument, 0, "run the specified hook after each worker enter the accepting phase (once per-instance)", uwsgi_opt_add_string_list, &uwsgi.hook_accepting_once, 0}, {"hook-accepting1-once", required_argument, 0, "run the specified hook after the first worker enters the accepting phase (once per instance)", uwsgi_opt_add_string_list, &uwsgi.hook_accepting1_once, 0}, - {"hook-as-vassal", required_argument, 0, "run the specified command before exec()ing the vassal", uwsgi_opt_add_string_list, &uwsgi.hook_as_vassal, 0}, - {"hook-as-emperor", required_argument, 0, "run the specified command in the emperor after the vassal has been started", uwsgi_opt_add_string_list, &uwsgi.hook_as_emperor, 0}, + {"hook-as-vassal", required_argument, 0, "run the specified hook before exec()ing the vassal", uwsgi_opt_add_string_list, &uwsgi.hook_as_vassal, 0}, + {"hook-as-emperor", required_argument, 0, "run the specified hook in the emperor after the vassal has been started", uwsgi_opt_add_string_list, &uwsgi.hook_as_emperor, 0}, + + {"hook-as-mule", required_argument, 0, "run the specified hook in each mule", uwsgi_opt_add_string_list, &uwsgi.hook_as_mule, 0}, {"exec-asap", required_argument, 0, "run the specified command as soon as possible", uwsgi_opt_add_string_list, &uwsgi.exec_asap, 0}, {"exec-pre-jail", required_argument, 0, "run the specified command before jailing", uwsgi_opt_add_string_list, &uwsgi.exec_pre_jail, 0}, diff --git a/uwsgi.h b/uwsgi.h index 99862804..38478aeb 100644 --- a/uwsgi.h +++ b/uwsgi.h @@ -1999,6 +1999,7 @@ struct uwsgi_server { struct uwsgi_string_list *hook_as_vassal; struct uwsgi_string_list *hook_as_emperor; + struct uwsgi_string_list *hook_as_mule; struct uwsgi_string_list *exec_asap; From 0574a120bce71e55a9c88586d0934b5eccd94403 Mon Sep 17 00:00:00 2001 From: Roberto De Ioris Date: Thu, 28 Nov 2013 06:04:15 +0100 Subject: [PATCH 44/94] ported mjpeg stream to newer osx --- examples/mjpeg_stream.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/examples/mjpeg_stream.py b/examples/mjpeg_stream.py index 20c33438..0df77daa 100644 --- a/examples/mjpeg_stream.py +++ b/examples/mjpeg_stream.py @@ -1,5 +1,4 @@ import uwsgi - import os def application(env, start_response): @@ -15,12 +14,12 @@ def application(env, start_response): ] ) - yield "--%s\n" % boundary while 1: yield "Content-Type: image/jpeg\r\n\r\n" - uwsgi.sendfile('screenshot.jpg') + print os.system('screencapture -t jpg -m -T 1 screenshot.jpg') + f = open('screenshot.jpg') + yield env['wsgi.file_wrapper'](f) yield "--%s\n" % boundary #os.system('./isightcapture -w 640 -h 480 screenshot.jpg') - #os.system('screencapture -m -T 1 screenshot.jpg') From 150f0c631e6309ba48b47a0a9ab89671aa67e8df Mon Sep 17 00:00:00 2001 From: Roberto De Ioris Date: Thu, 28 Nov 2013 06:47:36 +0100 Subject: [PATCH 45/94] fixed plain (without suspend engine) async mode --- core/async.c | 8 ++++---- plugins/psgi/psgi_response.c | 17 +++++++---------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/core/async.c b/core/async.c index 547e3a96..7f93dbe7 100644 --- a/core/async.c +++ b/core/async.c @@ -295,9 +295,8 @@ void async_schedule_to_req(void) { // a trick to avoid calling routes again uwsgi.wsgi_req->is_routing = 1; #endif - if (uwsgi.p[uwsgi.wsgi_req->uh->modifier1]->request(uwsgi.wsgi_req) <= UWSGI_OK) { - goto end; - } + uwsgi.wsgi_req->async_status = uwsgi.p[uwsgi.wsgi_req->uh->modifier1]->request(uwsgi.wsgi_req); + if (uwsgi.wsgi_req->async_status <= UWSGI_OK) goto end; if (uwsgi.schedule_to_main) { uwsgi.schedule_to_main(uwsgi.wsgi_req); @@ -478,7 +477,8 @@ void async_loop() { // remove fd from event poll and fd proto table uwsgi.async_proto_fd_table[interesting_fd] = NULL; event_queue_del_fd(uwsgi.async_queue, interesting_fd, event_queue_read()); - // put request in the runqueue + // put request in the runqueue (set it as UWSGI_OK to signal the first run) + uwsgi.wsgi_req->async_status = UWSGI_OK; runqueue_push(uwsgi.wsgi_req); continue; } diff --git a/plugins/psgi/psgi_response.c b/plugins/psgi/psgi_response.c index e1faaadd..ecedc97d 100644 --- a/plugins/psgi/psgi_response.c +++ b/plugins/psgi/psgi_response.c @@ -22,15 +22,13 @@ int psgi_response(struct wsgi_request *wsgi_req, AV *response) { return UWSGI_OK; } - if (wsgi_req->async_force_again) { - SvREFCNT_dec(chunk); - return UWSGI_AGAIN; - } - chitem = SvPV( chunk, hlen); if (hlen <= 0) { SvREFCNT_dec(chunk); + if (wsgi_req->async_force_again) { + return UWSGI_AGAIN; + } SV *closed = uwsgi_perl_obj_call(wsgi_req->async_placeholder, "close"); if (closed) { SvREFCNT_dec(closed); @@ -149,13 +147,12 @@ int psgi_response(struct wsgi_request *wsgi_req, AV *response) { } chitem = SvPV( chunk, hlen); - if (uwsgi.async > 1 && wsgi_req->async_force_again) { - SvREFCNT_dec(chunk); - wsgi_req->async_placeholder = (SV *) *hitem; - return UWSGI_AGAIN; - } if (hlen <= 0) { SvREFCNT_dec(chunk); + if (uwsgi.async > 1 && wsgi_req->async_force_again) { + wsgi_req->async_placeholder = (SV *) *hitem; + return UWSGI_AGAIN; + } break; } From 9cc58a3c13cb607569c5c1b0cd6cba40d881cc50 Mon Sep 17 00:00:00 2001 From: Roberto De Ioris Date: Thu, 28 Nov 2013 07:00:18 +0100 Subject: [PATCH 46/94] avoid segfault when async mode api is called without async mode --- core/async.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/core/async.c b/core/async.c index 7f93dbe7..dc11980b 100644 --- a/core/async.c +++ b/core/async.c @@ -173,6 +173,11 @@ static void async_expire_timeouts(uint64_t now) { int async_add_fd_read(struct wsgi_request *wsgi_req, int fd, int timeout) { + if (uwsgi.async < 2){ + uwsgi_log_verbose("ASYNC call without async mode !!!\n"); + return -1; + } + struct uwsgi_async_fd *last_uad = NULL, *uad = wsgi_req->waiting_fds; if (fd < 0) @@ -226,6 +231,11 @@ static int async_wait_fd_read(int fd, int timeout) { void async_add_timeout(struct wsgi_request *wsgi_req, int timeout) { + if (uwsgi.async < 2) { + uwsgi_log_verbose("ASYNC call without async mode !!!\n"); + return; + } + wsgi_req->async_ready_fd = 0; if (timeout > 0 && wsgi_req->async_timeout == NULL) { @@ -236,6 +246,11 @@ void async_add_timeout(struct wsgi_request *wsgi_req, int timeout) { int async_add_fd_write(struct wsgi_request *wsgi_req, int fd, int timeout) { + if (uwsgi.async < 2) { + uwsgi_log_verbose("ASYNC call without async mode !!!\n"); + return -1; + } + struct uwsgi_async_fd *last_uad = NULL, *uad = wsgi_req->waiting_fds; if (fd < 0) From cb1370b1488cbf15c7899ee7525e8c42c6837fa0 Mon Sep 17 00:00:00 2001 From: Roberto De Ioris Date: Thu, 28 Nov 2013 07:12:38 +0100 Subject: [PATCH 47/94] avoid allocating memory structures for fd monitoring and timeouts when non-plain async mode is in place --- core/async.c | 22 ++++++++-------------- core/uwsgi.c | 16 ++++++++++------ 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/core/async.c b/core/async.c index dc11980b..fae63344 100644 --- a/core/async.c +++ b/core/async.c @@ -28,7 +28,6 @@ void uwsgi_async_queue_is_full(time_t now) { } void uwsgi_async_init() { - int i; uwsgi.async_queue = event_queue_init(); @@ -40,16 +39,9 @@ void uwsgi_async_init() { uwsgi.rb_async_timeouts = uwsgi_init_rb_timer(); - // a stack of unused cores - uwsgi.async_queue_unused = uwsgi_malloc(sizeof(struct wsgi_request *) * uwsgi.async); - - // fill it with default values - for (i = 0; i < uwsgi.async; i++) { - uwsgi.async_queue_unused[i] = &uwsgi.workers[uwsgi.mywid].cores[i].req; - } - - // the first available core is the last one - uwsgi.async_queue_unused_ptr = uwsgi.async - 1; + // optimization, this array maps file descriptor to requests + uwsgi.async_waiting_fd_table = uwsgi_calloc(sizeof(struct wsgi_request *) * uwsgi.max_fd); + uwsgi.async_proto_fd_table = uwsgi_calloc(sizeof(struct wsgi_request *) * uwsgi.max_fd); } @@ -173,7 +165,7 @@ static void async_expire_timeouts(uint64_t now) { int async_add_fd_read(struct wsgi_request *wsgi_req, int fd, int timeout) { - if (uwsgi.async < 2){ + if (uwsgi.async < 2 || !uwsgi.async_waiting_fd_table){ uwsgi_log_verbose("ASYNC call without async mode !!!\n"); return -1; } @@ -231,7 +223,7 @@ static int async_wait_fd_read(int fd, int timeout) { void async_add_timeout(struct wsgi_request *wsgi_req, int timeout) { - if (uwsgi.async < 2) { + if (uwsgi.async < 2 || !uwsgi.rb_async_timeouts) { uwsgi_log_verbose("ASYNC call without async mode !!!\n"); return; } @@ -246,7 +238,7 @@ void async_add_timeout(struct wsgi_request *wsgi_req, int timeout) { int async_add_fd_write(struct wsgi_request *wsgi_req, int fd, int timeout) { - if (uwsgi.async < 2) { + if (uwsgi.async < 2 || !uwsgi.async_waiting_fd_table) { uwsgi_log_verbose("ASYNC call without async mode !!!\n"); return -1; } @@ -380,6 +372,8 @@ void async_loop() { void *events = event_queue_alloc(64); struct uwsgi_socket *uwsgi_sock; + uwsgi_async_init(); + uwsgi.async_runqueue = NULL; uwsgi.wait_write_hook = async_wait_fd_write; diff --git a/core/uwsgi.c b/core/uwsgi.c index 23c57956..98a15f08 100644 --- a/core/uwsgi.c +++ b/core/uwsgi.c @@ -2526,9 +2526,6 @@ int uwsgi_start(void *v_argv) { } } uwsgi_log_initial("- async cores set to %d - fd table size: %d\n", uwsgi.async, (int) uwsgi.max_fd); - // optimization, this array maps file descriptor to requests - uwsgi.async_waiting_fd_table = uwsgi_calloc(sizeof(struct wsgi_request *) * uwsgi.max_fd); - uwsgi.async_proto_fd_table = uwsgi_calloc(sizeof(struct wsgi_request *) * uwsgi.max_fd); } #ifdef UWSGI_DEBUG @@ -3167,10 +3164,17 @@ void uwsgi_worker_run() { // some apps could be mounted only on specific workers uwsgi_init_worker_mount_apps(); - //postpone the queue initialization as kevent - //do not pass kfd after fork() if (uwsgi.async > 1) { - uwsgi_async_init(); + // a stack of unused cores + uwsgi.async_queue_unused = uwsgi_malloc(sizeof(struct wsgi_request *) * uwsgi.async); + + // fill it with default values + for (i = 0; i < uwsgi.async; i++) { + uwsgi.async_queue_unused[i] = &uwsgi.workers[uwsgi.mywid].cores[i].req; + } + + // the first available core is the last one + uwsgi.async_queue_unused_ptr = uwsgi.async - 1; } // setup UNIX signals for the worker From 27c3532617e32e1869b5f3d2a11aaf91ccd576e8 Mon Sep 17 00:00:00 2001 From: Unbit Date: Thu, 28 Nov 2013 11:36:28 +0100 Subject: [PATCH 48/94] added S master fifo command --- core/fifo.c | 12 ++++++++++++ core/subscription.c | 2 ++ uwsgi.h | 1 + 3 files changed, 15 insertions(+) diff --git a/core/fifo.c b/core/fifo.c index aa2c39db..b6e4e748 100644 --- a/core/fifo.c +++ b/core/fifo.c @@ -37,6 +37,17 @@ static void uwsgi_fifo_set_slot_seven() { uwsgi.master_fifo_slot = 7; announce_f static void uwsgi_fifo_set_slot_eight() { uwsgi.master_fifo_slot = 8; announce_fifo; } static void uwsgi_fifo_set_slot_nine() { uwsgi.master_fifo_slot = 9; announce_fifo; } +static void subscriptions_blocker() { + if (uwsgi.subscriptions_blocked) { + uwsgi_log_verbose("subscriptions re-enabled\n"); + uwsgi.subscriptions_blocked = 0; + } + else { + uwsgi.subscriptions_blocked = 1; + uwsgi_log_verbose("subscriptions blocked\n"); + } +} + /* this is called as soon as possibile allowing plugins (or hooks) to override it @@ -73,6 +84,7 @@ void uwsgi_master_fifo_prepare() { uwsgi_fifo_table['r'] = grace_them_all; uwsgi_fifo_table['R'] = reap_them_all; uwsgi_fifo_table['s'] = stats; + uwsgi_fifo_table['S'] = subscriptions_blocker; uwsgi_fifo_table['w'] = uwsgi_reload_workers; uwsgi_fifo_table['W'] = uwsgi_brutally_reload_workers; diff --git a/core/subscription.c b/core/subscription.c index 3cf9926c..bf07314a 100644 --- a/core/subscription.c +++ b/core/subscription.c @@ -812,6 +812,8 @@ end: } void uwsgi_subscribe_all(uint8_t cmd, int verbose) { + + if (!uwsgi.subscriptions_blocked) return; // -- subscribe struct uwsgi_string_list *subscriptions = uwsgi.subscriptions; while (subscriptions) { diff --git a/uwsgi.h b/uwsgi.h index 38478aeb..175e3917 100644 --- a/uwsgi.h +++ b/uwsgi.h @@ -2520,6 +2520,7 @@ struct uwsgi_server { struct uwsgi_rpc *rpc_table; // subscription client + int subscriptions_blocked; int subscribe_freq; int subscription_tolerance; int unsubscribe_on_graceful_reload; From 0715031b0068695574ebae1596e6efa300cf1fd0 Mon Sep 17 00:00:00 2001 From: Roberto De Ioris Date: Fri, 29 Nov 2013 07:39:12 +0100 Subject: [PATCH 49/94] prepare for the new sharedarea api --- core/uwsgi.c | 10 +-- plugins/python/python_plugin.c | 2 +- plugins/python/uwsgi_pymodule.c | 153 ++------------------------------ uwsgi.h | 18 +++- uwsgiconfig.py | 2 +- 5 files changed, 28 insertions(+), 157 deletions(-) diff --git a/core/uwsgi.c b/core/uwsgi.c index 98a15f08..a9567138 100644 --- a/core/uwsgi.c +++ b/core/uwsgi.c @@ -259,7 +259,7 @@ static struct uwsgi_option uwsgi_base_options[] = { {"lock-engine", required_argument, 0, "set the lock engine", uwsgi_opt_set_str, &uwsgi.lock_engine, 0}, {"ftok", required_argument, 0, "set the ipcsem key via ftok() for avoiding duplicates", uwsgi_opt_set_str, &uwsgi.ftok, 0}, {"persistent-ipcsem", no_argument, 0, "do not remove ipcsem's on shutdown", uwsgi_opt_true, &uwsgi.persistent_ipcsem, 0}, - {"sharedarea", required_argument, 'A', "create a raw shared memory area of specified pages", uwsgi_opt_set_int, &uwsgi.sharedareasize, 0}, + {"sharedarea", required_argument, 'A', "create a raw shared memory area of specified pages (note: it supports keyval too)", uwsgi_opt_add_string_list, &uwsgi.sharedareas_list, 0}, {"safe-fd", required_argument, 0, "do not close the specified file descriptor", uwsgi_opt_safe_fd, NULL, 0}, {"fd-safe", required_argument, 0, "do not close the specified file descriptor", uwsgi_opt_safe_fd, NULL, 0}, @@ -2548,12 +2548,8 @@ int uwsgi_start(void *v_argv) { // allocate rpc structures uwsgi_rpc_init(); - // setup sharedarea - if (uwsgi.sharedareasize > 0) { - uwsgi.sharedarea = uwsgi_calloc_shared(uwsgi.page_size * uwsgi.sharedareasize); - uwsgi_log("shared area mapped at %p, you can access it with uwsgi.sharedarea* functions.\n", uwsgi.sharedarea); - uwsgi.sa_lock = uwsgi_rwlock_init("sharedarea"); - } + // initialize sharedareas + uwsgi_sharedareas_init(); uwsgi.snmp_lock = uwsgi_lock_init("snmp"); diff --git a/plugins/python/python_plugin.c b/plugins/python/python_plugin.c index 95cbc726..b88c8295 100644 --- a/plugins/python/python_plugin.c +++ b/plugins/python/python_plugin.c @@ -833,7 +833,7 @@ void init_uwsgi_embedded_module() { init_uwsgi_module_spooler(new_uwsgi_module); } - if (uwsgi.sharedareasize > 0 && uwsgi.sharedarea) { + if (uwsgi.sharedareas) { init_uwsgi_module_sharedarea(new_uwsgi_module); } diff --git a/plugins/python/uwsgi_pymodule.c b/plugins/python/uwsgi_pymodule.c index c336b4e8..e20fb226 100644 --- a/plugins/python/uwsgi_pymodule.c +++ b/plugins/python/uwsgi_pymodule.c @@ -1429,33 +1429,11 @@ PyObject *py_uwsgi_extract(PyObject * self, PyObject * args) { PyObject *py_uwsgi_sharedarea_inclong(PyObject * self, PyObject * args) { uint64_t pos = 0; uint64_t value = 1; - uint64_t current_value = 0; - - if (uwsgi.sharedareasize <= 0) { - Py_INCREF(Py_None); - return Py_None; - } if (!PyArg_ParseTuple(args, "l|l:sharedarea_inclong", &pos, &value)) { return NULL; } - if (pos + 8 >= uwsgi.page_size * uwsgi.sharedareasize) { - Py_INCREF(Py_None); - return Py_None; - } - - UWSGI_RELEASE_GIL - - uwsgi_wlock(uwsgi.sa_lock); - - memcpy(¤t_value, uwsgi.sharedarea + pos, 8); - value = current_value + value; - memcpy(uwsgi.sharedarea + pos, &value, 8); - - uwsgi_rwunlock(uwsgi.sa_lock); - UWSGI_GET_GIL - return PyInt_FromLong(value); } @@ -1464,30 +1442,10 @@ PyObject *py_uwsgi_sharedarea_writelong(PyObject * self, PyObject * args) { uint64_t pos = 0; uint64_t value = 0; - if (uwsgi.sharedareasize <= 0) { - Py_INCREF(Py_None); - return Py_None; - } - if (!PyArg_ParseTuple(args, "ll:sharedarea_writelong", &pos, &value)) { return NULL; } - if (pos + 8 >= uwsgi.page_size * uwsgi.sharedareasize) { - Py_INCREF(Py_None); - return Py_None; - } - - UWSGI_RELEASE_GIL - - uwsgi_wlock(uwsgi.sa_lock); - - memcpy(uwsgi.sharedarea + pos, &value, 8); - - uwsgi_rwunlock(uwsgi.sa_lock); - - UWSGI_GET_GIL - return PyInt_FromLong(value); } @@ -1497,31 +1455,10 @@ PyObject *py_uwsgi_sharedarea_write(PyObject * self, PyObject * args) { char *value; Py_ssize_t value_len = 0; - if (uwsgi.sharedareasize <= 0) { - Py_INCREF(Py_None); - return Py_None; - } - if (!PyArg_ParseTuple(args, "ls#:sharedarea_write", &pos, &value, &value_len)) { return NULL; } - if (pos + value_len >= uwsgi.page_size * uwsgi.sharedareasize) { - Py_INCREF(Py_None); - return Py_None; - } - - UWSGI_RELEASE_GIL - - uwsgi_wlock(uwsgi.sa_lock); - - memcpy(uwsgi.sharedarea + pos, value, value_len); - - - uwsgi_rwunlock(uwsgi.sa_lock); - - UWSGI_GET_GIL - return PyInt_FromLong(value_len); @@ -1531,32 +1468,10 @@ PyObject *py_uwsgi_sharedarea_writebyte(PyObject * self, PyObject * args) { uint64_t pos = 0; char value; - if (uwsgi.sharedareasize <= 0) { - Py_INCREF(Py_None); - return Py_None; - } - - if (!PyArg_ParseTuple(args, "lb:sharedarea_writebyte", &pos, &value)) { return NULL; } - if (pos >= uwsgi.page_size * uwsgi.sharedareasize) { - Py_INCREF(Py_None); - return Py_None; - } - - UWSGI_RELEASE_GIL - - uwsgi_wlock(uwsgi.sa_lock); - - uwsgi.sharedarea[pos] = value; - - - uwsgi_rwunlock(uwsgi.sa_lock); - - UWSGI_GET_GIL - return PyInt_FromLong(value); } @@ -1565,30 +1480,11 @@ PyObject *py_uwsgi_sharedarea_readlong(PyObject * self, PyObject * args) { uint64_t pos = 0; uint64_t value; - if (uwsgi.sharedareasize <= 0) { - Py_INCREF(Py_None); - return Py_None; - } - if (!PyArg_ParseTuple(args, "l:sharedarea_readlong", &pos)) { return NULL; } - if (pos + 8 >= uwsgi.page_size * uwsgi.sharedareasize) { - Py_INCREF(Py_None); - return Py_None; - } - - UWSGI_RELEASE_GIL - - uwsgi_wlock(uwsgi.sa_lock); - - memcpy(&value, uwsgi.sharedarea + pos, 8); - - - uwsgi_rwunlock(uwsgi.sa_lock); - - UWSGI_GET_GIL + value = 1; return PyLong_FromLong(value); @@ -1597,32 +1493,15 @@ PyObject *py_uwsgi_sharedarea_readlong(PyObject * self, PyObject * args) { PyObject *py_uwsgi_sharedarea_readbyte(PyObject * self, PyObject * args) { uint64_t pos = 0; - - if (uwsgi.sharedareasize <= 0) { - Py_INCREF(Py_None); - return Py_None; - } + uint8_t byte; if (!PyArg_ParseTuple(args, "l:sharedarea_readbyte", &pos)) { return NULL; } - if (pos >= uwsgi.page_size * uwsgi.sharedareasize) { - Py_INCREF(Py_None); - return Py_None; - } + byte = 1; - UWSGI_RELEASE_GIL - - uwsgi_wlock(uwsgi.sa_lock); - - char value = uwsgi.sharedarea[pos]; - - uwsgi_rwunlock(uwsgi.sa_lock); - - UWSGI_GET_GIL - - return PyInt_FromLong(value); + return PyInt_FromLong(byte); } @@ -1630,38 +1509,20 @@ PyObject *py_uwsgi_sharedarea_read(PyObject * self, PyObject * args) { uint64_t pos = 0; uint64_t len = 1; - if (uwsgi.sharedareasize <= 0) { - Py_INCREF(Py_None); - return Py_None; - } - if (!PyArg_ParseTuple(args, "l|l:sharedarea_read", &pos, &len)) { return NULL; } - if (pos + len >= uwsgi.page_size * uwsgi.sharedareasize) { - Py_INCREF(Py_None); - return Py_None; - } - +/* PyObject *ret = PyString_FromStringAndSize(NULL, len); #ifdef PYTHREE char *storage = PyBytes_AsString(ret); #else char *storage = PyString_AS_STRING(ret); #endif +*/ - UWSGI_RELEASE_GIL - - uwsgi_wlock(uwsgi.sa_lock); - - memcpy(storage, uwsgi.sharedarea + pos, len); - - uwsgi_rwunlock(uwsgi.sa_lock); - - UWSGI_GET_GIL - - return ret; + return NULL; } PyObject *py_uwsgi_spooler_freq(PyObject * self, PyObject * args) { diff --git a/uwsgi.h b/uwsgi.h index 175e3917..82ed4ce6 100644 --- a/uwsgi.h +++ b/uwsgi.h @@ -711,6 +711,16 @@ struct uwsgi_hash_algo *uwsgi_hash_algo_get(char *); void uwsgi_hash_algo_register(char *, uint32_t(*)(char *, uint64_t)); void uwsgi_hash_algo_register_all(void); +struct uwsgi_sharedarea { + int id; + int pages; + int fd; + struct uwsgi_lock_item *lock; + uint64_t max_pos; + uint64_t updates; + uint64_t hits; +}; + // maintain alignment here !!! struct uwsgi_cache_item { // item specific flags @@ -2304,8 +2314,9 @@ struct uwsgi_server { int vec_size; // shared area - char *sharedarea; - uint64_t sharedareasize; + struct uwsgi_string_list *sharedareas_list; + int sharedareas_cnt; + struct uwsgi_sharedarea **sharedareas; // avoid thundering herd in threaded modes pthread_mutex_t thunder_mutex; @@ -2588,6 +2599,7 @@ struct uwsgi_server { int (*wait_write_hook) (int, int); int (*wait_read_hook) (int, int); + int (*wait_milliseconds_hook) (int); struct uwsgi_string_list *schemes; @@ -4546,6 +4558,8 @@ void uwsgi_protocols_register(void); void uwsgi_build_plugin(char *dir); +void uwsgi_sharedareas_init(); + #ifdef __cplusplus } #endif diff --git a/uwsgiconfig.py b/uwsgiconfig.py index 288704a9..9dfb06b5 100644 --- a/uwsgiconfig.py +++ b/uwsgiconfig.py @@ -573,7 +573,7 @@ class uConf(object): 'core/setup_utils', 'core/clock', 'core/init', 'core/buffer', 'core/reader', 'core/writer', 'core/alarm', 'core/cron', 'core/hooks', 'core/plugins', 'core/lock', 'core/cache', 'core/daemons', 'core/errors', 'core/hash', 'core/master_events', 'core/chunked', 'core/queue', 'core/event', 'core/signal', 'core/strings', 'core/progress', 'core/timebomb', 'core/ini', 'core/fsmon', 'core/mount', - 'core/metrics', 'core/plugins_builder', + 'core/metrics', 'core/plugins_builder', 'core/sharedarea', 'core/rpc', 'core/gateway', 'core/loop', 'core/cookie', 'core/querystring', 'core/rb_timers', 'core/transformations', 'core/uwsgi'] # add protocols self.gcc_list.append('proto/base') From 082d47e15647c1e56aece6abd5f39e8d9571a481 Mon Sep 17 00:00:00 2001 From: Roberto De Ioris Date: Fri, 29 Nov 2013 07:46:28 +0100 Subject: [PATCH 50/94] area is a better field name for sharedarea --- core/sharedarea.c | 118 ++++++++++++++++++++++++++++++++++++++++++++++ uwsgi.h | 1 + 2 files changed, 119 insertions(+) create mode 100644 core/sharedarea.c diff --git a/core/sharedarea.c b/core/sharedarea.c new file mode 100644 index 00000000..ca75bf3a --- /dev/null +++ b/core/sharedarea.c @@ -0,0 +1,118 @@ +#include + +extern struct uwsgi_server uwsgi; + +/* + +This is an high-performance memory area shared by all workers/cores/threads + +Contrary to the caching subsystem it is 1-copy (caching for non-c apps is 2-copy) + +Languages not allowing that kind of access should emulate it calling uwsgi_malloc and then copying it back to +the language object. + +The memory areas could be monitored for changes (read: cores can be suspended while waiting for values) + +You can configure multiple areas specifying multiple --sharedarea options + +This is a very low-level api, try to use it to build higher-level primitives or rely on the caching subsystem + +*/ + +struct uwsgi_sharedarea *uwsgi_sharedarea_get_by_id(int id, uint64_t pos) { + if (id > uwsgi.sharedareas_cnt-1) return NULL; + struct uwsgi_sharedarea *sa = uwsgi.sharedareas[id]; + if (pos > sa->max_pos) return NULL; + return sa; +} + +int uwsgi_sharedarea_read(int id, uint64_t pos, char *blob, uint64_t len) { + return -1; +} + +int uwsgi_sharedarea_write(int id, uint64_t pos, char *blob, uint64_t len) { + return -1; +} + +int uwsgi_sharedarea_read64(int id, uint64_t pos, int64_t *value) { + return -1; +} + +int uwsgi_sharedarea_write64(int id, uint64_t pos, int64_t value) { + return -1; +} + +/* + returns: + 0 -> on updates + -1 -> on error + -2 -> on timeout +*/ +int uwsgi_sharedarea_wait(int id, int freq, int timeout) { + int waiting = 0; + struct uwsgi_sharedarea *sa = uwsgi_sharedarea_get_by_id(id, 0); + if (!sa) return -1; + uwsgi_rlock(sa->lock); + uint64_t updates = sa->updates; + uwsgi_rwunlock(sa->lock); + while(timeout == 0 || (timeout > 0 && (waiting/1000) >= timeout)) { + uwsgi.wait_milliseconds_hook(freq); + waiting += freq; + // lock sa + uwsgi_rlock(sa->lock); + if (sa->updates != updates) return 0; + // unlock sa + uwsgi_rwunlock(sa->lock); + } + return -2; +} + +struct uwsgi_sharedarea *uwsgi_sharedarea_init(int id, int pages) { + uwsgi.sharedareas[id] = uwsgi_calloc_shared(uwsgi.page_size * (pages + 1)); + uwsgi.sharedareas[id]->area = ((char *) uwsgi.sharedareas[id]) + uwsgi.page_size; + uwsgi.sharedareas[id]->id = id; + uwsgi.sharedareas[id]->fd = -1; + uwsgi.sharedareas[id]->pages = pages; + uwsgi.sharedareas[id]->max_pos = (uwsgi.page_size * pages) -1; + char *id_str = uwsgi_num2str(id); + uwsgi.sharedareas[id]->lock = uwsgi_rwlock_init(uwsgi_concat2("sharedarea", id_str)); + free(id_str); + return uwsgi.sharedareas[id]; +} + +struct uwsgi_sharedarea *uwsgi_sharedarea_init_keyval(int id, char *arg) { + char *s_pages = NULL; + char *s_file = NULL; + char *s_fd = NULL; + char *s_size = NULL; + if (uwsgi_kvlist_parse(arg, strlen(arg), ',', '=', + "pages", &s_pages, + "file", &s_file, + "fd", &s_fd, + "size", &s_size, + NULL)) { + } + + return NULL; +} + + +void uwsgi_sharedareas_init() { + struct uwsgi_string_list *usl = NULL; + uwsgi_foreach(usl, uwsgi.sharedareas_list) { + uwsgi.sharedareas_cnt++; + } + uwsgi.sharedareas = uwsgi_calloc(sizeof(struct uwsgi_sharedarea *) * uwsgi.sharedareas_cnt); + int id = 0; + uwsgi_foreach(usl, uwsgi.sharedareas_list) { + char *is_keyval = strchr(usl->value, '='); + struct uwsgi_sharedarea *sa = NULL; + if (!is_keyval) { + sa = uwsgi_sharedarea_init(id, atoi(usl->value)); + } + else { + sa = uwsgi_sharedarea_init_keyval(id, usl->value); + } + id++; + } +} diff --git a/uwsgi.h b/uwsgi.h index 82ed4ce6..091bcbc3 100644 --- a/uwsgi.h +++ b/uwsgi.h @@ -716,6 +716,7 @@ struct uwsgi_sharedarea { int pages; int fd; struct uwsgi_lock_item *lock; + char *area; uint64_t max_pos; uint64_t updates; uint64_t hits; From 5182eaf485674787f3c75e27a95b4140d3f9b637 Mon Sep 17 00:00:00 2001 From: Unbit Date: Fri, 29 Nov 2013 12:47:59 +0000 Subject: [PATCH 51/94] another round of sharedarea implementations --- core/clock.c | 5 +++ core/init.c | 1 + core/sharedarea.c | 26 ++++++++++++-- core/websockets.c | 21 ++++++++--- plugins/psgi/uwsgi_plmodule.c | 65 +++++++++++++++++++++++++++++++++++ uwsgi.h | 6 ++++ 6 files changed, 116 insertions(+), 8 deletions(-) diff --git a/core/clock.c b/core/clock.c index 6fbf6c8b..74113c02 100644 --- a/core/clock.c +++ b/core/clock.c @@ -2,6 +2,11 @@ extern struct uwsgi_server uwsgi; +int uwsgi_simple_wait_milliseconds_hook(int timeout) { + return poll(NULL, 0, timeout); +} + + // in the future we will need to use the best clock source for each os/system time_t uwsgi_now() { return uwsgi.clock->seconds(); diff --git a/core/init.c b/core/init.c index 3129cdc2..466fd3c7 100644 --- a/core/init.c +++ b/core/init.c @@ -170,6 +170,7 @@ void uwsgi_init_default() { uwsgi.wait_read_hook = uwsgi_simple_wait_read_hook; uwsgi.wait_write_hook = uwsgi_simple_wait_write_hook; + uwsgi.wait_milliseconds_hook = uwsgi_simple_wait_milliseconds_hook; uwsgi_websockets_init(); diff --git a/core/sharedarea.c b/core/sharedarea.c index ca75bf3a..920f090d 100644 --- a/core/sharedarea.c +++ b/core/sharedarea.c @@ -27,11 +27,25 @@ struct uwsgi_sharedarea *uwsgi_sharedarea_get_by_id(int id, uint64_t pos) { } int uwsgi_sharedarea_read(int id, uint64_t pos, char *blob, uint64_t len) { - return -1; + struct uwsgi_sharedarea *sa = uwsgi_sharedarea_get_by_id(id, pos); + if (!sa) return -1; + if (pos + len > sa->max_pos + 1) return -1; + uwsgi_rlock(sa->lock); + memcpy(blob, sa->area + pos, len); + sa->hits++; + uwsgi_rwunlock(sa->lock); + return 0; } int uwsgi_sharedarea_write(int id, uint64_t pos, char *blob, uint64_t len) { - return -1; + struct uwsgi_sharedarea *sa = uwsgi_sharedarea_get_by_id(id, pos); + if (!sa) return -1; + if (pos + len > sa->max_pos + 1) return -1; + uwsgi_wlock(sa->lock); + memcpy(sa->area + pos, blob, len); + sa->updates++; + uwsgi_rwunlock(sa->lock); + return 0; } int uwsgi_sharedarea_read64(int id, uint64_t pos, int64_t *value) { @@ -60,7 +74,10 @@ int uwsgi_sharedarea_wait(int id, int freq, int timeout) { waiting += freq; // lock sa uwsgi_rlock(sa->lock); - if (sa->updates != updates) return 0; + if (sa->updates != updates) { + uwsgi_rwunlock(sa->lock); + return 0; + } // unlock sa uwsgi_rwunlock(sa->lock); } @@ -113,6 +130,9 @@ void uwsgi_sharedareas_init() { else { sa = uwsgi_sharedarea_init_keyval(id, usl->value); } + if (sa) { + uwsgi_log("sharedaread % initialized at %p\n", id, sa); + } id++; } } diff --git a/core/websockets.c b/core/websockets.c index aea5ac04..8bc7b15f 100644 --- a/core/websockets.c +++ b/core/websockets.c @@ -10,9 +10,9 @@ extern struct uwsgi_server uwsgi; -static struct uwsgi_buffer *uwsgi_websocket_message(char *msg, size_t len) { +static struct uwsgi_buffer *uwsgi_websocket_message(char *msg, size_t len, uint8_t opcode) { struct uwsgi_buffer *ub = uwsgi_buffer_new(10 + len); - if (uwsgi_buffer_u8(ub, 0x81)) goto error; + if (uwsgi_buffer_u8(ub, opcode)) goto error; if (len < 126) { if (uwsgi_buffer_u8(ub, len)) goto error; } @@ -66,8 +66,8 @@ static int uwsgi_websockets_check_pingpong(struct wsgi_request *wsgi_req) { return 0; } -static int uwsgi_websocket_send_do(struct wsgi_request *wsgi_req, char *msg, size_t len) { - struct uwsgi_buffer *ub = uwsgi_websocket_message(msg, len); +static int uwsgi_websocket_send_do(struct wsgi_request *wsgi_req, char *msg, size_t len, uint8_t opcode) { + struct uwsgi_buffer *ub = uwsgi_websocket_message(msg, len, opcode); if (!ub) return -1; ssize_t ret = uwsgi_response_write_body_do(wsgi_req, ub->buf, ub->pos); @@ -80,13 +80,24 @@ int uwsgi_websocket_send(struct wsgi_request *wsgi_req, char *msg, size_t len) { if (wsgi_req->websocket_closed) { return -1; } - ssize_t ret = uwsgi_websocket_send_do(wsgi_req, msg, len); + ssize_t ret = uwsgi_websocket_send_do(wsgi_req, msg, len, 0x81); if (ret < 0) { wsgi_req->websocket_closed = 1; } return ret; } +int uwsgi_websocket_send_binary(struct wsgi_request *wsgi_req, char *msg, size_t len) { + if (wsgi_req->websocket_closed) { + return -1; + } + ssize_t ret = uwsgi_websocket_send_do(wsgi_req, msg, len, 0x82); + if (ret < 0) { + wsgi_req->websocket_closed = 1; + } + return ret; +} + static void uwsgi_websocket_parse_header(struct wsgi_request *wsgi_req) { uint8_t byte1 = wsgi_req->websocket_buf->buf[0]; uint8_t byte2 = wsgi_req->websocket_buf->buf[1]; diff --git a/plugins/psgi/uwsgi_plmodule.c b/plugins/psgi/uwsgi_plmodule.c index 6d25f4c9..c3a241b1 100644 --- a/plugins/psgi/uwsgi_plmodule.c +++ b/plugins/psgi/uwsgi_plmodule.c @@ -498,6 +498,26 @@ XS(XS_websocket_send) { XSRETURN_UNDEF; } +XS(XS_websocket_send_binary) { + dXSARGS; + + char *message = NULL; + STRLEN message_len = 0; + + psgi_check_args(1); + + message = SvPV(ST(0), message_len); + + struct wsgi_request *wsgi_req = current_wsgi_req(); + + if (uwsgi_websocket_send_binary(wsgi_req, message, message_len)) { + croak("unable to send websocket binary message"); + } + + XSRETURN_UNDEF; +} + + XS(XS_websocket_recv) { dXSARGS; @@ -668,6 +688,47 @@ XS(XS_metric_get) { XSRETURN(1); } +XS(XS_sharedarea_wait) { + dXSARGS; + int id; + + psgi_check_args(1); + + id = SvIV(ST(0)); + + if (uwsgi_sharedarea_wait(id, 100, 0)) { + croak("unable to wait for sharedarea %d", id); + XSRETURN_UNDEF; + } + XSRETURN_YES; +} + +XS(XS_sharedarea_read) { + dXSARGS; + int id; + uint64_t pos; + uint64_t len = 1; + psgi_check_args(2); + + id = SvIV(ST(0)); + pos = SvIV(ST(1)); + + if (items > 2) { + len = SvIV(ST(2)); + } + + char *buf = uwsgi_malloc(len); + if (uwsgi_sharedarea_read(id, pos, buf, len)) { + free(buf); + croak("unable to read from sharedarea %d", id); + XSRETURN_UNDEF; + } + + ST(0) = sv_newmortal(); + sv_usepvn(ST(0), buf,len); + XSRETURN(1); +} + XS(XS_chunked_read) { dXSARGS; int timeout = 0; @@ -740,6 +801,7 @@ void init_perl_embedded_module() { psgi_xs(websocket_recv); psgi_xs(websocket_recv_nb); psgi_xs(websocket_send); + psgi_xs(websocket_send_binary); psgi_xs(postfork); psgi_xs(atexit); @@ -757,5 +819,8 @@ void init_perl_embedded_module() { psgi_xs(chunked_read); psgi_xs(chunked_read_nb); + + psgi_xs(sharedarea_read); + psgi_xs(sharedarea_wait); } diff --git a/uwsgi.h b/uwsgi.h index 091bcbc3..a4c0a8c9 100644 --- a/uwsgi.h +++ b/uwsgi.h @@ -4165,6 +4165,7 @@ void uwsgi_subscribe_all(uint8_t, int); void uwsgi_websockets_init(void); int uwsgi_websocket_send(struct wsgi_request *, char *, size_t); +int uwsgi_websocket_send_binary(struct wsgi_request *, char *, size_t); struct uwsgi_buffer *uwsgi_websocket_recv(struct wsgi_request *); struct uwsgi_buffer *uwsgi_websocket_recv_nb(struct wsgi_request *); @@ -4188,6 +4189,7 @@ struct uwsgi_buffer *uwsgi_proto_base_add_header(struct wsgi_request *, char *, int uwsgi_simple_wait_write_hook(int, int); int uwsgi_simple_wait_read_hook(int, int); +int uwsgi_simple_wait_milliseconds_hook(int); int uwsgi_response_write_headers_do(struct wsgi_request *); char *uwsgi_request_body_read(struct wsgi_request *, ssize_t , ssize_t *); char *uwsgi_request_body_readline(struct wsgi_request *, ssize_t, ssize_t *); @@ -4561,6 +4563,10 @@ void uwsgi_build_plugin(char *dir); void uwsgi_sharedareas_init(); +int uwsgi_sharedarea_read(int, uint64_t, char *, uint64_t); +int uwsgi_sharedarea_write(int, uint64_t, char *, uint64_t); +int uwsgi_sharedarea_wait(int, int, int); + #ifdef __cplusplus } #endif From e4d3d209c8efdfa6e0d1edb881405c03d17a0fd7 Mon Sep 17 00:00:00 2001 From: Unbit Date: Fri, 29 Nov 2013 13:32:30 +0000 Subject: [PATCH 52/94] allows sharedarea_wait to specify the freq --- core/sharedarea.c | 1 + plugins/psgi/uwsgi_plmodule.c | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/core/sharedarea.c b/core/sharedarea.c index 920f090d..d7d094c2 100644 --- a/core/sharedarea.c +++ b/core/sharedarea.c @@ -66,6 +66,7 @@ int uwsgi_sharedarea_wait(int id, int freq, int timeout) { int waiting = 0; struct uwsgi_sharedarea *sa = uwsgi_sharedarea_get_by_id(id, 0); if (!sa) return -1; + if (!freq) freq = 100; uwsgi_rlock(sa->lock); uint64_t updates = sa->updates; uwsgi_rwunlock(sa->lock); diff --git a/plugins/psgi/uwsgi_plmodule.c b/plugins/psgi/uwsgi_plmodule.c index c3a241b1..3b2ec978 100644 --- a/plugins/psgi/uwsgi_plmodule.c +++ b/plugins/psgi/uwsgi_plmodule.c @@ -691,12 +691,16 @@ XS(XS_metric_get) { XS(XS_sharedarea_wait) { dXSARGS; int id; + int timeout = 0; psgi_check_args(1); id = SvIV(ST(0)); + if (items > 1) { + timeout = SvIV(ST(1)); + } - if (uwsgi_sharedarea_wait(id, 100, 0)) { + if (uwsgi_sharedarea_wait(id, timeout, 0)) { croak("unable to wait for sharedarea %d", id); XSRETURN_UNDEF; } From f6923728d869f34ee8129cb10d9b7df8db43f359 Mon Sep 17 00:00:00 2001 From: Unbit Date: Fri, 29 Nov 2013 16:07:27 +0100 Subject: [PATCH 53/94] more sharedarea api functions --- core/sharedarea.c | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/core/sharedarea.c b/core/sharedarea.c index d7d094c2..73cc9df2 100644 --- a/core/sharedarea.c +++ b/core/sharedarea.c @@ -49,11 +49,36 @@ int uwsgi_sharedarea_write(int id, uint64_t pos, char *blob, uint64_t len) { } int uwsgi_sharedarea_read64(int id, uint64_t pos, int64_t *value) { - return -1; + return uwsgi_sharedarea_read(id, pos, (char *) value, 8); } -int uwsgi_sharedarea_write64(int id, uint64_t pos, int64_t value) { - return -1; +int uwsgi_sharedarea_write64(int id, uint64_t pos, int64_t *value) { + return uwsgi_sharedarea_write(id, pos, (char *) value, 8); +} + +int uwsgi_sharedarea_read8(int id, uint64_t pos, int8_t *value) { + return uwsgi_sharedarea_read(id, pos, (char *) value, 1); +} + +int uwsgi_sharedarea_write8(int id, uint64_t pos, int8_t *value) { + return uwsgi_sharedarea_write(id, pos, (char *) value, 1); +} + +int uwsgi_sharedarea_read16(int id, uint64_t pos, int16_t *value) { + return uwsgi_sharedarea_read(id, pos, (char *) value, 2); +} + +int uwsgi_sharedarea_write16(int id, uint64_t pos, int16_t *value) { + return uwsgi_sharedarea_write(id, pos, (char *) value, 2); +} + + +int uwsgi_sharedarea_read32(int id, uint64_t pos, int32_t *value) { + return uwsgi_sharedarea_read(id, pos, (char *) value, 4); +} + +int uwsgi_sharedarea_write32(int id, uint64_t pos, int32_t *value) { + return uwsgi_sharedarea_write(id, pos, (char *) value, 4); } /* From 6fdeecce0f0923ed0c2468647f3e7afbc0592d90 Mon Sep 17 00:00:00 2001 From: Unbit Date: Fri, 29 Nov 2013 16:33:39 +0100 Subject: [PATCH 54/94] added milliseconds wait hook to Coro::AnyEvent --- plugins/coroae/coroae.c | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/plugins/coroae/coroae.c b/plugins/coroae/coroae.c index 1e05e6e5..e3616a42 100644 --- a/plugins/coroae/coroae.c +++ b/plugins/coroae/coroae.c @@ -65,6 +65,33 @@ SV * coroae_coro_new(CV *block) { return newobj; } +static int coroae_wait_milliseconds(int timeout) { + int ret = -1; + dSP; + ENTER; + SAVETMPS; + PUSHMARK(SP); + XPUSHs(newSVnv(((double)timeout)/1000.0)); + PUTBACK; + call_pv("Coro::AnyEvent::sleep", G_SCALAR); + SPAGAIN; + if(SvTRUE(ERRSV)) { + uwsgi_log("[uwsgi-perl error] %s", SvPV_nolen(ERRSV)); + } + else { + SV *p_ret = POPs; + if (SvTRUE(p_ret)) { + ret = 0; + } + } + PUTBACK; + FREETMPS; + LEAVE; + + return ret; +} + + static int coroae_wait_fd_read(int fd, int timeout) { int ret = 0; dSP; @@ -377,6 +404,7 @@ static void coroae_loop() { uwsgi.current_wsgi_req = coroae_current_wsgi_req; uwsgi.wait_write_hook = coroae_wait_fd_write; uwsgi.wait_read_hook = coroae_wait_fd_read; + uwsgi.wait_milliseconds_hook = coroae_wait_milliseconds; I_CORO_API("uwsgi::coroae"); From 88b5f968a480c7dbd58b24a755a95f371d6cf98e Mon Sep 17 00:00:00 2001 From: Unbit Date: Fri, 29 Nov 2013 16:43:07 +0100 Subject: [PATCH 55/94] added milliseconds hook support to gevent --- plugins/gevent/gevent.c | 1 + plugins/gevent/gevent.h | 1 + plugins/gevent/hooks.c | 39 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+) diff --git a/plugins/gevent/gevent.c b/plugins/gevent/gevent.c index e50a8f9a..d4df77ef 100644 --- a/plugins/gevent/gevent.c +++ b/plugins/gevent/gevent.c @@ -361,6 +361,7 @@ static void gevent_loop() { uwsgi.wait_write_hook = uwsgi_gevent_wait_write_hook; uwsgi.wait_read_hook = uwsgi_gevent_wait_read_hook; + uwsgi.wait_milliseconds_hook = uwsgi_gevent_wait_milliseconds_hook; struct uwsgi_socket *uwsgi_sock = uwsgi.sockets; diff --git a/plugins/gevent/gevent.h b/plugins/gevent/gevent.h index 8f794c0c..f7687d88 100644 --- a/plugins/gevent/gevent.h +++ b/plugins/gevent/gevent.h @@ -2,6 +2,7 @@ int uwsgi_gevent_wait_write_hook(int, int); int uwsgi_gevent_wait_read_hook(int, int); +int uwsgi_gevent_wait_milliseconds_hook(int); #define GEVENT_SWITCH PyObject *gswitch = python_call(ugevent.greenlet_switch, ugevent.greenlet_switch_args, 0, NULL); Py_DECREF(gswitch) #define GET_CURRENT_GREENLET python_call(ugevent.get_current, ugevent.get_current_args, 0, NULL) diff --git a/plugins/gevent/hooks.c b/plugins/gevent/hooks.c index 1c213905..e96d0fe4 100644 --- a/plugins/gevent/hooks.c +++ b/plugins/gevent/hooks.c @@ -98,3 +98,42 @@ int uwsgi_gevent_wait_read_hook(int fd, int timeout) { return 1; } +int uwsgi_gevent_wait_milliseconds_hook(int timeout) { + + PyObject *ret = NULL; + + PyObject *timer = PyObject_CallMethod(ugevent.hub_loop, "timer", "f", ((double) timeout)/1000.0); + if (!timer) return -1; + + PyObject *current_greenlet = GET_CURRENT_GREENLET; + PyObject *current = PyObject_GetAttrString(current_greenlet, "switch"); + + ret = PyObject_CallMethod(timer, "start", "OO", current, timer); + if (!ret) { + Py_DECREF(current); Py_DECREF(current_greenlet); + Py_DECREF(timer); + return -1; + } + Py_DECREF(ret); + + ret = PyObject_CallMethod(ugevent.hub, "switch", NULL); + if (!ret) { + ret = PyObject_CallMethod(timer, "stop", NULL); + if (ret) { Py_DECREF(ret); } + Py_DECREF(current); Py_DECREF(current_greenlet); + Py_DECREF(timer); + return -1; + } + Py_DECREF(ret); + + if (ret == timer) { + ret = PyObject_CallMethod(timer, "stop", NULL); + if (ret) { Py_DECREF(ret); } + Py_DECREF(current); Py_DECREF(current_greenlet); + Py_DECREF(timer); + return 0; + } + + return -1; +} + From f55c52eaff1eb20a7e25296fc3f95ff7faa31ddb Mon Sep 17 00:00:00 2001 From: Unbit Date: Fri, 29 Nov 2013 16:47:33 +0100 Subject: [PATCH 56/94] added milliseconds hook support for rbthreads --- plugins/rbthreads/rbthreads.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/plugins/rbthreads/rbthreads.c b/plugins/rbthreads/rbthreads.c index 8cf65b4f..6ec15f51 100644 --- a/plugins/rbthreads/rbthreads.c +++ b/plugins/rbthreads/rbthreads.c @@ -37,6 +37,7 @@ struct uwsgi_rbthreads { int rbthreads; int (*orig_wait_write_hook) (int, int); int (*orig_wait_read_hook) (int, int); + int (*orig_wait_milliseconds_hook) (int); } urbts; static struct uwsgi_option rbthreads_options[] = { @@ -142,6 +143,20 @@ static int rbthreads_wait_fd_read(int fd, int timeout) { return urbt.ret; } +static void *rbthreads_wait_milliseconds_do(void *arg) { + struct uwsgi_rbthread *urbt = (struct uwsgi_rbthread *) arg; + urbt->ret = urbts.orig_wait_milliseconds_hook(urbt->timeout); + return NULL; +} + +static int rbthreads_wait_milliseconds(int timeout) { + struct uwsgi_rbthread urbt; + urbt.timeout = timeout; + rb_thread_call_without_gvl(rbthreads_wait_milliseconds_do, &urbt, NULL, NULL); + return urbt.ret; +} + + static void rbthreads_loop() { struct uwsgi_plugin *rup = uwsgi_plugin_get("rack"); // disable init_thread warning @@ -152,8 +167,10 @@ static void rbthreads_loop() { // override read/write nb hooks urbts.orig_wait_write_hook = uwsgi.wait_write_hook; urbts.orig_wait_read_hook = uwsgi.wait_read_hook; + urbts.orig_wait_milliseconds_hook = uwsgi.wait_milliseconds_hook; uwsgi.wait_write_hook = rbthreads_wait_fd_write; uwsgi.wait_read_hook = rbthreads_wait_fd_read; + uwsgi.wait_milliseconds_hook = rbthreads_wait_milliseconds; int i; for(i=1;i Date: Fri, 29 Nov 2013 17:07:11 +0100 Subject: [PATCH 57/94] split setup from running for easy library integration --- core/uwsgi.c | 23 ++++++++++++++--------- lib/linux_ns.c | 7 ++++++- uwsgi.h | 3 +++ 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/core/uwsgi.c b/core/uwsgi.c index a9567138..0bbcb755 100644 --- a/core/uwsgi.c +++ b/core/uwsgi.c @@ -1852,14 +1852,19 @@ void uwsgi_init_random() { #ifdef UWSGI_AS_SHARED_LIBRARY int uwsgi_init(int argc, char *argv[], char *envp[]) { +#else +int main(int argc, char *argv[], char *envp[]) { +#endif + uwsgi_setup(argc, argv, envp); + return uwsgi_run(); +} +void uwsgi_setup(int argc, char *argv[], char *envp[]) { +#ifdef UWSGI_AS_SHARED_LIBRARY #ifdef __APPLE__ char ***envPtr = _NSGetEnviron(); environ = *envPtr; #endif - -#else -int main(int argc, char *argv[], char *envp[]) { #endif int i; @@ -2318,7 +2323,6 @@ int main(int argc, char *argv[], char *envp[]) { } } - // TODO pluginize basic Linux namespace support #if defined(__linux__) && !defined(__ia64__) if (uwsgi.ns) { @@ -2331,10 +2335,6 @@ int main(int argc, char *argv[], char *envp[]) { #if defined(__linux__) && !defined(__ia64__) } #endif - - - // never here - return 0; } @@ -3044,9 +3044,14 @@ next2: // END OF INITIALIZATION + return 0; + +} + +int uwsgi_run() { // !!! from now on, we could be in the master or in a worker !!! - + int i; if (getpid() == masterpid && uwsgi.master_process == 1) { #ifdef UWSGI_AS_SHARED_LIBRARY diff --git a/lib/linux_ns.c b/lib/linux_ns.c index 474c2529..65ee105e 100644 --- a/lib/linux_ns.c +++ b/lib/linux_ns.c @@ -52,6 +52,11 @@ int uwsgi_is_a_keep_mount(char *mp) { } +static int uwsgi_ns_start(void *v_argv) { + uwsgi_start(v_argv); + return uwsgi_run(); +} + void linux_namespace_start(void *argv) { for (;;) { char stack[PTHREAD_STACK_MIN]; @@ -61,7 +66,7 @@ void linux_namespace_start(void *argv) { if (uwsgi.ns_net) { clone_flags |= CLONE_NEWNET; } - pid_t pid = clone(uwsgi_start, stack + PTHREAD_STACK_MIN, clone_flags, (void *) argv); + pid_t pid = clone(uwsgi_ns_start, stack + PTHREAD_STACK_MIN, clone_flags, (void *) argv); if (pid == -1) { uwsgi_error("clone()"); exit(1); diff --git a/uwsgi.h b/uwsgi.h index a4c0a8c9..fb27e14f 100644 --- a/uwsgi.h +++ b/uwsgi.h @@ -4567,6 +4567,9 @@ int uwsgi_sharedarea_read(int, uint64_t, char *, uint64_t); int uwsgi_sharedarea_write(int, uint64_t, char *, uint64_t); int uwsgi_sharedarea_wait(int, int, int); +void uwsgi_setup(int, char **, char **); +int uwsgi_run(void); + #ifdef __cplusplus } #endif From a69c775cd5ea045121bdff23f6a5693994a0838c Mon Sep 17 00:00:00 2001 From: Unbit Date: Fri, 29 Nov 2013 17:17:36 +0100 Subject: [PATCH 58/94] added inc variants for sharedarea --- core/sharedarea.c | 51 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/core/sharedarea.c b/core/sharedarea.c index 73cc9df2..f161b31c 100644 --- a/core/sharedarea.c +++ b/core/sharedarea.c @@ -81,6 +81,57 @@ int uwsgi_sharedarea_write32(int id, uint64_t pos, int32_t *value) { return uwsgi_sharedarea_write(id, pos, (char *) value, 4); } +int uwsgi_sharedarea_inc8(int id, uint64_t pos, int8_t amount) { + struct uwsgi_sharedarea *sa = uwsgi_sharedarea_get_by_id(id, pos); + if (!sa) return -1; + if (pos + 1 > sa->max_pos + 1) return -1; + uwsgi_wlock(sa->lock); + uint8_t *n_ptr = (uint8_t *) (sa->area + pos); + *n_ptr+=amount; + sa->updates++; + uwsgi_rwunlock(sa->lock); + return 0; +} + +int uwsgi_sharedarea_inc16(int id, uint64_t pos, int16_t amount) { + struct uwsgi_sharedarea *sa = uwsgi_sharedarea_get_by_id(id, pos); + if (!sa) return -1; + if (pos + 2 > sa->max_pos + 1) return -1; + uwsgi_wlock(sa->lock); + uint16_t *n_ptr = (uint16_t *) (sa->area + pos); + *n_ptr+=amount; + sa->updates++; + uwsgi_rwunlock(sa->lock); + return 0; +} + +int uwsgi_sharedarea_inc32(int id, uint64_t pos, int32_t amount) { + struct uwsgi_sharedarea *sa = uwsgi_sharedarea_get_by_id(id, pos); + if (!sa) return -1; + if (pos + 4 > sa->max_pos + 1) return -1; + uwsgi_wlock(sa->lock); + uint32_t *n_ptr = (uint32_t *) (sa->area + pos); + *n_ptr+=amount; + sa->updates++; + uwsgi_rwunlock(sa->lock); + return 0; +} + +int uwsgi_sharedarea_inc64(int id, uint64_t pos, int64_t amount) { + struct uwsgi_sharedarea *sa = uwsgi_sharedarea_get_by_id(id, pos); + if (!sa) return -1; + if (pos + 4 > sa->max_pos + 1) return -1; + uwsgi_wlock(sa->lock); + uint32_t *n_ptr = (uint32_t *) (sa->area + pos); + *n_ptr+=amount; + sa->updates++; + uwsgi_rwunlock(sa->lock); + return 0; +} + + + + /* returns: 0 -> on updates From d17d947cd1a293719864175f0fbf18dec1352eef Mon Sep 17 00:00:00 2001 From: Unbit Date: Fri, 29 Nov 2013 17:21:15 +0100 Subject: [PATCH 59/94] added dec variants for sharedarea --- core/sharedarea.c | 48 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/core/sharedarea.c b/core/sharedarea.c index f161b31c..be58de3f 100644 --- a/core/sharedarea.c +++ b/core/sharedarea.c @@ -130,6 +130,54 @@ int uwsgi_sharedarea_inc64(int id, uint64_t pos, int64_t amount) { } +int uwsgi_sharedarea_dec8(int id, uint64_t pos, int8_t amount) { + struct uwsgi_sharedarea *sa = uwsgi_sharedarea_get_by_id(id, pos); + if (!sa) return -1; + if (pos + 1 > sa->max_pos + 1) return -1; + uwsgi_wlock(sa->lock); + uint8_t *n_ptr = (uint8_t *) (sa->area + pos); + *n_ptr-=amount; + sa->updates++; + uwsgi_rwunlock(sa->lock); + return 0; +} + +int uwsgi_sharedarea_dec16(int id, uint64_t pos, int16_t amount) { + struct uwsgi_sharedarea *sa = uwsgi_sharedarea_get_by_id(id, pos); + if (!sa) return -1; + if (pos + 2 > sa->max_pos + 1) return -1; + uwsgi_wlock(sa->lock); + uint16_t *n_ptr = (uint16_t *) (sa->area + pos); + *n_ptr-=amount; + sa->updates++; + uwsgi_rwunlock(sa->lock); + return 0; +} + +int uwsgi_sharedarea_dec32(int id, uint64_t pos, int32_t amount) { + struct uwsgi_sharedarea *sa = uwsgi_sharedarea_get_by_id(id, pos); + if (!sa) return -1; + if (pos + 4 > sa->max_pos + 1) return -1; + uwsgi_wlock(sa->lock); + uint32_t *n_ptr = (uint32_t *) (sa->area + pos); + *n_ptr-=amount; + sa->updates++; + uwsgi_rwunlock(sa->lock); + return 0; +} + +int uwsgi_sharedarea_dec64(int id, uint64_t pos, int64_t amount) { + struct uwsgi_sharedarea *sa = uwsgi_sharedarea_get_by_id(id, pos); + if (!sa) return -1; + if (pos + 4 > sa->max_pos + 1) return -1; + uwsgi_wlock(sa->lock); + uint32_t *n_ptr = (uint32_t *) (sa->area + pos); + *n_ptr-=amount; + sa->updates++; + uwsgi_rwunlock(sa->lock); + return 0; +} + /* From 890e0004b6e9d5031f04f1dadecbd74fea39cfcd Mon Sep 17 00:00:00 2001 From: Unbit Date: Fri, 29 Nov 2013 17:28:05 +0100 Subject: [PATCH 60/94] fixed signs in sharedarea --- core/sharedarea.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/core/sharedarea.c b/core/sharedarea.c index be58de3f..765a6026 100644 --- a/core/sharedarea.c +++ b/core/sharedarea.c @@ -86,7 +86,7 @@ int uwsgi_sharedarea_inc8(int id, uint64_t pos, int8_t amount) { if (!sa) return -1; if (pos + 1 > sa->max_pos + 1) return -1; uwsgi_wlock(sa->lock); - uint8_t *n_ptr = (uint8_t *) (sa->area + pos); + int8_t *n_ptr = (int8_t *) (sa->area + pos); *n_ptr+=amount; sa->updates++; uwsgi_rwunlock(sa->lock); @@ -98,7 +98,7 @@ int uwsgi_sharedarea_inc16(int id, uint64_t pos, int16_t amount) { if (!sa) return -1; if (pos + 2 > sa->max_pos + 1) return -1; uwsgi_wlock(sa->lock); - uint16_t *n_ptr = (uint16_t *) (sa->area + pos); + int16_t *n_ptr = (int16_t *) (sa->area + pos); *n_ptr+=amount; sa->updates++; uwsgi_rwunlock(sa->lock); @@ -110,7 +110,7 @@ int uwsgi_sharedarea_inc32(int id, uint64_t pos, int32_t amount) { if (!sa) return -1; if (pos + 4 > sa->max_pos + 1) return -1; uwsgi_wlock(sa->lock); - uint32_t *n_ptr = (uint32_t *) (sa->area + pos); + int32_t *n_ptr = (int32_t *) (sa->area + pos); *n_ptr+=amount; sa->updates++; uwsgi_rwunlock(sa->lock); @@ -122,7 +122,7 @@ int uwsgi_sharedarea_inc64(int id, uint64_t pos, int64_t amount) { if (!sa) return -1; if (pos + 4 > sa->max_pos + 1) return -1; uwsgi_wlock(sa->lock); - uint32_t *n_ptr = (uint32_t *) (sa->area + pos); + int64_t *n_ptr = (int64_t *) (sa->area + pos); *n_ptr+=amount; sa->updates++; uwsgi_rwunlock(sa->lock); @@ -135,7 +135,7 @@ int uwsgi_sharedarea_dec8(int id, uint64_t pos, int8_t amount) { if (!sa) return -1; if (pos + 1 > sa->max_pos + 1) return -1; uwsgi_wlock(sa->lock); - uint8_t *n_ptr = (uint8_t *) (sa->area + pos); + int8_t *n_ptr = (int8_t *) (sa->area + pos); *n_ptr-=amount; sa->updates++; uwsgi_rwunlock(sa->lock); @@ -147,7 +147,7 @@ int uwsgi_sharedarea_dec16(int id, uint64_t pos, int16_t amount) { if (!sa) return -1; if (pos + 2 > sa->max_pos + 1) return -1; uwsgi_wlock(sa->lock); - uint16_t *n_ptr = (uint16_t *) (sa->area + pos); + int16_t *n_ptr = (int16_t *) (sa->area + pos); *n_ptr-=amount; sa->updates++; uwsgi_rwunlock(sa->lock); @@ -159,7 +159,7 @@ int uwsgi_sharedarea_dec32(int id, uint64_t pos, int32_t amount) { if (!sa) return -1; if (pos + 4 > sa->max_pos + 1) return -1; uwsgi_wlock(sa->lock); - uint32_t *n_ptr = (uint32_t *) (sa->area + pos); + int32_t *n_ptr = (int32_t *) (sa->area + pos); *n_ptr-=amount; sa->updates++; uwsgi_rwunlock(sa->lock); @@ -171,7 +171,7 @@ int uwsgi_sharedarea_dec64(int id, uint64_t pos, int64_t amount) { if (!sa) return -1; if (pos + 4 > sa->max_pos + 1) return -1; uwsgi_wlock(sa->lock); - uint32_t *n_ptr = (uint32_t *) (sa->area + pos); + int64_t *n_ptr = (int64_t *) (sa->area + pos); *n_ptr-=amount; sa->updates++; uwsgi_rwunlock(sa->lock); From 217f2f99ff837da7276c88a27d3d3e83920cff2b Mon Sep 17 00:00:00 2001 From: Roberto De Ioris Date: Sat, 30 Nov 2013 06:47:52 +0100 Subject: [PATCH 61/94] added declarations for sharedarea api --- uwsgi.h | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/uwsgi.h b/uwsgi.h index fb27e14f..a47f713f 100644 --- a/uwsgi.h +++ b/uwsgi.h @@ -4565,6 +4565,22 @@ void uwsgi_sharedareas_init(); int uwsgi_sharedarea_read(int, uint64_t, char *, uint64_t); int uwsgi_sharedarea_write(int, uint64_t, char *, uint64_t); +int uwsgi_sharedarea_read64(int, uint64_t, int64_t *); +int uwsgi_sharedarea_write64(int, uint64_t, int64_t *); +int uwsgi_sharedarea_read8(int, uint64_t, int8_t *); +int uwsgi_sharedarea_write8(int, uint64_t, int8_t *); +int uwsgi_sharedarea_read16(int, uint64_t, int16_t *); +int uwsgi_sharedarea_write16(int, uint64_t, int16_t *); +int uwsgi_sharedarea_read32(int, uint64_t, int32_t *); +int uwsgi_sharedarea_write32(int, uint64_t, int32_t *); +int uwsgi_sharedarea_inc8(int, uint64_t, int8_t); +int uwsgi_sharedarea_inc16(int, uint64_t, int16_t); +int uwsgi_sharedarea_inc32(int, uint64_t, int32_t); +int uwsgi_sharedarea_inc64(int, uint64_t, int64_t); +int uwsgi_sharedarea_dec8(int, uint64_t, int8_t); +int uwsgi_sharedarea_dec16(int, uint64_t, int16_t); +int uwsgi_sharedarea_dec32(int, uint64_t, int32_t); +int uwsgi_sharedarea_dec64(int, uint64_t, int64_t); int uwsgi_sharedarea_wait(int, int, int); void uwsgi_setup(int, char **, char **); From cabb6d72d9ab67e394024cde645c4d01ee2bd146 Mon Sep 17 00:00:00 2001 From: Roberto De Ioris Date: Sat, 30 Nov 2013 06:49:15 +0100 Subject: [PATCH 62/94] added uwsgi.websocket_send_binary_message to cpython --- plugins/python/uwsgi_pymodule.c | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/plugins/python/uwsgi_pymodule.c b/plugins/python/uwsgi_pymodule.c index e20fb226..f3be7ea4 100644 --- a/plugins/python/uwsgi_pymodule.c +++ b/plugins/python/uwsgi_pymodule.c @@ -1066,6 +1066,27 @@ PyObject *py_uwsgi_websocket_send(PyObject * self, PyObject * args) { return Py_None; } +PyObject *py_uwsgi_websocket_send_binary(PyObject * self, PyObject * args) { + char *message = NULL; + Py_ssize_t message_len = 0; + + if (!PyArg_ParseTuple(args, "s#:websocket_send_binary", &message, &message_len)) { + return NULL; + } + + struct wsgi_request *wsgi_req = py_current_wsgi_req(); + + UWSGI_RELEASE_GIL + int ret = uwsgi_websocket_send_binary(wsgi_req, message, message_len); + UWSGI_GET_GIL + if (ret < 0) { + return PyErr_Format(PyExc_IOError, "unable to send websocket binary message"); + } + Py_INCREF(Py_None); + return Py_None; +} + + PyObject *py_uwsgi_chunked_read(PyObject * self, PyObject * args) { int timeout = 0; if (!PyArg_ParseTuple(args, "|i:chunked_read", &timeout)) { @@ -2380,11 +2401,11 @@ static PyMethodDef uwsgi_advanced_methods[] = { {"ready", py_uwsgi_ready, METH_VARARGS, ""}, {"set_user_harakiri", py_uwsgi_set_user_harakiri, METH_VARARGS, ""}, - //{"call_hook", py_uwsgi_call_hook, METH_VARARGS, ""}, {"websocket_recv", py_uwsgi_websocket_recv, METH_VARARGS, ""}, {"websocket_recv_nb", py_uwsgi_websocket_recv_nb, METH_VARARGS, ""}, {"websocket_send", py_uwsgi_websocket_send, METH_VARARGS, ""}, + {"websocket_send_binary", py_uwsgi_websocket_send_binary, METH_VARARGS, ""}, {"websocket_handshake", py_uwsgi_websocket_handshake, METH_VARARGS, ""}, {"chunked_read", py_uwsgi_chunked_read, METH_VARARGS, ""}, From 98541dbc6574a0e643af715712e11a7fa44afc4a Mon Sep 17 00:00:00 2001 From: Roberto De Ioris Date: Sat, 30 Nov 2013 07:23:05 +0100 Subject: [PATCH 63/94] more sharedarea improvements --- core/sharedarea.c | 3 + plugins/psgi/uwsgi_plmodule.c | 24 +++++- plugins/python/uwsgi_pymodule.c | 146 +++++++++++++++++++++++--------- 3 files changed, 130 insertions(+), 43 deletions(-) diff --git a/core/sharedarea.c b/core/sharedarea.c index 765a6026..aca579e4 100644 --- a/core/sharedarea.c +++ b/core/sharedarea.c @@ -30,6 +30,7 @@ int uwsgi_sharedarea_read(int id, uint64_t pos, char *blob, uint64_t len) { struct uwsgi_sharedarea *sa = uwsgi_sharedarea_get_by_id(id, pos); if (!sa) return -1; if (pos + len > sa->max_pos + 1) return -1; + if (len == 0) len = (sa->max_pos + 1) - pos; uwsgi_rlock(sa->lock); memcpy(blob, sa->area + pos, len); sa->hits++; @@ -226,11 +227,13 @@ struct uwsgi_sharedarea *uwsgi_sharedarea_init_keyval(int id, char *arg) { char *s_pages = NULL; char *s_file = NULL; char *s_fd = NULL; + char *s_ptr = NULL; char *s_size = NULL; if (uwsgi_kvlist_parse(arg, strlen(arg), ',', '=', "pages", &s_pages, "file", &s_file, "fd", &s_fd, + "ptr", &s_ptr, "size", &s_size, NULL)) { } diff --git a/plugins/psgi/uwsgi_plmodule.c b/plugins/psgi/uwsgi_plmodule.c index 3b2ec978..6835484c 100644 --- a/plugins/psgi/uwsgi_plmodule.c +++ b/plugins/psgi/uwsgi_plmodule.c @@ -711,7 +711,7 @@ XS(XS_sharedarea_read) { dXSARGS; int id; uint64_t pos; - uint64_t len = 1; + uint64_t len = 0; psgi_check_args(2); id = SvIV(ST(0)); @@ -733,6 +733,27 @@ XS(XS_sharedarea_read) { XSRETURN(1); } +XS(XS_sharedarea_write) { + dXSARGS; + int id; + uint64_t pos; + STRLEN vallen; + + psgi_check_args(3); + + id = SvIV(ST(0)); + pos = SvIV(ST(1)); + char *value = SvPV(ST(2), vallen); + + if (uwsgi_sharedarea_write(id, pos, value, vallen)) { + croak("unable to write to sharedarea %d", id); + XSRETURN_UNDEF; + } + + XSRETURN_YES; +} + + XS(XS_chunked_read) { dXSARGS; int timeout = 0; @@ -825,6 +846,7 @@ void init_perl_embedded_module() { psgi_xs(chunked_read_nb); psgi_xs(sharedarea_read); + psgi_xs(sharedarea_write); psgi_xs(sharedarea_wait); } diff --git a/plugins/python/uwsgi_pymodule.c b/plugins/python/uwsgi_pymodule.c index f3be7ea4..8fd735a6 100644 --- a/plugins/python/uwsgi_pymodule.c +++ b/plugins/python/uwsgi_pymodule.c @@ -1447,103 +1447,160 @@ PyObject *py_uwsgi_extract(PyObject * self, PyObject * args) { } -PyObject *py_uwsgi_sharedarea_inclong(PyObject * self, PyObject * args) { +PyObject *py_uwsgi_sharedarea_inc64(PyObject * self, PyObject * args) { + int id; uint64_t pos = 0; - uint64_t value = 1; + int64_t value = 1; - if (!PyArg_ParseTuple(args, "l|l:sharedarea_inclong", &pos, &value)) { + if (!PyArg_ParseTuple(args, "il|l:sharedarea_inc64", &id, &pos, &value)) { return NULL; } - return PyInt_FromLong(value); + UWSGI_RELEASE_GIL + int ret = uwsgi_sharedarea_inc64(id, pos, value); + UWSGI_GET_GIL + + if (ret) { + return PyErr_Format(PyExc_ValueError, "error calling uwsgi_sharedarea_inc64()"); + } + + Py_INCREF(Py_None); + return Py_None; } -PyObject *py_uwsgi_sharedarea_writelong(PyObject * self, PyObject * args) { +PyObject *py_uwsgi_sharedarea_write64(PyObject * self, PyObject * args) { + int id; uint64_t pos = 0; - uint64_t value = 0; + int64_t value = 0; - if (!PyArg_ParseTuple(args, "ll:sharedarea_writelong", &pos, &value)) { + if (!PyArg_ParseTuple(args, "ill:sharedarea_write64", &id, &pos, &value)) { return NULL; } - return PyInt_FromLong(value); + UWSGI_RELEASE_GIL + int ret = uwsgi_sharedarea_write64(id, pos, &value); + UWSGI_GET_GIL + if (ret) { + return PyErr_Format(PyExc_ValueError, "error calling uwsgi_sharedarea_write64()"); + } + + Py_INCREF(Py_None); + return Py_None; } PyObject *py_uwsgi_sharedarea_write(PyObject * self, PyObject * args) { + int id; uint64_t pos = 0; char *value; Py_ssize_t value_len = 0; - if (!PyArg_ParseTuple(args, "ls#:sharedarea_write", &pos, &value, &value_len)) { + if (!PyArg_ParseTuple(args, "ils#:sharedarea_write", &id, &pos, &value, &value_len)) { return NULL; } - return PyInt_FromLong(value_len); - + UWSGI_RELEASE_GIL + int ret = uwsgi_sharedarea_write(id, pos, value, value_len); + UWSGI_GET_GIL + + if (ret) { + return PyErr_Format(PyExc_ValueError, "error calling uwsgi_sharedarea_write()"); + } + + Py_INCREF(Py_None); + return Py_None; +} + +PyObject *py_uwsgi_sharedarea_write8(PyObject * self, PyObject * args) { + int id; + uint64_t pos = 0; + int8_t value; + + if (!PyArg_ParseTuple(args, "ilb:sharedarea_write8", &id, &pos, &value)) { + return NULL; + } + + UWSGI_RELEASE_GIL + int ret = uwsgi_sharedarea_write8(id, pos, &value); + UWSGI_GET_GIL + + if (ret) { + return PyErr_Format(PyExc_ValueError, "error calling uwsgi_sharedarea_write8()"); + } + + Py_INCREF(Py_None); + return Py_None; } -PyObject *py_uwsgi_sharedarea_writebyte(PyObject * self, PyObject * args) { +PyObject *py_uwsgi_sharedarea_read64(PyObject * self, PyObject * args) { + int id; uint64_t pos = 0; - char value; + int64_t value; - if (!PyArg_ParseTuple(args, "lb:sharedarea_writebyte", &pos, &value)) { + if (!PyArg_ParseTuple(args, "il:sharedarea_read64", &id, &pos)) { return NULL; } - return PyInt_FromLong(value); + UWSGI_RELEASE_GIL + int ret = uwsgi_sharedarea_read64(id, pos, &value); + UWSGI_GET_GIL -} - -PyObject *py_uwsgi_sharedarea_readlong(PyObject * self, PyObject * args) { - uint64_t pos = 0; - uint64_t value; - - if (!PyArg_ParseTuple(args, "l:sharedarea_readlong", &pos)) { - return NULL; - } - - value = 1; + if (ret) { + return PyErr_Format(PyExc_ValueError, "error calling uwsgi_sharedarea_read64()"); + } return PyLong_FromLong(value); } - -PyObject *py_uwsgi_sharedarea_readbyte(PyObject * self, PyObject * args) { +PyObject *py_uwsgi_sharedarea_read8(PyObject * self, PyObject * args) { + int id; uint64_t pos = 0; - uint8_t byte; + int8_t byte; - if (!PyArg_ParseTuple(args, "l:sharedarea_readbyte", &pos)) { + if (!PyArg_ParseTuple(args, "il:sharedarea_read8", &id, &pos)) { return NULL; } - byte = 1; + UWSGI_RELEASE_GIL + int ret = uwsgi_sharedarea_read8(id, pos, &byte); + UWSGI_GET_GIL + + if (ret) { + return PyErr_Format(PyExc_ValueError, "error calling uwsgi_sharedarea_read8()"); + } return PyInt_FromLong(byte); - } PyObject *py_uwsgi_sharedarea_read(PyObject * self, PyObject * args) { + int id; uint64_t pos = 0; - uint64_t len = 1; + uint64_t len = 0; - if (!PyArg_ParseTuple(args, "l|l:sharedarea_read", &pos, &len)) { + if (!PyArg_ParseTuple(args, "il|l:sharedarea_read", &id, &pos, &len)) { return NULL; } -/* PyObject *ret = PyString_FromStringAndSize(NULL, len); #ifdef PYTHREE char *storage = PyBytes_AsString(ret); #else char *storage = PyString_AS_STRING(ret); #endif -*/ - return NULL; + UWSGI_RELEASE_GIL + int r = uwsgi_sharedarea_read(id, pos, storage, len); + UWSGI_GET_GIL + + if (r) { + Py_DECREF(ret); + return PyErr_Format(PyExc_ValueError, "error calling uwsgi_sharedarea_read()"); + } + + return ret; } PyObject *py_uwsgi_spooler_freq(PyObject * self, PyObject * args) { @@ -2420,11 +2477,16 @@ static PyMethodDef uwsgi_advanced_methods[] = { static PyMethodDef uwsgi_sa_methods[] = { {"sharedarea_read", py_uwsgi_sharedarea_read, METH_VARARGS, ""}, {"sharedarea_write", py_uwsgi_sharedarea_write, METH_VARARGS, ""}, - {"sharedarea_readbyte", py_uwsgi_sharedarea_readbyte, METH_VARARGS, ""}, - {"sharedarea_writebyte", py_uwsgi_sharedarea_writebyte, METH_VARARGS, ""}, - {"sharedarea_readlong", py_uwsgi_sharedarea_readlong, METH_VARARGS, ""}, - {"sharedarea_writelong", py_uwsgi_sharedarea_writelong, METH_VARARGS, ""}, - {"sharedarea_inclong", py_uwsgi_sharedarea_inclong, METH_VARARGS, ""}, + {"sharedarea_readbyte", py_uwsgi_sharedarea_read8, METH_VARARGS, ""}, + {"sharedarea_writebyte", py_uwsgi_sharedarea_write8, METH_VARARGS, ""}, + {"sharedarea_read8", py_uwsgi_sharedarea_read8, METH_VARARGS, ""}, + {"sharedarea_write8", py_uwsgi_sharedarea_write8, METH_VARARGS, ""}, + {"sharedarea_readlong", py_uwsgi_sharedarea_read64, METH_VARARGS, ""}, + {"sharedarea_writelong", py_uwsgi_sharedarea_write64, METH_VARARGS, ""}, + {"sharedarea_read64", py_uwsgi_sharedarea_read64, METH_VARARGS, ""}, + {"sharedarea_write64", py_uwsgi_sharedarea_write64, METH_VARARGS, ""}, + {"sharedarea_inclong", py_uwsgi_sharedarea_inc64, METH_VARARGS, ""}, + {"sharedarea_inc64", py_uwsgi_sharedarea_inc64, METH_VARARGS, ""}, {NULL, NULL}, }; From 9993e3710cc42bafaef266fc53866ce5aebdb323 Mon Sep 17 00:00:00 2001 From: Roberto De Ioris Date: Sat, 30 Nov 2013 07:31:57 +0100 Subject: [PATCH 64/94] first attempt of uwsgi_sharedarea_readfast --- plugins/psgi/uwsgi_plmodule.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/plugins/psgi/uwsgi_plmodule.c b/plugins/psgi/uwsgi_plmodule.c index 6835484c..cb5721b7 100644 --- a/plugins/psgi/uwsgi_plmodule.c +++ b/plugins/psgi/uwsgi_plmodule.c @@ -733,6 +733,30 @@ XS(XS_sharedarea_read) { XSRETURN(1); } +XS(XS_sharedarea_readfast) { + dXSARGS; + int id; + uint64_t pos; + uint64_t len = 0; + psgi_check_args(3); + + id = SvIV(ST(0)); + pos = SvIV(ST(1)); + char *buf = SvPV_nolen(ST(2)); + + if (items > 3) { + len = SvIV(ST(3)); + } + + if (uwsgi_sharedarea_read(id, pos, buf, len)) { + croak("unable to (fast) read from sharedarea %d", id); + XSRETURN_UNDEF; + } + + XSRETURN_YES; +} + + XS(XS_sharedarea_write) { dXSARGS; int id; @@ -846,6 +870,7 @@ void init_perl_embedded_module() { psgi_xs(chunked_read_nb); psgi_xs(sharedarea_read); + psgi_xs(sharedarea_readfast); psgi_xs(sharedarea_write); psgi_xs(sharedarea_wait); } From ace79b1179758180f335340d80145d8f51a5271d Mon Sep 17 00:00:00 2001 From: Unbit Date: Sat, 30 Nov 2013 10:41:56 +0000 Subject: [PATCH 65/94] various websockets and sharedarea optimizations --- core/sharedarea.c | 58 ++++++++++++++++++++++++--------- core/utils.c | 3 ++ core/websockets.c | 20 +++++++----- core/writer.c | 3 ++ plugins/psgi/uwsgi_plmodule.c | 9 ++++- plugins/python/uwsgi_pymodule.c | 8 +++++ uwsgi.h | 6 ++++ 7 files changed, 83 insertions(+), 24 deletions(-) diff --git a/core/sharedarea.c b/core/sharedarea.c index aca579e4..0a14a680 100644 --- a/core/sharedarea.c +++ b/core/sharedarea.c @@ -210,7 +210,30 @@ int uwsgi_sharedarea_wait(int id, int freq, int timeout) { return -2; } -struct uwsgi_sharedarea *uwsgi_sharedarea_init(int id, int pages) { +int uwsgi_sharedarea_new_id() { + int id = uwsgi.sharedareas_cnt; + uwsgi.sharedareas_cnt++; + if (!uwsgi.sharedareas) { + uwsgi.sharedareas = uwsgi_malloc(sizeof(struct uwsgi_sharedarea *)); + } + else { + struct uwsgi_sharedarea **usa = realloc(uwsgi.sharedareas, ((sizeof(struct uwsgi_sharedarea *)) * uwsgi.sharedareas_cnt)); + if (!usa) { + uwsgi_error("uwsgi_sharedarea_init()/realloc()"); + exit(1); + } + uwsgi.sharedareas = usa; + } + return id; +} + +static struct uwsgi_sharedarea *announce_sa(struct uwsgi_sharedarea *sa) { + uwsgi_log("sharedarea %d created at %p (%d pages, area at %p)\n", sa->id, sa, sa->pages, sa->area); + return sa; +} + +struct uwsgi_sharedarea *uwsgi_sharedarea_init(int pages) { + int id = uwsgi_sharedarea_new_id(); uwsgi.sharedareas[id] = uwsgi_calloc_shared(uwsgi.page_size * (pages + 1)); uwsgi.sharedareas[id]->area = ((char *) uwsgi.sharedareas[id]) + uwsgi.page_size; uwsgi.sharedareas[id]->id = id; @@ -220,10 +243,25 @@ struct uwsgi_sharedarea *uwsgi_sharedarea_init(int id, int pages) { char *id_str = uwsgi_num2str(id); uwsgi.sharedareas[id]->lock = uwsgi_rwlock_init(uwsgi_concat2("sharedarea", id_str)); free(id_str); - return uwsgi.sharedareas[id]; + return announce_sa(uwsgi.sharedareas[id]); } -struct uwsgi_sharedarea *uwsgi_sharedarea_init_keyval(int id, char *arg) { +struct uwsgi_sharedarea *uwsgi_sharedarea_init_ptr(char *area, uint64_t len) { + int id = uwsgi_sharedarea_new_id(); + uwsgi.sharedareas[id] = uwsgi_calloc_shared(sizeof(struct uwsgi_sharedarea)); + uwsgi.sharedareas[id]->area = area; + uwsgi.sharedareas[id]->id = id; + uwsgi.sharedareas[id]->fd = -1; + uwsgi.sharedareas[id]->pages = len / uwsgi.page_size; + if (len % uwsgi.page_size != 0) uwsgi.sharedareas[id]->pages++; + uwsgi.sharedareas[id]->max_pos = len-1; + char *id_str = uwsgi_num2str(id); + uwsgi.sharedareas[id]->lock = uwsgi_rwlock_init(uwsgi_concat2("sharedarea", id_str)); + free(id_str); + return announce_sa(uwsgi.sharedareas[id]); +} + +struct uwsgi_sharedarea *uwsgi_sharedarea_init_keyval(char *arg) { char *s_pages = NULL; char *s_file = NULL; char *s_fd = NULL; @@ -244,23 +282,13 @@ struct uwsgi_sharedarea *uwsgi_sharedarea_init_keyval(int id, char *arg) { void uwsgi_sharedareas_init() { struct uwsgi_string_list *usl = NULL; - uwsgi_foreach(usl, uwsgi.sharedareas_list) { - uwsgi.sharedareas_cnt++; - } - uwsgi.sharedareas = uwsgi_calloc(sizeof(struct uwsgi_sharedarea *) * uwsgi.sharedareas_cnt); - int id = 0; uwsgi_foreach(usl, uwsgi.sharedareas_list) { char *is_keyval = strchr(usl->value, '='); - struct uwsgi_sharedarea *sa = NULL; if (!is_keyval) { - sa = uwsgi_sharedarea_init(id, atoi(usl->value)); + uwsgi_sharedarea_init(atoi(usl->value)); } else { - sa = uwsgi_sharedarea_init_keyval(id, usl->value); + uwsgi_sharedarea_init_keyval(usl->value); } - if (sa) { - uwsgi_log("sharedaread % initialized at %p\n", id, sa); - } - id++; } } diff --git a/core/utils.c b/core/utils.c index ea94ebf8..6b5eb9b6 100644 --- a/core/utils.c +++ b/core/utils.c @@ -1138,6 +1138,9 @@ void uwsgi_close_request(struct wsgi_request *wsgi_req) { if (wsgi_req->websocket_buf) { uwsgi_buffer_destroy(wsgi_req->websocket_buf); } + if (wsgi_req->websocket_send_buf) { + uwsgi_buffer_destroy(wsgi_req->websocket_send_buf); + } // reset request diff --git a/core/websockets.c b/core/websockets.c index 8bc7b15f..f4f629ec 100644 --- a/core/websockets.c +++ b/core/websockets.c @@ -10,8 +10,16 @@ extern struct uwsgi_server uwsgi; -static struct uwsgi_buffer *uwsgi_websocket_message(char *msg, size_t len, uint8_t opcode) { - struct uwsgi_buffer *ub = uwsgi_buffer_new(10 + len); +static struct uwsgi_buffer *uwsgi_websocket_message(struct wsgi_request *wsgi_req, char *msg, size_t len, uint8_t opcode) { + struct uwsgi_buffer *ub = wsgi_req->websocket_send_buf; + if (!ub) { + wsgi_req->websocket_send_buf = uwsgi_buffer_new(10 + len); + ub = wsgi_req->websocket_send_buf; + } + else { + // reset the buffer + ub->pos = 0; + } if (uwsgi_buffer_u8(ub, opcode)) goto error; if (len < 126) { if (uwsgi_buffer_u8(ub, len)) goto error; @@ -29,7 +37,6 @@ static struct uwsgi_buffer *uwsgi_websocket_message(char *msg, size_t len, uint8 return ub; error: - uwsgi_buffer_destroy(ub); return NULL; } @@ -67,13 +74,10 @@ static int uwsgi_websockets_check_pingpong(struct wsgi_request *wsgi_req) { } static int uwsgi_websocket_send_do(struct wsgi_request *wsgi_req, char *msg, size_t len, uint8_t opcode) { - struct uwsgi_buffer *ub = uwsgi_websocket_message(msg, len, opcode); + struct uwsgi_buffer *ub = uwsgi_websocket_message(wsgi_req, msg, len, opcode); if (!ub) return -1; - ssize_t ret = uwsgi_response_write_body_do(wsgi_req, ub->buf, ub->pos); - uwsgi_buffer_destroy(ub); - return ret; - + return uwsgi_response_write_body_do(wsgi_req, ub->buf, ub->pos); } int uwsgi_websocket_send(struct wsgi_request *wsgi_req, char *msg, size_t len) { diff --git a/core/writer.c b/core/writer.c index e7ee2475..603f57a3 100644 --- a/core/writer.c +++ b/core/writer.c @@ -221,6 +221,7 @@ int uwsgi_response_write_headers_do(struct wsgi_request *wsgi_req) { if (ret == UWSGI_OK) { break; } + if (!uwsgi_is_again()) continue; ret = uwsgi_wait_write_req(wsgi_req); if (ret < 0) { wsgi_req->write_errors++; return -1;} if (ret == 0) { @@ -301,6 +302,7 @@ sendbody: if (ret == UWSGI_OK) { break; } + if (!uwsgi_is_again()) continue; ret = uwsgi_wait_write_req(wsgi_req); if (ret < 0) { wsgi_req->write_errors++; return -1;} if (ret == 0) { @@ -465,6 +467,7 @@ int uwsgi_simple_write(struct wsgi_request *wsgi_req, char *buf, size_t len) { if (ret == UWSGI_OK) { break; } + if (!uwsgi_is_again()) continue; ret = uwsgi_wait_write_req(wsgi_req); if (ret < 0) { wsgi_req->write_errors++; return -1;} if (ret == 0) { diff --git a/plugins/psgi/uwsgi_plmodule.c b/plugins/psgi/uwsgi_plmodule.c index cb5721b7..aaebe3ab 100644 --- a/plugins/psgi/uwsgi_plmodule.c +++ b/plugins/psgi/uwsgi_plmodule.c @@ -720,6 +720,14 @@ XS(XS_sharedarea_read) { if (items > 2) { len = SvIV(ST(2)); } + else { + struct uwsgi_sharedarea *sa = uwsgi_sharedarea_get_by_id(id, pos); + if (!sa) { + croak("unable to read from sharedarea %d", id); + XSRETURN_UNDEF; + } + len = sa->max_pos+1; + } char *buf = uwsgi_malloc(len); if (uwsgi_sharedarea_read(id, pos, buf, len)) { @@ -874,4 +882,3 @@ void init_perl_embedded_module() { psgi_xs(sharedarea_write); psgi_xs(sharedarea_wait); } - diff --git a/plugins/python/uwsgi_pymodule.c b/plugins/python/uwsgi_pymodule.c index 8fd735a6..862e10e3 100644 --- a/plugins/python/uwsgi_pymodule.c +++ b/plugins/python/uwsgi_pymodule.c @@ -1584,6 +1584,14 @@ PyObject *py_uwsgi_sharedarea_read(PyObject * self, PyObject * args) { return NULL; } + if (!len) { + struct uwsgi_sharedarea *sa = uwsgi_sharedarea_get_by_id(id, pos); + if (!sa) { + return PyErr_Format(PyExc_ValueError, "error calling uwsgi_sharedarea_read()"); + } + len = sa->max_pos+1; + } + PyObject *ret = PyString_FromStringAndSize(NULL, len); #ifdef PYTHREE char *storage = PyBytes_AsString(ret); diff --git a/uwsgi.h b/uwsgi.h index a47f713f..bdfcce86 100644 --- a/uwsgi.h +++ b/uwsgi.h @@ -1495,6 +1495,7 @@ struct wsgi_request { struct uwsgi_string_list *remove_headers; struct uwsgi_buffer *websocket_buf; + struct uwsgi_buffer *websocket_send_buf; size_t websocket_need; int websocket_phase; uint8_t websocket_opcode; @@ -4563,6 +4564,9 @@ void uwsgi_build_plugin(char *dir); void uwsgi_sharedareas_init(); +struct uwsgi_sharedarea *uwsgi_sharedarea_init(int); +struct uwsgi_sharedarea *uwsgi_sharedarea_init_ptr(char *, uint64_t); + int uwsgi_sharedarea_read(int, uint64_t, char *, uint64_t); int uwsgi_sharedarea_write(int, uint64_t, char *, uint64_t); int uwsgi_sharedarea_read64(int, uint64_t, int64_t *); @@ -4583,6 +4587,8 @@ int uwsgi_sharedarea_dec32(int, uint64_t, int32_t); int uwsgi_sharedarea_dec64(int, uint64_t, int64_t); int uwsgi_sharedarea_wait(int, int, int); +struct uwsgi_sharedarea *uwsgi_sharedarea_get_by_id(int, uint64_t); + void uwsgi_setup(int, char **, char **); int uwsgi_run(void); From 60b54c4d7c37c43e49fc1c28ea99cc6bb69ea871 Mon Sep 17 00:00:00 2001 From: Unbit Date: Sat, 30 Nov 2013 11:19:28 +0000 Subject: [PATCH 66/94] allows sending websockets messages directly from a sharedarea --- core/websockets.c | 35 ++++++++++++++++++++++++ plugins/psgi/uwsgi_plmodule.c | 48 ++++++++++++++++++++++++++++++++- plugins/python/uwsgi_pymodule.c | 2 +- uwsgi.h | 2 ++ 4 files changed, 85 insertions(+), 2 deletions(-) diff --git a/core/websockets.c b/core/websockets.c index f4f629ec..bdb8ad12 100644 --- a/core/websockets.c +++ b/core/websockets.c @@ -80,6 +80,19 @@ static int uwsgi_websocket_send_do(struct wsgi_request *wsgi_req, char *msg, siz return uwsgi_response_write_body_do(wsgi_req, ub->buf, ub->pos); } +static int uwsgi_websocket_send_from_sharedarea_do(struct wsgi_request *wsgi_req, int id, uint64_t pos, uint64_t len, uint8_t opcode) { + struct uwsgi_sharedarea *sa = uwsgi_sharedarea_get_by_id(id, pos); + if (!sa) return -1; + if (!len) len = ((sa->max_pos+1)-pos); + uwsgi_rlock(sa->lock); + sa->hits++; + struct uwsgi_buffer *ub = uwsgi_websocket_message(wsgi_req, sa->area, len, opcode); + uwsgi_rwunlock(sa->lock); + if (!ub) return -1; + + return uwsgi_response_write_body_do(wsgi_req, ub->buf, ub->pos); +} + int uwsgi_websocket_send(struct wsgi_request *wsgi_req, char *msg, size_t len) { if (wsgi_req->websocket_closed) { return -1; @@ -91,6 +104,17 @@ int uwsgi_websocket_send(struct wsgi_request *wsgi_req, char *msg, size_t len) { return ret; } +int uwsgi_websocket_send_from_sharedarea(struct wsgi_request *wsgi_req, int id, uint64_t pos, uint64_t len) { + if (wsgi_req->websocket_closed) { + return -1; + } + ssize_t ret = uwsgi_websocket_send_from_sharedarea_do(wsgi_req, id, pos, len, 0x81); + if (ret < 0) { + wsgi_req->websocket_closed = 1; + } + return ret; +} + int uwsgi_websocket_send_binary(struct wsgi_request *wsgi_req, char *msg, size_t len) { if (wsgi_req->websocket_closed) { return -1; @@ -102,6 +126,17 @@ int uwsgi_websocket_send_binary(struct wsgi_request *wsgi_req, char *msg, size_t return ret; } +int uwsgi_websocket_send_binary_from_sharedarea(struct wsgi_request *wsgi_req, int id, uint64_t pos, uint64_t len) { + if (wsgi_req->websocket_closed) { + return -1; + } + ssize_t ret = uwsgi_websocket_send_from_sharedarea_do(wsgi_req, id, pos, len, 0x82); + if (ret < 0) { + wsgi_req->websocket_closed = 1; + } + return ret; +} + static void uwsgi_websocket_parse_header(struct wsgi_request *wsgi_req) { uint8_t byte1 = wsgi_req->websocket_buf->buf[0]; uint8_t byte2 = wsgi_req->websocket_buf->buf[1]; diff --git a/plugins/psgi/uwsgi_plmodule.c b/plugins/psgi/uwsgi_plmodule.c index aaebe3ab..37d83f0a 100644 --- a/plugins/psgi/uwsgi_plmodule.c +++ b/plugins/psgi/uwsgi_plmodule.c @@ -498,6 +498,28 @@ XS(XS_websocket_send) { XSRETURN_UNDEF; } +XS(XS_websocket_send_from_sharedarea) { + dXSARGS; + + psgi_check_args(2); + int id = SvIV(ST(0)); + uint64_t pos = SvIV(ST(1)); + uint64_t len = 0; + + if (items > 2) { + len = SvIV(ST(2)); + } + + struct wsgi_request *wsgi_req = current_wsgi_req(); + + if (uwsgi_websocket_send_from_sharedarea(wsgi_req, id, pos, len)) { + croak("unable to send websocket message from sharedarea"); + } + + XSRETURN_UNDEF; +} + + XS(XS_websocket_send_binary) { dXSARGS; @@ -517,6 +539,28 @@ XS(XS_websocket_send_binary) { XSRETURN_UNDEF; } +XS(XS_websocket_send_binary_from_sharedarea) { + dXSARGS; + + psgi_check_args(2); + int id = SvIV(ST(0)); + uint64_t pos = SvIV(ST(1)); + uint64_t len = 0; + + if (items > 2) { + len = SvIV(ST(2)); + } + + struct wsgi_request *wsgi_req = current_wsgi_req(); + + if (uwsgi_websocket_send_binary_from_sharedarea(wsgi_req, id, pos, len)) { + croak("unable to send websocket binary message from sharedarea"); + } + + XSRETURN_UNDEF; +} + + XS(XS_websocket_recv) { dXSARGS; @@ -726,7 +770,7 @@ XS(XS_sharedarea_read) { croak("unable to read from sharedarea %d", id); XSRETURN_UNDEF; } - len = sa->max_pos+1; + len = (sa->max_pos+1)-pos; } char *buf = uwsgi_malloc(len); @@ -858,7 +902,9 @@ void init_perl_embedded_module() { psgi_xs(websocket_recv); psgi_xs(websocket_recv_nb); psgi_xs(websocket_send); + psgi_xs(websocket_send_from_sharedarea); psgi_xs(websocket_send_binary); + psgi_xs(websocket_send_binary_from_sharedarea); psgi_xs(postfork); psgi_xs(atexit); diff --git a/plugins/python/uwsgi_pymodule.c b/plugins/python/uwsgi_pymodule.c index 862e10e3..aa819d84 100644 --- a/plugins/python/uwsgi_pymodule.c +++ b/plugins/python/uwsgi_pymodule.c @@ -1589,7 +1589,7 @@ PyObject *py_uwsgi_sharedarea_read(PyObject * self, PyObject * args) { if (!sa) { return PyErr_Format(PyExc_ValueError, "error calling uwsgi_sharedarea_read()"); } - len = sa->max_pos+1; + len = (sa->max_pos+1)-pos; } PyObject *ret = PyString_FromStringAndSize(NULL, len); diff --git a/uwsgi.h b/uwsgi.h index bdfcce86..3366efd1 100644 --- a/uwsgi.h +++ b/uwsgi.h @@ -4588,6 +4588,8 @@ int uwsgi_sharedarea_dec64(int, uint64_t, int64_t); int uwsgi_sharedarea_wait(int, int, int); struct uwsgi_sharedarea *uwsgi_sharedarea_get_by_id(int, uint64_t); +int uwsgi_websocket_send_from_sharedarea(struct wsgi_request *, int, uint64_t, uint64_t); +int uwsgi_websocket_send_binary_from_sharedarea(struct wsgi_request *, int, uint64_t, uint64_t); void uwsgi_setup(int, char **, char **); int uwsgi_run(void); From d226f65890f9921a7dba11df66625ea59cc75f30 Mon Sep 17 00:00:00 2001 From: Unbit Date: Sat, 30 Nov 2013 12:23:13 +0100 Subject: [PATCH 67/94] optimized sendfile --- core/writer.c | 1 + 1 file changed, 1 insertion(+) diff --git a/core/writer.c b/core/writer.c index 603f57a3..c1f6f27a 100644 --- a/core/writer.c +++ b/core/writer.c @@ -402,6 +402,7 @@ sendfile: if (ret == UWSGI_OK) { break; } + if (!uwsgi_is_again()) continue; ret = uwsgi_wait_write_req(wsgi_req); if (ret < 0) { wsgi_req->write_errors++; From b972c2d6bb660ac3119cb12c7730da160b79a121 Mon Sep 17 00:00:00 2001 From: Unbit Date: Sat, 30 Nov 2013 12:33:24 +0100 Subject: [PATCH 68/94] another round of non-blocking optimizations --- core/io.c | 9 +++++---- core/writer.c | 4 ++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/core/io.c b/core/io.c index 1b06bc68..f5c98e07 100644 --- a/core/io.c +++ b/core/io.c @@ -756,11 +756,12 @@ int uwsgi_write_true_nb(int fd, char *buf, size_t remains, int timeout) { int ret; while(remains > 0) { + errno = 0; ssize_t len = write(fd, ptr, remains); if (len > 0) goto written; if (len == 0) return -1; if (len < 0) { - if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINPROGRESS) goto wait; + if (uwsgi_is_again()) goto wait; return -1; } wait: @@ -911,7 +912,7 @@ ssize_t uwsgi_read_true_nb(int fd, char *buf, size_t len, int timeout) { } if (rlen == 0) return -1; if (rlen < 0) { - if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINPROGRESS) goto wait; + if (uwsgi_is_again()) goto wait; } return -1; wait: @@ -961,7 +962,7 @@ int uwsgi_read_with_realloc(int fd, char **buffer, size_t *rlen, int timeout, ui if (len > 0) goto readok; if (len == 0) return -1; if (len < 0) { - if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINPROGRESS) goto wait; + if (uwsgi_is_again()) goto wait; return -1; } wait: @@ -1003,7 +1004,7 @@ readok: if (len > 0) goto readok2; if (len == 0) return -1; if (len < 0) { - if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINPROGRESS) goto wait2; + if (uwsgi_is_again()) goto wait2; return -1; } wait2: diff --git a/core/writer.c b/core/writer.c index c1f6f27a..b7a2a32c 100644 --- a/core/writer.c +++ b/core/writer.c @@ -210,6 +210,7 @@ int uwsgi_response_write_headers_do(struct wsgi_request *wsgi_req) { if (wsgi_req->socket->proto_fix_headers(wsgi_req)) { wsgi_req->write_errors++ ; return -1;} for(;;) { + errno = 0; int ret = wsgi_req->socket->proto_write_headers(wsgi_req, wsgi_req->headers->buf, wsgi_req->headers->pos); if (ret < 0) { if (!uwsgi.ignore_write_errors) { @@ -291,6 +292,7 @@ sendbody: if (len == 0) return UWSGI_OK; for(;;) { + errno = 0; int ret = wsgi_req->socket->proto_write(wsgi_req, buf, len); if (ret < 0) { if (!uwsgi.ignore_write_errors) { @@ -390,6 +392,7 @@ sendfile: wsgi_req->via = UWSGI_VIA_SENDFILE; for(;;) { + errno = 0; int ret = wsgi_req->socket->proto_sendfile(wsgi_req, fd, pos, len); if (ret < 0) { if (!uwsgi.ignore_write_errors) { @@ -457,6 +460,7 @@ int uwsgi_simple_write(struct wsgi_request *wsgi_req, char *buf, size_t len) { wsgi_req->write_pos = 0; for(;;) { + errno = 0; int ret = wsgi_req->socket->proto_write(wsgi_req, buf, len); if (ret < 0) { if (!uwsgi.ignore_write_errors) { From c99f5d08752a402fd34fdf981b6d8a9c39edc950 Mon Sep 17 00:00:00 2001 From: Unbit Date: Sat, 30 Nov 2013 12:46:17 +0100 Subject: [PATCH 69/94] fixed sharedarea_wait perl api --- plugins/psgi/uwsgi_plmodule.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/plugins/psgi/uwsgi_plmodule.c b/plugins/psgi/uwsgi_plmodule.c index 37d83f0a..3de0a253 100644 --- a/plugins/psgi/uwsgi_plmodule.c +++ b/plugins/psgi/uwsgi_plmodule.c @@ -735,16 +735,17 @@ XS(XS_metric_get) { XS(XS_sharedarea_wait) { dXSARGS; int id; + int freq = 0; int timeout = 0; psgi_check_args(1); id = SvIV(ST(0)); if (items > 1) { - timeout = SvIV(ST(1)); + freq = SvIV(ST(1)); } - if (uwsgi_sharedarea_wait(id, timeout, 0)) { + if (uwsgi_sharedarea_wait(id, freq, timeout)) { croak("unable to wait for sharedarea %d", id); XSRETURN_UNDEF; } From f1b3522c2ad1724a143220e4e0b3cc9c2dffee97 Mon Sep 17 00:00:00 2001 From: Unbit Date: Sat, 30 Nov 2013 13:49:21 +0000 Subject: [PATCH 70/94] allows building coroae on raspberrypi --- plugins/coroae/uwsgiplugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/coroae/uwsgiplugin.py b/plugins/coroae/uwsgiplugin.py index 28ecfac4..6d35564e 100644 --- a/plugins/coroae/uwsgiplugin.py +++ b/plugins/coroae/uwsgiplugin.py @@ -14,7 +14,7 @@ if not coroapi: NAME='coroae' CFLAGS = os.popen('perl -MExtUtils::Embed -e ccopts').read().rstrip().split() -CFLAGS += ['-Wno-int-to-pointer-cast', '-Wno-error=int-to-pointer-cast', '-I%s/Coro' % coroapi] +CFLAGS += ['-Wno-int-to-pointer-cast', '-Wno-error=format', '-Wno-error=int-to-pointer-cast', '-I%s/Coro' % coroapi] LDFLAGS = [] LIBS = [] From 46cceb0465c5d582d59ce680de13d4e61f25dfe3 Mon Sep 17 00:00:00 2001 From: Unbit Date: Sat, 30 Nov 2013 14:30:22 +0000 Subject: [PATCH 71/94] croak if the psgi streamer fails --- plugins/psgi/psgi_loader.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/psgi/psgi_loader.c b/plugins/psgi/psgi_loader.c index ead460c7..420b71d3 100644 --- a/plugins/psgi/psgi_loader.c +++ b/plugins/psgi/psgi_loader.c @@ -149,7 +149,7 @@ XS(XS_streaming_write) { uwsgi_response_write_body_do(wsgi_req, body, blen); uwsgi_pl_check_write_errors { - // noop + croak("error while streaming PSGI response"); } XSRETURN(0); From 92b6c62cac80ae1f812270736e5b579f392f8f7b Mon Sep 17 00:00:00 2001 From: Unbit Date: Sat, 30 Nov 2013 15:43:17 +0100 Subject: [PATCH 72/94] improved mjpeg example --- examples/mjpeg_stream.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/mjpeg_stream.py b/examples/mjpeg_stream.py index 0df77daa..761887e8 100644 --- a/examples/mjpeg_stream.py +++ b/examples/mjpeg_stream.py @@ -10,16 +10,16 @@ def application(env, start_response): ('Cache-Control', 'no-cache'), ('Cache-Control', 'private'), ('Pragma', 'no-cache'), - ('Content-Type', 'multipart/x-mixed-replace; boundary=' + boundary), + ('Content-Type', 'multipart/x-mixed-replace; boundary=--' + boundary), ] ) - yield "--%s\n" % boundary + yield "--%s\r\n" % boundary while 1: yield "Content-Type: image/jpeg\r\n\r\n" print os.system('screencapture -t jpg -m -T 1 screenshot.jpg') f = open('screenshot.jpg') yield env['wsgi.file_wrapper'](f) - yield "--%s\n" % boundary + yield "--%s\r\n" % boundary #os.system('./isightcapture -w 640 -h 480 screenshot.jpg') From b934ac958825a0d610aee356396abdc6a3608fce Mon Sep 17 00:00:00 2001 From: Unbit Date: Sat, 30 Nov 2013 18:30:52 +0000 Subject: [PATCH 73/94] refactored sharedarea read --- core/sharedarea.c | 6 +++--- core/websockets.c | 4 +++- plugins/psgi/uwsgi_plmodule.c | 5 +++-- plugins/python/uwsgi_pymodule.c | 7 +++++-- plugins/python/uwsgi_python.h | 6 ++++++ uwsgi.h | 4 +++- 6 files changed, 23 insertions(+), 9 deletions(-) diff --git a/core/sharedarea.c b/core/sharedarea.c index 0a14a680..1593b739 100644 --- a/core/sharedarea.c +++ b/core/sharedarea.c @@ -26,16 +26,16 @@ struct uwsgi_sharedarea *uwsgi_sharedarea_get_by_id(int id, uint64_t pos) { return sa; } -int uwsgi_sharedarea_read(int id, uint64_t pos, char *blob, uint64_t len) { +int64_t uwsgi_sharedarea_read(int id, uint64_t pos, char *blob, uint64_t len) { struct uwsgi_sharedarea *sa = uwsgi_sharedarea_get_by_id(id, pos); if (!sa) return -1; if (pos + len > sa->max_pos + 1) return -1; - if (len == 0) len = (sa->max_pos + 1) - pos; + if (len == 0) len = sa->honour_used ? sa->used-pos : (sa->max_pos + 1) - pos; uwsgi_rlock(sa->lock); memcpy(blob, sa->area + pos, len); sa->hits++; uwsgi_rwunlock(sa->lock); - return 0; + return len; } int uwsgi_sharedarea_write(int id, uint64_t pos, char *blob, uint64_t len) { diff --git a/core/websockets.c b/core/websockets.c index bdb8ad12..e03df308 100644 --- a/core/websockets.c +++ b/core/websockets.c @@ -83,7 +83,9 @@ static int uwsgi_websocket_send_do(struct wsgi_request *wsgi_req, char *msg, siz static int uwsgi_websocket_send_from_sharedarea_do(struct wsgi_request *wsgi_req, int id, uint64_t pos, uint64_t len, uint8_t opcode) { struct uwsgi_sharedarea *sa = uwsgi_sharedarea_get_by_id(id, pos); if (!sa) return -1; - if (!len) len = ((sa->max_pos+1)-pos); + if (!len) { + len = sa->honour_used ? sa->used-pos : ((sa->max_pos+1)-pos); + } uwsgi_rlock(sa->lock); sa->hits++; struct uwsgi_buffer *ub = uwsgi_websocket_message(wsgi_req, sa->area, len, opcode); diff --git a/plugins/psgi/uwsgi_plmodule.c b/plugins/psgi/uwsgi_plmodule.c index 3de0a253..1ad10647 100644 --- a/plugins/psgi/uwsgi_plmodule.c +++ b/plugins/psgi/uwsgi_plmodule.c @@ -775,14 +775,15 @@ XS(XS_sharedarea_read) { } char *buf = uwsgi_malloc(len); - if (uwsgi_sharedarea_read(id, pos, buf, len)) { + int64_t rlen = uwsgi_sharedarea_read(id, pos, buf, len); + if (rlen < 0) { free(buf); croak("unable to read from sharedarea %d", id); XSRETURN_UNDEF; } ST(0) = sv_newmortal(); - sv_usepvn(ST(0), buf,len); + sv_usepvn(ST(0), buf, rlen); XSRETURN(1); } diff --git a/plugins/python/uwsgi_pymodule.c b/plugins/python/uwsgi_pymodule.c index aa819d84..a717532d 100644 --- a/plugins/python/uwsgi_pymodule.c +++ b/plugins/python/uwsgi_pymodule.c @@ -1600,14 +1600,17 @@ PyObject *py_uwsgi_sharedarea_read(PyObject * self, PyObject * args) { #endif UWSGI_RELEASE_GIL - int r = uwsgi_sharedarea_read(id, pos, storage, len); + int64_t rlen = uwsgi_sharedarea_read(id, pos, storage, len); UWSGI_GET_GIL - if (r) { + if (rlen < 0) { Py_DECREF(ret); return PyErr_Format(PyExc_ValueError, "error calling uwsgi_sharedarea_read()"); } + // HACK: we are safe as rlen can only be lower or equal to len + Py_SIZE(ret) = rlen; + return ret; } diff --git a/plugins/python/uwsgi_python.h b/plugins/python/uwsgi_python.h index fdf8e97e..ce4a54ba 100644 --- a/plugins/python/uwsgi_python.h +++ b/plugins/python/uwsgi_python.h @@ -30,6 +30,12 @@ #define PYTHREE #endif +#if (PY_VERSION_HEX < 0x02060000) +#ifndef Py_SIZE +#define Py_SIZE(ob) (((PyVarObject*)(ob))->ob_size) +#endif +#endif + #define UWSGI_GET_GIL up.gil_get(); #define UWSGI_RELEASE_GIL up.gil_release(); diff --git a/uwsgi.h b/uwsgi.h index 3366efd1..ffe522aa 100644 --- a/uwsgi.h +++ b/uwsgi.h @@ -720,6 +720,8 @@ struct uwsgi_sharedarea { uint64_t max_pos; uint64_t updates; uint64_t hits; + uint8_t honour_used; + uint64_t used; }; // maintain alignment here !!! @@ -4567,7 +4569,7 @@ void uwsgi_sharedareas_init(); struct uwsgi_sharedarea *uwsgi_sharedarea_init(int); struct uwsgi_sharedarea *uwsgi_sharedarea_init_ptr(char *, uint64_t); -int uwsgi_sharedarea_read(int, uint64_t, char *, uint64_t); +int64_t uwsgi_sharedarea_read(int, uint64_t, char *, uint64_t); int uwsgi_sharedarea_write(int, uint64_t, char *, uint64_t); int uwsgi_sharedarea_read64(int, uint64_t, int64_t *); int uwsgi_sharedarea_write64(int, uint64_t, int64_t *); From cc3e43eb3da0913f66b48ab72a204def24dcff37 Mon Sep 17 00:00:00 2001 From: Unbit Date: Sun, 1 Dec 2013 04:40:39 +0000 Subject: [PATCH 74/94] fixed uwsgi_sharedarea_read --- core/sharedarea.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/sharedarea.c b/core/sharedarea.c index 1593b739..03aa8ea4 100644 --- a/core/sharedarea.c +++ b/core/sharedarea.c @@ -30,7 +30,8 @@ int64_t uwsgi_sharedarea_read(int id, uint64_t pos, char *blob, uint64_t len) { struct uwsgi_sharedarea *sa = uwsgi_sharedarea_get_by_id(id, pos); if (!sa) return -1; if (pos + len > sa->max_pos + 1) return -1; - if (len == 0) len = sa->honour_used ? sa->used-pos : (sa->max_pos + 1) - pos; + if (len == 0) len = (sa->max_pos + 1) - pos; + if (sa->honour_used) len = sa->used-pos; uwsgi_rlock(sa->lock); memcpy(blob, sa->area + pos, len); sa->hits++; From bfd699dac5ebb6a5d91fe3db1e2546fcfcdc1caa Mon Sep 17 00:00:00 2001 From: Roberto De Ioris Date: Sun, 1 Dec 2013 05:44:36 +0100 Subject: [PATCH 75/94] another mjpeg fix --- core/sharedarea.c | 2 +- examples/mjpeg_stream.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/sharedarea.c b/core/sharedarea.c index 03aa8ea4..680ed0e4 100644 --- a/core/sharedarea.c +++ b/core/sharedarea.c @@ -31,7 +31,7 @@ int64_t uwsgi_sharedarea_read(int id, uint64_t pos, char *blob, uint64_t len) { if (!sa) return -1; if (pos + len > sa->max_pos + 1) return -1; if (len == 0) len = (sa->max_pos + 1) - pos; - if (sa->honour_used) len = sa->used-pos; + if (sa->honour_used && sa->used-pos < len) len = sa->used-pos ; uwsgi_rlock(sa->lock); memcpy(blob, sa->area + pos, len); sa->hits++; diff --git a/examples/mjpeg_stream.py b/examples/mjpeg_stream.py index 761887e8..ae9e5474 100644 --- a/examples/mjpeg_stream.py +++ b/examples/mjpeg_stream.py @@ -10,7 +10,7 @@ def application(env, start_response): ('Cache-Control', 'no-cache'), ('Cache-Control', 'private'), ('Pragma', 'no-cache'), - ('Content-Type', 'multipart/x-mixed-replace; boundary=--' + boundary), + ('Content-Type', 'multipart/x-mixed-replace; boundary=' + boundary), ] ) @@ -21,5 +21,5 @@ def application(env, start_response): print os.system('screencapture -t jpg -m -T 1 screenshot.jpg') f = open('screenshot.jpg') yield env['wsgi.file_wrapper'](f) - yield "--%s\r\n" % boundary + yield "\r\n--%s\r\n" % boundary #os.system('./isightcapture -w 640 -h 480 screenshot.jpg') From 9a36b2b8913d8ff8a3cc6444382c3caf6b6f1adb Mon Sep 17 00:00:00 2001 From: Unbit Date: Sun, 1 Dec 2013 08:12:32 +0100 Subject: [PATCH 76/94] added writev api --- core/writer.c | 124 +++++++++++++++++++++++++++++++++++++++++++++++++ proto/base.c | 51 ++++++++++++++++++++ proto/http.c | 1 + proto/puwsgi.c | 1 + proto/scgi.c | 2 + proto/uwsgi.c | 1 + uwsgi.h | 4 ++ 7 files changed, 184 insertions(+) diff --git a/core/writer.c b/core/writer.c index b7a2a32c..06a7bfed 100644 --- a/core/writer.c +++ b/core/writer.c @@ -321,6 +321,130 @@ sendbody: return UWSGI_OK; } +int uwsgi_response_writev_body_do(struct wsgi_request *wsgi_req, struct iovec *iov, size_t len) { + + if (wsgi_req->write_errors) return -1; + if (wsgi_req->ignore_body) return UWSGI_OK; + +#ifdef UWSGI_ROUTING + // special case here, we could need to set transformations before + if (!wsgi_req->headers_sent) { + // apply response routes + if (uwsgi_apply_response_routes(wsgi_req) == UWSGI_ROUTE_BREAK) { + // from now on ignore write body requests... + wsgi_req->ignore_body = 1; + return -1; + } + wsgi_req->is_response_routing = 0; + } +#endif + + size_t i; + int buffering = 0; + // transformations apply to every vector + for(i=0;itransformed_chunk_len == 0 && wsgi_req->transformations) { + int t_ret = uwsgi_apply_transformations(wsgi_req, iov[i].iov_base, iov[i].iov_len); + if (t_ret == 0) { + iov[i].iov_base = wsgi_req->transformed_chunk; + iov[i].iov_len = wsgi_req->transformed_chunk_len; + // reset transformation + wsgi_req->transformed_chunk = NULL; + wsgi_req->transformed_chunk_len = 0; + goto write; + } + if (t_ret == 1) { + buffering = 1; + continue; + } + wsgi_req->write_errors++; + return -1; + } + } + + if (buffering) return UWSGI_OK; + +write: + // send headers if not already sent + if (!wsgi_req->headers_sent) { + int ret = uwsgi_response_write_headers_do(wsgi_req); + if (ret == UWSGI_OK) goto sendbody; + if (ret == UWSGI_AGAIN) return UWSGI_AGAIN; + wsgi_req->write_errors++; + return -1; + } + +sendbody: + + if (len == 0) return UWSGI_OK; + // unfortunately vector based I/O cannot be accomplished on all protocols + if (!wsgi_req->socket->proto_writev) goto fallback; + + // we use a copy to avoid mess + size_t iov_len = len; + for(;;) { + errno = 0; + int ret = wsgi_req->socket->proto_writev(wsgi_req, iov, &iov_len); + if (ret < 0) { + if (!uwsgi.ignore_write_errors) { + uwsgi_req_error("uwsgi_response_writev_body_do()"); + } + wsgi_req->write_errors++; + return -1; + } + if (ret == UWSGI_OK) { + break; + } + if (!uwsgi_is_again()) continue; + ret = uwsgi_wait_write_req(wsgi_req); + if (ret < 0) { wsgi_req->write_errors++; return -1;} + if (ret == 0) { + uwsgi_log("uwsgi_response_writev_body_do() TIMEOUT !!!\n"); + wsgi_req->write_errors++; + return -1; + } + } + + goto done; + +fallback: + + for(i=0;isocket->proto_write(wsgi_req, iov[i].iov_base, iov[i].iov_len); + if (ret < 0) { + if (!uwsgi.ignore_write_errors) { + uwsgi_req_error("uwsgi_response_writev_body_do()"); + } + wsgi_req->write_errors++; + return -1; + } + if (ret == UWSGI_OK) { + break; + } + if (!uwsgi_is_again()) continue; + ret = uwsgi_wait_write_req(wsgi_req); + if (ret < 0) { wsgi_req->write_errors++; return -1;} + if (ret == 0) { + uwsgi_log("uwsgi_response_writev_body_do() TIMEOUT !!!\n"); + wsgi_req->write_errors++; + return -1; + } + } + } + +done: + + wsgi_req->response_size += wsgi_req->write_pos; + // reset for the next write + wsgi_req->write_pos = 0; + + return UWSGI_OK; +} + + int uwsgi_response_sendfile_do(struct wsgi_request *wsgi_req, int fd, size_t pos, size_t len) { return uwsgi_response_sendfile_do_can_close(wsgi_req, fd, pos, len, 1); } diff --git a/proto/base.c b/proto/base.c index 5ea3643b..a15480a4 100644 --- a/proto/base.c +++ b/proto/base.c @@ -208,6 +208,57 @@ int uwsgi_proto_base_write(struct wsgi_request * wsgi_req, char *buf, size_t len return -1; } +/* + NOTE: len is a pointer as it could be changed on the fly +*/ +int uwsgi_proto_base_writev(struct wsgi_request * wsgi_req, struct iovec *iov, size_t *len) { + size_t i,needed = 0; + // count the number of bytes to write + for(i=0;i<*len;i++) needed += iov[i].iov_len; + ssize_t wlen = writev(wsgi_req->fd, iov, *len); + if (wlen > 0) { + wsgi_req->write_pos += wlen; + if ((size_t)wlen == needed) { + return UWSGI_OK; + } + // now the complex part, we need to rebuild iovec and len... + size_t orig_len = *len; + // first remove the consumed items + size_t first_iov = 0; + size_t skip_bytes = 0; + for(i=0;iproto_fix_headers = uwsgi_proto_base_fix_headers; uwsgi_sock->proto_read_body = uwsgi_proto_base_read_body; uwsgi_sock->proto_write = uwsgi_proto_base_write; + uwsgi_sock->proto_writev = uwsgi_proto_base_writev; uwsgi_sock->proto_write_headers = uwsgi_proto_base_write; uwsgi_sock->proto_sendfile = uwsgi_proto_base_sendfile; uwsgi_sock->proto_close = uwsgi_proto_base_close; diff --git a/proto/puwsgi.c b/proto/puwsgi.c index 83729bed..72840bea 100644 --- a/proto/puwsgi.c +++ b/proto/puwsgi.c @@ -98,6 +98,7 @@ void uwsgi_proto_puwsgi_setup(struct uwsgi_socket *uwsgi_sock) { uwsgi_sock->proto_fix_headers = uwsgi_proto_base_fix_headers; uwsgi_sock->proto_read_body = uwsgi_proto_noop_read_body; uwsgi_sock->proto_write = uwsgi_proto_base_write; + uwsgi_sock->proto_writev = uwsgi_proto_base_writev; uwsgi_sock->proto_write_headers = uwsgi_proto_base_write; uwsgi_sock->proto_sendfile = uwsgi_proto_base_sendfile; uwsgi_sock->proto_close = uwsgi_proto_puwsgi_close; diff --git a/proto/scgi.c b/proto/scgi.c index a94428be..a5afec4a 100644 --- a/proto/scgi.c +++ b/proto/scgi.c @@ -115,6 +115,7 @@ void uwsgi_proto_scgi_setup(struct uwsgi_socket *uwsgi_sock) { uwsgi_sock->proto_fix_headers = uwsgi_proto_base_fix_headers; uwsgi_sock->proto_read_body = uwsgi_proto_base_read_body; uwsgi_sock->proto_write = uwsgi_proto_base_write; + uwsgi_sock->proto_writev = uwsgi_proto_base_writev; uwsgi_sock->proto_write_headers = uwsgi_proto_base_write; uwsgi_sock->proto_sendfile = uwsgi_proto_base_sendfile; uwsgi_sock->proto_close = uwsgi_proto_base_close; @@ -128,6 +129,7 @@ void uwsgi_proto_scgi_nph_setup(struct uwsgi_socket *uwsgi_sock) { uwsgi_sock->proto_fix_headers = uwsgi_proto_base_fix_headers; uwsgi_sock->proto_read_body = uwsgi_proto_base_read_body; uwsgi_sock->proto_write = uwsgi_proto_base_write; + uwsgi_sock->proto_writev = uwsgi_proto_base_writev; uwsgi_sock->proto_write_headers = uwsgi_proto_base_write; uwsgi_sock->proto_sendfile = uwsgi_proto_base_sendfile; uwsgi_sock->proto_close = uwsgi_proto_base_close; diff --git a/proto/uwsgi.c b/proto/uwsgi.c index 42dc10a0..07f3aaa1 100644 --- a/proto/uwsgi.c +++ b/proto/uwsgi.c @@ -219,6 +219,7 @@ void uwsgi_proto_uwsgi_setup(struct uwsgi_socket *uwsgi_sock) { uwsgi_sock->proto_fix_headers = uwsgi_proto_base_fix_headers; uwsgi_sock->proto_read_body = uwsgi_proto_base_read_body; uwsgi_sock->proto_write = uwsgi_proto_base_write; + uwsgi_sock->proto_writev = uwsgi_proto_base_writev; uwsgi_sock->proto_write_headers = uwsgi_proto_base_write; uwsgi_sock->proto_sendfile = uwsgi_proto_base_sendfile; uwsgi_sock->proto_close = uwsgi_proto_base_close; diff --git a/uwsgi.h b/uwsgi.h index ffe522aa..b29c3a34 100644 --- a/uwsgi.h +++ b/uwsgi.h @@ -940,6 +940,8 @@ struct uwsgi_socket { void (*proto_close) (struct wsgi_request *); // special hook to call (if needed) in multithread mode void (*proto_thread_fixup) (struct uwsgi_socket *, int); + // optimization for vectors + int (*proto_writev) (struct wsgi_request *, struct iovec *, size_t *); int edge_trigger; int *retry; @@ -3237,6 +3239,7 @@ size_t uwsgi_str_num(char *, int); size_t uwsgi_str_occurence(char *, size_t, char); int uwsgi_proto_base_write(struct wsgi_request *, char *, size_t); +int uwsgi_proto_base_writev(struct wsgi_request *, struct iovec *, size_t *); #ifdef UWSGI_SSL int uwsgi_proto_ssl_write(struct wsgi_request *, char *, size_t); #endif @@ -4201,6 +4204,7 @@ void uwsgi_request_body_seek(struct wsgi_request *, off_t); struct uwsgi_buffer *uwsgi_proto_base_prepare_headers(struct wsgi_request *, char *, uint16_t); struct uwsgi_buffer *uwsgi_proto_base_cgi_prepare_headers(struct wsgi_request *, char *, uint16_t); int uwsgi_response_write_body_do(struct wsgi_request *, char *, size_t); +int uwsgi_response_writev_body_do(struct wsgi_request *, struct iovec *, size_t); int uwsgi_proto_base_sendfile(struct wsgi_request *, int, size_t, size_t); #ifdef UWSGI_SSL From e6c3c04e64b8d9d0dbc125a6965a18b5a44ea641 Mon Sep 17 00:00:00 2001 From: Unbit Date: Sun, 1 Dec 2013 08:47:43 +0100 Subject: [PATCH 77/94] completed vector based I/O --- proto/base.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/proto/base.c b/proto/base.c index a15480a4..a8d85059 100644 --- a/proto/base.c +++ b/proto/base.c @@ -223,20 +223,22 @@ int uwsgi_proto_base_writev(struct wsgi_request * wsgi_req, struct iovec *iov, s } // now the complex part, we need to rebuild iovec and len... size_t orig_len = *len; + size_t new_len = orig_len; // first remove the consumed items size_t first_iov = 0; size_t skip_bytes = 0; for(i=0;i Date: Sun, 1 Dec 2013 09:11:33 +0100 Subject: [PATCH 78/94] enable writev optimization by default --- core/writer.c | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/core/writer.c b/core/writer.c index 06a7bfed..aaf40982 100644 --- a/core/writer.c +++ b/core/writer.c @@ -179,7 +179,7 @@ int uwsgi_response_add_header_force(struct wsgi_request *wsgi_req, char *key, ui return uwsgi_response_add_header_do(wsgi_req, key, key_len, value, value_len); } -int uwsgi_response_write_headers_do(struct wsgi_request *wsgi_req) { +static int uwsgi_response_write_headers_do0(struct wsgi_request *wsgi_req) { if (wsgi_req->headers_sent || !wsgi_req->headers || wsgi_req->response_size || wsgi_req->write_errors) { return UWSGI_OK; } @@ -209,6 +209,14 @@ int uwsgi_response_write_headers_do(struct wsgi_request *wsgi_req) { if (wsgi_req->socket->proto_fix_headers(wsgi_req)) { wsgi_req->write_errors++ ; return -1;} + return UWSGI_AGAIN; +} + +int uwsgi_response_write_headers_do(struct wsgi_request *wsgi_req) { + + int ret = uwsgi_response_write_headers_do0(wsgi_req); + if (ret != UWSGI_AGAIN) return ret; + for(;;) { errno = 0; int ret = wsgi_req->socket->proto_write_headers(wsgi_req, wsgi_req->headers->buf, wsgi_req->headers->pos); @@ -240,6 +248,57 @@ int uwsgi_response_write_headers_do(struct wsgi_request *wsgi_req) { return UWSGI_OK; } +/* + private function for highly optimized writes (1 single syscall for headers and body) +*/ +static int uwsgi_response_writev_headers_and_body_do(struct wsgi_request *wsgi_req, char *buf, size_t len) { + + struct iovec iov[2]; + + int ret = uwsgi_response_write_headers_do0(wsgi_req); + if (ret != UWSGI_AGAIN) return ret; + + iov[0].iov_base = wsgi_req->headers->buf; + iov[0].iov_len = wsgi_req->headers->pos; + iov[1].iov_base = buf; + iov[1].iov_len = len; + + size_t iov_len = 2; + for(;;) { + errno = 0; + int ret = wsgi_req->socket->proto_writev(wsgi_req, iov, &iov_len); + if (ret < 0) { + if (!uwsgi.ignore_write_errors) { + uwsgi_req_error("uwsgi_response_writev_headers_and_body_do()"); + } + wsgi_req->write_errors++; + return -1; + } + if (ret == UWSGI_OK) { + break; + } + if (!uwsgi_is_again()) continue; + ret = uwsgi_wait_write_req(wsgi_req); + if (ret < 0) { wsgi_req->write_errors++; return -1;} + if (ret == 0) { + uwsgi_log("uwsgi_response_writev_headers_and_body_do() TIMEOUT !!!\n"); + wsgi_req->write_errors++; + return -1; + } + } + + wsgi_req->headers_size += wsgi_req->headers->pos; + wsgi_req->response_size += len; + wsgi_req->headers_sent = 1; + + // reset for the next write + wsgi_req->write_pos = 0; + + return UWSGI_OK; + + +} + // this is the function called by all request plugins to send chunks to the client int uwsgi_response_write_body_do(struct wsgi_request *wsgi_req, char *buf, size_t len) { @@ -280,6 +339,9 @@ int uwsgi_response_write_body_do(struct wsgi_request *wsgi_req, char *buf, size_ write: // send headers if not already sent if (!wsgi_req->headers_sent) { + if (wsgi_req->socket->proto_writev && len > 0) { + return uwsgi_response_writev_headers_and_body_do(wsgi_req, buf, len); + } int ret = uwsgi_response_write_headers_do(wsgi_req); if (ret == UWSGI_OK) goto sendbody; if (ret == UWSGI_AGAIN) return UWSGI_AGAIN; From 49c5206d86b84d6c189d7ace2d6bde367ee3bcaf Mon Sep 17 00:00:00 2001 From: Roberto De Ioris Date: Mon, 2 Dec 2013 04:58:08 +0100 Subject: [PATCH 79/94] fixed writev optimization --- core/writer.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/writer.c b/core/writer.c index aaf40982..9a6210ee 100644 --- a/core/writer.c +++ b/core/writer.c @@ -296,7 +296,6 @@ static int uwsgi_response_writev_headers_and_body_do(struct wsgi_request *wsgi_r return UWSGI_OK; - } // this is the function called by all request plugins to send chunks to the client @@ -339,7 +338,7 @@ int uwsgi_response_write_body_do(struct wsgi_request *wsgi_req, char *buf, size_ write: // send headers if not already sent if (!wsgi_req->headers_sent) { - if (wsgi_req->socket->proto_writev && len > 0) { + if (wsgi_req->socket->proto_writev && len > 0 && wsgi_req->headers) { return uwsgi_response_writev_headers_and_body_do(wsgi_req, buf, len); } int ret = uwsgi_response_write_headers_do(wsgi_req); From 248a22b06434c0e140a84373d9b683fd1670a71e Mon Sep 17 00:00:00 2001 From: Unbit Date: Mon, 2 Dec 2013 07:01:44 +0100 Subject: [PATCH 80/94] added another optimization for writev and a test for it --- core/writer.c | 44 +++++++++++++++++++++++++++++++ t/core/apps/read_body_and_send.pl | 5 ++++ t/core/read_body_and_send.pl | 35 ++++++++++++++++++++++++ 3 files changed, 84 insertions(+) create mode 100644 t/core/apps/read_body_and_send.pl create mode 100644 t/core/read_body_and_send.pl diff --git a/core/writer.c b/core/writer.c index 9a6210ee..3a10d9ba 100644 --- a/core/writer.c +++ b/core/writer.c @@ -266,6 +266,17 @@ static int uwsgi_response_writev_headers_and_body_do(struct wsgi_request *wsgi_r size_t iov_len = 2; for(;;) { errno = 0; + // no need to use writev if a single iovec remains + if (iov_len == 1) { + buf = iov[0].iov_base; + len = iov[0].iov_len; + // update counters + wsgi_req->headers_size += wsgi_req->headers->pos; + wsgi_req->headers_sent = 1; + wsgi_req->response_size += wsgi_req->write_pos - wsgi_req->headers_size; + wsgi_req->write_pos = 0; + goto fallback; + } int ret = wsgi_req->socket->proto_writev(wsgi_req, iov, &iov_len); if (ret < 0) { if (!uwsgi.ignore_write_errors) { @@ -296,6 +307,39 @@ static int uwsgi_response_writev_headers_and_body_do(struct wsgi_request *wsgi_r return UWSGI_OK; +fallback: + + for(;;) { + errno = 0; + int ret = wsgi_req->socket->proto_write(wsgi_req, buf, len); + if (ret < 0) { + if (!uwsgi.ignore_write_errors) { + // here we use the parent name + uwsgi_req_error("uwsgi_response_write_body_do()"); + } + wsgi_req->write_errors++; + return -1; + } + if (ret == UWSGI_OK) { + break; + } + if (!uwsgi_is_again()) continue; + ret = uwsgi_wait_write_req(wsgi_req); + if (ret < 0) { wsgi_req->write_errors++; return -1;} + if (ret == 0) { + // here we use the parent name + uwsgi_log("uwsgi_response_write_body_do() TIMEOUT !!!\n"); + wsgi_req->write_errors++; + return -1; + } + } + + wsgi_req->response_size += wsgi_req->write_pos; + // reset for the next write + wsgi_req->write_pos = 0; + + return UWSGI_OK; + } // this is the function called by all request plugins to send chunks to the client diff --git a/t/core/apps/read_body_and_send.pl b/t/core/apps/read_body_and_send.pl new file mode 100644 index 00000000..bd088ec5 --- /dev/null +++ b/t/core/apps/read_body_and_send.pl @@ -0,0 +1,5 @@ +my $app = sub { + my $env = shift; + $env->{'psgi.input'}->read(my $body, $env->{CONTENT_LENGTH}); + return [200, ['Content-Type' => 'x-application/binary'], [$body]]; +}; diff --git a/t/core/read_body_and_send.pl b/t/core/read_body_and_send.pl new file mode 100644 index 00000000..36cdbaf0 --- /dev/null +++ b/t/core/read_body_and_send.pl @@ -0,0 +1,35 @@ +#!/usr/bin/perl + +#uwsgi --http-socket :9090 --psgi t/core/apps/read_body_and_send.pl + +use IO::Socket::INET; + +my @chars = ("A".."Z", "a".."z"); + +foreach(0..100) { + $size = int(rand(8*1024*1024)); + print "testing: round ".$_." body size ".$size."\n"; + my $req = "POST /foobar HTTP/1.0\r\n"; + $req .= 'Content-Length: '.$size."\r\n\r\n"; + my $body = ''; + $body .= $chars[rand @chars] for 1..($size); + $req .= $body; + + my $s = IO::Socket::INET->new(PeerAddr => $ARGV[0]); + $s->send($req); + + my $response = ''; + while(1) { + $s->recv(my $buf, 4096); + last unless length($buf); + $response .= $buf; + } + $s->close; + + if ($response ne "HTTP/1.0 200 OK\r\nContent-Type: x-application/binary\r\n\r\n".$body) { + print "TEST FOR ROUND ".$_." FAILED\n"; + exit; + } +} + +print "test result: SUCCESS\n"; From 57ce416ae117b228b7814a658f4ef06004a289ae Mon Sep 17 00:00:00 2001 From: Unbit Date: Tue, 3 Dec 2013 14:26:49 +0100 Subject: [PATCH 81/94] fixed unbit CPU smashing --- core/lock.c | 1 + 1 file changed, 1 insertion(+) diff --git a/core/lock.c b/core/lock.c index fc9bd4da..d3681486 100644 --- a/core/lock.c +++ b/core/lock.c @@ -647,6 +647,7 @@ void *uwsgi_robust_mutexes_watchdog_loop(void *arg) { for(;;) { uwsgi_lock(uwsgi.the_thunder_lock); uwsgi_unlock(uwsgi.the_thunder_lock); + sleep(1); } return NULL; } From 3d156604076f9ea202abbb24ae46b7cc7b9698e8 Mon Sep 17 00:00:00 2001 From: Unbit Date: Tue, 3 Dec 2013 19:34:15 +0100 Subject: [PATCH 82/94] added a skel hook for waiting for 2 file descriptors --- core/init.c | 1 + core/reader.c | 33 +++++++++++++++++++++++++++++++++ uwsgi.h | 2 ++ 3 files changed, 36 insertions(+) diff --git a/core/init.c b/core/init.c index 466fd3c7..a81e9538 100644 --- a/core/init.c +++ b/core/init.c @@ -171,6 +171,7 @@ void uwsgi_init_default() { uwsgi.wait_read_hook = uwsgi_simple_wait_read_hook; uwsgi.wait_write_hook = uwsgi_simple_wait_write_hook; uwsgi.wait_milliseconds_hook = uwsgi_simple_wait_milliseconds_hook; + uwsgi.wait_read2_hook = uwsgi_simple_wait_read2_hook; uwsgi_websockets_init(); diff --git a/core/reader.c b/core/reader.c index 70441cf7..b994511a 100644 --- a/core/reader.c +++ b/core/reader.c @@ -24,6 +24,39 @@ int uwsgi_simple_wait_read_hook(int fd, int timeout) { return ret; } +int uwsgi_simple_wait_read2_hook(int fd0, int fd1, int timeout, int *fd) { + struct pollfd upoll[2]; + timeout = timeout * 1000; + + upoll[0].fd = fd0; + upoll[0].events = POLLIN; + upoll[0].revents = 0; + + upoll[0].fd = fd0; + upoll[0].events = POLLIN; + upoll[0].revents = 0; + + int ret = poll(upoll, 2, timeout); + + if (ret > 0) { + if (upoll[0].revents & POLLIN) { + *fd = fd0; + return 1; + } + if (upoll[1].revents & POLLIN) { + *fd = fd1; + return 1; + } + return -1; + } + if (ret < 0) { + uwsgi_error("uwsgi_simple_wait_read_hook2()/poll()"); + } + + return ret; +} + + /* seek()/rewind() language-independent implementations. */ diff --git a/uwsgi.h b/uwsgi.h index b29c3a34..fece647a 100644 --- a/uwsgi.h +++ b/uwsgi.h @@ -2606,6 +2606,7 @@ struct uwsgi_server { int (*wait_write_hook) (int, int); int (*wait_read_hook) (int, int); int (*wait_milliseconds_hook) (int); + int (*wait_read2_hook) (int, int, int, int *); struct uwsgi_string_list *schemes; @@ -4195,6 +4196,7 @@ struct uwsgi_buffer *uwsgi_proto_base_add_header(struct wsgi_request *, char *, int uwsgi_simple_wait_write_hook(int, int); int uwsgi_simple_wait_read_hook(int, int); +int uwsgi_simple_wait_read2_hook(int, int, int, int *); int uwsgi_simple_wait_milliseconds_hook(int); int uwsgi_response_write_headers_do(struct wsgi_request *); char *uwsgi_request_body_read(struct wsgi_request *, ssize_t , ssize_t *); From 0217766dd1a8a4896df6579309dd0b1b59bb04af Mon Sep 17 00:00:00 2001 From: Unbit Date: Tue, 3 Dec 2013 22:25:30 +0100 Subject: [PATCH 83/94] read2 hook for async mode --- core/async.c | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/core/async.c b/core/async.c index fae63344..af3122a6 100644 --- a/core/async.c +++ b/core/async.c @@ -221,6 +221,40 @@ static int async_wait_fd_read(int fd, int timeout) { return 1; } +static int async_wait_fd_read2(int fd0, int fd1, int timeout, int *fd) { + + struct wsgi_request *wsgi_req = current_wsgi_req(); + + wsgi_req->async_ready_fd = 0; + + if (async_add_fd_read(wsgi_req, fd0, timeout)) { + return -1; + } + + if (async_add_fd_read(wsgi_req, fd1, timeout)) { + // reset already registered fd + async_reset_request(wsgi_req); + return -1; + } + + if (uwsgi.schedule_to_main) { + uwsgi.schedule_to_main(wsgi_req); + } + + if (wsgi_req->async_timed_out) { + wsgi_req->async_timed_out = 0; + return 0; + } + + if (wsgi_req->async_ready_fd) { + *fd = wsgi_req->async_last_ready_fd; + return 1; + } + + return -1; +} + + void async_add_timeout(struct wsgi_request *wsgi_req, int timeout) { if (uwsgi.async < 2 || !uwsgi.rb_async_timeouts) { @@ -378,6 +412,7 @@ void async_loop() { uwsgi.wait_write_hook = async_wait_fd_write; uwsgi.wait_read_hook = async_wait_fd_read; + uwsgi.wait_read2_hook = async_wait_fd_read2; if (uwsgi.signal_socket > -1) { event_queue_add_fd_read(uwsgi.async_queue, uwsgi.signal_socket); From 31c31c4ce23bb4ccaaf5d929d43dfc877c7214ec Mon Sep 17 00:00:00 2001 From: Unbit Date: Tue, 3 Dec 2013 22:52:51 +0100 Subject: [PATCH 84/94] fixed read2 hook in non async mode --- core/reader.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/reader.c b/core/reader.c index b994511a..57359579 100644 --- a/core/reader.c +++ b/core/reader.c @@ -32,9 +32,9 @@ int uwsgi_simple_wait_read2_hook(int fd0, int fd1, int timeout, int *fd) { upoll[0].events = POLLIN; upoll[0].revents = 0; - upoll[0].fd = fd0; - upoll[0].events = POLLIN; - upoll[0].revents = 0; + upoll[1].fd = fd1; + upoll[1].events = POLLIN; + upoll[1].revents = 0; int ret = poll(upoll, 2, timeout); From 29418654d735dfdd494c202e90331394a5a4e854 Mon Sep 17 00:00:00 2001 From: Unbit Date: Wed, 4 Dec 2013 12:03:47 +0100 Subject: [PATCH 85/94] added support for mountpoints in the symcall plugin --- plugins/symcall/symcall_plugin.c | 55 ++++++++++++++++++++++++++------ 1 file changed, 45 insertions(+), 10 deletions(-) diff --git a/plugins/symcall/symcall_plugin.c b/plugins/symcall/symcall_plugin.c index 7bb57886..114f0241 100644 --- a/plugins/symcall/symcall_plugin.c +++ b/plugins/symcall/symcall_plugin.c @@ -3,9 +3,8 @@ extern struct uwsgi_server uwsgi; struct uwsgi_symcall { - char *symcall_function_name; + struct uwsgi_string_list *symcall_function_name; int (*symcall_function)(struct wsgi_request *); - struct uwsgi_string_list *rpc; struct uwsgi_string_list *post_fork; } usym; @@ -13,24 +12,40 @@ struct uwsgi_symcall { struct uwsgi_plugin symcall_plugin; static struct uwsgi_option uwsgi_symcall_options[] = { - {"symcall", required_argument, 0, "load the specified C symbol as the symcall request handler", uwsgi_opt_set_str, &usym.symcall_function_name, 0}, + {"symcall", required_argument, 0, "load the specified C symbol as the symcall request handler (supports too)", uwsgi_opt_add_string_list, &usym.symcall_function_name, 0}, {"symcall-register-rpc", required_argument, 0, "load the specified C symbol as an RPC function (syntax: name function)", uwsgi_opt_add_string_list, &usym.rpc, 0}, {"symcall-post-fork", required_argument, 0, "call the specified C symbol after each fork()", uwsgi_opt_add_string_list, &usym.post_fork, 0}, {0, 0, 0, 0}, }; static void uwsgi_symcall_init(){ - if (usym.symcall_function_name) { - usym.symcall_function = dlsym(RTLD_DEFAULT, usym.symcall_function_name); - if (!usym.symcall_function) { - uwsgi_log("unable to find symbol \"%s\" in process address space\n", usym.symcall_function_name); + struct uwsgi_string_list *usl = NULL; + int has_mountpoints = 0; + uwsgi_foreach(usl, usym.symcall_function_name) { + char *func = usl->value, *mountpoint = ""; + char *equal = strchr(usl->value, '='); + if (equal) { + *equal = 0; + func = equal+1; + mountpoint = usl->value; + has_mountpoints = 1; + } + usl->custom_ptr = dlsym(RTLD_DEFAULT, func); + if (!usl->custom_ptr) { + uwsgi_log("unable to find symbol \"%s\" in process address space\n", func); exit(1); } - uwsgi_log("symcall function ptr: %p\n", usym.symcall_function); + int id = uwsgi_apps_cnt; + struct uwsgi_app *ua = uwsgi_add_app(id, symcall_plugin.modifier1, mountpoint, strlen(mountpoint), usl->custom_ptr, NULL); + uwsgi_log("symcall app %d (mountpoint: \"%.*s\") mapped to function ptr: %p\n", id, ua->mountpoint_len, ua->mountpoint, usl->custom_ptr); + if (equal) *equal = '='; } - struct uwsgi_string_list *usl = usym.rpc; - while(usl) { + if (!has_mountpoints && usym.symcall_function_name) { + usym.symcall_function = usym.symcall_function_name->custom_ptr; + } + + uwsgi_foreach(usl, usym.rpc) { char *space = strchr(usl->value, ' '); if (!space) { uwsgi_log("invalid symcall RPC syntax, must be: rpcname symbol\n"); @@ -55,6 +70,26 @@ static int uwsgi_symcall_request(struct wsgi_request *wsgi_req) { if (usym.symcall_function) { return usym.symcall_function(wsgi_req); } + + if (uwsgi_parse_vars(wsgi_req)) return -1; + + wsgi_req->app_id = uwsgi_get_app_id(wsgi_req, wsgi_req->appid, wsgi_req->appid_len, symcall_plugin.modifier1); + if (wsgi_req->app_id == -1 && !uwsgi.no_default_app && uwsgi.default_app > -1) { + if (uwsgi_apps[uwsgi.default_app].modifier1 == symcall_plugin.modifier1) { + wsgi_req->app_id = uwsgi.default_app; + } + } + + if (wsgi_req->app_id == -1) { + uwsgi_404(wsgi_req); + return UWSGI_OK; + } + + struct uwsgi_app *ua = &uwsgi_apps[wsgi_req->app_id]; + if (ua->interpreter) { + int (*func)(struct wsgi_request *) = (int (*)(struct wsgi_request *)) ua->interpreter; + return func(wsgi_req); + } return UWSGI_OK; } From a57360217127369265cbb47fb8b4f66f7ce7709d Mon Sep 17 00:00:00 2001 From: Unbit Date: Wed, 4 Dec 2013 12:56:31 +0100 Subject: [PATCH 86/94] added tmsecs and tmicros log vars --- core/logging.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/core/logging.c b/core/logging.c index b48ac14c..d0714e97 100644 --- a/core/logging.c +++ b/core/logging.c @@ -1099,6 +1099,16 @@ static ssize_t uwsgi_lf_ftime(struct wsgi_request * wsgi_req, char **buf) { return ret; } +static ssize_t uwsgi_lf_tmsecs(struct wsgi_request * wsgi_req, char **buf) { + *buf = uwsgi_64bit2str(wsgi_req->start_of_request / (int64_t) 1000); + return strlen(*buf); +} + +static ssize_t uwsgi_lf_tmicros(struct wsgi_request * wsgi_req, char **buf) { + *buf = uwsgi_64bit2str(wsgi_req->start_of_request); + return strlen(*buf); +} + static ssize_t uwsgi_lf_micros(struct wsgi_request * wsgi_req, char **buf) { *buf = uwsgi_num2str(wsgi_req->end_of_request - wsgi_req->start_of_request); return strlen(*buf); @@ -1290,6 +1300,16 @@ void uwsgi_add_logchunk(int variable, int pos, char *ptr, size_t len) { logchunk->func = uwsgi_lf_msecs; logchunk->free = 1; } + else if (!uwsgi_strncmp(ptr, len, "tmsecs", 6)) { + logchunk->type = 3; + logchunk->func = uwsgi_lf_tmsecs; + logchunk->free = 1; + } + else if (!uwsgi_strncmp(ptr, len, "tmicros", 7)) { + logchunk->type = 3; + logchunk->func = uwsgi_lf_tmicros; + logchunk->free = 1; + } else if (!uwsgi_strncmp(ptr, len, "time", 4)) { logchunk->type = 3; logchunk->func = uwsgi_lf_time; From 70dd9fca32a340353fb35f7d4392afb94e9c1e2b Mon Sep 17 00:00:00 2001 From: Unbit Date: Wed, 4 Dec 2013 21:33:27 +0100 Subject: [PATCH 87/94] allows building plugin from a single file --- core/plugins_builder.c | 1 - core/utils.c | 7 +++++++ core/uwsgi.c | 12 ++++++++++++ uwsgi.h | 2 ++ uwsgiconfig.py | 28 ++++++++++++++++++---------- 5 files changed, 39 insertions(+), 11 deletions(-) diff --git a/core/plugins_builder.c b/core/plugins_builder.c index b4341f6c..f0fd38c9 100644 --- a/core/plugins_builder.c +++ b/core/plugins_builder.c @@ -15,7 +15,6 @@ */ void uwsgi_build_plugin(char *directory) { - uwsgi_log("building uWSGI plugin %s...\n", directory); if (!uwsgi_file_exists(UWSGI_BUILD_DIR)) { if (mkdir(UWSGI_BUILD_DIR, S_IRWXU) < 0) { diff --git a/core/utils.c b/core/utils.c index 6b5eb9b6..75307989 100644 --- a/core/utils.c +++ b/core/utils.c @@ -1074,6 +1074,13 @@ void uwsgi_close_request(struct wsgi_request *wsgi_req) { if (!wsgi_req->is_raw && uwsgi.p[wsgi_req->uh->modifier1]->after_request) uwsgi.p[wsgi_req->uh->modifier1]->after_request(wsgi_req); + // after_request custom hooks + struct uwsgi_string_list *usl = NULL; + uwsgi_foreach(usl, uwsgi.after_request_hooks) { + void (*func)(struct wsgi_request *) = (void (*)(struct wsgi_request *)) usl->custom_ptr; + func(wsgi_req); + } + if (uwsgi.threads > 1) { // now the thread can die... pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &tmp_id); diff --git a/core/uwsgi.c b/core/uwsgi.c index 0bbcb755..c4196556 100644 --- a/core/uwsgi.c +++ b/core/uwsgi.c @@ -383,6 +383,9 @@ static struct uwsgi_option uwsgi_base_options[] = { {"hook-as-mule", required_argument, 0, "run the specified hook in each mule", uwsgi_opt_add_string_list, &uwsgi.hook_as_mule, 0}, + {"after-request-hook", required_argument, 0, "run the specified function/symbol after each request", uwsgi_opt_add_string_list, &uwsgi.after_request_hooks, 0}, + {"after-request-call", required_argument, 0, "run the specified function/symbol after each request", uwsgi_opt_add_string_list, &uwsgi.after_request_hooks, 0}, + {"exec-asap", required_argument, 0, "run the specified command as soon as possible", uwsgi_opt_add_string_list, &uwsgi.exec_asap, 0}, {"exec-pre-jail", required_argument, 0, "run the specified command before jailing", uwsgi_opt_add_string_list, &uwsgi.exec_pre_jail, 0}, {"exec-post-jail", required_argument, 0, "run the specified command after jailing", uwsgi_opt_add_string_list, &uwsgi.exec_post_jail, 0}, @@ -2901,6 +2904,15 @@ next: } } + // initialize after_request hooks + uwsgi_foreach(usl, uwsgi.after_request_hooks) { + usl->custom_ptr = dlsym(RTLD_DEFAULT, usl->value); + if (!usl->custom_ptr) { + uwsgi_log("unable to find symbol/function \"%s\"\n", usl->value); + exit(1); + } + uwsgi_log("added \"%s(struct wsgi_request *)\" to the after-request chain\n", usl->value); + } if (uwsgi.daemonize2) { masterpid = uwsgi_daemonize2(); diff --git a/uwsgi.h b/uwsgi.h index fece647a..aabf95d3 100644 --- a/uwsgi.h +++ b/uwsgi.h @@ -2068,6 +2068,8 @@ struct uwsgi_server { struct uwsgi_string_list *umount_as_vassal; struct uwsgi_string_list *umount_as_emperor; + struct uwsgi_string_list *after_request_hooks; + struct uwsgi_string_list *wait_for_interface; int wait_for_interface_timeout; diff --git a/uwsgiconfig.py b/uwsgiconfig.py index 9dfb06b5..52e2b3a4 100644 --- a/uwsgiconfig.py +++ b/uwsgiconfig.py @@ -1207,18 +1207,26 @@ class uConf(object): def build_plugin(path, uc, cflags, ldflags, libs, name = None): path = path.rstrip('/') - if not os.path.isdir(path): + up = {} + + if os.path.isfile(path): + bname = os.path.basename(path) + # override path + path = os.path.dirname(path) + up['GCC_LIST'] = [bname] + up['NAME'] = bname.split('.')[0] + if not path: path = '.' + elif os.path.isdir(path): + try: + execfile('%s/uwsgiplugin.py' % path, up) + except: + f = open('%s/uwsgiplugin.py' % path) + exec(f.read(), up) + f.close() + else: print("Error: unable to find directory '%s'" % path) sys.exit(1) - up = {} - try: - execfile('%s/uwsgiplugin.py' % path, up) - except: - f = open('%s/uwsgiplugin.py' % path) - exec(f.read(), up) - f.close() - requires = [] p_cflags = cflags[:] @@ -1469,7 +1477,7 @@ if __name__ == "__main__": print("*** uWSGI building and linking plugin %s ***" % options.plugin[0] ) build_plugin(options.plugin[0], uc, cflags, ldflags, libs, name) elif options.extra_plugin: - print("*** uWSGI building and linking plugin %s ***" % options.extra_plugin[0]) + print("*** uWSGI building and linking plugin from %s ***" % options.extra_plugin[0]) cflags = os.environ['UWSGI_PLUGINS_BUILDER_CFLAGS'].split() + os.environ.get("CFLAGS", "").split() cflags.append('-I.uwsgi_plugins_builder/') ldflags = os.environ.get("LDFLAGS", "").split() From 3519fcbcc6b617609d0fe293532138b6b6e522b0 Mon Sep 17 00:00:00 2001 From: unbit Date: Fri, 6 Dec 2013 17:37:51 +0100 Subject: [PATCH 88/94] Update README --- README | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README b/README index 5816d789..7bb52cd9 100644 --- a/README +++ b/README @@ -1,3 +1,5 @@ surprise i am empty ! Look in the online documentation on https://uwsgi-docs.readthedocs.org/en/latest/ + +For commercial support check: http://unbit.com/ From 4eb3b76c3598ca3178a07a142a2b6e35fc5eb852 Mon Sep 17 00:00:00 2001 From: unbit Date: Fri, 6 Dec 2013 17:38:52 +0100 Subject: [PATCH 89/94] Update README --- README | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README b/README index 7bb52cd9..6cf48d80 100644 --- a/README +++ b/README @@ -1,5 +1,5 @@ -surprise i am empty ! +The uWSGI project -Look in the online documentation on https://uwsgi-docs.readthedocs.org/en/latest/ +For official documentation check: https://uwsgi-docs.readthedocs.org/en/latest/ For commercial support check: http://unbit.com/ From ce07c1a454d29b26817e23a3891caabbcc8306cc Mon Sep 17 00:00:00 2001 From: Unbit Date: Sat, 7 Dec 2013 07:33:38 +0100 Subject: [PATCH 90/94] report total build time --- uwsgiconfig.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/uwsgiconfig.py b/uwsgiconfig.py index 52e2b3a4..9154e331 100644 --- a/uwsgiconfig.py +++ b/uwsgiconfig.py @@ -55,6 +55,8 @@ if CPUCOUNT < 1: binary_list = [] +started_at = time.time() + # this is used for reporting (at the end of the build) # the server configuration report = { @@ -521,6 +523,8 @@ def build_uwsgi(uc, print_only=False, gcll=None): print("") print("############## end of uWSGI configuration #############") + print("total build time: %d seconds" % (time.time() - started_at)) + if bin_name.find("/") < 0: bin_name = './' + bin_name if uc.get('as_shared_library'): From c7161139e7718bc2c3a1ba89f6c2917c997ebbb5 Mon Sep 17 00:00:00 2001 From: Unbit Date: Sat, 7 Dec 2013 08:16:43 +0000 Subject: [PATCH 91/94] report build time even for plugins --- uwsgiconfig.py | 1 + 1 file changed, 1 insertion(+) diff --git a/uwsgiconfig.py b/uwsgiconfig.py index 9154e331..cc5f0f37 100644 --- a/uwsgiconfig.py +++ b/uwsgiconfig.py @@ -1390,6 +1390,7 @@ def build_plugin(path, uc, cflags, ldflags, libs, name = None): if post_build: post_build(uc) + print("build time: %d seconds" % (time.time() - started_at)) print("*** %s plugin built and available in %s ***" % (name, plugin_dest + '.so')) def vararg_callback(option, opt_str, value, parser): From a6b9e653a408f0c58a26cf5d6f9c647631c0b41f Mon Sep 17 00:00:00 2001 From: Unbit Date: Sat, 7 Dec 2013 08:26:01 +0000 Subject: [PATCH 92/94] fix plugin build time --- uwsgiconfig.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/uwsgiconfig.py b/uwsgiconfig.py index cc5f0f37..ada0e28a 100644 --- a/uwsgiconfig.py +++ b/uwsgiconfig.py @@ -1211,6 +1211,8 @@ class uConf(object): def build_plugin(path, uc, cflags, ldflags, libs, name = None): path = path.rstrip('/') + plugin_started_at = time.time() + up = {} if os.path.isfile(path): @@ -1390,7 +1392,7 @@ def build_plugin(path, uc, cflags, ldflags, libs, name = None): if post_build: post_build(uc) - print("build time: %d seconds" % (time.time() - started_at)) + print("build time: %d seconds" % (time.time() - plugin_started_at)) print("*** %s plugin built and available in %s ***" % (name, plugin_dest + '.so')) def vararg_callback(option, opt_str, value, parser): From 441013b71baeab83099cf0ca934ed34fefb86e78 Mon Sep 17 00:00:00 2001 From: Unbit Date: Sat, 7 Dec 2013 09:07:42 +0000 Subject: [PATCH 93/94] added mule support for symcall and more robust check for psgi response --- plugins/psgi/psgi_plugin.c | 7 +++++-- plugins/psgi/psgi_response.c | 2 +- plugins/symcall/symcall_plugin.c | 17 ++++++++++++++++- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/plugins/psgi/psgi_plugin.c b/plugins/psgi/psgi_plugin.c index a89cf842..5db391be 100644 --- a/plugins/psgi/psgi_plugin.c +++ b/plugins/psgi/psgi_plugin.c @@ -194,9 +194,12 @@ int uwsgi_perl_obj_can(SV *obj, char *method, size_t len) { XPUSHs(sv_2mortal(newSVpv(method, len))); PUTBACK; - call_method( "can", G_SCALAR); + call_method( "can", G_SCALAR|G_EVAL); SPAGAIN; + if(SvTRUE(ERRSV)) { + uwsgi_log("%s", SvPV_nolen(ERRSV)); + } ret = SvROK(POPs); PUTBACK; @@ -253,7 +256,7 @@ SV *uwsgi_perl_obj_call(SV *obj, char *method) { SPAGAIN; if(SvTRUE(ERRSV)) { - uwsgi_log("%s\n", SvPV_nolen(ERRSV)); + uwsgi_log("%s", SvPV_nolen(ERRSV)); } else { ret = SvREFCNT_inc(POPs); diff --git a/plugins/psgi/psgi_response.c b/plugins/psgi/psgi_response.c index ecedc97d..f91e47b4 100644 --- a/plugins/psgi/psgi_response.c +++ b/plugins/psgi/psgi_response.c @@ -101,7 +101,7 @@ int psgi_response(struct wsgi_request *wsgi_req, AV *response) { if (!SvRV(*hitem)) { uwsgi_log("invalid PSGI response body\n") ; return UWSGI_OK; } - if (SvTYPE(SvRV(*hitem)) == SVt_PVGV || SvTYPE(SvRV(*hitem)) == SVt_PVHV || SvTYPE(SvRV(*hitem)) == SVt_PVMG) { + if (SvROK(*hitem) && (SvTYPE(SvRV(*hitem)) == SVt_PVGV || SvTYPE(SvRV(*hitem)) == SVt_PVHV || SvTYPE(SvRV(*hitem)) == SVt_PVMG)) { // respond to fileno ? if (uwsgi.async < 2) { diff --git a/plugins/symcall/symcall_plugin.c b/plugins/symcall/symcall_plugin.c index 114f0241..49750a50 100644 --- a/plugins/symcall/symcall_plugin.c +++ b/plugins/symcall/symcall_plugin.c @@ -117,6 +117,21 @@ static void uwsgi_symcall_post_fork() { } } +static int uwsgi_symcall_mule(char *opt) { + if (uwsgi_endswith(opt, "()")) { + char *func_name = uwsgi_concat2n(opt, strlen(opt)-2, "", 0); + void (*func)() = dlsym(RTLD_DEFAULT, func_name); + if (!func) { + uwsgi_log("unable to find symbol \"%s\" in process address space\n", func_name); + exit(1); + } + free(func_name); + func(); + return 1; + } + return 0; +} + struct uwsgi_plugin symcall_plugin = { .name = "symcall", @@ -127,6 +142,6 @@ struct uwsgi_plugin symcall_plugin = { .after_request = uwsgi_symcall_after_request, .rpc = uwsgi_symcall_rpc, .post_fork = uwsgi_symcall_post_fork, - + .mule = uwsgi_symcall_mule, }; From d81330c514306ebc155044ffff0929eebd79d747 Mon Sep 17 00:00:00 2001 From: Unbit Date: Sat, 7 Dec 2013 09:52:51 +0000 Subject: [PATCH 94/94] support IO object is perl async mode too --- plugins/psgi/psgi_response.c | 58 ++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/plugins/psgi/psgi_response.c b/plugins/psgi/psgi_response.c index f91e47b4..e20cfc4d 100644 --- a/plugins/psgi/psgi_response.c +++ b/plugins/psgi/psgi_response.c @@ -100,41 +100,40 @@ int psgi_response(struct wsgi_request *wsgi_req, AV *response) { } if (!SvRV(*hitem)) { uwsgi_log("invalid PSGI response body\n") ; return UWSGI_OK; } - - if (SvROK(*hitem) && (SvTYPE(SvRV(*hitem)) == SVt_PVGV || SvTYPE(SvRV(*hitem)) == SVt_PVHV || SvTYPE(SvRV(*hitem)) == SVt_PVMG)) { - // respond to fileno ? - if (uwsgi.async < 2) { - // check for fileno() method, IO class or GvIO - if (uwsgi_perl_obj_can(*hitem, "fileno", 6) || uwsgi_perl_obj_isa(*hitem, "IO") || (uwsgi_perl_obj_isa(*hitem, "GLOB") && GvIO(SvRV(*hitem))) ) { - SV *fn = uwsgi_perl_obj_call(*hitem, "fileno"); - if (fn) { - if (SvTYPE(fn) == SVt_IV && SvIV(fn) >= 0) { - wsgi_req->sendfile_fd = SvIV(fn); - SvREFCNT_dec(fn); - uwsgi_response_sendfile_do(wsgi_req, wsgi_req->sendfile_fd, 0, 0); - // no need to close here as perl GC will do the close() - uwsgi_pl_check_write_errors { - // noop - } - return UWSGI_OK; - } + if (!SvROK(*hitem)) goto unsupported; + + if (SvTYPE(SvRV(*hitem)) == SVt_PVGV || SvTYPE(SvRV(*hitem)) == SVt_PVHV || SvTYPE(SvRV(*hitem)) == SVt_PVMG) { + + // check for fileno() method, IO class or GvIO + if (uwsgi_perl_obj_can(*hitem, "fileno", 6) || uwsgi_perl_obj_isa(*hitem, "IO") || (uwsgi_perl_obj_isa(*hitem, "GLOB") && GvIO(SvRV(*hitem))) ) { + SV *fn = uwsgi_perl_obj_call(*hitem, "fileno"); + if (fn) { + if (SvTYPE(fn) == SVt_IV && SvIV(fn) >= 0) { + wsgi_req->sendfile_fd = SvIV(fn); SvREFCNT_dec(fn); + uwsgi_response_sendfile_do(wsgi_req, wsgi_req->sendfile_fd, 0, 0); + // no need to close here as perl GC will do the close() + uwsgi_pl_check_write_errors { + // noop + } + return UWSGI_OK; } + SvREFCNT_dec(fn); } + } - // check for path method - if (uwsgi_perl_obj_can(*hitem, "path", 4)) { - SV *p = uwsgi_perl_obj_call(*hitem, "path"); - int fd = open(SvPV_nolen(p), O_RDONLY); - SvREFCNT_dec(p); - // the following function will close fd - uwsgi_response_sendfile_do(wsgi_req, fd, 0, 0); - uwsgi_pl_check_write_errors { - // noop - } - return UWSGI_OK; + // check for path method + if (uwsgi_perl_obj_can(*hitem, "path", 4)) { + SV *p = uwsgi_perl_obj_call(*hitem, "path"); + int fd = open(SvPV_nolen(p), O_RDONLY); + SvREFCNT_dec(p); + // the following function will close fd + uwsgi_response_sendfile_do(wsgi_req, fd, 0, 0); + uwsgi_pl_check_write_errors { + // noop } + return UWSGI_OK; } for(;;) { @@ -191,6 +190,7 @@ int psgi_response(struct wsgi_request *wsgi_req, AV *response) { } else { +unsupported: uwsgi_log("unsupported response body type: %d\n", SvTYPE(SvRV(*hitem))); }