From c1682f4ee43ce1c59e55c773b42189de1d27d4f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Mierzwa?= Date: Sun, 20 Jan 2013 22:24:07 +0100 Subject: [PATCH] reload/cheap mercy for invidual workers --- core/init.c | 2 ++ core/master.c | 10 ++++++++++ core/master_utils.c | 3 +++ core/uwsgi.c | 4 +++- uwsgi.h | 3 +++ 5 files changed, 21 insertions(+), 1 deletion(-) diff --git a/core/init.c b/core/init.c index b6137651..cbb287b4 100644 --- a/core/init.c +++ b/core/init.c @@ -107,6 +107,8 @@ void uwsgi_init_default() { uwsgi.log_master_bufsize = 8192; + uwsgi.worker_reload_mercy = 60; + uwsgi.max_vars = MAX_VARS; uwsgi.vec_size = 4 + 1 + (4 * MAX_VARS); diff --git a/core/master.c b/core/master.c index 0488fa06..de5c9582 100644 --- a/core/master.c +++ b/core/master.c @@ -706,6 +706,13 @@ int master_loop(char **argv, char **environ) { return -1; } + for (i = 1; i <= uwsgi.numproc; i++) { + if (uwsgi.workers[i].stopped_at > 0 && uwsgi.workers[i].pid > 0 && (uwsgi_now() - uwsgi.workers[i].stopped_at >= uwsgi.worker_reload_mercy)) { + uwsgi_log("worker %d is taking too much time to die (%ds), sending SIGKILL\n",i, (int) (uwsgi_now() - uwsgi.workers[i].stopped_at)); + kill(uwsgi.workers[i].pid, SIGKILL); + } + } + diedpid = waitpid(WAIT_ANY, &waitpid_status, WNOHANG); if (diedpid == -1) { if (errno == ECHILD) { @@ -1398,6 +1405,9 @@ next: uwsgi_log("DAMN ! worker %d (pid: %d) died :( trying respawn ...\n", uwsgi.mywid, (int) diedpid); } } + else if (uwsgi.workers[uwsgi.mywid].stopped_at > 0) { + uwsgi_log("worker %d killed successfully (pid: %d)\n", uwsgi.mywid, (int) diedpid); + } // manage_next_request is zero, but killed by signal... else if (WIFSIGNALED(waitpid_status)) { uwsgi_log("DAMN ! worker %d (pid: %d) MISTERIOUSLY killed by signal %d :( trying respawn ...\n", uwsgi.mywid, (int) diedpid, (int) WTERMSIG(waitpid_status)); diff --git a/core/master_utils.c b/core/master_utils.c index ce4a290a..e4c44253 100644 --- a/core/master_utils.c +++ b/core/master_utils.c @@ -78,6 +78,7 @@ int uwsgi_calc_cheaper(void) { #endif uwsgi.workers[oldest_worker].cheaped = 1; uwsgi.workers[oldest_worker].manage_next_request = 0; + uwsgi.workers[oldest_worker].stopped_at = now; // wakeup task in case of wait (void) kill(uwsgi.workers[oldest_worker].pid, SIGWINCH); } @@ -504,6 +505,8 @@ int uwsgi_respawn_worker(int wid) { uwsgi.workers[wid].pending_harakiri = 0; uwsgi.workers[wid].rss_size = 0; uwsgi.workers[wid].vsz_size = 0; + // ... reset stopped_at + uwsgi.workers[wid].stopped_at = 0; // internal statuses should be reset too diff --git a/core/uwsgi.c b/core/uwsgi.c index 46ca5b80..1418c9e9 100644 --- a/core/uwsgi.c +++ b/core/uwsgi.c @@ -161,7 +161,8 @@ static struct uwsgi_option uwsgi_base_options[] = { {"vassal-sos-backlog", required_argument, 0, "ask emperor for sos if backlog queue has more items than the value specified", uwsgi_opt_set_int, &uwsgi.vassal_sos_backlog, 0}, {"heartbeat", required_argument, 0, "announce healtness to the emperor", uwsgi_opt_set_int, &uwsgi.heartbeat, 0}, {"auto-snapshot", optional_argument, 0, "automatically make workers snaphost after reload", uwsgi_opt_set_int, &uwsgi.auto_snapshot, UWSGI_OPT_LAZY}, - {"reload-mercy", required_argument, 0, "set the maximum time (in seconds) a worker can take to reload/shutdown", uwsgi_opt_set_int, &uwsgi.reload_mercy, 0}, + {"reload-mercy", required_argument, 0, "set the maximum time (in seconds) we wait for workers and other processes to die during reload/shutdown", uwsgi_opt_set_int, &uwsgi.reload_mercy, 0}, + {"worker-reload-mercy", required_argument, 0, "set the maximum time (in seconds) a worker can take to reload/shutdown (default is 60)", uwsgi_opt_set_int, &uwsgi.worker_reload_mercy, 0}, {"exit-on-reload", no_argument, 0, "force exit even if a reload is requested", uwsgi_opt_true, &uwsgi.exit_on_reload, 0}, {"die-on-term", no_argument, 0, "exit instead of brutal reload on SIGTERM", uwsgi_opt_true, &uwsgi.die_on_term, 0}, {"help", no_argument, 'h', "show this help", uwsgi_help, NULL, UWSGI_OPT_IMMEDIATE}, @@ -918,6 +919,7 @@ void simple_goodbye_cruel_world() { } void goodbye_cruel_world() { + uwsgi.workers[uwsgi.mywid].stopped_at = uwsgi_now(); if (!uwsgi.gbcw_hook) { simple_goodbye_cruel_world(); diff --git a/uwsgi.h b/uwsgi.h index afa6dfd9..bdb061f6 100644 --- a/uwsgi.h +++ b/uwsgi.h @@ -1523,6 +1523,7 @@ struct uwsgi_server { int cpu_affinity; int reload_mercy; + int worker_reload_mercy; // map reloads to death int exit_on_reload; @@ -2372,6 +2373,8 @@ struct uwsgi_worker { int sig; uint8_t signum; + time_t stopped_at; + // signals managed by this worker uint64_t signals;