Merge branch 'master' of https://github.com/unbit/uwsgi

core/init.c

@@ -107,6 +107,8 @@ void uwsgi_init_default() {
 
 	uwsgi.log_master_bufsize = 8192;
 
+	uwsgi.worker_reload_mercy = 60;
+
 	uwsgi.max_vars = MAX_VARS;
 	uwsgi.vec_size = 4 + 1 + (4 * MAX_VARS);
 

core/master.c

@@ -706,6 +706,13 @@ int master_loop(char **argv, char **environ) {
 			return -1;
 		}
 
+		for (i = 1; i <= uwsgi.numproc; i++) {
+			if (uwsgi.workers[i].stopped_at > 0 && uwsgi.workers[i].pid > 0 && (uwsgi_now() - uwsgi.workers[i].stopped_at >= uwsgi.worker_reload_mercy)) {
+				uwsgi_log("worker %d is taking too much time to die (%ds), sending SIGKILL\n",i, (int) (uwsgi_now() - uwsgi.workers[i].stopped_at));
+				kill(uwsgi.workers[i].pid, SIGKILL);
+			}
+		}
+
 		diedpid = waitpid(WAIT_ANY, &waitpid_status, WNOHANG);
 		if (diedpid == -1) {
 			if (errno == ECHILD) {
@@ -1398,6 +1405,9 @@ next:
 				uwsgi_log("DAMN ! worker %d (pid: %d) died :( trying respawn ...\n", uwsgi.mywid, (int) diedpid);
 			}
 		}
+		else if (uwsgi.workers[uwsgi.mywid].stopped_at > 0) {
+			uwsgi_log("worker %d killed successfully (pid: %d)\n", uwsgi.mywid, (int) diedpid);
+		}
 		// manage_next_request is zero, but killed by signal...
 		else if (WIFSIGNALED(waitpid_status)) {
 			uwsgi_log("DAMN ! worker %d (pid: %d) MISTERIOUSLY killed by signal %d :( trying respawn ...\n", uwsgi.mywid, (int) diedpid, (int) WTERMSIG(waitpid_status));

core/master_utils.c

@@ -78,6 +78,7 @@ int uwsgi_calc_cheaper(void) {
 #endif
 			uwsgi.workers[oldest_worker].cheaped = 1;
 			uwsgi.workers[oldest_worker].manage_next_request = 0;
+			uwsgi.workers[oldest_worker].stopped_at = now;
 			// wakeup task in case of wait
 			(void) kill(uwsgi.workers[oldest_worker].pid, SIGWINCH);
 		}
@@ -504,6 +505,8 @@ int uwsgi_respawn_worker(int wid) {
 	uwsgi.workers[wid].pending_harakiri = 0;
 	uwsgi.workers[wid].rss_size = 0;
 	uwsgi.workers[wid].vsz_size = 0;
+	// ... reset stopped_at
+	uwsgi.workers[wid].stopped_at = 0;
 
 	// internal statuses should be reset too
 

core/uwsgi.c

@@ -161,7 +161,8 @@ static struct uwsgi_option uwsgi_base_options[] = {
 	{"vassal-sos-backlog", required_argument, 0, "ask emperor for sos if backlog queue has more items than the value specified", uwsgi_opt_set_int, &uwsgi.vassal_sos_backlog, 0},
 	{"heartbeat", required_argument, 0, "announce healtness to the emperor", uwsgi_opt_set_int, &uwsgi.heartbeat, 0},
 	{"auto-snapshot", optional_argument, 0, "automatically make workers snaphost after reload", uwsgi_opt_set_int, &uwsgi.auto_snapshot, UWSGI_OPT_LAZY},
-	{"reload-mercy", required_argument, 0, "set the maximum time (in seconds) a worker can take to reload/shutdown", uwsgi_opt_set_int, &uwsgi.reload_mercy, 0},
+	{"reload-mercy", required_argument, 0, "set the maximum time (in seconds) we wait for workers and other processes to die during reload/shutdown", uwsgi_opt_set_int, &uwsgi.reload_mercy, 0},
+	{"worker-reload-mercy", required_argument, 0, "set the maximum time (in seconds) a worker can take to reload/shutdown (default is 60)", uwsgi_opt_set_int, &uwsgi.worker_reload_mercy, 0},
 	{"exit-on-reload", no_argument, 0, "force exit even if a reload is requested", uwsgi_opt_true, &uwsgi.exit_on_reload, 0},
 	{"die-on-term", no_argument, 0, "exit instead of brutal reload on SIGTERM", uwsgi_opt_true, &uwsgi.die_on_term, 0},
 	{"help", no_argument, 'h', "show this help", uwsgi_help, NULL, UWSGI_OPT_IMMEDIATE},
@@ -918,6 +919,7 @@ void simple_goodbye_cruel_world() {
 }
 
 void goodbye_cruel_world() {
+	uwsgi.workers[uwsgi.mywid].stopped_at = uwsgi_now();
 
 	if (!uwsgi.gbcw_hook) {
 		simple_goodbye_cruel_world();

uwsgi.h

@@ -1523,6 +1523,7 @@ struct uwsgi_server {
 	int cpu_affinity;
 
 	int reload_mercy;
+	int worker_reload_mercy;
 	// map reloads to death
 	int exit_on_reload;
 
@@ -2372,6 +2373,8 @@ struct uwsgi_worker {
         int sig;
 	uint8_t signum;
 
+	time_t stopped_at;
+
 	// signals managed by this worker
         uint64_t signals;