From 13277d9d71ffaca15fdfcb61e1a872793d9af65c Mon Sep 17 00:00:00 2001 From: "roberto@quantal64" Date: Thu, 6 Sep 2012 17:10:03 +0200 Subject: [PATCH] improved cheaper_busyness --- plugins/cheaper_busyness/cheaper_busyness.c | 66 ++++++++++++++++----- 1 file changed, 51 insertions(+), 15 deletions(-) diff --git a/plugins/cheaper_busyness/cheaper_busyness.c b/plugins/cheaper_busyness/cheaper_busyness.c index f1d56f31..758c1081 100644 --- a/plugins/cheaper_busyness/cheaper_busyness.c +++ b/plugins/cheaper_busyness/cheaper_busyness.c @@ -22,8 +22,10 @@ struct uwsgi_cheaper_busyness_global { int last_action; // 1 - spawn workers ; 2 - cheap worker int verbose; // 1 - show debug logs, 0 - only important uint64_t tolerance_counter; // used to keep track of what to do if min <= busyness <= max for few cycles in row + int emergency_workers; // counts the number of running emergency workers #ifdef __linux__ int backlog_alert; + uint64_t backlog_multi; // multiplier used to cheap emergency workers #endif } uwsgi_cheaper_busyness_global; @@ -52,6 +54,9 @@ struct uwsgi_option uwsgi_cheaper_busyness_options[] = { {"cheaper-busyness-backlog-alert", required_argument, 0, "spawn emergency worker if anytime listen queue is higher than this value (default 33)", uwsgi_opt_set_int, &uwsgi_cheaper_busyness_global.backlog_alert, 0}, + {"cheaper-busyness-backlog-multiplier", required_argument, 0, + "set cheaper multiplier used for emergency workers (default 3)", + uwsgi_opt_set_64bit, &uwsgi_cheaper_busyness_global.backlog_multi, 0}, #endif {0, 0, 0, 0, 0, 0 ,0}, @@ -63,10 +68,22 @@ struct uwsgi_option uwsgi_cheaper_busyness_options[] = { void set_next_cheap_time(void) { uint64_t now = uwsgi_micros(); - // we will start workers now so we will set time when workers can be cheaped to - // some time in the future, so that workers are cheaped only if long term busyness - // is low enough - uwsgi_cheaper_busyness_global.next_cheap = now + uwsgi.cheaper_overload*uwsgi_cheaper_busyness_global.cheap_multi*1000000; +#ifdef __linux__ + if (uwsgi_cheaper_busyness_global.emergency_workers > 0) { + // we have some emergency workers running, we will use minimum delay (2 cycles) to cheap workers + // to have quicker recovery from big but short load spikes + // otherwise we might wait a lot before cheaping all emergency workers + if (uwsgi_cheaper_busyness_global.verbose) + uwsgi_log("[busyness] %d emergency worker(s) running, using %d seconds cheaper timer\n", + uwsgi_cheaper_busyness_global.emergency_workers, uwsgi.cheaper_overload*uwsgi_cheaper_busyness_global.backlog_multi); + uwsgi_cheaper_busyness_global.next_cheap = now + uwsgi.cheaper_overload*uwsgi_cheaper_busyness_global.backlog_multi*1000000; + } else { +#endif + // no emergency workers running, we use normal math for setting timer + uwsgi_cheaper_busyness_global.next_cheap = now + uwsgi.cheaper_overload*uwsgi_cheaper_busyness_global.cheap_multi*1000000; +#ifdef __linux__ + } +#endif } @@ -79,6 +96,25 @@ void decrease_multi(void) { } +#ifdef __linux__ +void spawn_emergency_worker(int backlog) { + // reset cheaper multiplier to minimum value so we can start cheaping workers sooner + // if this was just random spike + uwsgi_cheaper_busyness_global.cheap_multi = uwsgi_cheaper_busyness_global.min_multi; + + // set last action to spawn + uwsgi_cheaper_busyness_global.last_action = 1; + + uwsgi_cheaper_busyness_global.emergency_workers++; + + set_next_cheap_time(); + + uwsgi_log("[busyness] %d requests in listen queue, spawning emergency worker (%d)!\n", + backlog, uwsgi_cheaper_busyness_global.emergency_workers); +} +#endif + + int cheaper_busyness_algo(void) { int i; @@ -98,6 +134,7 @@ int cheaper_busyness_algo(void) { #ifdef __linux__ if (!uwsgi_cheaper_busyness_global.backlog_alert) uwsgi_cheaper_busyness_global.backlog_alert = 33; + if (!uwsgi_cheaper_busyness_global.backlog_multi) uwsgi_cheaper_busyness_global.backlog_multi = 3; #endif if (!uwsgi_cheaper_busyness_global.min_multi) { @@ -195,11 +232,7 @@ int cheaper_busyness_algo(void) { #ifdef __linux__ } else if (backlog > uwsgi_cheaper_busyness_global.backlog_alert && active_workers < uwsgi.numproc) { - // reset counters - set_next_cheap_time(); - uwsgi_cheaper_busyness_global.last_action = 1; - - uwsgi_log("[busyness] %d requests in listen queue, spawning emergency worker!\n", backlog); + spawn_emergency_worker(backlog); return 1; #endif @@ -226,6 +259,9 @@ int cheaper_busyness_algo(void) { // store information that last action performed was cheaping worker uwsgi_cheaper_busyness_global.last_action = 2; + if (uwsgi_cheaper_busyness_global.emergency_workers > 0) + uwsgi_cheaper_busyness_global.emergency_workers--; + return -1; } else if (uwsgi_cheaper_busyness_global.verbose) uwsgi_log("[busyness] need to wait %d more second(s) to cheap worker\n", (uwsgi_cheaper_busyness_global.next_cheap - now)/1000000); @@ -235,6 +271,11 @@ int cheaper_busyness_algo(void) { // with only 1 worker running there is no point in doing all that magic if (active_workers == 1) return 0; + if (uwsgi_cheaper_busyness_global.emergency_workers > 0) + // we had emergency workers running and we went down to the busyness + // level that is high enough to slow down cheaping workers at extra speed + uwsgi_cheaper_busyness_global.emergency_workers--; + // we have min <= busyness <= max we need to check what happened before uwsgi_cheaper_busyness_global.tolerance_counter++; @@ -260,12 +301,7 @@ int cheaper_busyness_algo(void) { #ifdef __linux__ } else if (backlog > uwsgi_cheaper_busyness_global.backlog_alert && active_workers < uwsgi.numproc) { // we check for backlog overload every cycle - - // reset counters - set_next_cheap_time(); - uwsgi_cheaper_busyness_global.last_action = 1; - - uwsgi_log("[busyness] %d requests in listen queue, spawning emergency worker!\n", backlog); + spawn_emergency_worker(backlog); return 1; #endif }