X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;f=include%2Fpoller.php;h=3a28b177c343e152c55f2f671d888fccfcb2003a;hb=3df235e5bd2666fdb4043e374d1df15414a355bd;hp=ab7deb6e0ac028c669a869945607e0b31daaaafd;hpb=ed8cdc7d8502b03ca9096b4d4a4c5fd9d149d08f;p=friendica.git diff --git a/include/poller.php b/include/poller.php index ab7deb6e0a..3a28b177c3 100644 --- a/include/poller.php +++ b/include/poller.php @@ -26,16 +26,16 @@ function poller_run(&$argv, &$argc){ unset($db_host, $db_user, $db_pass, $db_data); }; - if(function_exists('sys_getloadavg')) { - $maxsysload = intval(get_config('system','maxloadavg')); - if($maxsysload < 1) - $maxsysload = 50; + if (poller_max_connections_reached()) + return; - $load = sys_getloadavg(); - if(intval($load[0]) > $maxsysload) { - logger('system: load ' . $load[0] . ' too high. poller deferred to next scheduled run.'); - return; - } + if (App::maxload_reached()) + return; + + // Checking the number of workers + if (poller_too_much_workers(1)) { + poller_kill_stale_workers(); + return; } if(($argc <= 1) OR ($argv[1] != "no_cron")) { @@ -46,34 +46,25 @@ function poller_run(&$argv, &$argc){ proc_run("php","include/cronhooks.php"); // Cleaning dead processes - $r = q("SELECT DISTINCT(`pid`) FROM `workerqueue` WHERE `executed` != '0000-00-00 00:00:00'"); - foreach($r AS $pid) - if (!posix_kill($pid["pid"], 0)) - q("UPDATE `workerqueue` SET `executed` = '0000-00-00 00:00:00', `pid` = 0 WHERE `pid` = %d", - intval($pid["pid"])); - else { - // To-Do: Kill long running processes - // But: Update processes (like the database update) mustn't be killed - } - + poller_kill_stale_workers(); } else - // Sleep two seconds before checking for running processes to avoid having too many workers + // Sleep four seconds before checking for running processes again to avoid having too many workers sleep(4); // Checking number of workers - if (poller_too_much_workers()) + if (poller_too_much_workers(2)) return; $starttime = time(); while ($r = q("SELECT * FROM `workerqueue` WHERE `executed` = '0000-00-00 00:00:00' ORDER BY `created` LIMIT 1")) { - // Quit the poller once every hour - if (time() > ($starttime + 3600)) + // Constantly check the number of available database connections to let the frontend be accessible at any time + if (poller_max_connections_reached()) return; // Count active workers and compare them with a maximum value that depends on the load - if (poller_too_much_workers()) + if (poller_too_much_workers(3)) return; q("UPDATE `workerqueue` SET `executed` = '%s', `pid` = %d WHERE `id` = %d AND `executed` = '0000-00-00 00:00:00'", @@ -86,7 +77,7 @@ function poller_run(&$argv, &$argc){ intval($r[0]["id"]), intval(getmypid())); if (!$id) { - logger("Queue item ".$r[0]["id"]." was executed multiple times - quitting this one", LOGGER_DEBUG); + logger("Queue item ".$r[0]["id"]." was executed multiple times - skip this execution", LOGGER_DEBUG); continue; } @@ -108,19 +99,135 @@ function poller_run(&$argv, &$argc){ $funcname=str_replace(".php", "", basename($argv[0]))."_run"; if (function_exists($funcname)) { - logger("Process ".getmypid().": ".$funcname." ".$r[0]["parameter"]); + logger("Process ".getmypid()." - ID ".$r[0]["id"].": ".$funcname." ".$r[0]["parameter"]); $funcname($argv, $argc); - logger("Process ".getmypid().": ".$funcname." - done"); + logger("Process ".getmypid()." - ID ".$r[0]["id"].": ".$funcname." - done"); q("DELETE FROM `workerqueue` WHERE `id` = %d", intval($r[0]["id"])); } else logger("Function ".$funcname." does not exist"); + + // Quit the poller once every hour + if (time() > ($starttime + 3600)) + return; + } + +} + +/** + * @brief Checks if the number of database connections has reached a critical limit. + * + * @return bool Are more than 3/4 of the maximum connections used? + */ +function poller_max_connections_reached() { + + // Fetch the max value from the config. This is needed when the system cannot detect the correct value by itself. + $max = get_config("system", "max_connections"); + + // Fetch the percentage level where the poller will get active + $maxlevel = get_config("system", "max_connections_level"); + if ($maxlevel == 0) + $maxlevel = 75; + + if ($max == 0) { + // the maximum number of possible user connections can be a system variable + $r = q("SHOW VARIABLES WHERE `variable_name` = 'max_user_connections'"); + if ($r) + $max = $r[0]["Value"]; + + // Or it can be granted. This overrides the system variable + $r = q("SHOW GRANTS"); + if ($r) + foreach ($r AS $grants) { + $grant = array_pop($grants); + if (stristr($grant, "GRANT USAGE ON")) + if (preg_match("/WITH MAX_USER_CONNECTIONS (\d*)/", $grant, $match)) + $max = $match[1]; + } } + // If $max is set we will use the processlist to determine the current number of connections + // The processlist only shows entries of the current user + if ($max != 0) { + $r = q("SHOW PROCESSLIST"); + if (!$r) + return false; + + $used = count($r); + + logger("Connection usage (user values): ".$used."/".$max, LOGGER_DEBUG); + + $level = ($used / $max) * 100; + + if ($level >= $maxlevel) { + logger("Maximum level (".$maxlevel."%) of user connections reached: ".$used."/".$max); + return true; + } + } + + // We will now check for the system values. + // This limit could be reached although the user limits are fine. + $r = q("SHOW VARIABLES WHERE `variable_name` = 'max_connections'"); + if (!$r) + return false; + + $max = intval($r[0]["Value"]); + if ($max == 0) + return false; + + $r = q("SHOW STATUS WHERE `variable_name` = 'Threads_connected'"); + if (!$r) + return false; + + $used = intval($r[0]["Value"]); + if ($used == 0) + return false; + + logger("Connection usage (system values): ".$used."/".$max, LOGGER_DEBUG); + + $level = $used / $max * 100; + + if ($level < $maxlevel) + return false; + + logger("Maximum level (".$level."%) of system connections reached: ".$used."/".$max); + return true; +} + +/** + * @brief fix the queue entry if the worker process died + * + */ +function poller_kill_stale_workers() { + $r = q("SELECT `pid`, `executed` FROM `workerqueue` WHERE `executed` != '0000-00-00 00:00:00'"); + + if (!is_array($r) || count($r) == 0) { + // No processing here needed + return; + } + + foreach($r AS $pid) + if (!posix_kill($pid["pid"], 0)) + q("UPDATE `workerqueue` SET `executed` = '0000-00-00 00:00:00', `pid` = 0 WHERE `pid` = %d", + intval($pid["pid"])); + else { + // Kill long running processes + $duration = (time() - strtotime($pid["executed"])) / 60; + if ($duration > 180) { + logger("Worker process ".$pid["pid"]." took more than 3 hours. It will be killed now."); + posix_kill($pid["pid"], SIGTERM); + + // Question: If a process is stale: Should we remove it or should we reschedule it? + // By now we rescheduling it. It's maybe not the wisest decision? + q("UPDATE `workerqueue` SET `executed` = '0000-00-00 00:00:00', `pid` = 0 WHERE `pid` = %d", + intval($pid["pid"])); + } else + logger("Worker process ".$pid["pid"]." now runs for ".round($duration)." minutes. That's okay.", LOGGER_DEBUG); + } } -function poller_too_much_workers() { +function poller_too_much_workers($stage) { $queues = get_config("system", "worker_queues"); @@ -130,9 +237,8 @@ function poller_too_much_workers() { $active = poller_active_workers(); // Decrease the number of workers at higher load - if(function_exists('sys_getloadavg')) { - $load = max(sys_getloadavg()); - + $load = current_load(); + if($load) { $maxsysload = intval(get_config('system','maxloadavg')); if($maxsysload < 1) $maxsysload = 50; @@ -144,7 +250,7 @@ function poller_too_much_workers() { $slope = $maxworkers / pow($maxsysload, $exponent); $queues = ceil($slope * pow(max(0, $maxsysload - $load), $exponent)); - logger("Current load: ".$load." - maximum: ".$maxsysload." - current queues: ".$active." - maximum: ".$queues, LOGGER_DEBUG); + logger("Current load stage ".$stage.": ".$load." - maximum: ".$maxsysload." - current queues: ".$active." - maximum: ".$queues, LOGGER_DEBUG); }