X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;f=include%2Fpoller.php;h=ec6653b4882b83f3abaf5664793b7918c927b15f;hb=18cbe9f6ff275cbb363fd9a0a5014bc8fe3d9f7b;hp=3b348531c5b6ed47806b1fa821189bb7d0262b17;hpb=54085508e5a6d7971158771f60845aa293de1d46;p=friendica.git diff --git a/include/poller.php b/include/poller.php index 3b348531c5..ec6653b488 100644 --- a/include/poller.php +++ b/include/poller.php @@ -10,6 +10,9 @@ if (!file_exists("boot.php") AND (sizeof($_SERVER["argv"]) != 0)) { chdir($directory); } +use \Friendica\Core\Config; +use \Friendica\Core\PConfig; + require_once("boot.php"); function poller_run(&$argv, &$argc){ @@ -26,16 +29,19 @@ function poller_run(&$argv, &$argc){ unset($db_host, $db_user, $db_pass, $db_data); }; - if(function_exists('sys_getloadavg')) { - $maxsysload = intval(get_config('system','maxloadavg')); - if($maxsysload < 1) - $maxsysload = 50; + if ($a->max_processes_reached()) + return; - $load = sys_getloadavg(); - if(intval($load[0]) > $maxsysload) { - logger('system: load ' . $load[0] . ' too high. poller deferred to next scheduled run.'); - return; - } + if (poller_max_connections_reached()) + return; + + if (App::maxload_reached()) + return; + + // Checking the number of workers + if (poller_too_much_workers()) { + poller_kill_stale_workers(); + return; } if(($argc <= 1) OR ($argv[1] != "no_cron")) { @@ -46,32 +52,32 @@ function poller_run(&$argv, &$argc){ proc_run("php","include/cronhooks.php"); // Cleaning dead processes - $r = q("SELECT DISTINCT(`pid`) FROM `workerqueue` WHERE `executed` != '0000-00-00 00:00:00'"); - foreach($r AS $pid) - if (!posix_kill($pid["pid"], 0)) - q("UPDATE `workerqueue` SET `executed` = '0000-00-00 00:00:00', `pid` = 0 WHERE `pid` = %d", - intval($pid["pid"])); - else { - // To-Do: Kill long running processes - // But: Update processes (like the database update) mustn't be killed - } - - } else { - // Checking the number of workers - if (poller_too_much_workers(1)) - return; - + poller_kill_stale_workers(); + } else // Sleep four seconds before checking for running processes again to avoid having too many workers sleep(4); - } // Checking number of workers - if (poller_too_much_workers(2)) + if (poller_too_much_workers()) return; + $cooldown = Config::get("system", "worker_cooldown", 0); + $starttime = time(); - while ($r = q("SELECT * FROM `workerqueue` WHERE `executed` = '0000-00-00 00:00:00' ORDER BY `created` LIMIT 1")) { + while ($r = q("SELECT * FROM `workerqueue` WHERE `executed` = '0000-00-00 00:00:00' ORDER BY `priority`, `created` LIMIT 1")) { + + // Constantly check the number of parallel database processes + if ($a->max_processes_reached()) + return; + + // Constantly check the number of available database connections to let the frontend be accessible at any time + if (poller_max_connections_reached()) + return; + + // Count active workers and compare them with a maximum value that depends on the load + if (poller_too_much_workers()) + return; q("UPDATE `workerqueue` SET `executed` = '%s', `pid` = %d WHERE `id` = %d AND `executed` = '0000-00-00 00:00:00'", dbesc(datetime_convert()), @@ -102,13 +108,18 @@ function poller_run(&$argv, &$argc){ require_once($include); - $funcname=str_replace(".php", "", basename($argv[0]))."_run"; + $funcname = str_replace(".php", "", basename($argv[0]))."_run"; if (function_exists($funcname)) { - logger("Process ".getmypid().": ".$funcname." ".$r[0]["parameter"]); + logger("Process ".getmypid()." - Prio ".$r[0]["priority"]." - ID ".$r[0]["id"].": ".$funcname." ".$r[0]["parameter"]); $funcname($argv, $argc); - logger("Process ".getmypid().": ".$funcname." - done"); + if ($cooldown > 0) { + logger("Process ".getmypid()." - Prio ".$r[0]["priority"]." - ID ".$r[0]["id"].": ".$funcname." - in cooldown for ".$cooldown." seconds"); + sleep($cooldown); + } + + logger("Process ".getmypid()." - Prio ".$r[0]["priority"]." - ID ".$r[0]["id"].": ".$funcname." - done"); q("DELETE FROM `workerqueue` WHERE `id` = %d", intval($r[0]["id"])); } else @@ -117,27 +128,137 @@ function poller_run(&$argv, &$argc){ // Quit the poller once every hour if (time() > ($starttime + 3600)) return; + } - // Count active workers and compare them with a maximum value that depends on the load - if (poller_too_much_workers(3)) - return; +} + +/** + * @brief Checks if the number of database connections has reached a critical limit. + * + * @return bool Are more than 3/4 of the maximum connections used? + */ +function poller_max_connections_reached() { + + // Fetch the max value from the config. This is needed when the system cannot detect the correct value by itself. + $max = get_config("system", "max_connections"); + + // Fetch the percentage level where the poller will get active + $maxlevel = get_config("system", "max_connections_level"); + if ($maxlevel == 0) + $maxlevel = 75; + + if ($max == 0) { + // the maximum number of possible user connections can be a system variable + $r = q("SHOW VARIABLES WHERE `variable_name` = 'max_user_connections'"); + if ($r) + $max = $r[0]["Value"]; + + // Or it can be granted. This overrides the system variable + $r = q("SHOW GRANTS"); + if ($r) + foreach ($r AS $grants) { + $grant = array_pop($grants); + if (stristr($grant, "GRANT USAGE ON")) + if (preg_match("/WITH MAX_USER_CONNECTIONS (\d*)/", $grant, $match)) + $max = $match[1]; + } + } + + // If $max is set we will use the processlist to determine the current number of connections + // The processlist only shows entries of the current user + if ($max != 0) { + $r = q("SHOW PROCESSLIST"); + if (!$r) + return false; + + $used = count($r); + + logger("Connection usage (user values): ".$used."/".$max, LOGGER_DEBUG); + + $level = ($used / $max) * 100; + + if ($level >= $maxlevel) { + logger("Maximum level (".$maxlevel."%) of user connections reached: ".$used."/".$max); + return true; + } } + // We will now check for the system values. + // This limit could be reached although the user limits are fine. + $r = q("SHOW VARIABLES WHERE `variable_name` = 'max_connections'"); + if (!$r) + return false; + + $max = intval($r[0]["Value"]); + if ($max == 0) + return false; + + $r = q("SHOW STATUS WHERE `variable_name` = 'Threads_connected'"); + if (!$r) + return false; + + $used = intval($r[0]["Value"]); + if ($used == 0) + return false; + + logger("Connection usage (system values): ".$used."/".$max, LOGGER_DEBUG); + + $level = $used / $max * 100; + + if ($level < $maxlevel) + return false; + + logger("Maximum level (".$level."%) of system connections reached: ".$used."/".$max); + return true; } -function poller_too_much_workers($stage) { +/** + * @brief fix the queue entry if the worker process died + * + */ +function poller_kill_stale_workers() { + $r = q("SELECT `pid`, `executed` FROM `workerqueue` WHERE `executed` != '0000-00-00 00:00:00'"); + + if (!dbm::is_result($r)) { + // No processing here needed + return; + } + + foreach($r AS $pid) + if (!posix_kill($pid["pid"], 0)) + q("UPDATE `workerqueue` SET `executed` = '0000-00-00 00:00:00', `pid` = 0 WHERE `pid` = %d", + intval($pid["pid"])); + else { + // Kill long running processes + $duration = (time() - strtotime($pid["executed"])) / 60; + if ($duration > 180) { + logger("Worker process ".$pid["pid"]." took more than 3 hours. It will be killed now."); + posix_kill($pid["pid"], SIGTERM); + + // Question: If a process is stale: Should we remove it or should we reschedule it? + // By now we rescheduling it. It's maybe not the wisest decision? + q("UPDATE `workerqueue` SET `executed` = '0000-00-00 00:00:00', `pid` = 0 WHERE `pid` = %d", + intval($pid["pid"])); + } else + logger("Worker process ".$pid["pid"]." now runs for ".round($duration)." minutes. That's okay.", LOGGER_DEBUG); + } +} + +function poller_too_much_workers() { + $queues = get_config("system", "worker_queues"); if ($queues == 0) $queues = 4; + $maxqueues = $queues; + $active = poller_active_workers(); // Decrease the number of workers at higher load - if(function_exists('sys_getloadavg')) { - $load = max(sys_getloadavg()); - + $load = current_load(); + if($load) { $maxsysload = intval(get_config('system','maxloadavg')); if($maxsysload < 1) $maxsysload = 50; @@ -149,7 +270,10 @@ function poller_too_much_workers($stage) { $slope = $maxworkers / pow($maxsysload, $exponent); $queues = ceil($slope * pow(max(0, $maxsysload - $load), $exponent)); - logger("Current load stage ".$stage.": ".$load." - maximum: ".$maxsysload." - current queues: ".$active." - maximum: ".$queues, LOGGER_DEBUG); + $s = q("SELECT COUNT(*) AS `total` FROM `workerqueue` WHERE `executed` = '0000-00-00 00:00:00'"); + $entries = $s[0]["total"]; + + logger("Current load: ".$load." - maximum: ".$maxsysload." - current queues: ".$active."/".$entries." - maximum: ".$queues."/".$maxqueues, LOGGER_DEBUG); }