X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;f=src%2FCore%2FWorker.php;h=2e7bb483e8a170813cc7e017c3550338334804e5;hb=7c470f0be9a0b4a9f8625333345c8b994487b504;hp=e1787071cbb15fa90ea13d0f68f3b494ad80e170;hpb=cd1724eb9ef4e07ced4c9019dd7bad55dc21abce;p=friendica.git diff --git a/src/Core/Worker.php b/src/Core/Worker.php index e1787071cb..2e7bb483e8 100644 --- a/src/Core/Worker.php +++ b/src/Core/Worker.php @@ -1,14 +1,17 @@ ($starttime + 300)) { logger('Process lifetime reached, quitting.', LOGGER_DEBUG); return; @@ -140,9 +145,10 @@ class Worker { * * @return integer Number of non executed entries in the worker queue */ - private static function totalEntries() { + private static function totalEntries() + { $s = dba::fetch_first("SELECT COUNT(*) AS `total` FROM `workerqueue` WHERE `executed` <= ? AND NOT `done`", NULL_DATE); - if (dbm::is_result($s)) { + if (DBM::is_result($s)) { return $s["total"]; } else { return 0; @@ -152,12 +158,13 @@ class Worker { /** * @brief Returns the highest priority in the worker queue that isn't executed * - * @return integer Number of active poller processes + * @return integer Number of active worker processes */ - private static function highestPriority() { + private static function highestPriority() + { $condition = array("`executed` <= ? AND NOT `done`", NULL_DATE); $s = dba::select('workerqueue', array('priority'), $condition, array('limit' => 1, 'order' => array('priority'))); - if (dbm::is_result($s)) { + if (DBM::is_result($s)) { return $s["priority"]; } else { return 0; @@ -171,7 +178,8 @@ class Worker { * * @return integer Is there a process running with that priority? */ - private static function processWithPriorityActive($priority) { + private static function processWithPriorityActive($priority) + { $condition = array("`priority` <= ? AND `executed` > ? AND NOT `done`", $priority, NULL_DATE); return dba::exists('workerqueue', $condition); } @@ -183,7 +191,8 @@ class Worker { * * @return boolean "true" if further processing should be stopped */ - public static function execute($queue) { + public static function execute($queue) + { $a = get_app(); $mypid = getmypid(); @@ -211,6 +220,34 @@ class Worker { // Check for existance and validity of the include file $include = $argv[0]; + if (method_exists(sprintf('Friendica\Worker\%s', $include), 'execute')) { + // We constantly update the "executed" date every minute to avoid being killed too soon + if (!isset(self::$last_update)) { + self::$last_update = strtotime($queue["executed"]); + } + + $age = (time() - self::$last_update) / 60; + self::$last_update = time(); + + if ($age > 1) { + $stamp = (float)microtime(true); + dba::update('workerqueue', array('executed' => datetime_convert()), array('pid' => $mypid, 'done' => false)); + self::$db_duration += (microtime(true) - $stamp); + } + + array_shift($argv); + + self::execFunction($queue, $include, $argv, true); + + $stamp = (float)microtime(true); + if (dba::update('workerqueue', array('done' => true), array('id' => $queue["id"]))) { + Config::set('system', 'last_poller_execution', datetime_convert()); + } + self::$db_duration = (microtime(true) - $stamp); + + return true; + } + // The script could be provided as full path or only with the function name if ($include == basename($include)) { $include = "include/".$include.".php"; @@ -222,12 +259,11 @@ class Worker { return true; } - require_once($include); + require_once $include; $funcname = str_replace(".php", "", basename($argv[0]))."_run"; if (function_exists($funcname)) { - // We constantly update the "executed" date every minute to avoid being killed too soon if (!isset(self::$last_update)) { self::$last_update = strtotime($queue["executed"]); @@ -242,7 +278,7 @@ class Worker { self::$db_duration += (microtime(true) - $stamp); } - self::execFunction($queue, $funcname, $argv); + self::execFunction($queue, $funcname, $argv, false); $stamp = (float)microtime(true); if (dba::update('workerqueue', array('done' => true), array('id' => $queue["id"]))) { @@ -260,11 +296,14 @@ class Worker { /** * @brief Execute a function from the queue * - * @param array $queue Workerqueue entry - * @param string $funcname name of the function - * @param array $argv Array of values to be passed to the function + * @param array $queue Workerqueue entry + * @param string $funcname name of the function + * @param array $argv Array of values to be passed to the function + * @param boolean $method_call boolean + * @return void */ - private static function execFunction($queue, $funcname, $argv) { + private static function execFunction($queue, $funcname, $argv, $method_call) + { $a = get_app(); $mypid = getmypid(); @@ -303,7 +342,11 @@ class Worker { // Reset global data to avoid interferences unset($_SESSION); - $funcname($argv, $argc); + if ($method_call) { + call_user_func_array(sprintf('Friendica\Worker\%s::execute', $funcname), $argv); + } else { + $funcname($argv, $argc); + } $a->process_id = $old_process_id; unset($a->queue); @@ -317,10 +360,14 @@ class Worker { * The execution time is the productive time. * By changing parameters like the maximum number of workers we can check the effectivness. */ - logger('DB: '.number_format(self::$db_duration, 2). + logger( + 'DB: '.number_format(self::$db_duration, 2). ' - Lock: '.number_format(self::$lock_duration, 2). ' - Rest: '.number_format($up_duration - self::$db_duration - self::$lock_duration, 2). - ' - Execution: '.number_format($duration, 2), LOGGER_DEBUG); + ' - Execution: '.number_format($duration, 2), + LOGGER_DEBUG + ); + self::$lock_duration = 0; if ($duration > 3600) { @@ -342,7 +389,7 @@ class Worker { if (Config::get("rendertime", "callstack")) { if (isset($a->callstack["database"])) { $o = "\nDatabase Read:\n"; - foreach ($a->callstack["database"] AS $func => $time) { + foreach ($a->callstack["database"] as $func => $time) { $time = round($time, 3); if ($time > 0) { $o .= $func.": ".$time."\n"; @@ -351,7 +398,7 @@ class Worker { } if (isset($a->callstack["database_write"])) { $o .= "\nDatabase Write:\n"; - foreach ($a->callstack["database_write"] AS $func => $time) { + foreach ($a->callstack["database_write"] as $func => $time) { $time = round($time, 3); if ($time > 0) { $o .= $func.": ".$time."\n"; @@ -360,7 +407,7 @@ class Worker { } if (isset($a->callstack["network"])) { $o .= "\nNetwork:\n"; - foreach ($a->callstack["network"] AS $func => $time) { + foreach ($a->callstack["network"] as $func => $time) { $time = round($time, 3); if ($time > 0) { $o .= $func.": ".$time."\n"; @@ -371,14 +418,18 @@ class Worker { $o = ''; } - logger("ID ".$queue["id"].": ".$funcname.": ".sprintf("DB: %s/%s, Net: %s, I/O: %s, Other: %s, Total: %s".$o, - number_format($a->performance["database"] - $a->performance["database_write"], 2), - number_format($a->performance["database_write"], 2), - number_format($a->performance["network"], 2), - number_format($a->performance["file"], 2), - number_format($duration - ($a->performance["database"] + $a->performance["network"] + $a->performance["file"]), 2), - number_format($duration, 2)), - LOGGER_DEBUG); + logger( + "ID ".$queue["id"].": ".$funcname.": ".sprintf( + "DB: %s/%s, Net: %s, I/O: %s, Other: %s, Total: %s".$o, + number_format($a->performance["database"] - $a->performance["database_write"], 2), + number_format($a->performance["database_write"], 2), + number_format($a->performance["network"], 2), + number_format($a->performance["file"], 2), + number_format($duration - ($a->performance["database"] + $a->performance["network"] + $a->performance["file"]), 2), + number_format($duration, 2) + ), + LOGGER_DEBUG + ); } $cooldown = Config::get("system", "worker_cooldown", 0); @@ -394,18 +445,18 @@ class Worker { * * @return bool Are more than 3/4 of the maximum connections used? */ - private static function maxConnectionsReached() { - + private static function maxConnectionsReached() + { // Fetch the max value from the config. This is needed when the system cannot detect the correct value by itself. $max = Config::get("system", "max_connections"); - // Fetch the percentage level where the poller will get active + // Fetch the percentage level where the worker will get active $maxlevel = Config::get("system", "max_connections_level", 75); if ($max == 0) { // the maximum number of possible user connections can be a system variable $r = dba::fetch_first("SHOW VARIABLES WHERE `variable_name` = 'max_user_connections'"); - if (dbm::is_result($r)) { + if (DBM::is_result($r)) { $max = $r["Value"]; } // Or it can be granted. This overrides the system variable @@ -441,7 +492,7 @@ class Worker { // We will now check for the system values. // This limit could be reached although the user limits are fine. $r = dba::fetch_first("SHOW VARIABLES WHERE `variable_name` = 'max_connections'"); - if (!dbm::is_result($r)) { + if (!DBM::is_result($r)) { return false; } $max = intval($r["Value"]); @@ -449,7 +500,7 @@ class Worker { return false; } $r = dba::fetch_first("SHOW STATUS WHERE `variable_name` = 'Threads_connected'"); - if (!dbm::is_result($r)) { + if (!DBM::is_result($r)) { return false; } $used = intval($r["Value"]); @@ -469,16 +520,24 @@ class Worker { /** * @brief fix the queue entry if the worker process died - * + * @return void */ - private static function killStaleWorkers() { - $entries = dba::select('workerqueue', array('id', 'pid', 'executed', 'priority', 'parameter'), - array('`executed` > ? AND NOT `done` AND `pid` != 0', NULL_DATE), - array('order' => array('priority', 'created'))); + private static function killStaleWorkers() + { + $entries = dba::select( + 'workerqueue', + array('id', 'pid', 'executed', 'priority', 'parameter'), + array('`executed` > ? AND NOT `done` AND `pid` != 0', NULL_DATE), + array('order' => array('priority', 'created')) + ); + while ($entry = dba::fetch($entries)) { if (!posix_kill($entry["pid"], 0)) { - dba::update('workerqueue', array('executed' => NULL_DATE, 'pid' => 0), - array('id' => $entry["id"])); + dba::update( + 'workerqueue', + array('executed' => NULL_DATE, 'pid' => 0), + array('id' => $entry["id"]) + ); } else { // Kill long running processes // Check if the priority is in a valid range @@ -509,9 +568,11 @@ class Worker { } elseif ($entry["priority"] != PRIORITY_CRITICAL) { $new_priority = PRIORITY_NEGLIGIBLE; } - dba::update('workerqueue', - array('executed' => NULL_DATE, 'created' => datetime_convert(), 'priority' => $new_priority, 'pid' => 0), - array('id' => $entry["id"])); + dba::update( + 'workerqueue', + array('executed' => NULL_DATE, 'created' => datetime_convert(), 'priority' => $new_priority, 'pid' => 0), + array('id' => $entry["id"]) + ); } else { logger("Worker process ".$entry["pid"]." (".implode(" ", $argv).") now runs for ".round($duration)." of ".$max_duration." allowed minutes. That's okay.", LOGGER_DEBUG); } @@ -524,7 +585,8 @@ class Worker { * * @return bool Are there too much workers running? */ - public static function tooMuchWorkers() { + public static function tooMuchWorkers() + { $queues = Config::get("system", "worker_queues", 4); $maxqueues = $queues; @@ -548,9 +610,13 @@ class Worker { $listitem = array(); // Adding all processes with no workerqueue entry - $processes = dba::p("SELECT COUNT(*) AS `running` FROM `process` WHERE NOT EXISTS + $processes = dba::p( + "SELECT COUNT(*) AS `running` FROM `process` WHERE NOT EXISTS (SELECT id FROM `workerqueue` - WHERE `workerqueue`.`pid` = `process`.`pid` AND NOT `done` AND `pid` != ?)", getmypid()); + WHERE `workerqueue`.`pid` = `process`.`pid` AND NOT `done` AND `pid` != ?)", + getmypid() + ); + if ($process = dba::fetch($processes)) { $listitem[0] = "0:".$process["running"]; } @@ -569,7 +635,7 @@ class Worker { $intervals = array(1, 10, 60); $jobs_per_minute = array(); - foreach ($intervals AS $interval) { + foreach ($intervals as $interval) { $jobs = dba::p("SELECT COUNT(*) AS `jobs` FROM `workerqueue` WHERE `done` AND `executed` > UTC_TIMESTAMP() - INTERVAL ".intval($interval)." MINUTE"); if ($job = dba::fetch($jobs)) { $jobs_per_minute[$interval] = number_format($job['jobs'] / $interval, 0); @@ -604,11 +670,12 @@ class Worker { } /** - * @brief Returns the number of active poller processes + * @brief Returns the number of active worker processes * - * @return integer Number of active poller processes + * @return integer Number of active worker processes */ - private static function activeWorkers() { + private static function activeWorkers() + { $workers = dba::fetch_first("SELECT COUNT(*) AS `processes` FROM `process` WHERE `command` = 'Worker.php'"); return $workers["processes"]; @@ -623,15 +690,18 @@ class Worker { * @param string $highest_priority Returns the currently highest priority * @return bool We let pass a slower process than $highest_priority */ - private static function passingSlow(&$highest_priority) { + private static function passingSlow(&$highest_priority) + { $highest_priority = 0; - $r = dba::p("SELECT `priority` + $r = dba::p( + "SELECT `priority` FROM `process` - INNER JOIN `workerqueue` ON `workerqueue`.`pid` = `process`.`pid` AND NOT `done`"); + INNER JOIN `workerqueue` ON `workerqueue`.`pid` = `process`.`pid` AND NOT `done`" + ); // No active processes at all? Fine - if (!dbm::is_result($r)) { + if (!DBM::is_result($r)) { return false; } $priorities = array(); @@ -652,7 +722,7 @@ class Worker { return false; } $high = 0; - foreach ($priorities AS $priority) { + foreach ($priorities as $priority) { if ($priority == $highest_priority) { ++$high; } @@ -672,7 +742,8 @@ class Worker { * @param boolean $passing_slow Returns if we had passed low priority processes * @return boolean Have we found something? */ - private static function findWorkerProcesses(&$passing_slow) { + private static function findWorkerProcesses(&$passing_slow) + { $mypid = getmypid(); // Check if we should pass some low priority process @@ -696,8 +767,12 @@ class Worker { if (self::passingSlow($highest_priority)) { // Are there waiting processes with a higher priority than the currently highest? - $result = dba::select('workerqueue', array('id'), array("`executed` <= ? AND `priority` < ? AND NOT `done`", NULL_DATE, $highest_priority), - array('limit' => $limit, 'order' => array('priority', 'created'), 'only_query' => true)); + $result = dba::select( + 'workerqueue', + array('id'), + array("`executed` <= ? AND `priority` < ? AND NOT `done`", NULL_DATE, $highest_priority), + array('limit' => $limit, 'order' => array('priority', 'created'), 'only_query' => true) + ); while ($id = dba::fetch($result)) { $ids[] = $id["id"]; @@ -708,8 +783,12 @@ class Worker { if (!$found) { // Give slower processes some processing time - $result = dba::select('workerqueue', array('id'), array("`executed` <= ? AND `priority` > ? AND NOT `done`", NULL_DATE, $highest_priority), - array('limit' => $limit, 'order' => array('priority', 'created'), 'only_query' => true)); + $result = dba::select( + 'workerqueue', + array('id'), + array("`executed` <= ? AND `priority` > ? AND NOT `done`", NULL_DATE, $highest_priority), + array('limit' => $limit, 'order' => array('priority', 'created'), 'only_query' => true) + ); while ($id = dba::fetch($result)) { $ids[] = $id["id"]; @@ -723,8 +802,12 @@ class Worker { // If there is no result (or we shouldn't pass lower processes) we check without priority limit if (!$found) { - $result = dba::select('workerqueue', array('id'), array("`executed` <= ? AND NOT `done`", NULL_DATE), - array('limit' => $limit, 'order' => array('priority', 'created'), 'only_query' => true)); + $result = dba::select( + 'workerqueue', + array('id'), + array("`executed` <= ? AND NOT `done`", NULL_DATE), + array('limit' => $limit, 'order' => array('priority', 'created'), 'only_query' => true) + ); while ($id = dba::fetch($result)) { $ids[] = $id["id"]; @@ -749,12 +832,13 @@ class Worker { * @param boolean $passing_slow Returns if we had passed low priority processes * @return string SQL statement */ - public static function workerProcess(&$passing_slow) { + public static function workerProcess(&$passing_slow) + { $stamp = (float)microtime(true); // There can already be jobs for us in the queue. $r = dba::select('workerqueue', array(), array('pid' => getmypid(), 'done' => false)); - if (dbm::is_result($r)) { + if (DBM::is_result($r)) { self::$db_duration += (microtime(true) - $stamp); return dba::inArray($r); } @@ -781,8 +865,10 @@ class Worker { /** * @brief Removes a workerqueue entry from the current process + * @return void */ - public static function unclaimProcess() { + public static function unclaimProcess() + { $mypid = getmypid(); dba::update('workerqueue', array('executed' => NULL_DATE, 'pid' => 0), array('pid' => $mypid, 'done' => false)); @@ -790,8 +876,10 @@ class Worker { /** * @brief Call the front end worker + * @return void */ - public static function callWorker() { + public static function callWorker() + { if (!Config::get("system", "frontend_worker")) { return; } @@ -802,13 +890,15 @@ class Worker { /** * @brief Call the front end worker if there aren't any active + * @return void */ - public static function executeIfIdle() { + public static function executeIfIdle() + { if (!Config::get("system", "frontend_worker")) { return; } - // Do we have "proc_open"? Then we can fork the poller + // Do we have "proc_open"? Then we can fork the worker if (function_exists("proc_open")) { // When was the last time that we called the worker? // Less than one minute? Then we quit @@ -829,7 +919,7 @@ class Worker { self::runCron(); - logger('Call poller', LOGGER_DEBUG); + logger('Call worker', LOGGER_DEBUG); self::spawnWorker(); return; } @@ -850,37 +940,45 @@ class Worker { /** * @brief Removes long running worker processes + * @return void */ - public static function clearProcesses() { + public static function clearProcesses() + { $timeout = Config::get("system", "frontend_worker_timeout", 10); /// @todo We should clean up the corresponding workerqueue entries as well $condition = array("`created` < ? AND `command` = 'worker.php'", - datetime_convert('UTC','UTC',"now - ".$timeout." minutes")); + datetime_convert('UTC', 'UTC', "now - ".$timeout." minutes")); dba::delete('process', $condition); } /** * @brief Runs the cron processes + * @return void */ - private static function runCron() { + private static function runCron() + { logger('Add cron entries', LOGGER_DEBUG); // Check for spooled items - self::add(PRIORITY_HIGH, "spool_post"); + self::add(PRIORITY_HIGH, "SpoolPost"); // Run the cron job that calls all other jobs - self::add(PRIORITY_MEDIUM, "cron"); + self::add(PRIORITY_MEDIUM, "Cron"); // Run the cronhooks job separately from cron for being able to use a different timing - self::add(PRIORITY_MEDIUM, "cronhooks"); + self::add(PRIORITY_MEDIUM, "CronHooks"); // Cleaning dead processes self::killStaleWorkers(); } - public static function spawnWorker() { - $args = array("include/poller.php", "no_cron"); + /** + * @return void + */ + public static function spawnWorker() + { + $args = array("scripts/worker.php", "no_cron"); get_app()->proc_run($args); } @@ -890,8 +988,8 @@ class Worker { * @param (integer|array) priority or parameter array, strings are deprecated and are ignored * * next args are passed as $cmd command line - * or: Worker::add(PRIORITY_HIGH, "notifier", "drop", $drop_id); - * or: Worker::add(array('priority' => PRIORITY_HIGH, 'dont_fork' => true), "create_shadowentry", $post_id); + * or: Worker::add(PRIORITY_HIGH, "Notifier", "drop", $drop_id); + * or: Worker::add(array('priority' => PRIORITY_HIGH, 'dont_fork' => true), "CreateShadowEntry", $post_id); * * @note $cmd and string args are surrounded with "" * @@ -900,7 +998,8 @@ class Worker { * * @return boolean "false" if proc_run couldn't be executed */ - public static function add($cmd) { + public static function add($cmd) + { $proc_args = func_get_args(); $args = array(); @@ -967,7 +1066,7 @@ class Worker { dba::insert('workerqueue', array('parameter' => $parameters, 'created' => $created, 'priority' => $priority)); } - // Should we quit and wait for the poller to be called as a cronjob? + // Should we quit and wait for the worker to be called as a cronjob? if ($dont_fork) { return true; } @@ -985,7 +1084,7 @@ class Worker { return true; } - // Now call the poller to execute the jobs that we just added to the queue + // Now call the worker to execute the jobs that we just added to the queue self::spawnWorker(); return true;