Merge pull request #2347 from annando/1602-dfrn-forum

[friendica.git] / include / poller.php

diff --git a/include/poller.php b/include/poller.php
index b1d6099ad343271e8f4c2d2ca65a879406c98a74..9976e76db2cd619b85f171eeaa72fd2fb7721a42 100644 (file)
--- a/include/poller.php
+++ b/include/poller.php
@@ -26,6 +26,9 @@ function poller_run(&$argv, &$argc){
                unset($db_host, $db_user, $db_pass, $db_data);
        };
 
+       if (poller_max_connections_reached())
+               return;
+
        $load = current_load();
        if($load) {
                $maxsysload = intval(get_config('system','maxloadavg'));
@@ -39,8 +42,10 @@ function poller_run(&$argv, &$argc){
        }
 
        // Checking the number of workers
-       if (poller_too_much_workers(1))
+       if (poller_too_much_workers(1)) {
+               poller_kill_stale_workers();
                return;
+       }
 
        if(($argc <= 1) OR ($argv[1] != "no_cron")) {
                // Run the cron job that calls all other jobs
@@ -50,16 +55,7 @@ function poller_run(&$argv, &$argc){
                proc_run("php","include/cronhooks.php");
 
                // Cleaning dead processes
-               $r = q("SELECT DISTINCT(`pid`) FROM `workerqueue` WHERE `executed` != '0000-00-00 00:00:00'");
-               foreach($r AS $pid)
-                       if (!posix_kill($pid["pid"], 0))
-                               q("UPDATE `workerqueue` SET `executed` = '0000-00-00 00:00:00', `pid` = 0 WHERE `pid` = %d",
-                                       intval($pid["pid"]));
-                       else {
-                               // To-Do: Kill long running processes
-                               // But: Update processes (like the database update) mustn't be killed
-                       }
-
+               poller_kill_stale_workers();
        } else
                // Sleep four seconds before checking for running processes again to avoid having too many workers
                sleep(4);
@@ -124,6 +120,78 @@ function poller_run(&$argv, &$argc){
 
 }
 
+/**
+ * @brief Checks if the number of database connections has reached a critical limit.
+ *
+ * @return bool Are more than 3/4 of the maximum connections used?
+ */
+function poller_max_connections_reached() {
+       $r = q("SHOW VARIABLES WHERE `variable_name` = 'max_connections'");
+       if (!$r)
+               return false;
+
+       // Fetch the max value from the config. This is needed when the system cannot detect the correct value by itself.
+       // In that case we use the processlist to determine the current number of connections
+       $max = get_config("system", "max_connections");
+
+       if ($max == 0) {
+               $max = intval($r[0]["Value"]);
+               if ($max == 0)
+                       return false;
+
+               $r = q("SHOW STATUS WHERE `variable_name` = 'Threads_connected'");
+               if (!$r)
+                       return false;
+
+               $connected = intval($r[0]["Value"]);
+               if ($connected == 0)
+                       return false;
+       } else {
+               $r = q("SHOW PROCESSLIST");
+               if (!$r)
+                       return false;
+
+               $connected = count($r);
+       }
+
+       $level = $connected / $max;
+
+       logger("Connection usage: ".$connected."/".$max, LOGGER_DEBUG);
+
+       if ($level < (3/4))
+               return false;
+
+       logger("Maximum level (3/4) of connections reached: ".$connected."/".$max);
+       return true;
+
+}
+
+/**
+ * @brief fix the queue entry if the worker process died
+ *
+ */
+function poller_kill_stale_workers() {
+       $r = q("SELECT `pid`, `executed` FROM `workerqueue` WHERE `executed` != '0000-00-00 00:00:00'");
+       foreach($r AS $pid)
+               if (!posix_kill($pid["pid"], 0))
+                       q("UPDATE `workerqueue` SET `executed` = '0000-00-00 00:00:00', `pid` = 0 WHERE `pid` = %d",
+                               intval($pid["pid"]));
+               else {
+                       // Kill long running processes
+                       $duration = (time() - strtotime($pid["executed"])) / 60;
+                       if ($duration > 180) {
+                               logger("Worker process ".$pid["pid"]." took more than 3 hours. It will be killed now.");
+                               posix_kill($pid["pid"], SIGTERM);
+
+                               // Question: If a process is stale: Should we remove it or should we reschedule it?
+                               // By now we rescheduling it. It's maybe not the wisest decision?
+                               q("UPDATE `workerqueue` SET `executed` = '0000-00-00 00:00:00', `pid` = 0 WHERE `pid` = %d",
+                                       intval($pid["pid"]));
+                       } else
+                               logger("Worker process ".$pid["pid"]." now runs for ".round($duration)." minutes. That's okay.", LOGGER_DEBUG);
+               }
+}
+
 function poller_too_much_workers($stage) {
 
        $queues = get_config("system", "worker_queues");