More cooldown calls for worker processes

diff --git a/bin/daemon.php b/bin/daemon.php
index 880115d0d01fb8a8ff94b103c3288d115062d483..f1eb077adde604de570dfa1d6e37017b32098425 100755 (executable)
--- a/bin/daemon.php
+++ b/bin/daemon.php
@@ -76,8 +76,8 @@ DI::config()->load();
 if (empty(DI::config()->get('system', 'pidfile'))) {
        die(<<<TXT
 Please set system.pidfile in config/local.config.php. For example:
-    
-    'system' => [ 
+
+    'system' => [
         'pidfile' => '/path/to/daemon.pid',
     ],
 TXT
@@ -199,6 +199,7 @@ while (true) {
        }
 
        if ($do_cron || (!DI::system()->isMaxLoadReached() && Worker::entriesExists() && Worker::isReady())) {
+               Worker::coolDown();
                Worker::spawnWorker($do_cron);
        } else {
                Logger::info('Cool down for 5 seconds', ['pid' => $pid]);

diff --git a/src/Core/Worker.php b/src/Core/Worker.php
index 830e02983210d208f55db7e83520f943a5bcc0c6..4496caf920039e67c7eb2144e9763abcbc43bfbd 100644 (file)
--- a/src/Core/Worker.php
+++ b/src/Core/Worker.php
@@ -445,42 +445,68 @@ class Worker
        }
 
        /**
-        * Execute a function from the queue
+        * Slow the execution down if the system load is too high
         *
-        * @param array   $queue       Workerqueue entry
-        * @param string  $funcname    name of the function
-        * @param array   $argv        Array of values to be passed to the function
-        * @param boolean $method_call boolean
         * @return void
-        * @throws \Friendica\Network\HTTPException\InternalServerErrorException
         */
-       private static function execFunction(array $queue, string $funcname, array $argv, bool $method_call)
+       public static function coolDown()
        {
-               $a = DI::app();
+               $load_cooldown      = DI::config()->get('system', 'worker_load_cooldown');
+               $processes_cooldown = DI::config()->get('system', 'worker_processes_cooldown');
 
-               $cooldown = DI::config()->get('system', 'worker_cooldown', 0);
-               if ($cooldown > 0) {
-                       Logger::debug('Pre execution cooldown.', ['cooldown' => $cooldown, 'id' => $queue['id'], 'priority' => $queue['priority'], 'command' => $queue['command']]);
-                       sleep($cooldown);
+               if (($load_cooldown == 0) && ($processes_cooldown == 0)) {
+                       return;
                }
 
-               $load_cooldown      = DI::config()->get('system', 'worker_load_cooldown');
-               $processes_cooldown = DI::config()->get('system', 'worker_processes_cooldown');
+               $sleeping = false;
 
-               while ((($load_cooldown > 0) || ($processes_cooldown > 0)) && ($load = System::getLoadAvg())) {
+               while ($load = System::getLoadAvg()) {
                        if (($load_cooldown > 0) && ($load['average1'] > $load_cooldown)) {
-                               Logger::debug('Load induced pre execution cooldown.', ['max' => $load_cooldown, 'load' => $load, 'id' => $queue['id'], 'priority' => $queue['priority'], 'command' => $queue['command']]);
+                               if (!$sleeping) {
+                                       Logger::notice('Load induced pre execution cooldown.', ['max' => $load_cooldown, 'load' => $load, 'called-by' => System::callstack(1)]);
+                                       $sleeping = true;
+                               }
                                sleep(1);
                                continue;
                        }
                        if (($processes_cooldown > 0) && ($load['scheduled'] > $processes_cooldown)) {
-                               Logger::debug('Process induced pre execution cooldown.', ['max' => $processes_cooldown, 'load' => $load, 'id' => $queue['id'], 'priority' => $queue['priority'], 'command' => $queue['command']]);
+                               if (!$sleeping) {
+                                       Logger::notice('Process induced pre execution cooldown.', ['max' => $processes_cooldown, 'load' => $load, 'called-by' => System::callstack(1)]);
+                                       $sleeping = true;
+                               }
                                sleep(1);
                                continue;
                        }
                        break;
                }
 
+               if ($sleeping) {
+                       Logger::notice('Cooldown ended.', ['max-load' => $load_cooldown, 'max-processes' => $processes_cooldown, 'load' => $load, 'called-by' => System::callstack(1)]);
+               }
+       }
+
+       /**
+        * Execute a function from the queue
+        *
+        * @param array   $queue       Workerqueue entry
+        * @param string  $funcname    name of the function
+        * @param array   $argv        Array of values to be passed to the function
+        * @param boolean $method_call boolean
+        * @return void
+        * @throws \Friendica\Network\HTTPException\InternalServerErrorException
+        */
+       private static function execFunction(array $queue, string $funcname, array $argv, bool $method_call)
+       {
+               $a = DI::app();
+
+               $cooldown = DI::config()->get('system', 'worker_cooldown', 0);
+               if ($cooldown > 0) {
+                       Logger::notice('Pre execution cooldown.', ['cooldown' => $cooldown, 'id' => $queue['id'], 'priority' => $queue['priority'], 'command' => $queue['command']]);
+                       sleep($cooldown);
+               }
+
+               self::coolDown();
+
                Logger::enableWorker($funcname);
 
                Logger::info('Process start.', ['priority' => $queue['priority'], 'id' => $queue['id']]);
@@ -527,6 +553,8 @@ class Worker
 
                Logger::info('Performance:', ['state' => self::$state, 'count' => $dbcount, 'stat' => $dbstat, 'write' => $dbwrite, 'lock' => $dblock, 'total' => $dbtotal, 'rest' => $rest, 'exec' => $exec]);
 
+               self::coolDown();
+
                self::$up_start = microtime(true);
                self::$db_duration = 0;
                self::$db_duration_count = 0;

diff --git a/src/Protocol/ActivityPub/Delivery.php b/src/Protocol/ActivityPub/Delivery.php
index 9aa45d0419a93dbb87e03b2f2612e684430fb56a..49a2b5aa4486d27e5121af73d1804c87fd25a997 100644 (file)
--- a/src/Protocol/ActivityPub/Delivery.php
+++ b/src/Protocol/ActivityPub/Delivery.php
@@ -22,6 +22,7 @@
 namespace Friendica\Protocol\ActivityPub;
 
 use Friendica\Core\Logger;
+use Friendica\Core\Worker;
 use Friendica\Database\DBA;
 use Friendica\DI;
 use Friendica\Model\Contact;
@@ -55,6 +56,7 @@ class Delivery
                                        Logger::notice('Inbox delivery has a server failure', ['inbox' => $inbox]);
                                        $serverfail = true;
                                }
+                               Worker::coolDown();
                        }
 
                        if ($serverfail || (!$result['success'] && !$result['drop'])) {

diff --git a/src/Worker/Notifier.php b/src/Worker/Notifier.php
index 7921097d4a466eb707439823095b22998c02e9e3..b236abc1951a012ea069837d5120c03ad3a20ec1 100644 (file)
--- a/src/Worker/Notifier.php
+++ b/src/Worker/Notifier.php
@@ -573,6 +573,7 @@ class Notifier
                        if (Worker::add($deliver_options, 'Delivery', $cmd, $post_uriid, (int)$contact['id'], $sender_uid)) {
                                $delivery_queue_count++;
                        }
+                       Worker::coolDown();
                }
                return $delivery_queue_count;
        }
@@ -695,6 +696,7 @@ class Notifier
                        Logger::info('Account removal via ActivityPub', ['uid' => $self_user_id, 'inbox' => $inbox]);
                        Worker::add(['priority' => PRIORITY_NEGLIGIBLE, 'created' => $created, 'dont_fork' => true],
                                'APDelivery', Delivery::REMOVAL, 0, $inbox, $self_user_id, $receivers);
+                       Worker::coolDown();
                }
 
                return true;
@@ -818,6 +820,7 @@ class Notifier
                                        $delivery_queue_count++;
                                }
                        }
+                       Worker::coolDown();
                }
 
                // We deliver posts to relay servers slightly delayed to priorize the direct delivery
@@ -833,6 +836,7 @@ class Notifier
                                        $delivery_queue_count++;
                                }
                        }
+                       Worker::coolDown();
                }
 
                return ['count' => $delivery_queue_count, 'contacts' => $contacts];

diff --git a/src/Worker/PollContacts.php b/src/Worker/PollContacts.php
index b61f5aebb82afb194f93de7f6c1972146c829f33..b4312ef0c3022cab852d8b97f57ab3facde3a411 100644 (file)
--- a/src/Worker/PollContacts.php
+++ b/src/Worker/PollContacts.php
@@ -81,6 +81,7 @@ class PollContacts
                        Logger::notice("Polling " . $contact["network"] . " " . $contact["id"] . " " . $contact['priority'] . " " . $contact["nick"] . " " . $contact["name"]);
 
                        Worker::add(['priority' => $priority, 'dont_fork' => true, 'force_priority' => true], 'OnePoll', (int)$contact['id']);
+                       Worker::coolDown();
                }
                DBA::close($contacts);
        }

diff --git a/src/Worker/UpdateContacts.php b/src/Worker/UpdateContacts.php
index 31426e79f22245c9ef65210f5d4a49b1d3d29e6b..7e352a27caf9f683bee524c8205cf8cf3993a5f4 100644 (file)
--- a/src/Worker/UpdateContacts.php
+++ b/src/Worker/UpdateContacts.php
@@ -62,6 +62,7 @@ class UpdateContacts
                        if (Worker::add(['priority' => PRIORITY_LOW, 'dont_fork' => true], "UpdateContact", $contact['id'])) {
                                ++$count;
                        }
+                       Worker::coolDown();
                }
                DBA::close($contacts);
 

diff --git a/src/Worker/UpdateGServers.php b/src/Worker/UpdateGServers.php
index e53754584661cd5ac04e9b8ee02cf3eaeb71f4df..108482eaecef4035336e8b1c7797c3f8ddacde29 100644 (file)
--- a/src/Worker/UpdateGServers.php
+++ b/src/Worker/UpdateGServers.php
@@ -72,6 +72,7 @@ class UpdateGServers
                                        $count++;
                                }
                        }
+                       Worker::coolDown();
                }
                DBA::close($gservers);
                Logger::info('Updated servers', ['count' => $count]);

diff --git a/src/Worker/UpdateServerPeers.php b/src/Worker/UpdateServerPeers.php
index 09c88499ec63fa2e985449ab78f1b01584158334..8c916504eddd75b1c80302de2c37d9c8f9384e11 100644 (file)
--- a/src/Worker/UpdateServerPeers.php
+++ b/src/Worker/UpdateServerPeers.php
@@ -63,6 +63,7 @@ class UpdateServerPeers
                        // This endpoint doesn't offer the schema. So we assume that it is HTTPS.
                        GServer::add('https://' . $peer);
                        ++$added;
+                       Worker::coolDown();
                }
                Logger::info('Server peer update ended', ['total' => $total, 'added' => $added, 'url' => $url]);
        }