]> git.mxchange.org Git - friendica.git/blobdiff - src/Core/Worker.php
Don't transmit content to already archived contacts
[friendica.git] / src / Core / Worker.php
index efb90ae6b6ca570090cdf850ac32d1aebcf9f641..f24df5252d4479bf9c4105039b706c2d0bd061f5 100644 (file)
@@ -5,6 +5,7 @@
 namespace Friendica\Core;
 
 use Friendica\BaseObject;
+use Friendica\Core;
 use Friendica\Database\DBA;
 use Friendica\Model\Process;
 use Friendica\Util\DateTimeFormat;
@@ -21,6 +22,14 @@ use Friendica\Util\Network;
  */
 class Worker
 {
+       const STATE_STARTUP    = 1; // Worker is in startup. This takes most time.
+       const STATE_LONG_LOOP  = 2; // Worker is processing the whole - long - loop.
+       const STATE_REFETCH    = 3; // Worker had refetched jobs in the execution loop.
+       const STATE_SHORT_LOOP = 4; // Worker is processing preassigned jobs, thus saving much time.
+
+       const FAST_COMMANDS = ['APDelivery', 'Delivery', 'CreateShadowEntry'];
+
+
        private static $up_start;
        private static $db_duration = 0;
        private static $db_duration_count = 0;
@@ -28,6 +37,7 @@ class Worker
        private static $db_duration_stat = 0;
        private static $lock_duration = 0;
        private static $last_update;
+       private static $state;
 
        /**
         * @brief Processes the tasks that are in the workerqueue table
@@ -91,9 +101,11 @@ class Worker
                }
 
                $starttime = time();
+               self::$state = self::STATE_STARTUP;
 
                // We fetch the next queue entry that is about to be executed
                while ($r = self::workerProcess()) {
+                       $refetched = false;
                        foreach ($r as $entry) {
                                // Assure that the priority is an integer value
                                $entry['priority'] = (int)$entry['priority'];
@@ -104,34 +116,43 @@ class Worker
                                        return;
                                }
 
-                               // If possible we will fetch new jobs for this worker
-                               if (!self::getWaitingJobForPID() && Lock::acquire('worker_process', 0)) {
+                               // Trying to fetch new processes - but only once when successful
+                               if (!$refetched && Lock::acquire('worker_process', 0)) {
                                        self::findWorkerProcesses();
                                        Lock::release('worker_process');
+                                       self::$state = self::STATE_REFETCH;
+                                       $refetched = true;
+                               } else {
+                                       self::$state = self::STATE_SHORT_LOOP;
                                }
                        }
 
                        // To avoid the quitting of multiple workers only one worker at a time will execute the check
-                       if (Lock::acquire('worker', 0)) {
+                       if (!self::getWaitingJobForPID()) {
+                               self::$state = self::STATE_LONG_LOOP;
+
+                               if (Lock::acquire('worker', 0)) {
                                // Count active workers and compare them with a maximum value that depends on the load
-                               if (self::tooMuchWorkers()) {
-                                       Logger::log('Active worker limit reached, quitting.', Logger::DEBUG);
-                                       Lock::release('worker');
-                                       return;
-                               }
+                                       if (self::tooMuchWorkers()) {
+                                               Logger::log('Active worker limit reached, quitting.', Logger::DEBUG);
+                                               Lock::release('worker');
+                                               return;
+                                       }
 
-                               // Check free memory
-                               if ($a->isMinMemoryReached()) {
-                                       Logger::log('Memory limit reached, quitting.', Logger::DEBUG);
+                                       // Check free memory
+                                       if ($a->isMinMemoryReached()) {
+                                               Logger::log('Memory limit reached, quitting.', Logger::DEBUG);
+                                               Lock::release('worker');
+                                               return;
+                                       }
                                        Lock::release('worker');
-                                       return;
                                }
-                               Lock::release('worker');
                        }
 
-                       // Quit the worker once every 5 minutes
-                       if (time() > ($starttime + 300)) {
-                               Logger::log('Process lifetime reached, quitting.', Logger::DEBUG);
+                       // Quit the worker once every cron interval
+                       if (time() > ($starttime + (Config::get('system', 'cron_interval') * 60))) {
+                               Logger::info('Process lifetime reached, respawning.');
+                               self::spawnWorker();
                                return;
                        }
                }
@@ -166,7 +187,7 @@ class Worker
        private static function deferredEntries()
        {
                $stamp = (float)microtime(true);
-               $count = DBA::count('workerqueue', ["NOT `done` AND `pid` = 0 AND `next_try` > ?", DateTimeFormat::utcNow()]);
+               $count = DBA::count('workerqueue', ["NOT `done` AND `pid` = 0 AND `retrial` > ?", 0]);
                self::$db_duration += (microtime(true) - $stamp);
                self::$db_duration_count += (microtime(true) - $stamp);
                return $count;
@@ -355,39 +376,18 @@ class Worker
        {
                $a = \get_app();
 
-               $mypid = getmypid();
-
                $argc = count($argv);
 
-               // Currently deactivated, since the new logger doesn't support this
-               //$new_process_id = System::processID("wrk");
-               $new_process_id = '';
+               Logger::enableWorker($funcname);
 
-               Logger::log("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." ".$queue["parameter"]." - Process PID: ".$new_process_id);
+               Logger::info("Process start.", ['priority' => $queue["priority"], 'id' => $queue["id"]]);
 
                $stamp = (float)microtime(true);
 
                // We use the callstack here to analyze the performance of executed worker entries.
                // For this reason the variables have to be initialized.
-               if (Config::get("system", "profiler")) {
-                       $a->performance["start"] = microtime(true);
-                       $a->performance["database"] = 0;
-                       $a->performance["database_write"] = 0;
-                       $a->performance["cache"] = 0;
-                       $a->performance["cache_write"] = 0;
-                       $a->performance["network"] = 0;
-                       $a->performance["file"] = 0;
-                       $a->performance["rendering"] = 0;
-                       $a->performance["parser"] = 0;
-                       $a->performance["marktime"] = 0;
-                       $a->performance["markstart"] = microtime(true);
-                       $a->callstack = [];
-               }
-
-               // For better logging create a new process id for every worker call
-               // But preserve the old one for the worker
-               $old_process_id = $a->process_id;
-               $a->process_id = $new_process_id;
+               $a->getProfiler()->reset();
+
                $a->queue = $queue;
 
                $up_duration = microtime(true) - self::$up_start;
@@ -395,13 +395,15 @@ class Worker
                // Reset global data to avoid interferences
                unset($_SESSION);
 
+               // Set the workerLogger as new default logger
                if ($method_call) {
                        call_user_func_array(sprintf('Friendica\Worker\%s::execute', $funcname), $argv);
                } else {
                        $funcname($argv, $argc);
                }
 
-               $a->process_id = $old_process_id;
+               Logger::disableWorker();
+
                unset($a->queue);
 
                $duration = (microtime(true) - $stamp);
@@ -411,16 +413,16 @@ class Worker
                 * The execution time is the productive time.
                 * By changing parameters like the maximum number of workers we can check the effectivness.
                */
-               Logger::log(
-                       'DB: '.number_format(self::$db_duration - (self::$db_duration_count + self::$db_duration_write + self::$db_duration_stat), 4).
-                       ' - DB-Count: '.number_format(self::$db_duration_count, 4).
-                       ' - DB-Stat: '.number_format(self::$db_duration_stat, 4).
-                       ' - DB-Write: '.number_format(self::$db_duration_write, 4).
-                       ' - Lock: '.number_format(self::$lock_duration, 4).
-                       ' - Rest: '.number_format(max(0, $up_duration - (self::$db_duration + self::$lock_duration)), 4).
-                       ' - Execution: '.number_format($duration, 4),
-                       Logger::DEBUG
-               );
+               $dbtotal = round(self::$db_duration, 2);
+               $dbread  = round(self::$db_duration - (self::$db_duration_count + self::$db_duration_write + self::$db_duration_stat), 2);
+               $dbcount = round(self::$db_duration_count, 2);
+               $dbstat  = round(self::$db_duration_stat, 2);
+               $dbwrite = round(self::$db_duration_write, 2);
+               $dblock  = round(self::$lock_duration, 2);
+               $rest    = round(max(0, $up_duration - (self::$db_duration + self::$lock_duration)), 2);
+               $exec    = round($duration, 2);
+
+               Logger::info('Performance:', ['state' => self::$state, 'count' => $dbcount, 'stat' => $dbstat, 'write' => $dbwrite, 'lock' => $dblock, 'total' => $dbtotal, 'rest' => $rest, 'exec' => $exec]);
 
                self::$up_start = microtime(true);
                self::$db_duration = 0;
@@ -430,92 +432,23 @@ class Worker
                self::$lock_duration = 0;
 
                if ($duration > 3600) {
-                       Logger::log("Prio ".$queue["priority"].": ".$queue["parameter"]." - longer than 1 hour (".round($duration/60, 3).")", Logger::DEBUG);
+                       Logger::info('Longer than 1 hour.', ['priority' => $queue["priority"], 'id' => $queue["id"], 'duration' => round($duration/60, 3)]);
                } elseif ($duration > 600) {
-                       Logger::log("Prio ".$queue["priority"].": ".$queue["parameter"]." - longer than 10 minutes (".round($duration/60, 3).")", Logger::DEBUG);
+                       Logger::info('Longer than 10 minutes.', ['priority' => $queue["priority"], 'id' => $queue["id"], 'duration' => round($duration/60, 3)]);
                } elseif ($duration > 300) {
-                       Logger::log("Prio ".$queue["priority"].": ".$queue["parameter"]." - longer than 5 minutes (".round($duration/60, 3).")", Logger::DEBUG);
+                       Logger::info('Longer than 5 minutes.', ['priority' => $queue["priority"], 'id' => $queue["id"], 'duration' => round($duration/60, 3)]);
                } elseif ($duration > 120) {
-                       Logger::log("Prio ".$queue["priority"].": ".$queue["parameter"]." - longer than 2 minutes (".round($duration/60, 3).")", Logger::DEBUG);
+                       Logger::info('Longer than 2 minutes.', ['priority' => $queue["priority"], 'id' => $queue["id"], 'duration' => round($duration/60, 3)]);
                }
 
-               Logger::log("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." - done in ".number_format($duration, 4)." seconds. Process PID: ".$new_process_id);
+               Logger::info('Process done.', ['priority' => $queue["priority"], 'id' => $queue["id"], 'duration' => round($duration, 3)]);
 
-               // Write down the performance values into the log
-               if (Config::get("system", "profiler")) {
-                       $duration = microtime(true)-$a->performance["start"];
-
-                       $o = '';
-                       if (Config::get("rendertime", "callstack")) {
-                               if (isset($a->callstack["database"])) {
-                                       $o .= "\nDatabase Read:\n";
-                                       foreach ($a->callstack["database"] as $func => $time) {
-                                               $time = round($time, 3);
-                                               if ($time > 0) {
-                                                       $o .= $func.": ".$time."\n";
-                                               }
-                                       }
-                               }
-                               if (isset($a->callstack["database_write"])) {
-                                       $o .= "\nDatabase Write:\n";
-                                       foreach ($a->callstack["database_write"] as $func => $time) {
-                                               $time = round($time, 3);
-                                               if ($time > 0) {
-                                                       $o .= $func.": ".$time."\n";
-                                               }
-                                       }
-                               }
-                               if (isset($a->callstack["dache"])) {
-                                       $o .= "\nCache Read:\n";
-                                       foreach ($a->callstack["dache"] as $func => $time) {
-                                               $time = round($time, 3);
-                                               if ($time > 0) {
-                                                       $o .= $func.": ".$time."\n";
-                                               }
-                                       }
-                               }
-                               if (isset($a->callstack["dache_write"])) {
-                                       $o .= "\nCache Write:\n";
-                                       foreach ($a->callstack["dache_write"] as $func => $time) {
-                                               $time = round($time, 3);
-                                               if ($time > 0) {
-                                                       $o .= $func.": ".$time."\n";
-                                               }
-                                       }
-                               }
-                               if (isset($a->callstack["network"])) {
-                                       $o .= "\nNetwork:\n";
-                                       foreach ($a->callstack["network"] as $func => $time) {
-                                               $time = round($time, 3);
-                                               if ($time > 0) {
-                                                       $o .= $func.": ".$time."\n";
-                                               }
-                                       }
-                               }
-                       }
-
-                       Logger::log(
-                               "ID ".$queue["id"].": ".$funcname.": ".sprintf(
-                                       "DB: %s/%s, Cache: %s/%s, Net: %s, I/O: %s, Other: %s, Total: %s".$o,
-                                       number_format($a->performance["database"] - $a->performance["database_write"], 2),
-                                       number_format($a->performance["database_write"], 2),
-                                       number_format($a->performance["cache"], 2),
-                                       number_format($a->performance["cache_write"], 2),
-                                       number_format($a->performance["network"], 2),
-                                       number_format($a->performance["file"], 2),
-                                       number_format($duration - ($a->performance["database"]
-                                               + $a->performance["cache"] + $a->performance["cache_write"]
-                                               + $a->performance["network"] + $a->performance["file"]), 2),
-                                       number_format($duration, 2)
-                               ),
-                               Logger::DEBUG
-                       );
-               }
+               $a->getProfiler()->saveLog($a->getLogger(), "ID " . $queue["id"] . ": " . $funcname);
 
                $cooldown = Config::get("system", "worker_cooldown", 0);
 
                if ($cooldown > 0) {
-                       Logger::log("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." - in cooldown for ".$cooldown." seconds");
+                       Logger::info('Cooldown.', ['priority' => $queue["priority"], 'id' => $queue["id"], 'cooldown' => $cooldown]);
                        sleep($cooldown);
                }
        }
@@ -683,7 +616,7 @@ class Worker
         */
        private static function tooMuchWorkers()
        {
-               $queues = Config::get("system", "worker_queues", 4);
+               $queues = Config::get("system", "worker_queues", 10);
 
                $maxqueues = $queues;
 
@@ -692,7 +625,7 @@ class Worker
                // Decrease the number of workers at higher load
                $load = System::currentLoad();
                if ($load) {
-                       $maxsysload = intval(Config::get("system", "maxloadavg", 50));
+                       $maxsysload = intval(Config::get("system", "maxloadavg", 20));
 
                        /* Default exponent 3 causes queues to rapidly decrease as load increases.
                         * If you have 20 max queues at idle, then you get only 5 queues at 37.1% of $maxsysload.
@@ -718,7 +651,7 @@ class Worker
                                        $stamp = (float)microtime(true);
                                        $jobs = DBA::p("SELECT COUNT(*) AS `jobs` FROM `workerqueue` WHERE `done` AND `executed` > UTC_TIMESTAMP() - INTERVAL ? MINUTE", $interval);
                                        self::$db_duration += (microtime(true) - $stamp);
-                                       //self::$db_duration_stat += (microtime(true) - $stamp);
+                                       self::$db_duration_stat += (microtime(true) - $stamp);
                                        if ($job = DBA::fetch($jobs)) {
                                                $jobs_per_minute[$interval] = number_format($job['jobs'] / $interval, 0);
                                        }
@@ -754,10 +687,8 @@ class Worker
                                        DBA::close($processes);
                                }
                                DBA::close($jobs);
-                               $entries = $deferred + $waiting_processes;
                        } else {
-                               $entries = self::totalEntries();
-                               $waiting_processes = max(0, $entries - $deferred);
+                               $waiting_processes =  self::totalEntries();
                                $stamp = (float)microtime(true);
                                $jobs = DBA::p("SELECT COUNT(*) AS `running`, `priority` FROM `process` INNER JOIN `workerqueue` ON `workerqueue`.`pid` = `process`.`pid` AND NOT `done` GROUP BY `priority` ORDER BY `priority`");
                                self::$db_duration += (microtime(true) - $stamp);
@@ -770,11 +701,13 @@ class Worker
                                DBA::close($jobs);
                        }
 
+                       $waiting_processes -= $deferred;
+
                        $listitem[0] = "0:" . max(0, $idle_workers);
 
                        $processlist .= ' ('.implode(', ', $listitem).')';
 
-                       if (Config::get("system", "worker_fastlane", false) && ($queues > 0) && self::entriesExists() && ($active >= $queues)) {
+                       if (Config::get("system", "worker_fastlane", false) && ($queues > 0) && ($active >= $queues) && self::entriesExists()) {
                                $top_priority = self::highestPriority();
                                $high_running = self::processWithPriorityActive($top_priority);
 
@@ -787,7 +720,7 @@ class Worker
                        Logger::log("Load: " . $load ."/" . $maxsysload . " - processes: " . $deferred . "/" . $active . "/" . $waiting_processes . $processlist . " - maximum: " . $queues . "/" . $maxqueues, Logger::DEBUG);
 
                        // Are there fewer workers running as possible? Then fork a new one.
-                       if (!Config::get("system", "worker_dont_fork", false) && ($queues > ($active + 1)) && ($entries > 1)) {
+                       if (!Config::get("system", "worker_dont_fork", false) && ($queues > ($active + 1)) && self::entriesExists()) {
                                Logger::log("Active workers: ".$active."/".$queues." Fork a new worker.", Logger::DEBUG);
                                if (Config::get('system', 'worker_daemon_mode', false)) {
                                        self::IPCSetJobState(true);
@@ -819,6 +752,12 @@ class Worker
                return $count;
        }
 
+       /**
+        * @brief Returns waiting jobs for the current process id
+        *
+        * @return array waiting workerqueue jobs
+        * @throws \Exception
+        */
        private static function getWaitingJobForPID()
        {
                $stamp = (float)microtime(true);
@@ -832,34 +771,47 @@ class Worker
                return false;
        }
 
+       /**
+        * @brief Returns the next jobs that should be executed
+        *
+        * @return array array with next jobs
+        * @throws \Exception
+        */
        private static function nextProcess()
        {
                $priority = self::nextPriority();
                if (empty($priority)) {
-                       Logger::log('No tasks found', Logger::DEBUG);
+                       Logger::info('No tasks found');
                        return [];
                }
 
-               if ($priority <= PRIORITY_MEDIUM) {
-                       $limit = Config::get('system', 'worker_fetch_limit', 1);
-               } else {
-                       $limit = 1;
-               }
+               $limit = Config::get('system', 'worker_fetch_limit', 1);
 
                $ids = [];
                $stamp = (float)microtime(true);
                $condition = ["`priority` = ? AND `pid` = 0 AND NOT `done` AND `next_try` < ?", $priority, DateTimeFormat::utcNow()];
-               $tasks = DBA::select('workerqueue', ['id'], $condition, ['limit' => $limit, 'order' => ['created']]);
+               $tasks = DBA::select('workerqueue', ['id', 'parameter'], $condition, ['limit' => $limit, 'order' => ['created']]);
                self::$db_duration += (microtime(true) - $stamp);
                while ($task = DBA::fetch($tasks)) {
                        $ids[] = $task['id'];
+                       // Only continue that loop while we are storing commands that can be processed quickly
+                       $command = json_decode($task['parameter'])[0];
+                       if (!in_array($command, self::FAST_COMMANDS)) {
+                               break;
+                       }
                }
                DBA::close($tasks);
 
-               Logger::log('Found task(s) ' . implode(', ', $ids) . ' with priority ' .$priority, Logger::DEBUG);
+               Logger::info('Found:', ['priority' => $priority, 'id' => $ids]);
                return $ids;
        }
 
+       /**
+        * @brief Returns the priority of the next workerqueue job
+        *
+        * @return string priority
+        * @throws \Exception
+        */
        private static function nextPriority()
        {
                $waiting = [];
@@ -891,7 +843,7 @@ class Worker
 
                foreach ($priorities as $priority) {
                        if (!empty($waiting[$priority]) && empty($running[$priority])) {
-                               Logger::log('No running worker found with priority ' . $priority . ' - assigning it.', Logger::DEBUG);
+                               Logger::info('No running worker found with priority {priority} - assigning it.', ['priority' => $priority]);
                                return $priority;
                        }
                }
@@ -913,14 +865,14 @@ class Worker
                $i = 0;
                foreach ($running as $priority => $workers) {
                        if ($workers < $limit[$i++]) {
-                               Logger::log('Priority ' . $priority . ' has got ' . $workers . ' workers out of a limit of ' . $limit[$i - 1], Logger::DEBUG);
+                               Logger::info('Priority {priority} has got {workers} workers out of a limit of {limit}', ['priority' => $priority, 'workers' => $workers, 'limit' => $limit[$i - 1]]);
                                return $priority;
                        }
                }
 
                if (!empty($waiting)) {
-                       $priority =  array_shift(array_keys($waiting));
-                       Logger::log('No underassigned priority found, now taking the highest priority (' . $priority . ').', Logger::DEBUG);
+                       $priority = array_keys($waiting)[0];
+                       Logger::info('No underassigned priority found, now taking the highest priority.', ['priority' => $priority]);
                        return $priority;
                }
 
@@ -941,27 +893,28 @@ class Worker
 
                // If there is no result we check without priority limit
                if (empty($ids)) {
+                       $limit = Config::get('system', 'worker_fetch_limit', 1);
+
                        $stamp = (float)microtime(true);
-                       $result = DBA::select(
-                               'workerqueue',
-                               ['id'],
-                               ["`pid` = 0 AND NOT `done` AND `next_try` < ?",
-                               DateTimeFormat::utcNow()],
-                               ['limit' => 1, 'order' => ['priority', 'created']]
-                       );
+                       $condition = ["`pid` = 0 AND NOT `done` AND `next_try` < ?", DateTimeFormat::utcNow()];
+                       $tasks = DBA::select('workerqueue', ['id', 'parameter'], $condition, ['limit' => $limit, 'order' => ['priority', 'created']]);
                        self::$db_duration += (microtime(true) - $stamp);
 
-                       while ($id = DBA::fetch($result)) {
-                               $ids[] = $id["id"];
+                       while ($task = DBA::fetch($tasks)) {
+                               $ids[] = $task['id'];
+                               // Only continue that loop while we are storing commands that can be processed quickly
+                               $command = json_decode($task['parameter'])[0];
+                               if (!in_array($command, self::FAST_COMMANDS)) {
+                                       break;
+                               }
                        }
-                       DBA::close($result);
+                       DBA::close($tasks);
                }
 
                if (!empty($ids)) {
                        $stamp = (float)microtime(true);
-                       $condition = "`id` IN (".substr(str_repeat("?, ", count($ids)), 0, -2).") AND `pid` = 0 AND NOT `done`";
-                       array_unshift($ids, $condition);
-                       DBA::update('workerqueue', ['executed' => DateTimeFormat::utcNow(), 'pid' => $mypid], $ids);
+                       $condition = ['id' => $ids, 'done' => false, 'pid' => 0];
+                       DBA::update('workerqueue', ['executed' => DateTimeFormat::utcNow(), 'pid' => $mypid], $condition);
                        self::$db_duration += (microtime(true) - $stamp);
                        self::$db_duration_write += (microtime(true) - $stamp);
                }
@@ -1029,7 +982,7 @@ class Worker
                }
 
                $url = System::baseUrl()."/worker";
-               Network::fetchUrl($url, false, $redirects, 1);
+               Network::fetchUrl($url, false, 1);
        }
 
        /**
@@ -1132,7 +1085,9 @@ class Worker
 
                $args = ['no_cron' => !$do_cron];
 
-               get_app()->proc_run($command, $args);
+               $a = get_app();
+               $process = new Core\Process($a->getLogger(), $a->getMode(), $a->getConfig(), $a->getBasePath());
+               $process->run($command, $args);
 
                // after spawning we have to remove the flag.
                if (Config::get('system', 'worker_daemon_mode', false)) {
@@ -1146,7 +1101,7 @@ class Worker
         * @param (integer|array) priority or parameter array, strings are deprecated and are ignored
         *
         * next args are passed as $cmd command line
-        * or: Worker::add(PRIORITY_HIGH, "Notifier", "drop", $drop_id);
+        * or: Worker::add(PRIORITY_HIGH, "Notifier", Delivery::DELETION, $drop_id);
         * or: Worker::add(array('priority' => PRIORITY_HIGH, 'dont_fork' => true), "CreateShadowEntry", $post_id);
         *
         * @return boolean "false" if proc_run couldn't be executed
@@ -1173,7 +1128,8 @@ class Worker
                }
 
                $priority = PRIORITY_MEDIUM;
-               $dont_fork = Config::get("system", "worker_dont_fork", false);
+               // Don't fork from frontend tasks by default
+               $dont_fork = Config::get("system", "worker_dont_fork", false) || !\get_app()->getMode()->isBackend();
                $created = DateTimeFormat::utcNow();
                $force_priority = false;
 
@@ -1240,13 +1196,40 @@ class Worker
                return true;
        }
 
+       /**
+        * Returns the next retrial level for worker jobs.
+        * This function will skip levels when jobs are older.
+        *
+        * @param array $queue Worker queue entry
+        * @param integer $max_level maximum retrial level
+        * @return integer the next retrial level value
+        */
+       private static function getNextRetrial($queue, $max_level)
+       {
+               $created = strtotime($queue['created']);
+               $retrial_time = time() - $created;
+
+               $new_retrial = $queue['retrial'] + 1;
+               $total = 0;
+               for ($retrial = 0; $retrial <= $max_level + 1; ++$retrial) {
+                       $delay = (($retrial + 3) ** 4) + (rand(1, 30) * ($retrial + 1));
+                       $total += $delay;
+                       if (($total < $retrial_time) && ($retrial > $queue['retrial'])) {
+                               $new_retrial = $retrial;
+                       }
+               }
+               Logger::info('New retrial for task', ['id' => $queue['id'], 'created' => $queue['created'], 'old' => $queue['retrial'], 'new' => $new_retrial]);
+               return $new_retrial;
+       }
+
        /**
         * Defers the current worker entry
+        * @return boolean had the entry been deferred?
         */
        public static function defer()
        {
                if (empty(BaseObject::getApp()->queue)) {
-                       return;
+                       return false;
                }
 
                $queue = BaseObject::getApp()->queue;
@@ -1255,30 +1238,36 @@ class Worker
                $id = $queue['id'];
                $priority = $queue['priority'];
 
-               if ($retrial > 14) {
-                       Logger::log('Id ' . $id . ' had been tried 14 times. We stop now.', Logger::DEBUG);
-                       return;
+               $max_level = Config::get('system', 'worker_defer_limit');
+
+               $new_retrial = self::getNextRetrial($queue, $max_level);
+
+               if ($new_retrial > $max_level) {
+                       Logger::info('The task exceeded the maximum retry count', ['id' => $id, 'created' => $queue['created'], 'old_prio' => $queue['priority'], 'old_retrial' => $queue['retrial'], 'max_level' => $max_level, 'retrial' => $new_retrial]);
+                       return false;
                }
 
                // Calculate the delay until the next trial
-               $delay = (($retrial + 3) ** 4) + (rand(1, 30) * ($retrial + 1));
+               $delay = (($new_retrial + 2) ** 4) + (rand(1, 30) * ($new_retrial));
                $next = DateTimeFormat::utc('now + ' . $delay . ' seconds');
 
-               if (($priority < PRIORITY_MEDIUM) && ($retrial > 2)) {
+               if (($priority < PRIORITY_MEDIUM) && ($new_retrial > 3)) {
                        $priority = PRIORITY_MEDIUM;
-               } elseif (($priority < PRIORITY_LOW) && ($retrial > 5)) {
+               } elseif (($priority < PRIORITY_LOW) && ($new_retrial > 6)) {
                        $priority = PRIORITY_LOW;
-               } elseif (($priority < PRIORITY_NEGLIGIBLE) && ($retrial > 7)) {
+               } elseif (($priority < PRIORITY_NEGLIGIBLE) && ($new_retrial > 8)) {
                        $priority = PRIORITY_NEGLIGIBLE;
                }
 
-               Logger::log('Defer execution ' . $retrial . ' of id ' . $id . ' to ' . $next . ' - priority old/new: ' . $queue['priority'] . '/' . $priority, Logger::DEBUG);
+               Logger::info('Deferred task', ['id' => $id, 'retrial' => $new_retrial, 'created' => $queue['created'], 'next_execution' => $next, 'old_prio' => $queue['priority'], 'new_prio' => $priority]);
 
                $stamp = (float)microtime(true);
-               $fields = ['retrial' => $retrial + 1, 'next_try' => $next, 'executed' => DBA::NULL_DATETIME, 'pid' => 0, 'priority' => $priority];
+               $fields = ['retrial' => $new_retrial, 'next_try' => $next, 'executed' => DBA::NULL_DATETIME, 'pid' => 0, 'priority' => $priority];
                DBA::update('workerqueue', $fields, ['id' => $id]);
                self::$db_duration += (microtime(true) - $stamp);
                self::$db_duration_write += (microtime(true) - $stamp);
+
+               return true;
        }
 
        /**