3 * @copyright Copyright (C) 2010-2022, the Friendica project
5 * @license GNU AGPL version 3 or any later version
7 * This program is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU Affero General Public License as
9 * published by the Free Software Foundation, either version 3 of the
10 * License, or (at your option) any later version.
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU Affero General Public License for more details.
17 * You should have received a copy of the GNU Affero General Public License
18 * along with this program. If not, see <https://www.gnu.org/licenses/>.
22 namespace Friendica\Core\Worker;
24 use Friendica\Core\Logger;
25 use Friendica\Core\Worker;
26 use Friendica\Database\DBA;
28 use Friendica\Model\Post;
29 use Friendica\Protocol\ActivityPub;
30 use Friendica\Util\DateTimeFormat;
33 * Contains the class for jobs that are executed in an interval
38 * Runs the cron processes
41 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
43 public static function run()
45 Logger::info('Add cron entries');
47 // Check for spooled items
48 Worker::add(['priority' => PRIORITY_HIGH, 'force_priority' => true], 'SpoolPost');
50 // Run the cron job that calls all other jobs
51 Worker::add(['priority' => PRIORITY_MEDIUM, 'force_priority' => true], 'Cron');
53 // Cleaning dead processes
54 self::killStaleWorkers();
56 // Remove old entries from the workerqueue
57 self::cleanWorkerQueue();
59 // Directly deliver or requeue posts
64 * fix the queue entry if the worker process died
69 public static function killStaleWorkers()
71 $entries = DBA::select(
73 ['id', 'pid', 'executed', 'priority', 'command', 'parameter'],
74 ['NOT `done` AND `pid` != 0'],
75 ['order' => ['priority', 'retrial', 'created']]
78 while ($entry = DBA::fetch($entries)) {
79 if (!posix_kill($entry["pid"], 0)) {
80 DBA::update('workerqueue', ['executed' => DBA::NULL_DATETIME, 'pid' => 0], ['id' => $entry["id"]]);
82 // Kill long running processes
84 // Define the maximum durations
85 $max_duration_defaults = [PRIORITY_CRITICAL => 720, PRIORITY_HIGH => 10, PRIORITY_MEDIUM => 60, PRIORITY_LOW => 180, PRIORITY_NEGLIGIBLE => 720];
86 $max_duration = $max_duration_defaults[$entry['priority']];
88 $argv = json_decode($entry['parameter'], true);
89 if (!empty($entry['command'])) {
90 $command = $entry['command'];
91 } elseif (!empty($argv)) {
92 $command = array_shift($argv);
97 $command = basename($command);
99 // How long is the process already running?
100 $duration = (time() - strtotime($entry["executed"])) / 60;
101 if ($duration > $max_duration) {
102 Logger::notice('Worker process took too much time - killed', ['duration' => number_format($duration, 3), 'max' => $max_duration, 'id' => $entry["id"], 'pid' => $entry["pid"], 'command' => $command]);
103 posix_kill($entry["pid"], SIGTERM);
105 // We killed the stale process.
106 // To avoid a blocking situation we reschedule the process at the beginning of the queue.
107 // Additionally we are lowering the priority. (But not PRIORITY_CRITICAL)
108 $new_priority = $entry['priority'];
109 if ($entry['priority'] == PRIORITY_HIGH) {
110 $new_priority = PRIORITY_MEDIUM;
111 } elseif ($entry['priority'] == PRIORITY_MEDIUM) {
112 $new_priority = PRIORITY_LOW;
113 } elseif ($entry['priority'] != PRIORITY_CRITICAL) {
114 $new_priority = PRIORITY_NEGLIGIBLE;
116 DBA::update('workerqueue', ['executed' => DBA::NULL_DATETIME, 'created' => DateTimeFormat::utcNow(), 'priority' => $new_priority, 'pid' => 0], ['id' => $entry["id"]]
119 Logger::info('Process runtime is okay', ['duration' => number_format($duration, 3), 'max' => $max_duration, 'id' => $entry["id"], 'pid' => $entry["pid"], 'command' => $command]);
123 DBA::close($entries);
127 * Remove old entries from the workerqueue
131 private static function cleanWorkerQueue()
133 DBA::delete('workerqueue', ["`done` AND `executed` < ?", DateTimeFormat::utc('now - 1 hour')]);
135 // Optimizing this table only last seconds
136 if (DI::config()->get('system', 'optimize_tables')) {
137 // We are acquiring the two locks from the worker to avoid locking problems
138 if (DI::lock()->acquire(Worker::LOCK_PROCESS, 10)) {
139 if (DI::lock()->acquire(Worker::LOCK_WORKER, 10)) {
140 DBA::e("OPTIMIZE TABLE `workerqueue`");
141 DBA::e("OPTIMIZE TABLE `process`");
142 DI::lock()->release(Worker::LOCK_WORKER);
144 DI::lock()->release(Worker::LOCK_PROCESS);
150 * Directly deliver AP messages or requeue them.
152 * This function is placed here as a safeguard. Even when the worker queue is completely blocked, messages will be delivered.
154 private static function deliverPosts()
156 $deliveries = DBA::p("SELECT `item-uri`.`uri` AS `inbox`, MAX(`failed`) AS `failed` FROM `post-delivery` INNER JOIN `item-uri` ON `item-uri`.`id` = `post-delivery`.`inbox-id` GROUP BY `inbox`");
157 while ($delivery = DBA::fetch($deliveries)) {
158 if ($delivery['failed'] == 0) {
159 $result = ActivityPub\Delivery::deliver($delivery['inbox']);
160 Logger::info('Directly deliver inbox', ['inbox' => $delivery['inbox'], 'result' => $result['success']]);
162 } elseif ($delivery['failed'] < 3) {
163 $priority = PRIORITY_HIGH;
164 } elseif ($delivery['failed'] < 6) {
165 $priority = PRIORITY_MEDIUM;
166 } elseif ($delivery['failed'] < 8) {
167 $priority = PRIORITY_LOW;
169 $priority = PRIORITY_NEGLIGIBLE;
172 if ($delivery['failed'] >= DI::config()->get('system', 'worker_defer_limit')) {
173 Logger::info('Removing failed deliveries', ['inbox' => $delivery['inbox'], 'failed' => $delivery['failed']]);
174 Post\Delivery::removeFailed($delivery['inbox']);
177 if (Worker::add($priority, 'APDelivery', '', 0, $delivery['inbox'], 0)) {
178 Logger::info('Missing APDelivery worker added for inbox', ['inbox' => $delivery['inbox'], 'failed' => $delivery['failed'], 'priority' => $priority]);