3 * @copyright Copyright (C) 2010-2023, the Friendica project
5 * @license GNU AGPL version 3 or any later version
7 * This program is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU Affero General Public License as
9 * published by the Free Software Foundation, either version 3 of the
10 * License, or (at your option) any later version.
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU Affero General Public License for more details.
17 * You should have received a copy of the GNU Affero General Public License
18 * along with this program. If not, see <https://www.gnu.org/licenses/>.
22 namespace Friendica\Core\Worker;
24 use Friendica\Core\Logger;
25 use Friendica\Core\Worker;
26 use Friendica\Database\DBA;
28 use Friendica\Model\Contact;
29 use Friendica\Model\GServer;
30 use Friendica\Model\Post;
31 use Friendica\Protocol\ActivityPub;
32 use Friendica\Protocol\Delivery;
33 use Friendica\Util\DateTimeFormat;
34 use Friendica\Util\Strings;
37 * Contains the class for jobs that are executed in an interval
42 * Runs the cron processes
45 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
47 public static function run()
49 Logger::info('Add cron entries');
51 // Check for spooled items
52 Worker::add(['priority' => Worker::PRIORITY_HIGH, 'force_priority' => true], 'SpoolPost');
54 // Run the cron job that calls all other jobs
55 Worker::add(['priority' => Worker::PRIORITY_MEDIUM, 'force_priority' => true], 'Cron');
57 // Cleaning dead processes
58 self::killStaleWorkers();
60 // Remove old entries from the workerqueue
61 self::cleanWorkerQueue();
63 // Directly deliver or requeue posts to ActivityPub systems
64 self::deliverAPPosts();
66 // Directly deliver or requeue posts to other systems
71 * fix the queue entry if the worker process died
76 public static function killStaleWorkers()
78 $entries = DBA::select(
80 ['id', 'pid', 'executed', 'priority', 'command', 'parameter'],
81 ['NOT `done` AND `pid` != 0'],
82 ['order' => ['priority', 'retrial', 'created']]
85 $max_duration_defaults = DI::config()->get('system', 'worker_max_duration');
87 while ($entry = DBA::fetch($entries)) {
88 if (!posix_kill($entry["pid"], 0)) {
89 DBA::update('workerqueue', ['executed' => DBA::NULL_DATETIME, 'pid' => 0], ['id' => $entry["id"]]);
91 // Kill long running processes
93 // Define the maximum durations
94 $max_duration = $max_duration_defaults[$entry['priority']] ?? 0;
95 if (empty($max_duration)) {
99 $argv = json_decode($entry['parameter'], true);
100 if (!empty($entry['command'])) {
101 $command = $entry['command'];
102 } elseif (!empty($argv)) {
103 $command = array_shift($argv);
108 $command = basename($command);
110 // How long is the process already running?
111 $duration = (time() - strtotime($entry["executed"])) / 60;
112 if ($duration > $max_duration) {
113 Logger::warning('Worker process took too much time - killed', ['duration' => number_format($duration, 3), 'max' => $max_duration, 'id' => $entry["id"], 'pid' => $entry["pid"], 'command' => $command]);
114 posix_kill($entry["pid"], SIGTERM);
116 // We killed the stale process.
117 // To avoid a blocking situation we reschedule the process at the beginning of the queue.
118 // Additionally we are lowering the priority. (But not PRIORITY_CRITICAL)
119 $new_priority = $entry['priority'];
120 if ($entry['priority'] == Worker::PRIORITY_HIGH) {
121 $new_priority = Worker::PRIORITY_MEDIUM;
122 } elseif ($entry['priority'] == Worker::PRIORITY_MEDIUM) {
123 $new_priority = Worker::PRIORITY_LOW;
124 } elseif ($entry['priority'] != Worker::PRIORITY_CRITICAL) {
125 $new_priority = Worker::PRIORITY_NEGLIGIBLE;
127 DBA::update('workerqueue', ['executed' => DBA::NULL_DATETIME, 'created' => DateTimeFormat::utcNow(), 'priority' => $new_priority, 'pid' => 0], ['id' => $entry["id"]]
130 Logger::info('Process runtime is okay', ['duration' => number_format($duration, 3), 'max' => $max_duration, 'id' => $entry["id"], 'pid' => $entry["pid"], 'command' => $command]);
134 DBA::close($entries);
138 * Remove old entries from the workerqueue
142 private static function cleanWorkerQueue()
144 DBA::delete('workerqueue', ["`done` AND `executed` < ?", DateTimeFormat::utc('now - 1 hour')]);
146 // Optimizing this table only last seconds
147 if (DI::config()->get('system', 'optimize_tables')) {
148 // We are acquiring the two locks from the worker to avoid locking problems
149 if (DI::lock()->acquire(Worker::LOCK_PROCESS, 10)) {
150 if (DI::lock()->acquire(Worker::LOCK_WORKER, 10)) {
151 DBA::e("OPTIMIZE TABLE `workerqueue`");
152 DBA::e("OPTIMIZE TABLE `process`");
153 DI::lock()->release(Worker::LOCK_WORKER);
155 DI::lock()->release(Worker::LOCK_PROCESS);
161 * Directly deliver AP messages or requeue them.
163 * This function is placed here as a safeguard. Even when the worker queue is completely blocked, messages will be delivered.
165 private static function deliverAPPosts()
167 $deliveries = DBA::p("SELECT `item-uri`.`uri` AS `inbox`, MAX(`gsid`) AS `gsid`, MAX(`failed`) AS `failed` FROM `post-delivery` INNER JOIN `item-uri` ON `item-uri`.`id` = `post-delivery`.`inbox-id` LEFT JOIN `inbox-status` ON `inbox-status`.`url` = `item-uri`.`uri` GROUP BY `inbox` ORDER BY RAND()");
168 while ($delivery = DBA::fetch($deliveries)) {
169 if ($delivery['failed'] > 0) {
170 Logger::info('Removing failed deliveries', ['inbox' => $delivery['inbox'], 'failed' => $delivery['failed']]);
171 Post\Delivery::removeFailed($delivery['inbox']);
173 if (($delivery['failed'] == 0) && !empty($delivery['gsid']) && GServer::isReachableById($delivery['gsid'])) {
174 $result = ActivityPub\Delivery::deliver($delivery['inbox']);
175 Logger::info('Directly deliver inbox', ['inbox' => $delivery['inbox'], 'result' => $result['success']]);
176 if (!$result['success']) {
177 GServer::setFailureById($delivery['gsid']);
180 } elseif ($delivery['failed'] < 3) {
181 $priority = Worker::PRIORITY_HIGH;
182 } elseif ($delivery['failed'] < 6) {
183 $priority = Worker::PRIORITY_MEDIUM;
184 } elseif ($delivery['failed'] < 8) {
185 $priority = Worker::PRIORITY_LOW;
187 $priority = Worker::PRIORITY_NEGLIGIBLE;
190 if (Worker::add(['priority' => $priority, 'force_priority' => true], 'APDelivery', '', 0, $delivery['inbox'], 0)) {
191 Logger::info('Priority for APDelivery worker adjusted', ['inbox' => $delivery['inbox'], 'failed' => $delivery['failed'], 'priority' => $priority]);
195 DBA::close($deliveries);
197 // Optimizing this table only last seconds
198 if (DI::config()->get('system', 'optimize_tables')) {
199 Logger::info('Optimize start');
200 DBA::e("OPTIMIZE TABLE `post-delivery`");
201 Logger::info('Optimize end');
206 * Directly deliver messages or requeue them.
208 private static function deliverPosts()
210 $deliveries = DBA::p("SELECT `gsid`, MAX(`failed`) AS `failed` FROM `delivery-queue` GROUP BY `gsid` ORDER BY RAND()");
211 while ($delivery = DBA::fetch($deliveries)) {
212 if ($delivery['failed'] > 0) {
213 Logger::info('Removing failed deliveries', ['gsid' => $delivery['gsid'], 'failed' => $delivery['failed']]);
214 Delivery::removeFailedQueue($delivery['gsid']);
217 if (($delivery['failed'] < 3) || GServer::isReachableById($delivery['gsid'])) {
218 $priority = Worker::PRIORITY_HIGH;
219 } elseif ($delivery['failed'] < 6) {
220 $priority = Worker::PRIORITY_MEDIUM;
221 } elseif ($delivery['failed'] < 8) {
222 $priority = Worker::PRIORITY_LOW;
224 $priority = Worker::PRIORITY_NEGLIGIBLE;
227 if (Worker::add(['priority' => $priority, 'force_priority' => true], 'BulkDelivery', $delivery['gsid'])) {
228 Logger::info('Priority for BulkDelivery worker adjusted', ['gsid' => $delivery['gsid'], 'failed' => $delivery['failed'], 'priority' => $priority]);
232 // Optimizing this table only last seconds
233 if (DI::config()->get('system', 'optimize_tables')) {
234 Logger::info('Optimize start');
235 DBA::e("OPTIMIZE TABLE `delivery-queue`");
236 Logger::info('Optimize end');
241 * Add missing "intro" records.
245 private static function addIntros()
247 $contacts = DBA::p("SELECT `uid`, `id`, `created` FROM `contact` WHERE `rel` = ? AND `pending` AND NOT `id` IN (SELECT `contact-id` FROM `intro`)", Contact::FOLLOWER);
248 while ($contact = DBA::fetch($contacts)) {
250 'uid' => $contact['uid'],
251 'contact-id' => $contact['id'],
252 'datetime' => $contact['created'],
253 'hash' => Strings::getRandomHex()
255 Logger::notice('Adding missing intro', ['fields' => $fields]);
256 DBA::insert('intro', $fields);