src/Core/Worker/Cron.php

   1 <?php
   2 /**
   3  * @copyright Copyright (C) 2010-2023, the Friendica project
   4  *
   5  * @license GNU AGPL version 3 or any later version
   6  *
   7  * This program is free software: you can redistribute it and/or modify
   8  * it under the terms of the GNU Affero General Public License as
   9  * published by the Free Software Foundation, either version 3 of the
  10  * License, or (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU Affero General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU Affero General Public License
  18  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  19  *
  20  */
  21
  22 namespace Friendica\Core\Worker;
  23
  24 use Friendica\Core\Logger;
  25 use Friendica\Core\Worker;
  26 use Friendica\Database\DBA;
  27 use Friendica\DI;
  28 use Friendica\Model\Contact;
  29 use Friendica\Model\GServer;
  30 use Friendica\Model\Post;
  31 use Friendica\Model\User;
  32 use Friendica\Protocol\ActivityPub;
  33 use Friendica\Util\DateTimeFormat;
  34 use Friendica\Util\Strings;
  35
  36 /**
  37  * Contains the class for jobs that are executed in an interval
  38  */
  39 class Cron
  40 {
  41         /**
  42          * Runs the cron processes
  43          *
  44          * @return void
  45          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
  46          */
  47         public static function run()
  48         {
  49                 Logger::info('Add cron entries');
  50
  51                 // Check for spooled items
  52                 Worker::add(['priority' => Worker::PRIORITY_HIGH, 'force_priority' => true], 'SpoolPost');
  53
  54                 // Run the cron job that calls all other jobs
  55                 Worker::add(['priority' => Worker::PRIORITY_MEDIUM, 'force_priority' => true], 'Cron');
  56
  57                 // Cleaning dead processes
  58                 self::killStaleWorkers();
  59
  60                 // Remove old entries from the workerqueue
  61                 self::cleanWorkerQueue();
  62
  63                 // Directly deliver or requeue posts to ActivityPub systems
  64                 self::deliverAPPosts();
  65
  66                 // Directly deliver or requeue posts to other systems
  67                 self::deliverPosts();
  68
  69                 // Automatically open/close the registration based on the user count
  70                 User::setRegisterMethodByUserCount();
  71         }
  72
  73         /**
  74          * fix the queue entry if the worker process died
  75          *
  76          * @return void
  77          * @throws \Exception
  78          */
  79         public static function killStaleWorkers()
  80         {
  81                 $entries = DBA::select(
  82                         'workerqueue',
  83                         ['id', 'pid', 'executed', 'priority', 'command', 'parameter'],
  84                         ['NOT `done` AND `pid` != 0'],
  85                         ['order' => ['priority', 'retrial', 'created']]
  86                 );
  87
  88                 $max_duration_defaults = DI::config()->get('system', 'worker_max_duration');
  89
  90                 while ($entry = DBA::fetch($entries)) {
  91                         if (!posix_kill($entry["pid"], 0)) {
  92                                 DBA::update('workerqueue', ['executed' => DBA::NULL_DATETIME, 'pid' => 0], ['id' => $entry["id"]]);
  93                         } else {
  94                                 // Kill long running processes
  95
  96                                 // Define the maximum durations
  97                                 $max_duration = $max_duration_defaults[$entry['priority']] ?? 0;
  98                                 if (empty($max_duration)) {
  99                                         continue;
 100                                 }
 101
 102                                 $argv = json_decode($entry['parameter'], true);
 103                                 if (!empty($entry['command'])) {
 104                                         $command = $entry['command'];
 105                                 } elseif (!empty($argv)) {
 106                                         $command = array_shift($argv);
 107                                 } else {
 108                                         return;
 109                                 }
 110
 111                                 $command = basename($command);
 112
 113                                 // How long is the process already running?
 114                                 $duration = (time() - strtotime($entry["executed"])) / 60;
 115                                 if ($duration > $max_duration) {
 116                                         Logger::warning('Worker process took too much time - killed', ['duration' => number_format($duration, 3), 'max' => $max_duration, 'id' => $entry["id"], 'pid' => $entry["pid"], 'command' => $command]);
 117                                         posix_kill($entry["pid"], SIGTERM);
 118
 119                                         // We killed the stale process.
 120                                         // To avoid a blocking situation we reschedule the process at the beginning of the queue.
 121                                         // Additionally we are lowering the priority. (But not PRIORITY_CRITICAL)
 122                                         $new_priority = $entry['priority'];
 123                                         if ($entry['priority'] == Worker::PRIORITY_HIGH) {
 124                                                 $new_priority = Worker::PRIORITY_MEDIUM;
 125                                         } elseif ($entry['priority'] == Worker::PRIORITY_MEDIUM) {
 126                                                 $new_priority = Worker::PRIORITY_LOW;
 127                                         } elseif ($entry['priority'] != Worker::PRIORITY_CRITICAL) {
 128                                                 $new_priority = Worker::PRIORITY_NEGLIGIBLE;
 129                                         }
 130                                         DBA::update('workerqueue', ['executed' => DBA::NULL_DATETIME, 'created' => DateTimeFormat::utcNow(), 'priority' => $new_priority, 'pid' => 0], ['id' => $entry["id"]]
 131                                         );
 132                                 } else {
 133                                         Logger::info('Process runtime is okay', ['duration' => number_format($duration, 3), 'max' => $max_duration, 'id' => $entry["id"], 'pid' => $entry["pid"], 'command' => $command]);
 134                                 }
 135                         }
 136                 }
 137                 DBA::close($entries);
 138         }
 139
 140         /**
 141          * Remove old entries from the workerqueue
 142          *
 143          * @return void
 144          */
 145         private static function cleanWorkerQueue()
 146         {
 147                 DBA::delete('workerqueue', ["`done` AND `executed` < ?", DateTimeFormat::utc('now - 1 hour')]);
 148
 149                 // Optimizing this table only last seconds
 150                 if (DI::config()->get('system', 'optimize_tables')) {
 151                         // We are acquiring the two locks from the worker to avoid locking problems
 152                         if (DI::lock()->acquire(Worker::LOCK_PROCESS, 10)) {
 153                                 if (DI::lock()->acquire(Worker::LOCK_WORKER, 10)) {
 154                                         DBA::optimizeTable('workerqueue');
 155                                         DBA::optimizeTable('process');
 156                                         DI::lock()->release(Worker::LOCK_WORKER);
 157                                 }
 158                                 DI::lock()->release(Worker::LOCK_PROCESS);
 159                         }
 160                 }
 161         }
 162
 163         /**
 164          * Directly deliver AP messages or requeue them.
 165          *
 166          * This function is placed here as a safeguard. Even when the worker queue is completely blocked, messages will be delivered.
 167          */
 168         private static function deliverAPPosts()
 169         {
 170                 $deliveries = DBA::p("SELECT `item-uri`.`uri` AS `inbox`, MAX(`gsid`) AS `gsid`, MAX(`shared`) AS `shared`, MAX(`failed`) AS `failed` FROM `post-delivery` INNER JOIN `item-uri` ON `item-uri`.`id` = `post-delivery`.`inbox-id` LEFT JOIN `inbox-status` ON `inbox-status`.`url` = `item-uri`.`uri` GROUP BY `inbox` ORDER BY RAND()");
 171                 while ($delivery = DBA::fetch($deliveries)) {
 172                         if ($delivery['failed'] > 0) {
 173                                 Logger::info('Removing failed deliveries', ['inbox' => $delivery['inbox'], 'failed' => $delivery['failed']]);
 174                                 Post\Delivery::removeFailed($delivery['inbox']);
 175                         }
 176                         if (($delivery['failed'] == 0) && $delivery['shared'] && !empty($delivery['gsid']) && GServer::isReachableById($delivery['gsid'])) {
 177                                 $result = ActivityPub\Delivery::deliver($delivery['inbox']);
 178                                 Logger::info('Directly deliver inbox', ['inbox' => $delivery['inbox'], 'result' => $result['success']]);
 179                                 continue;
 180                         } elseif ($delivery['failed'] < 3) {
 181                                 $priority = Worker::PRIORITY_HIGH;
 182                         } elseif ($delivery['failed'] < 6) {
 183                                 $priority = Worker::PRIORITY_MEDIUM;
 184                         } elseif ($delivery['failed'] < 8) {
 185                                 $priority = Worker::PRIORITY_LOW;
 186                         } else {
 187                                 $priority = Worker::PRIORITY_NEGLIGIBLE;
 188                         }
 189
 190                         if (Worker::add(['priority' => $priority, 'force_priority' => true], 'APDelivery', '', 0, $delivery['inbox'], 0)) {
 191                                 Logger::info('Priority for APDelivery worker adjusted', ['inbox' => $delivery['inbox'], 'failed' => $delivery['failed'], 'priority' => $priority]);
 192                         }
 193                 }
 194
 195                 DBA::close($deliveries);
 196
 197                 // Optimizing this table only last seconds
 198                 if (DI::config()->get('system', 'optimize_tables')) {
 199                         Logger::info('Optimize start');
 200                         DBA::optimizeTable('post-delivery');
 201                         Logger::info('Optimize end');
 202                 }
 203         }
 204
 205         /**
 206          * Directly deliver messages or requeue them.
 207          */
 208         private static function deliverPosts()
 209         {
 210                 foreach(DI::deliveryQueueItemRepo()->selectAggregateByServerId() as $delivery) {
 211                         if ($delivery->failed > 0) {
 212                                 Logger::info('Removing failed deliveries', ['gsid' => $delivery->targetServerId, 'failed' => $delivery->failed]);
 213                                 DI::deliveryQueueItemRepo()->removeFailedByServerId($delivery->targetServerId, DI::config()->get('system', 'worker_defer_limit'));
 214                         }
 215
 216                         if (($delivery->failed < 3) || GServer::isReachableById($delivery->targetServerId)) {
 217                                 $priority = Worker::PRIORITY_HIGH;
 218                         } elseif ($delivery->failed < 6) {
 219                                 $priority = Worker::PRIORITY_MEDIUM;
 220                         } elseif ($delivery->failed < 8) {
 221                                 $priority = Worker::PRIORITY_LOW;
 222                         } else {
 223                                 $priority = Worker::PRIORITY_NEGLIGIBLE;
 224                         }
 225
 226                         if (Worker::add(['priority' => $priority, 'force_priority' => true], 'BulkDelivery', $delivery->targetServerId)) {
 227                                 Logger::info('Priority for BulkDelivery worker adjusted', ['gsid' => $delivery->targetServerId, 'failed' => $delivery->failed, 'priority' => $priority]);
 228                         }
 229                 }
 230
 231                 // Optimizing this table only last seconds
 232                 if (DI::config()->get('system', 'optimize_tables')) {
 233                         Logger::info('Optimize start');
 234                         DI::deliveryQueueItemRepo()->optimizeStorage();
 235                         Logger::info('Optimize end');
 236                 }
 237         }
 238
 239         /**
 240          * Add missing "intro" records.
 241          *
 242          * @return void
 243          */
 244         private static function addIntros()
 245         {
 246                 $contacts = DBA::p("SELECT `uid`, `id`, `created` FROM `contact` WHERE `rel` = ? AND `pending` AND NOT `id` IN (SELECT `contact-id` FROM `intro`)", Contact::FOLLOWER);
 247                 while ($contact = DBA::fetch($contacts)) {
 248                         $fields = [
 249                                 'uid'        => $contact['uid'],
 250                                 'contact-id' => $contact['id'],
 251                                 'datetime'   => $contact['created'],
 252                                 'hash'       => Strings::getRandomHex()
 253                         ];
 254                         Logger::notice('Adding missing intro', ['fields' => $fields]);
 255                         DBA::insert('intro', $fields);
 256                 }
 257         }
 258 }