src/Core/Worker/Cron.php

   1 <?php
   2 /**
   3  * @copyright Copyright (C) 2010-2022, the Friendica project
   4  *
   5  * @license GNU AGPL version 3 or any later version
   6  *
   7  * This program is free software: you can redistribute it and/or modify
   8  * it under the terms of the GNU Affero General Public License as
   9  * published by the Free Software Foundation, either version 3 of the
  10  * License, or (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU Affero General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU Affero General Public License
  18  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  19  *
  20  */
  21
  22 namespace Friendica\Core\Worker;
  23
  24 use Friendica\Core\Logger;
  25 use Friendica\Core\Worker;
  26 use Friendica\Database\DBA;
  27 use Friendica\DI;
  28 use Friendica\Model\Post;
  29 use Friendica\Protocol\ActivityPub;
  30 use Friendica\Util\DateTimeFormat;
  31
  32 /**
  33  * Contains the class for jobs that are executed in an interval
  34  */
  35 class Cron
  36 {
  37         /**
  38          * Runs the cron processes
  39          *
  40          * @return void
  41          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
  42          */
  43         public static function run()
  44         {
  45                 Logger::info('Add cron entries');
  46
  47                 // Check for spooled items
  48                 Worker::add(['priority' => PRIORITY_HIGH, 'force_priority' => true], 'SpoolPost');
  49
  50                 // Run the cron job that calls all other jobs
  51                 Worker::add(['priority' => PRIORITY_MEDIUM, 'force_priority' => true], 'Cron');
  52
  53                 // Cleaning dead processes
  54                 self::killStaleWorkers();
  55
  56                 // Remove old entries from the workerqueue
  57                 self::cleanWorkerQueue();
  58
  59                 // Directly deliver or requeue posts
  60                 self::deliverPosts();
  61         }
  62
  63         /**
  64          * fix the queue entry if the worker process died
  65          *
  66          * @return void
  67          * @throws \Exception
  68          */
  69         public static function killStaleWorkers()
  70         {
  71                 $stamp = (float)microtime(true);
  72                 $entries = DBA::select(
  73                         'workerqueue',
  74                         ['id', 'pid', 'executed', 'priority', 'command', 'parameter'],
  75                         ['NOT `done` AND `pid` != 0'],
  76                         ['order' => ['priority', 'retrial', 'created']]
  77                 );
  78
  79                 while ($entry = DBA::fetch($entries)) {
  80                         if (!posix_kill($entry["pid"], 0)) {
  81                                 $stamp = (float)microtime(true);
  82                                 DBA::update(
  83                                         'workerqueue',
  84                                         ['executed' => DBA::NULL_DATETIME, 'pid' => 0],
  85                                         ['id' => $entry["id"]]
  86                                 );
  87                         } else {
  88                                 // Kill long running processes
  89
  90                                 // Define the maximum durations
  91                                 $max_duration_defaults = [PRIORITY_CRITICAL => 720, PRIORITY_HIGH => 10, PRIORITY_MEDIUM => 60, PRIORITY_LOW => 180, PRIORITY_NEGLIGIBLE => 720];
  92                                 $max_duration = $max_duration_defaults[$entry['priority']];
  93
  94                                 $argv = json_decode($entry['parameter'], true);
  95                                 if (!empty($entry['command'])) {
  96                                         $command = $entry['command'];
  97                                 } elseif (!empty($argv)) {
  98                                         $command = array_shift($argv);
  99                                 } else {
 100                                         return;
 101                                 }
 102
 103                                 $command = basename($command);
 104
 105                                 // How long is the process already running?
 106                                 $duration = (time() - strtotime($entry["executed"])) / 60;
 107                                 if ($duration > $max_duration) {
 108                                         Logger::notice('Worker process took too much time - killed', ['duration' => number_format($duration, 3), 'max' => $max_duration, 'id' => $entry["id"], 'pid' => $entry["pid"], 'command' => $command]);
 109                                         posix_kill($entry["pid"], SIGTERM);
 110
 111                                         // We killed the stale process.
 112                                         // To avoid a blocking situation we reschedule the process at the beginning of the queue.
 113                                         // Additionally we are lowering the priority. (But not PRIORITY_CRITICAL)
 114                                         $new_priority = $entry['priority'];
 115                                         if ($entry['priority'] == PRIORITY_HIGH) {
 116                                                 $new_priority = PRIORITY_MEDIUM;
 117                                         } elseif ($entry['priority'] == PRIORITY_MEDIUM) {
 118                                                 $new_priority = PRIORITY_LOW;
 119                                         } elseif ($entry['priority'] != PRIORITY_CRITICAL) {
 120                                                 $new_priority = PRIORITY_NEGLIGIBLE;
 121                                         }
 122                                         $stamp = (float)microtime(true);
 123                                         DBA::update(
 124                                                 'workerqueue',
 125                                                 ['executed' => DBA::NULL_DATETIME, 'created' => DateTimeFormat::utcNow(), 'priority' => $new_priority, 'pid' => 0],
 126                                                 ['id' => $entry["id"]]
 127                                         );
 128                                 } else {
 129                                         Logger::info('Process runtime is okay', ['duration' => number_format($duration, 3), 'max' => $max_duration, 'id' => $entry["id"], 'pid' => $entry["pid"], 'command' => $command]);
 130                                 }
 131                         }
 132                 }
 133                 DBA::close($entries);
 134         }
 135
 136         /**
 137          * Remove old entries from the workerqueue
 138          *
 139          * @return void
 140          */
 141         private static function cleanWorkerQueue()
 142         {
 143                 DBA::delete('workerqueue', ["`done` AND `executed` < ?", DateTimeFormat::utc('now - 1 hour')]);
 144
 145                 // Optimizing this table only last seconds
 146                 if (DI::config()->get('system', 'optimize_tables')) {
 147                         // We are acquiring the two locks from the worker to avoid locking problems
 148                         if (DI::lock()->acquire(Worker::LOCK_PROCESS, 10)) {
 149                                 if (DI::lock()->acquire(Worker::LOCK_WORKER, 10)) {
 150                                         DBA::e("OPTIMIZE TABLE `workerqueue`");
 151                                         DBA::e("OPTIMIZE TABLE `process`");
 152                                         DI::lock()->release(Worker::LOCK_WORKER);
 153                                 }
 154                                 DI::lock()->release(Worker::LOCK_PROCESS);
 155                         }
 156                 }
 157         }
 158
 159         /**
 160          * Directly deliver AP messages or requeue them.
 161          *
 162          * This function is placed here as a safeguard. Even when the worker queue is completely blocked, messages will be delivered.
 163          */
 164         private static function deliverPosts()
 165         {
 166                 $deliveries = DBA::p("SELECT `item-uri`.`uri` AS `inbox`, MAX(`failed`) AS `failed` FROM `post-delivery` INNER JOIN `item-uri` ON `item-uri`.`id` = `post-delivery`.`inbox-id` GROUP BY `inbox`");
 167                 while ($delivery = DBA::fetch($deliveries)) {
 168                         if ($delivery['failed'] == 0) {
 169                                 $result = ActivityPub\Delivery::deliver($delivery['inbox']);
 170                                 Logger::info('Drectly deliver inbox', ['inbox' => $delivery['inbox'], 'result' => $result['success']]);
 171                                 continue;
 172                         } elseif ($delivery['failed'] < 3) {
 173                                 $priority = PRIORITY_HIGH;
 174                         } elseif ($delivery['failed'] < 6) {
 175                                 $priority = PRIORITY_MEDIUM;
 176                         } elseif ($delivery['failed'] < 8) {
 177                                 $priority = PRIORITY_LOW;
 178                         } {
 179                                 $priority = PRIORITY_NEGLIGIBLE;
 180                         }
 181
 182                         if ($delivery['failed'] >= DI::config()->get('system', 'worker_defer_limit')) {
 183                                 Logger::info('Removing failed deliveries', ['inbox' => $delivery['inbox'], 'failed' => $delivery['failed']]);
 184                                 Post\Delivery::removeFailed($delivery['inbox']);
 185                         }
 186
 187                         if (Worker::add($priority, 'APDelivery', '', 0, $delivery['inbox'], 0)) {
 188                                 Logger::info('Missing APDelivery worker added for inbox', ['inbox' => $delivery['inbox'], 'failed' => $delivery['failed'], 'priority' => $priority]);
 189                         }
 190                 }
 191         }
 192 }