src/Protocol/ActivityPub/Queue.php

   1 <?php
   2 /**
   3  * @copyright Copyright (C) 2010-2023, the Friendica project
   4  *
   5  * @license GNU AGPL version 3 or any later version
   6  *
   7  * This program is free software: you can redistribute it and/or modify
   8  * it under the terms of the GNU Affero General Public License as
   9  * published by the Free Software Foundation, either version 3 of the
  10  * License, or (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU Affero General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU Affero General Public License
  18  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  19  *
  20  */
  21
  22 namespace Friendica\Protocol\ActivityPub;
  23
  24 use Friendica\Core\Logger;
  25 use Friendica\Database\Database;
  26 use Friendica\Database\DBA;
  27 use Friendica\DI;
  28 use Friendica\Model\ItemURI;
  29 use Friendica\Model\Post;
  30 use Friendica\Util\DateTimeFormat;
  31 use Friendica\Util\JsonLD;
  32
  33 /**
  34  * This class handles the processing of incoming posts
  35  */
  36 class Queue
  37 {
  38         /**
  39          * Add activity to the queue
  40          *
  41          * @param array $activity
  42          * @param string $type
  43          * @param integer $uid
  44          * @param string $http_signer
  45          * @param boolean $push
  46          * @return array
  47          */
  48         public static function add(array $activity, string $type, int $uid, string $http_signer, bool $push, bool $trust_source): array
  49         {
  50                 $fields = [
  51                         'activity-id' => $activity['id'],
  52                         'object-id'   => $activity['object_id'],
  53                         'type'        => $type,
  54                         'object-type' => $activity['object_type'],
  55                         'activity'    => json_encode($activity, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_PRETTY_PRINT),
  56                         'received'    => DateTimeFormat::utcNow(),
  57                         'push'        => $push,
  58                         'trust'       => $trust_source,
  59                 ];
  60
  61                 if (!empty($activity['reply-to-id'])) {
  62                         $fields['in-reply-to-id'] = $activity['reply-to-id'];
  63                 }
  64
  65                 if (!empty($activity['context'])) {
  66                         $fields['conversation'] = $activity['context'];
  67                 } elseif (!empty($activity['conversation'])) {
  68                         $fields['conversation'] = $activity['conversation'];
  69                 }
  70
  71                 if (!empty($activity['object_object_type'])) {
  72                         $fields['object-object-type'] = $activity['object_object_type'];
  73                 }
  74
  75                 if (!empty($http_signer)) {
  76                         $fields['signer'] = $http_signer;
  77                 }
  78
  79                 DBA::insert('inbox-entry', $fields, Database::INSERT_IGNORE);
  80
  81                 $queue = DBA::selectFirst('inbox-entry', ['id'], ['activity-id' => $activity['id']]);
  82                 if (!empty($queue['id'])) {
  83                         $activity['entry-id'] = $queue['id'];
  84                         DBA::insert('inbox-entry-receiver', ['queue-id' => $queue['id'], 'uid' => $uid], Database::INSERT_IGNORE);
  85                 }
  86                 return $activity;
  87         }
  88
  89         /**
  90          * Checks if an entry for a given url and type already exists
  91          *
  92          * @param string $url
  93          * @param string $type
  94          * @return boolean
  95          */
  96         public static function exists(string $url, string $type): bool
  97         {
  98                 return DBA::exists('inbox-entry', ['type' => $type, 'object-id' => $url]);
  99         }
 100
 101         /**
 102          * Remove activity from the queue
 103          *
 104          * @param array $activity
 105          * @return void
 106          */
 107         public static function remove(array $activity = [])
 108         {
 109                 if (empty($activity['entry-id'])) {
 110                         return;
 111                 }
 112                 DBA::delete('inbox-entry', ['id' => $activity['entry-id']]);
 113         }
 114
 115         /**
 116          * Delete all entries that depend on the given worker id
 117          *
 118          * @param integer $wid
 119          * @return void
 120          */
 121         public static function deleteByWorkerId(int $wid)
 122         {
 123                 $entries = DBA::select('inbox-entry', ['id'], ['wid' => $wid]);
 124                 while ($entry = DBA::fetch($entries)) {
 125                         self::deleteById($entry['id']);
 126                 }
 127                 DBA::close($entries);
 128         }
 129
 130         /**
 131          * Delete recursively an entry and all their children
 132          *
 133          * @param integer $id
 134          * @return void
 135          */
 136         public static function deleteById(int $id)
 137         {
 138                 $entry = DBA::selectFirst('inbox-entry', ['id', 'object-id'], ['id' => $id]);
 139                 if (empty($entry)) {
 140                         return;
 141                 }
 142
 143                 Logger::debug('Delete inbox-entry', ['id' => $entry['id']]);
 144
 145                 DBA::delete('inbox-entry', ['id' => $entry['id']]);
 146
 147                 $children = DBA::select('inbox-entry', ['id'], ['in-reply-to-id' => $entry['object-id']]);
 148                 while ($child = DBA::fetch($children)) {
 149                         self::deleteById($child['id']);
 150                 }
 151                 DBA::close($children);
 152         }
 153
 154         /**
 155          * Set the worker id for the queue entry
 156          *
 157          * @param int $entry_id
 158          * @param int $wid
 159          * @return void
 160          */
 161         public static function setWorkerId(int $entry_id, int $wid)
 162         {
 163                 if (empty($entry_id) || empty($wid)) {
 164                         return;
 165                 }
 166                 DBA::update('inbox-entry', ['wid' => $wid], ['id' => $entry_id]);
 167         }
 168
 169         /**
 170          * Check if there is an assigned worker task
 171          *
 172          * @param int $wid
 173          *
 174          * @return bool
 175          */
 176         public static function hasWorker(int $wid): bool
 177         {
 178                 if (empty($wid)) {
 179                         return false;
 180                 }
 181                 return DBA::exists('workerqueue', ['id' => $wid, 'done' => false]);
 182         }
 183
 184         /**
 185          * Process the activity with the given id
 186          *
 187          * @param integer $id
 188          * @param bool    $fetch_parents
 189          *
 190          * @return bool
 191          */
 192         public static function process(int $id, bool $fetch_parents = true): bool
 193         {
 194                 $entry = DBA::selectFirst('inbox-entry', [], ['id' => $id]);
 195                 if (empty($entry)) {
 196                         return false;
 197                 }
 198
 199                 if (!self::isProcessable($id)) {
 200                         Logger::debug('Other queue entries need to be processed first.', ['id' => $id]);
 201                         return false;
 202                 }
 203
 204                 if (!empty($entry['wid'])) {
 205                         $worker = DI::app()->getQueue();
 206                         $wid = $worker['id'] ?? 0;
 207                         if ($entry['wid'] != $wid) {
 208                                 $workerqueue = DBA::selectFirst('workerqueue', ['pid'], ['id' => $entry['wid'], 'done' => false]);
 209                                 if (!empty($workerqueue['pid']) && posix_kill($workerqueue['pid'], 0)) {
 210                                         Logger::notice('Entry is already processed via another process.', ['current' => $wid, 'processor' => $entry['wid']]);
 211                                         return false;
 212                                 }
 213                         }
 214                 }
 215
 216                 Logger::debug('Processing queue entry', ['id' => $entry['id'], 'type' => $entry['type'], 'object-type' => $entry['object-type'], 'uri' => $entry['object-id'], 'in-reply-to' => $entry['in-reply-to-id']]);
 217
 218                 $activity = json_decode($entry['activity'], true);
 219                 $type     = $entry['type'];
 220                 $push     = $entry['push'];
 221
 222                 $activity['entry-id']        = $entry['id'];
 223                 $activity['worker-id']       = $entry['wid'];
 224                 $activity['recursion-depth'] = 0;
 225
 226                 if (empty($activity['thread-children-type'])) {
 227                         $activity['thread-children-type'] = $type;
 228                 }
 229
 230                 $receivers = DBA::select('inbox-entry-receiver', ['uid'], ["`queue-id` = ? AND `uid` != ?", $entry['id'], 0]);
 231                 while ($receiver = DBA::fetch($receivers)) {
 232                         if (!in_array($receiver['uid'], $activity['receiver'])) {
 233                                 $activity['receiver'][] = $receiver['uid'];
 234                         }
 235                 }
 236                 DBA::close($receivers);
 237
 238                 if (!Receiver::routeActivities($activity, $type, $push, $fetch_parents, $activity['receiver'][0] ?? 0)) {
 239                         self::remove($activity);
 240                 }
 241
 242                 return true;
 243         }
 244
 245         /**
 246          * Process all activities
 247          *
 248          * @return void
 249          */
 250         public static function processAll()
 251         {
 252                 $entries = DBA::select('inbox-entry', ['id', 'type', 'object-type', 'object-id', 'in-reply-to-id'], ["`trust` AND `wid` IS NULL"], ['order' => ['id' => true]]);
 253                 while ($entry = DBA::fetch($entries)) {
 254                         // Don't process entries of items that are answer to nonexistent posts
 255                         if (!empty($entry['in-reply-to-id']) && !Post::exists(['uri' => $entry['in-reply-to-id']])) {
 256                                 continue;
 257                         }
 258                         // We don't need to process entries that depend on already existing entries.
 259                         if (!empty($entry['in-reply-to-id']) && DBA::exists('inbox-entry', ["`id` != ? AND `object-id` = ?", $entry['id'], $entry['in-reply-to-id']])) {
 260                                 continue;
 261                         }
 262                         Logger::debug('Process leftover entry', $entry);
 263                         self::process($entry['id'], false);
 264                 }
 265                 DBA::close($entries);
 266         }
 267
 268         public static function isProcessable(int $id): bool
 269         {
 270                 $entry = DBA::selectFirst('inbox-entry', [], ['id' => $id]);
 271                 if (empty($entry)) {
 272                         return false;
 273                 }
 274
 275                 if (!empty($entry['object-id']) && Post::exists(['uri' => $entry['object-id']])) {
 276                         // The object already exists, so processing can be done
 277                         return true;
 278                 }
 279
 280                 if (!empty($entry['conversation'])) {
 281                         $conv_id = ItemURI::getIdByURI($entry['conversation'], false);
 282                         if (DBA::exists('post-thread', ['conversation-id' => $conv_id])) {
 283                                 // We have got the conversation in the system, so the post can be processed
 284                                 return true;
 285                         }
 286                 }
 287
 288                 if (!empty($entry['object-id']) && !empty($entry['in-reply-to-id']) && ($entry['object-id'] != $entry['in-reply-to-id']) && DBA::exists('inbox-entry', ['object-id' => $entry['in-reply-to-id']])) {
 289                         // This entry belongs to some other entry that should be processed first
 290                         return false;
 291                 }
 292
 293                 return true;
 294         }
 295
 296         /**
 297          * Clear old activities
 298          *
 299          * @return void
 300          */
 301         public static function clear()
 302         {
 303                 // We delete all entries that aren't associated with a worker entry after seven days.
 304                 // The other entries are deleted when the worker deferred for too long.
 305                 $entries = DBA::select('inbox-entry', ['id'], ["`wid` IS NULL AND `received` < ?", DateTimeFormat::utc('now - 7 days')]);
 306                 while ($entry = DBA::fetch($entries)) {
 307                         self::deleteById($entry['id']);
 308                 }
 309                 DBA::close($entries);
 310
 311                 // Optimizing this table only last seconds
 312                 if (DI::config()->get('system', 'optimize_tables')) {
 313                         Logger::info('Optimize start');
 314                         DBA::optimizeTable('inbox-entry');
 315                         Logger::info('Optimize end');
 316                 }
 317         }
 318
 319         /**
 320          * Process all activities that are children of a given post url
 321          *
 322          * @param string $uri
 323          * @return int
 324          */
 325         public static function processReplyByUri(string $uri): int
 326         {
 327                 $count = 0;
 328                 $entries = DBA::select('inbox-entry', ['id'], ["`in-reply-to-id` = ? AND `object-id` != ?", $uri, $uri]);
 329                 while ($entry = DBA::fetch($entries)) {
 330                         $count += 1;
 331                         self::process($entry['id'], false);
 332                 }
 333                 DBA::close($entries);
 334                 return $count;
 335         }
 336
 337         /**
 338          * Checks if there are children of the given uri
 339          *
 340          * @param string $uri
 341          *
 342          * @return bool
 343          */
 344         public static function hasChildren(string $uri): bool
 345         {
 346                 return DBA::exists('inbox-entry', ["`in-reply-to-id` = ? AND `object-id` != ?", $uri, $uri]);
 347         }
 348
 349         /**
 350          * Prepare the queue entry.
 351          * This is a test function that is used solely for development.
 352          *
 353          * @param integer $id
 354          * @return array
 355          */
 356         public static function reprepareActivityById(int $id): array
 357         {
 358                 $entry = DBA::selectFirst('inbox-entry', [], ['id' => $id]);
 359                 if (empty($entry)) {
 360                         return [];
 361                 }
 362
 363                 $receiver = DBA::selectFirst('inbox-entry-receiver', ['uid'], ['queue-id' => $id]);
 364                 if (!empty($receiver)) {
 365                         $uid = $receiver['uid'];
 366                 } else {
 367                         $uid = 0;
 368                 }
 369
 370                 $trust_source = $entry['trust'];
 371
 372                 $data     = json_decode($entry['activity'], true);
 373                 $activity = json_decode($data['raw'], true);
 374
 375                 $ldactivity = JsonLD::compact($activity);
 376                 return [
 377                         'data'  => Receiver::prepareObjectData($ldactivity, $uid, $entry['push'], $trust_source),
 378                         'trust' => $trust_source
 379                 ];
 380         }
 381
 382         /**
 383          * Set the trust for all untrusted entries.
 384          * This is a test function that is used solely for development.
 385          *
 386          * @return void
 387          */
 388         public static function reprepareAll()
 389         {
 390                 $entries = DBA::select('inbox-entry', ['id'], ["NOT `trust` AND `wid` IS NULL"], ['order' => ['id' => true]]);
 391                 while ($entry = DBA::fetch($entries)) {
 392                         $data = self::reprepareActivityById($entry['id'], false);
 393                         if ($data['trust']) {
 394                                 DBA::update('inbox-entry', ['trust' => true], ['id' => $entry['id']]);
 395                         }
 396                 }
 397                 DBA::close($entries);
 398         }
 399 }