src/Protocol/ActivityPub/Queue.php

   1 <?php
   2 /**
   3  * @copyright Copyright (C) 2010-2023, the Friendica project
   4  *
   5  * @license GNU AGPL version 3 or any later version
   6  *
   7  * This program is free software: you can redistribute it and/or modify
   8  * it under the terms of the GNU Affero General Public License as
   9  * published by the Free Software Foundation, either version 3 of the
  10  * License, or (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU Affero General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU Affero General Public License
  18  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  19  *
  20  */
  21
  22 namespace Friendica\Protocol\ActivityPub;
  23
  24 use Friendica\Core\Logger;
  25 use Friendica\Core\Worker;
  26 use Friendica\Database\Database;
  27 use Friendica\Database\DBA;
  28 use Friendica\DI;
  29 use Friendica\Model\ItemURI;
  30 use Friendica\Model\Post;
  31 use Friendica\Util\DateTimeFormat;
  32 use Friendica\Util\JsonLD;
  33
  34 /**
  35  * This class handles the processing of incoming posts
  36  */
  37 class Queue
  38 {
  39         /**
  40          * Add activity to the queue
  41          *
  42          * @param array $activity
  43          * @param string $type
  44          * @param integer $uid
  45          * @param string $http_signer
  46          * @param boolean $push
  47          * @return array
  48          */
  49         public static function add(array $activity, string $type, int $uid, string $http_signer, bool $push, bool $trust_source): array
  50         {
  51                 $fields = [
  52                         'activity-id' => $activity['id'],
  53                         'object-id'   => $activity['object_id'],
  54                         'type'        => $type,
  55                         'object-type' => $activity['object_type'],
  56                         'activity'    => json_encode($activity, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_PRETTY_PRINT),
  57                         'received'    => DateTimeFormat::utcNow(),
  58                         'push'        => $push,
  59                         'trust'       => $trust_source,
  60                 ];
  61
  62                 if (!empty($activity['reply-to-id'])) {
  63                         $fields['in-reply-to-id'] = $activity['reply-to-id'];
  64                 }
  65
  66                 if (!empty($activity['context'])) {
  67                         $fields['conversation'] = $activity['context'];
  68                 } elseif (!empty($activity['conversation'])) {
  69                         $fields['conversation'] = $activity['conversation'];
  70                 }
  71
  72                 if (!empty($activity['object_object_type'])) {
  73                         $fields['object-object-type'] = $activity['object_object_type'];
  74                 }
  75
  76                 if (!empty($http_signer)) {
  77                         $fields['signer'] = $http_signer;
  78                 }
  79
  80                 DBA::insert('inbox-entry', $fields, Database::INSERT_IGNORE);
  81
  82                 $queue = DBA::selectFirst('inbox-entry', ['id'], ['activity-id' => $activity['id']]);
  83                 if (!empty($queue['id'])) {
  84                         $activity['entry-id'] = $queue['id'];
  85                         DBA::insert('inbox-entry-receiver', ['queue-id' => $queue['id'], 'uid' => $uid], Database::INSERT_IGNORE);
  86                 }
  87                 return $activity;
  88         }
  89
  90         /**
  91          * Checks if an entry for a given url and type already exists
  92          *
  93          * @param string $url
  94          * @param string $type
  95          * @return boolean
  96          */
  97         public static function exists(string $url, string $type): bool
  98         {
  99                 return DBA::exists('inbox-entry', ['type' => $type, 'object-id' => $url]);
 100         }
 101
 102         /**
 103          * Remove activity from the queue
 104          *
 105          * @param array $activity
 106          * @return void
 107          */
 108         public static function remove(array $activity = [])
 109         {
 110                 if (empty($activity['entry-id'])) {
 111                         return;
 112                 }
 113                 DBA::delete('inbox-entry', ['id' => $activity['entry-id']]);
 114         }
 115
 116         /**
 117          * Delete all entries that depend on the given worker id
 118          *
 119          * @param integer $wid
 120          * @return void
 121          */
 122         public static function deleteByWorkerId(int $wid)
 123         {
 124                 $entries = DBA::select('inbox-entry', ['id'], ['wid' => $wid]);
 125                 while ($entry = DBA::fetch($entries)) {
 126                         self::deleteById($entry['id']);
 127                 }
 128                 DBA::close($entries);
 129         }
 130
 131         /**
 132          * Delete recursively an entry and all their children
 133          *
 134          * @param integer $id
 135          * @return void
 136          */
 137         public static function deleteById(int $id)
 138         {
 139                 $entry = DBA::selectFirst('inbox-entry', ['id', 'object-id'], ['id' => $id]);
 140                 if (empty($entry)) {
 141                         return;
 142                 }
 143
 144                 Logger::debug('Delete inbox-entry', ['id' => $entry['id']]);
 145
 146                 DBA::delete('inbox-entry', ['id' => $entry['id']]);
 147
 148                 $children = DBA::select('inbox-entry', ['id'], ['in-reply-to-id' => $entry['object-id']]);
 149                 while ($child = DBA::fetch($children)) {
 150                         self::deleteById($child['id']);
 151                 }
 152                 DBA::close($children);
 153         }
 154
 155         /**
 156          * Set the worker id for the queue entry
 157          *
 158          * @param int $entry_id
 159          * @param int $wid
 160          * @return void
 161          */
 162         public static function setWorkerId(int $entry_id, int $wid)
 163         {
 164                 if (empty($entry_id) || empty($wid)) {
 165                         return;
 166                 }
 167                 DBA::update('inbox-entry', ['wid' => $wid], ['id' => $entry_id]);
 168         }
 169
 170         /**
 171          * Check if there is an assigned worker task
 172          *
 173          * @param int $wid
 174          *
 175          * @return bool
 176          */
 177         public static function hasWorker(int $wid): bool
 178         {
 179                 if (empty($wid)) {
 180                         return false;
 181                 }
 182                 return DBA::exists('workerqueue', ['id' => $wid, 'done' => false]);
 183         }
 184
 185         /**
 186          * Process the activity with the given id
 187          *
 188          * @param integer $id
 189          * @param bool    $fetch_parents
 190          *
 191          * @return bool
 192          */
 193         public static function process(int $id, bool $fetch_parents = true): bool
 194         {
 195                 $entry = DBA::selectFirst('inbox-entry', [], ['id' => $id]);
 196                 if (empty($entry)) {
 197                         return false;
 198                 }
 199
 200                 if (!self::isProcessable($id)) {
 201                         Logger::debug('Other queue entries need to be processed first.', ['id' => $id]);
 202                         return false;
 203                 }
 204
 205                 if (!empty($entry['wid'])) {
 206                         $worker = DI::app()->getQueue();
 207                         $wid = $worker['id'] ?? 0;
 208                         if ($entry['wid'] != $wid) {
 209                                 $workerqueue = DBA::selectFirst('workerqueue', ['pid'], ['id' => $entry['wid'], 'done' => false]);
 210                                 if (!empty($workerqueue['pid']) && posix_kill($workerqueue['pid'], 0)) {
 211                                         Logger::notice('Entry is already processed via another process.', ['current' => $wid, 'processor' => $entry['wid']]);
 212                                         return false;
 213                                 }
 214                         }
 215                 }
 216
 217                 Logger::debug('Processing queue entry', ['id' => $entry['id'], 'type' => $entry['type'], 'object-type' => $entry['object-type'], 'uri' => $entry['object-id'], 'in-reply-to' => $entry['in-reply-to-id']]);
 218
 219                 $activity = json_decode($entry['activity'], true);
 220                 $type     = $entry['type'];
 221                 $push     = $entry['push'];
 222
 223                 $activity['entry-id']        = $entry['id'];
 224                 $activity['worker-id']       = $entry['wid'];
 225                 $activity['recursion-depth'] = 0;
 226
 227                 if (empty($activity['thread-children-type'])) {
 228                         $activity['thread-children-type'] = $type;
 229                 }
 230
 231                 $receivers = DBA::select('inbox-entry-receiver', ['uid'], ["`queue-id` = ? AND `uid` != ?", $entry['id'], 0]);
 232                 while ($receiver = DBA::fetch($receivers)) {
 233                         if (!in_array($receiver['uid'], $activity['receiver'])) {
 234                                 $activity['receiver'][] = $receiver['uid'];
 235                         }
 236                 }
 237                 DBA::close($receivers);
 238
 239                 if (!Receiver::routeActivities($activity, $type, $push, $fetch_parents, $activity['receiver'][0] ?? 0)) {
 240                         self::remove($activity);
 241                 }
 242
 243                 return true;
 244         }
 245
 246         /**
 247          * Process all activities
 248          *
 249          * @return void
 250          */
 251         public static function processAll()
 252         {
 253                 $entries = DBA::select('inbox-entry', ['id', 'type', 'object-type', 'object-id', 'in-reply-to-id'], ["`trust` AND `wid` IS NULL"], ['order' => ['id' => true]]);
 254                 while ($entry = DBA::fetch($entries)) {
 255                         // Don't process entries of items that are answer to nonexistent posts
 256                         if (!empty($entry['in-reply-to-id']) && !Post::exists(['uri' => $entry['in-reply-to-id']])) {
 257                                 continue;
 258                         }
 259                         // We don't need to process entries that depend on already existing entries.
 260                         if (!empty($entry['in-reply-to-id']) && DBA::exists('inbox-entry', ["`id` != ? AND `object-id` = ?", $entry['id'], $entry['in-reply-to-id']])) {
 261                                 continue;
 262                         }
 263                         Logger::debug('Process leftover entry', $entry);
 264                         self::process($entry['id'], false);
 265                 }
 266                 DBA::close($entries);
 267         }
 268
 269         public static function isProcessable(int $id): bool
 270         {
 271                 $entry = DBA::selectFirst('inbox-entry', [], ['id' => $id]);
 272                 if (empty($entry)) {
 273                         return false;
 274                 }
 275
 276                 if (!empty($entry['object-id']) && Post::exists(['uri' => $entry['object-id']])) {
 277                         // The object already exists, so processing can be done
 278                         return true;
 279                 }
 280
 281                 if (!empty($entry['conversation'])) {
 282                         $conv_id = ItemURI::getIdByURI($entry['conversation'], false);
 283                         if (DBA::exists('post-thread', ['conversation-id' => $conv_id])) {
 284                                 // We have got the conversation in the system, so the post can be processed
 285                                 return true;
 286                         }
 287                 }
 288
 289                 if (!empty($entry['object-id']) && !empty($entry['in-reply-to-id']) && ($entry['object-id'] != $entry['in-reply-to-id'])) {
 290                         if (DBA::exists('inbox-entry', ['object-id' => $entry['in-reply-to-id']])) {
 291                                 // This entry belongs to some other entry that should be processed first
 292                                 return false;
 293                         }
 294                         if (!Post::exists(['uri' => $entry['in-reply-to-id']])) {
 295                                 // This entry belongs to some other entry that need to be fetched first
 296                                 if (Fetch::hasWorker($entry['in-reply-to-id'])) {
 297                                         Logger::debug('Fetching of the activity is already queued', ['id' => $entry['activity-id'], 'reply-to-id' => $entry['in-reply-to-id']]);
 298                                         return false;
 299                                 }
 300                                 Fetch::add($entry['in-reply-to-id']);
 301                                 $activity = json_decode($entry['activity'], true);
 302                                 $activity['recursion-depth'] = 0;
 303                                 $wid = Worker::add(Worker::PRIORITY_HIGH, 'FetchMissingActivity', $entry['in-reply-to-id'], $activity, '', Receiver::COMPLETION_ASYNC);
 304                                 Fetch::setWorkerId($entry['in-reply-to-id'], $wid);
 305                                 Logger::debug('Fetch missing activity', ['wid' => $wid, 'id' => $entry['activity-id'], 'reply-to-id' => $entry['in-reply-to-id']]);
 306                                 return false;
 307                         }
 308                 }
 309
 310                 return true;
 311         }
 312
 313         /**
 314          * Clear old activities
 315          *
 316          * @return void
 317          */
 318         public static function clear()
 319         {
 320                 // We delete all entries that aren't associated with a worker entry after seven days.
 321                 // The other entries are deleted when the worker deferred for too long.
 322                 $entries = DBA::select('inbox-entry', ['id'], ["`wid` IS NULL AND `received` < ?", DateTimeFormat::utc('now - 7 days')]);
 323                 while ($entry = DBA::fetch($entries)) {
 324                         self::deleteById($entry['id']);
 325                 }
 326                 DBA::close($entries);
 327
 328                 // Optimizing this table only last seconds
 329                 if (DI::config()->get('system', 'optimize_tables')) {
 330                         Logger::info('Optimize start');
 331                         DBA::optimizeTable('inbox-entry');
 332                         Logger::info('Optimize end');
 333                 }
 334         }
 335
 336         /**
 337          * Process all activities that are children of a given post url
 338          *
 339          * @param string $uri
 340          * @return int
 341          */
 342         public static function processReplyByUri(string $uri): int
 343         {
 344                 $count = 0;
 345                 $entries = DBA::select('inbox-entry', ['id'], ["`in-reply-to-id` = ? AND `object-id` != ?", $uri, $uri]);
 346                 while ($entry = DBA::fetch($entries)) {
 347                         $count += 1;
 348                         self::process($entry['id'], false);
 349                 }
 350                 DBA::close($entries);
 351                 return $count;
 352         }
 353
 354         /**
 355          * Checks if there are children of the given uri
 356          *
 357          * @param string $uri
 358          *
 359          * @return bool
 360          */
 361         public static function hasChildren(string $uri): bool
 362         {
 363                 return DBA::exists('inbox-entry', ["`in-reply-to-id` = ? AND `object-id` != ?", $uri, $uri]);
 364         }
 365
 366         /**
 367          * Prepare the queue entry.
 368          * This is a test function that is used solely for development.
 369          *
 370          * @param integer $id
 371          * @return array
 372          */
 373         public static function reprepareActivityById(int $id): array
 374         {
 375                 $entry = DBA::selectFirst('inbox-entry', [], ['id' => $id]);
 376                 if (empty($entry)) {
 377                         return [];
 378                 }
 379
 380                 $receiver = DBA::selectFirst('inbox-entry-receiver', ['uid'], ['queue-id' => $id]);
 381                 if (!empty($receiver)) {
 382                         $uid = $receiver['uid'];
 383                 } else {
 384                         $uid = 0;
 385                 }
 386
 387                 $trust_source = $entry['trust'];
 388
 389                 $data     = json_decode($entry['activity'], true);
 390                 $activity = json_decode($data['raw'], true);
 391
 392                 $ldactivity = JsonLD::compact($activity);
 393                 return [
 394                         'data'  => Receiver::prepareObjectData($ldactivity, $uid, $entry['push'], $trust_source),
 395                         'trust' => $trust_source
 396                 ];
 397         }
 398
 399         /**
 400          * Set the trust for all untrusted entries.
 401          * This is a test function that is used solely for development.
 402          *
 403          * @return void
 404          */
 405         public static function reprepareAll()
 406         {
 407                 $entries = DBA::select('inbox-entry', ['id'], ["NOT `trust` AND `wid` IS NULL"], ['order' => ['id' => true]]);
 408                 while ($entry = DBA::fetch($entries)) {
 409                         $data = self::reprepareActivityById($entry['id'], false);
 410                         if ($data['trust']) {
 411                                 DBA::update('inbox-entry', ['trust' => true], ['id' => $entry['id']]);
 412                         }
 413                 }
 414                 DBA::close($entries);
 415         }
 416 }