X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;ds=sidebyside;f=src%2FProtocol%2FActivityPub%2FProcessor.php;h=8e21a664469bc7a7e274dcb4bbc01f9e67c84f60;hb=71a7f2d5034211a3e640c24663e0bad7a24c466a;hp=421f2d4c0c36ba47bb63cdc9428260d6ba0393ab;hpb=a27274b3b4ae1ddb980ee3d72e784bfb8d365d3d;p=friendica.git diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php index 421f2d4c0c..8e21a66446 100644 --- a/src/Protocol/ActivityPub/Processor.php +++ b/src/Protocol/ActivityPub/Processor.php @@ -58,13 +58,40 @@ use Friendica\Worker\Delivery; class Processor { const CACHEKEY_FETCH_ACTIVITY = 'processor:fetchMissingActivity:'; + const CACHEKEY_JUST_FETCHED = 'processor:isJustFetched:'; + + /** + * Add an object id to the list of processed ids + * + * @param string $id + * + * @return void + */ + private static function addActivityId(string $id) + { + DBA::delete('fetched-activity', ["`received` < ?", DateTimeFormat::utc('now - 5 minutes')]); + DBA::insert('fetched-activity', ['object-id' => $id, 'received' => DateTimeFormat::utcNow()]); + } + + /** + * Checks if the given object id has just been fetched + * + * @param string $id + * + * @return boolean + */ + private static function isFetched(string $id): bool + { + return DBA::exists('fetched-activity', ['object-id' => $id]); + } + /** * Extracts the tag character (#, @, !) from mention links * * @param string $body * @return string */ - protected static function normalizeMentionLinks(string $body): string + public static function normalizeMentionLinks(string $body): string { return preg_replace('%\[url=([^\[\]]*)]([#@!])(.*?)\[/url]%ism', '$2[url=$1]$3[/url]', $body); } @@ -143,7 +170,7 @@ class Processor } /** - * Stire attachment data + * Store attachment data * * @param array $activity * @param array $item @@ -160,7 +187,7 @@ class Processor } /** - * Store attachment data + * Store question data * * @param array $activity * @param array $item @@ -199,11 +226,12 @@ class Processor */ public static function updateItem(array $activity) { - $item = Post::selectFirst(['uri', 'uri-id', 'thr-parent', 'gravity', 'post-type'], ['uri' => $activity['id']]); + $item = Post::selectFirst(['uri', 'uri-id', 'thr-parent', 'gravity', 'post-type', 'private'], ['uri' => $activity['id']]); if (!DBA::isResult($item)) { - Logger::warning('No existing item, item will be created', ['uri' => $activity['id']]); - $item = self::createItem($activity); + Logger::notice('No existing item, item will be created', ['uri' => $activity['id']]); + $item = self::createItem($activity, false); if (empty($item)) { + Queue::remove($activity); return; } @@ -215,14 +243,14 @@ class Processor $item['edited'] = DateTimeFormat::utc($activity['updated']); $item = self::processContent($activity, $item); - - self::storeAttachments($activity, $item); - self::storeQuestion($activity, $item); - if (empty($item)) { + Queue::remove($activity); return; } + self::storeAttachments($activity, $item); + self::storeQuestion($activity, $item); + Post\History::add($item['uri-id'], $item); Item::update($item, ['uri' => $activity['id']]); @@ -265,29 +293,32 @@ class Processor /** * Prepares data for a message * - * @param array $activity Activity array + * @param array $activity Activity array + * @param bool $fetch_parents + * * @return array Internal item + * * @throws \Friendica\Network\HTTPException\InternalServerErrorException * @throws \ImagickException */ - public static function createItem(array $activity): array + public static function createItem(array $activity, bool $fetch_parents): array { $item = []; $item['verb'] = Activity::POST; $item['thr-parent'] = $activity['reply-to-id']; if ($activity['reply-to-id'] == $activity['id']) { - $item['gravity'] = GRAVITY_PARENT; + $item['gravity'] = Item::GRAVITY_PARENT; $item['object-type'] = Activity\ObjectType::NOTE; } else { - $item['gravity'] = GRAVITY_COMMENT; + $item['gravity'] = Item::GRAVITY_COMMENT; $item['object-type'] = Activity\ObjectType::COMMENT; } - if (!empty($activity['context'])) { - $item['conversation'] = $activity['context']; - } elseif (!empty($activity['conversation'])) { + if (!empty($activity['conversation'])) { $item['conversation'] = $activity['conversation']; + } elseif (!empty($activity['context'])) { + $item['conversation'] = $activity['context']; } if (!empty($item['conversation'])) { @@ -295,54 +326,41 @@ class Processor if (!empty($conversation)) { Logger::debug('Got conversation', ['conversation' => $item['conversation'], 'parent' => $conversation]); $item['parent-uri'] = $conversation['uri']; + $item['parent-uri-id'] = ItemURI::getIdByURI($item['parent-uri']); } } else { $conversation = []; } - if (empty($activity['directmessage']) && ($activity['id'] != $activity['reply-to-id']) && !Post::exists(['uri' => $activity['reply-to-id']])) { - $recursion_depth = $activity['recursion-depth'] ?? 0; - Logger::notice('Parent not found. Try to refetch it.', ['parent' => $activity['reply-to-id'], 'recursion-depth' => $recursion_depth]); - if ($recursion_depth < 10) { - $result = self::fetchMissingActivity($activity['reply-to-id'], $activity, '', Receiver::COMPLETION_AUTO); - if (empty($result) && self::isActivityGone($activity['reply-to-id'])) { - // Recursively delete this and all depending entries - Queue::deleteById($activity['entry-id']); - return []; - } - $fetch_by_worker = empty($result); - } else { - Logger::notice('Recursion level is too high.', ['parent' => $activity['reply-to-id'], 'recursion-depth' => $recursion_depth]); - $fetch_by_worker = true; - } + Logger::debug('Create Item', ['id' => $activity['id'], 'conversation' => $item['conversation'] ?? '']); + if (empty($activity['author']) && empty($activity['actor'])) { + Logger::notice('Missing author and actor. We quit here.', ['activity' => $activity]); + Queue::remove($activity); + return []; + } - if ($fetch_by_worker && Queue::hasWorker($activity)) { - Logger::notice('There is already a worker task to fetch the post.', ['id' => $activity['id'], 'parent' => $activity['reply-to-id']]); - $fetch_by_worker = false; - if (!empty($conversation)) { - return []; - } - } + if (!in_array(0, $activity['receiver']) || !DI::config()->get('system', 'fetch_parents')) { + $fetch_parents = false; + } - if ($fetch_by_worker) { - Logger::notice('Fetching is done by worker.', ['parent' => $activity['reply-to-id'], 'recursion-depth' => $recursion_depth]); - $activity['recursion-depth'] = 0; - $wid = Worker::add(PRIORITY_HIGH, 'FetchMissingActivity', $activity['reply-to-id'], $activity, '', Receiver::COMPLETION_AUTO); - Queue::setWorkerId($activity, $wid); - if (!empty($conversation)) { - return []; - } - } elseif (!empty($result)) { + if ($fetch_parents && empty($activity['directmessage']) && ($activity['id'] != $activity['reply-to-id']) && !Post::exists(['uri' => $activity['reply-to-id']])) { + $result = self::fetchParent($activity, !empty($conversation)); + if (!empty($result)) { if (($item['thr-parent'] != $result) && Post::exists(['uri' => $result])) { $item['thr-parent'] = $result; } + } elseif (empty($conversation)) { + return []; } } $item['diaspora_signed_text'] = $activity['diaspora:comment'] ?? ''; - if (empty($conversation) && empty($activity['directmessage']) && ($item['gravity'] != GRAVITY_PARENT) && !Post::exists(['uri' => $item['thr-parent']])) { - Logger::info('Parent not found, message will be discarded.', ['thr-parent' => $item['thr-parent']]); + if (empty($conversation) && empty($activity['directmessage']) && ($item['gravity'] != Item::GRAVITY_PARENT) && !Post::exists(['uri' => $item['thr-parent']])) { + Logger::notice('Parent not found, message will be discarded.', ['thr-parent' => $item['thr-parent']]); + if (!$fetch_parents) { + Queue::remove($activity); + } return []; } @@ -362,11 +380,12 @@ class Processor if (!empty($activity['raw'])) { $item['source'] = $activity['raw']; - $item['protocol'] = Conversation::PARCEL_ACTIVITYPUB; + } - if (isset($activity['push'])) { - $item['direction'] = $activity['push'] ? Conversation::PUSH : Conversation::PULL; - } + $item['protocol'] = Conversation::PARCEL_ACTIVITYPUB; + + if (isset($activity['push'])) { + $item['direction'] = $activity['push'] ? Conversation::PUSH : Conversation::PULL; } if (!empty($activity['from-relay'])) { @@ -412,7 +431,7 @@ class Processor $item['owner-id'] = $item['author-id']; } else { $actor = APContact::getByURL($item['owner-link'], false); - $item['isForum'] = ($actor['type'] == 'Group'); + $item['isForum'] = ($actor['type'] ?? 'Person') == 'Group'; } $item['uri'] = $activity['id']; @@ -432,9 +451,12 @@ class Processor return []; } + $item['thr-parent-id'] = ItemURI::getIdByURI($item['thr-parent']); + $item = self::processContent($activity, $item); if (empty($item)) { Logger::info('Message was not processed'); + Queue::remove($activity); return []; } @@ -459,6 +481,86 @@ class Processor return $item; } + /** + * Fetch and process parent posts for the given activity + * + * @param array $activity + * @param bool $in_background + * + * @return string + */ + private static function fetchParent(array $activity, bool $in_background = false): string + { + if (self::isFetched($activity['reply-to-id'])) { + Logger::info('Id is already fetched', ['id' => $activity['reply-to-id']]); + return ''; + } + + self::addActivityId($activity['reply-to-id']); + + if (!DI::config()->get('system', 'fetch_by_worker')) { + $in_background = false; + } + + $recursion_depth = $activity['recursion-depth'] ?? 0; + + if (!$in_background && ($recursion_depth < DI::config()->get('system', 'max_recursion_depth'))) { + Logger::notice('Parent not found. Try to refetch it.', ['parent' => $activity['reply-to-id'], 'recursion-depth' => $recursion_depth]); + $result = self::fetchMissingActivity($activity['reply-to-id'], $activity, '', Receiver::COMPLETION_AUTO); + if (empty($result) && self::isActivityGone($activity['reply-to-id'])) { + Logger::notice('The activity is gone, the queue entry will be deleted', ['parent' => $activity['reply-to-id']]); + if (!empty($activity['entry-id'])) { + Queue::deleteById($activity['entry-id']); + } + return ''; + } elseif (!empty($result)) { + $exists = Post::exists(['uri' => [$result, $activity['reply-to-id']]]); + if ($exists) { + Logger::notice('The activity has been fetched and created.', ['parent' => $result]); + return $result; + } elseif (DI::config()->get('system', 'fetch_by_worker') || DI::config()->get('system', 'decoupled_receiver')) { + Logger::notice('The activity has been fetched and will hopefully be created later.', ['parent' => $result]); + } else { + Logger::notice('The activity exists but has not been created, the queue entry will be deleted.', ['parent' => $result]); + if (!empty($activity['entry-id'])) { + Queue::deleteById($activity['entry-id']); + } + } + return ''; + } + if (empty($result) && !DI::config()->get('system', 'fetch_by_worker')) { + return ''; + } + } elseif (self::isActivityGone($activity['reply-to-id'])) { + Logger::notice('The activity is gone. We will not spawn a worker. The queue entry will be deleted', ['parent' => $activity['reply-to-id']]); + if ($in_background) { + // fetching in background is done for all activities where we have got the conversation + // There we only delete the single activity and not the whole thread since we can store the + // other posts in the thread even with missing posts. + Queue::remove($activity); + } elseif (!empty($activity['entry-id'])) { + Queue::deleteById($activity['entry-id']); + } + return ''; + } elseif ($in_background) { + Logger::notice('Fetching is done in the background.', ['parent' => $activity['reply-to-id']]); + } else { + Logger::notice('Recursion level is too high.', ['parent' => $activity['reply-to-id'], 'recursion-depth' => $recursion_depth]); + } + + if (!Fetch::hasWorker($activity['reply-to-id'])) { + Logger::notice('Fetching is done by worker.', ['parent' => $activity['reply-to-id'], 'recursion-depth' => $recursion_depth]); + Fetch::add($activity['reply-to-id']); + $activity['recursion-depth'] = 0; + $wid = Worker::add(Worker::PRIORITY_HIGH, 'FetchMissingActivity', $activity['reply-to-id'], $activity, '', Receiver::COMPLETION_AUTO); + Fetch::setWorkerId($activity['reply-to-id'], $wid); + } else { + Logger::debug('Activity will already be fetched via a worker.', ['url' => $activity['reply-to-id']]); + } + + return ''; + } + /** * Check if a given activity is no longer available * @@ -475,16 +577,23 @@ class Processor } // @todo To ensure that the remote system is working correctly, we can check if the "Content-Type" contains JSON - if (in_array($curlResult->getReturnCode(), [404])) { + if (in_array($curlResult->getReturnCode(), [401, 404])) { return true; } - $object = json_decode($curlResult->getBody(), true); - if (!empty($object)) { - $activity = JsonLD::compact($object); - if (JsonLD::fetchElement($activity, '@type') == 'as:Tombstone') { + if ($curlResult->isSuccess()) { + $object = json_decode($curlResult->getBody(), true); + if (!empty($object)) { + $activity = JsonLD::compact($object); + if (JsonLD::fetchElement($activity, '@type') == 'as:Tombstone') { + return true; + } + } + } elseif ($curlResult->getReturnCode() == 0) { + $host = parse_url($url, PHP_URL_HOST); + if (!(filter_var($host, FILTER_VALIDATE_IP) || @dns_get_record($host . '.', DNS_A + DNS_AAAA))) { return true; - } + } } return false; @@ -546,14 +655,15 @@ class Processor public static function createActivity(array $activity, string $verb) { $activity['reply-to-id'] = $activity['object_id']; - $item = self::createItem($activity); + $item = self::createItem($activity, false); if (empty($item)) { + Logger::debug('Activity was not prepared', ['id' => $activity['object_id']]); return; } $item['verb'] = $verb; $item['thr-parent'] = $activity['object_id']; - $item['gravity'] = GRAVITY_ACTIVITY; + $item['gravity'] = Item::GRAVITY_ACTIVITY; unset($item['post-type']); $item['object-type'] = Activity\ObjectType::NOTE; @@ -570,7 +680,7 @@ class Processor * Fetch the Uri-Id of a post for the "featured" collection * * @param array $activity - * @return null|int + * @return null|array */ private static function getUriIdForFeaturedCollection(array $activity) { @@ -588,7 +698,7 @@ class Processor } } - $parent = Post::selectFirst(['uri-id'], ['uri' => $activity['object_id']]); + $parent = Post::selectFirst(['uri-id', 'author-id'], ['uri' => $activity['object_id']]); if (empty($parent['uri-id'])) { if (self::fetchMissingActivity($activity['object_id'], $activity, '', Receiver::COMPLETION_AUTO)) { $parent = Post::selectFirst(['uri-id'], ['uri' => $activity['object_id']]); @@ -596,7 +706,7 @@ class Processor } if (!empty($parent['uri-id'])) { - return $parent['uri-id']; + $parent; } return null; @@ -609,14 +719,14 @@ class Processor */ public static function addToFeaturedCollection(array $activity) { - $uriid = self::getUriIdForFeaturedCollection($activity); - if (empty($uriid)) { + $post = self::getUriIdForFeaturedCollection($activity); + if (empty($post)) { return; } - Logger::debug('Add post to featured collection', ['uri-id' => $uriid]); + Logger::debug('Add post to featured collection', ['post' => $post]); - Post\Collection::add($uriid, Post\Collection::FEATURED); + Post\Collection::add($post['uri-id'], Post\Collection::FEATURED, $post['author-id']); Queue::remove($activity); } @@ -627,14 +737,14 @@ class Processor */ public static function removeFromFeaturedCollection(array $activity) { - $uriid = self::getUriIdForFeaturedCollection($activity); - if (empty($uriid)) { + $post = self::getUriIdForFeaturedCollection($activity); + if (empty($post)) { return; } - Logger::debug('Remove post from featured collection', ['uri-id' => $uriid]); + Logger::debug('Remove post from featured collection', ['post' => $post]); - Post\Collection::remove($uriid, Post\Collection::FEATURED); + Post\Collection::remove($post['uri-id'], Post\Collection::FEATURED); Queue::remove($activity); } @@ -650,7 +760,7 @@ class Processor public static function createEvent(array $activity, array $item): int { $event['summary'] = HTML::toBBCode($activity['name'] ?: $activity['summary']); - $event['desc'] = HTML::toBBCode($activity['content']); + $event['desc'] = HTML::toBBCode($activity['content'] ?? ''); if (!empty($activity['start-time'])) { $event['start'] = DateTimeFormat::utc($activity['start-time']); } @@ -717,19 +827,35 @@ class Processor $content = self::addMentionLinks($content, $activity['tags']); + if (!empty($activity['quote-url'])) { + $id = Item::fetchByLink($activity['quote-url']); + if ($id) { + $shared_item = Post::selectFirst(['uri-id'], ['id' => $id]); + $item['quote-uri-id'] = $shared_item['uri-id']; + } else { + Logger::info('Quote was not fetched', ['guid' => $item['guid'], 'uri-id' => $item['uri-id'], 'quote' => $activity['quote-url']]); + } + } + if (!empty($activity['source'])) { $item['body'] = $activity['source']; $item['raw-body'] = $content; - $item['body'] = Item::improveSharedDataInBody($item); + + $quote_uri_id = Item::getQuoteUriId($item['body']); + if (empty($item['quote-uri-id']) && !empty($quote_uri_id)) { + $item['quote-uri-id'] = $quote_uri_id; + } + + $item['body'] = BBCode::removeSharedData($item['body']); } else { $parent_uri = $item['parent-uri'] ?? $item['thr-parent']; - if (empty($activity['directmessage']) && ($parent_uri != $item['uri']) && ($item['gravity'] == GRAVITY_COMMENT)) { + if (empty($activity['directmessage']) && ($parent_uri != $item['uri']) && ($item['gravity'] == Item::GRAVITY_COMMENT)) { $parent = Post::selectFirst(['id', 'uri-id', 'private', 'author-link', 'alias'], ['uri' => $parent_uri]); if (!DBA::isResult($parent)) { Logger::warning('Unknown parent item.', ['uri' => $parent_uri]); return false; } - if (($item['private'] == Item::PRIVATE) && ($parent['private'] != Item::PRIVATE)) { + if (!empty($activity['type']) && in_array($activity['type'], Receiver::CONTENT_TYPES) && ($item['private'] == Item::PRIVATE) && ($parent['private'] != Item::PRIVATE)) { Logger::warning('Item is private but the parent is not. Dropping.', ['item-uri' => $item['uri'], 'thr-parent' => $item['thr-parent']]); return false; } @@ -823,7 +949,7 @@ class Processor return true; } - if ($item['gravity'] != GRAVITY_PARENT) { + if ($item['gravity'] != Item::GRAVITY_PARENT) { // We cannot reliably check at this point if a comment or activity belongs to an accepted post or needs to be fetched // This can possibly be improved in the future. Logger::debug('Message is no parent - accepted', ['uri-id' => $item['uri-id'], 'guid' => $item['guid'], 'url' => $item['uri']]); @@ -831,7 +957,7 @@ class Processor } $tags = array_column(Tag::getByURIId($item['uri-id'], [Tag::HASHTAG]), 'name'); - if (Relay::isSolicitedPost($tags, $item['body'], $item['author-id'], $item['uri'], Protocol::ACTIVITYPUB)) { + if (Relay::isSolicitedPost($tags, $item['body'], $item['author-id'], $item['uri'], Protocol::ACTIVITYPUB, $activity['thread-completion'] ?? 0)) { Logger::debug('Post is accepted because of the relay settings', ['uri-id' => $item['uri-id'], 'guid' => $item['guid'], 'url' => $item['uri']]); return true; } else { @@ -859,6 +985,9 @@ class Processor if (!self::isSolicitedMessage($activity, $item)) { DBA::delete('item-uri', ['id' => $item['uri-id']]); + if (!empty($activity['entry-id'])) { + Queue::deleteById($activity['entry-id']); + } return; } @@ -867,6 +996,14 @@ class Processor continue; } + if (($receiver != 0) && empty($item['parent-uri-id']) && !empty($item['thr-parent-id'])) { + $parent = Post::selectFirst(['parent-uri-id', 'parent-uri'], ['uri-id' => $item['thr-parent-id'], 'uid' => [0, $receiver]]); + if (!empty($parent['parent-uri-id'])) { + $item['parent-uri-id'] = $parent['parent-uri-id']; + $item['parent-uri'] = $parent['parent-uri']; + } + } + $item['uid'] = $receiver; $type = $activity['reception_type'][$receiver] ?? Receiver::TARGET_UNKNOWN; @@ -896,10 +1033,21 @@ class Processor $item['post-reason'] = Item::PR_NONE; } - if (!empty($activity['from-relay'])) { - $item['post-reason'] = Item::PR_RELAY; - } elseif (!empty($activity['thread-completion'])) { - $item['post-reason'] = Item::PR_FETCHED; + $item['post-reason'] = Item::getPostReason($item); + + if (in_array($item['post-reason'], [Item::PR_GLOBAL, Item::PR_NONE])) { + if (!empty($activity['from-relay'])) { + $item['post-reason'] = Item::PR_RELAY; + } elseif (!empty($activity['thread-completion'])) { + $item['post-reason'] = Item::PR_FETCHED; + } elseif (!empty($activity['push'])) { + $item['post-reason'] = Item::PR_PUSHED; + } + } elseif (($item['post-reason'] == Item::PR_FOLLOWER) && !empty($activity['from-relay'])) { + // When a post arrives via a relay and we follow the author, we have to override the causer. + // Otherwise the system assumes that we follow the relay. (See "addRowInformation") + Logger::debug('Relay post for follower', ['receiver' => $receiver, 'guid' => $item['guid'], 'relay' => $activity['from-relay']]); + $item['causer-id'] = ($item['gravity'] == Item::GRAVITY_PARENT) ? $item['owner-id'] : $item['author-id']; } if ($item['isForum'] ?? false) { @@ -917,48 +1065,42 @@ class Processor continue; } - if (!($item['isForum'] ?? false) && ($receiver != 0) && ($item['gravity'] == GRAVITY_PARENT) && !Contact::isSharingByURL($activity['author'], $receiver)) { - if ($item['post-reason'] == Item::PR_BCC) { - Logger::info('Top level post via BCC from a non sharer, ignoring', ['uid' => $receiver, 'contact' => $item['contact-id']]); - continue; - } + if (($receiver != 0) && ($item['gravity'] == Item::GRAVITY_PARENT) && !in_array($item['post-reason'], [Item::PR_FOLLOWER, Item::PR_TAG, item::PR_TO, Item::PR_CC])) { + if (!($item['isForum'] ?? false)) { + if ($item['post-reason'] == Item::PR_BCC) { + Logger::info('Top level post via BCC from a non sharer, ignoring', ['uid' => $receiver, 'contact' => $item['contact-id'], 'url' => $item['uri']]); + continue; + } - if ( - !empty($activity['thread-children-type']) - && in_array($activity['thread-children-type'], Receiver::ACTIVITY_TYPES) - && DI::pConfig()->get($receiver, 'system', 'accept_only_sharer') != Item::COMPLETION_LIKE - ) { - Logger::info('Top level post from thread completion from a non sharer had been initiated via an activity, ignoring', - ['type' => $activity['thread-children-type'], 'user' => $item['uid'], 'causer' => $item['causer-link'], 'author' => $activity['author'], 'url' => $item['uri']]); - continue; + if ((DI::pConfig()->get($receiver, 'system', 'accept_only_sharer') != Item::COMPLETION_LIKE) + && in_array($activity['thread-children-type'] ?? '', Receiver::ACTIVITY_TYPES)) { + Logger::info('Top level post from thread completion from a non sharer had been initiated via an activity, ignoring', + ['type' => $activity['thread-children-type'], 'user' => $item['uid'], 'causer' => $item['causer-link'], 'author' => $activity['author'], 'url' => $item['uri']]); + continue; + } } - } - - $is_forum = false; - if ($receiver != 0) { + $is_forum = false; $user = User::getById($receiver, ['account-type']); if (!empty($user['account-type'])) { $is_forum = ($user['account-type'] == User::ACCOUNT_TYPE_COMMUNITY); } - } - - if (!$is_forum && DI::pConfig()->get($receiver, 'system', 'accept_only_sharer') == Item::COMPLETION_NONE && ($receiver != 0) && ($item['gravity'] == GRAVITY_PARENT)) { - $skip = !Contact::isSharingByURL($activity['author'], $receiver); - - if ($skip && (($activity['type'] == 'as:Announce') || ($item['isForum'] ?? false))) { - $skip = !Contact::isSharingByURL($activity['actor'], $receiver); - } - if ($skip) { - Logger::info('Skipping post', ['uid' => $receiver, 'url' => $item['uri']]); + if ((DI::pConfig()->get($receiver, 'system', 'accept_only_sharer') == Item::COMPLETION_NONE) + && ((!$is_forum && !($item['isForum'] ?? false) && ($activity['type'] != 'as:Announce')) + || !Contact::isSharingByURL($activity['actor'], $receiver))) { + Logger::info('Actor is a non sharer, is no forum or it is no announce', ['uid' => $receiver, 'actor' => $activity['actor'], 'url' => $item['uri'], 'type' => $activity['type']]); continue; } Logger::info('Accepting post', ['uid' => $receiver, 'url' => $item['uri']]); } - if (($item['gravity'] != GRAVITY_ACTIVITY) && ($activity['object_type'] == 'as:Event')) { + if (!self::hasParents($item, $receiver)) { + continue; + } + + if (($item['gravity'] != Item::GRAVITY_ACTIVITY) && ($activity['object_type'] == 'as:Event')) { $event_id = self::createEvent($activity, $item); $item = Event::getItemArrayForImportedId($event_id, $item); @@ -970,8 +1112,9 @@ class Processor $success = true; } else { Logger::notice('Item insertion aborted', ['uri' => $item['uri'], 'uid' => $item['uid']]); - if (Item::isTooOld($item) || !Item::isValid($item)) { - Queue::remove($activity); + if (($item['uid'] == 0) && (count($activity['receiver']) > 1)) { + Logger::info('Public item was aborted. We skip for all users.', ['uri' => $item['uri']]); + break; } } @@ -980,16 +1123,14 @@ class Processor } } - if ($success) { - Queue::remove($activity); + Queue::remove($activity); - if (Queue::hasChildren($item['uri'])) { - Worker::add(PRIORITY_HIGH, 'ProcessReplyByUri', $item['uri']); - } + if ($success && Queue::hasChildren($item['uri']) && Post::exists(['uri' => $item['uri']])) { + Queue::processReplyByUri($item['uri']); } // Store send a follow request for every reshare - but only when the item had been stored - if ($stored && ($item['private'] != Item::PRIVATE) && ($item['gravity'] == GRAVITY_PARENT) && !empty($item['author-link']) && ($item['author-link'] != $item['owner-link'])) { + if ($stored && ($item['private'] != Item::PRIVATE) && ($item['gravity'] == Item::GRAVITY_PARENT) && !empty($item['author-link']) && ($item['author-link'] != $item['owner-link'])) { $author = APContact::getByURL($item['owner-link'], false); // We send automatic follow requests for reshared messages. (We don't need though for forum posts) if ($author['type'] != 'Group') { @@ -999,6 +1140,88 @@ class Processor } } + /** + * Checks if there are parent posts for the given receiver. + * If not, then the system will try to add them. + * + * @param array $item + * @param integer $receiver + * @return boolean + */ + private static function hasParents(array $item, int $receiver) + { + if (($receiver == 0) || ($item['gravity'] == Item::GRAVITY_PARENT)) { + return true; + } + + $fields = ['causer-id' => $item['causer-id'] ?? $item['author-id'], 'post-reason' => Item::PR_FETCHED]; + + $add_parent = true; + + if ($item['verb'] != Activity::ANNOUNCE) { + switch (DI::pConfig()->get($receiver, 'system', 'accept_only_sharer')) { + case Item::COMPLETION_COMMENT: + $add_parent = ($item['gravity'] != Item::GRAVITY_ACTIVITY); + break; + + case Item::COMPLETION_NONE: + $add_parent = false; + break; + } + } + + if ($add_parent) { + $add_parent = Contact::isSharing($fields['causer-id'], $receiver); + if (!$add_parent && ($item['author-id'] != $fields['causer-id'])) { + $add_parent = Contact::isSharing($item['author-id'], $receiver); + } + if (!$add_parent && !in_array($item['owner-id'], [$fields['causer-id'], $item['author-id']])) { + $add_parent = Contact::isSharing($item['owner-id'], $receiver); + } + } + + $has_parents = false; + + if (!empty($item['parent-uri-id'])) { + if (Post::exists(['uri-id' => $item['parent-uri-id'], 'uid' => $receiver])) { + $has_parents = true; + } elseif ($add_parent && Post::exists(['uri-id' => $item['parent-uri-id'], 'uid' => 0])) { + $stored = Item::storeForUserByUriId($item['parent-uri-id'], $receiver, $fields); + $has_parents = (bool)$stored; + if ($stored) { + Logger::notice('Inserted missing parent post', ['stored' => $stored, 'uid' => $receiver, 'parent' => $item['parent-uri']]); + } else { + Logger::notice('Parent could not be added.', ['uid' => $receiver, 'uri' => $item['uri'], 'parent' => $item['parent-uri']]); + return false; + } + } elseif ($add_parent) { + Logger::debug('Parent does not exist.', ['uid' => $receiver, 'uri' => $item['uri'], 'parent' => $item['parent-uri']]); + } else { + Logger::debug('Parent should not be added.', ['uid' => $receiver, 'gravity' => $item['gravity'], 'verb' => $item['verb'], 'guid' => $item['guid'], 'uri' => $item['uri'], 'parent' => $item['parent-uri']]); + } + } + + if (empty($item['parent-uri-id']) || ($item['thr-parent-id'] != $item['parent-uri-id'])) { + if (Post::exists(['uri-id' => $item['thr-parent-id'], 'uid' => $receiver])) { + $has_parents = true; + } elseif (($has_parents || $add_parent) && Post::exists(['uri-id' => $item['thr-parent-id'], 'uid' => 0])) { + $stored = Item::storeForUserByUriId($item['thr-parent-id'], $receiver, $fields); + $has_parents = $has_parents || (bool)$stored; + if ($stored) { + Logger::notice('Inserted missing thread parent post', ['stored' => $stored, 'uid' => $receiver, 'thread-parent' => $item['thr-parent']]); + } else { + Logger::notice('Thread parent could not be added.', ['uid' => $receiver, 'uri' => $item['uri'], 'thread-parent' => $item['thr-parent']]); + } + } elseif ($add_parent) { + Logger::debug('Thread parent does not exist.', ['uid' => $receiver, 'uri' => $item['uri'], 'thread-parent' => $item['thr-parent']]); + } else { + Logger::debug('Thread parent should not be added.', ['uid' => $receiver, 'gravity' => $item['gravity'], 'verb' => $item['verb'], 'guid' => $item['guid'], 'uri' => $item['uri'], 'thread-parent' => $item['thr-parent']]); + } + } + + return $has_parents; + } + /** * Store tags and mentions into the tag table * @@ -1072,7 +1295,7 @@ class Processor */ private static function postMail(array $activity, array $item) { - if (($item['gravity'] != GRAVITY_PARENT) && !DBA::exists('mail', ['uri' => $item['thr-parent'], 'uid' => $item['uid']])) { + if (($item['gravity'] != Item::GRAVITY_PARENT) && !DBA::exists('mail', ['uri' => $item['thr-parent'], 'uid' => $item['uid']])) { Logger::info('Parent not found, mail will be discarded.', ['uid' => $item['uid'], 'uri' => $item['thr-parent']]); return false; } @@ -1143,7 +1366,7 @@ class Processor $pcid = Contact::getIdForURL($url, 0, false); if (empty($pcid)) { - Logger::info('Contact not found', ['contact' => $url]); + Logger::notice('Contact not found', ['contact' => $url]); return; } @@ -1174,10 +1397,10 @@ class Processor } $id = Item::fetchByLink($post['id']); if (!empty($id)) { - $item = Post::selectFirst(['uri-id', 'featured'], ['id' => $id]); + $item = Post::selectFirst(['uri-id', 'featured', 'author-id'], ['id' => $id]); if (!empty($item['uri-id'])) { if (!$item['featured']) { - Post\Collection::add($item['uri-id'], Post\Collection::FEATURED); + Post\Collection::add($item['uri-id'], Post\Collection::FEATURED, $item['author-id']); Logger::debug('Added featured post', ['uri-id' => $item['uri-id'], 'contact' => $url]); $new++; } else { @@ -1201,6 +1424,39 @@ class Processor Logger::info('Fetched featured posts', ['new' => $new, 'old' => $old, 'contact' => $url]); } + public static function fetchCachedActivity(string $url, int $uid): array + { + $cachekey = self::CACHEKEY_FETCH_ACTIVITY . $uid . ':' . hash('sha256', $url); + $object = DI::cache()->get($cachekey); + + if (!is_null($object)) { + if (!empty($object)) { + Logger::debug('Fetch from cache', ['url' => $url, 'uid' => $uid]); + } else { + Logger::debug('Fetch from negative cache', ['url' => $url, 'uid' => $uid]); + } + return $object; + } + + $object = ActivityPub::fetchContent($url, $uid); + if (empty($object)) { + Logger::notice('Activity was not fetchable, aborting.', ['url' => $url, 'uid' => $uid]); + // We perform negative caching. + DI::cache()->set($cachekey, [], Duration::FIVE_MINUTES); + return []; + } + + if (empty($object['id'])) { + Logger::notice('Activity has got not id, aborting. ', ['url' => $url, 'object' => $object]); + return []; + } + DI::cache()->set($cachekey, $object, Duration::FIVE_MINUTES); + + Logger::debug('Activity was fetched successfully', ['url' => $url, 'uid' => $uid]); + + return $object; + } + /** * Fetches missing posts * @@ -1214,29 +1470,9 @@ class Processor */ public static function fetchMissingActivity(string $url, array $child = [], string $relay_actor = '', int $completion = Receiver::COMPLETION_MANUAL): string { - if (!empty($child['receiver'])) { - $uid = ActivityPub\Receiver::getFirstUserFromReceivers($child['receiver']); - } else { - $uid = 0; - } - - $cachekey = self::CACHEKEY_FETCH_ACTIVITY . $url; - $object = DI::cache()->get($cachekey); - - if (is_null($object)) { - $object = ActivityPub::fetchContent($url, $uid); - if (empty($object)) { - Logger::notice('Activity was not fetchable, aborting.', ['url' => $url, 'uid' => $uid]); - return ''; - } - - if (empty($object['id'])) { - Logger::notice('Activity has got not id, aborting. ', ['url' => $url, 'object' => $object]); - return ''; - } - DI::cache()->set($cachekey, $object, Duration::FIVE_MINUTES); - } else { - Logger::debug('Fetch from cache', ['url' => $url]); + $object = self::fetchCachedActivity($url, 0); + if (empty($object)) { + return ''; } $signer = []; @@ -1247,7 +1483,7 @@ class Processor $compacted = JsonLD::compact($object); $attributed_to = JsonLD::fetchElement($compacted, 'as:attributedTo', '@id'); } - $signer[] = $attributed_to; + $signer[] = $attributed_to; } if (!empty($object['actor'])) { @@ -1289,7 +1525,7 @@ class Processor $ldactivity = JsonLD::compact($activity); - $ldactivity['recursion-depth'] = !empty($child['recursion-depth']) ? $child['recursion-depth'] + 1 : 1; + $ldactivity['recursion-depth'] = !empty($child['recursion-depth']) ? $child['recursion-depth'] + 1 : 0; if (!empty($relay_actor)) { $ldactivity['thread-completion'] = $ldactivity['from-relay'] = Contact::getIdForURL($relay_actor); @@ -1302,17 +1538,25 @@ class Processor $ldactivity['completion-mode'] = $completion; } - if (!empty($child['type'])) { + if (!empty($child['thread-children-type'])) { + $ldactivity['thread-children-type'] = $child['thread-children-type']; + } elseif (!empty($child['type'])) { $ldactivity['thread-children-type'] = $child['type']; + } else { + $ldactivity['thread-children-type'] = 'as:Create'; } if (!empty($relay_actor) && !self::acceptIncomingMessage($ldactivity, $object['id'])) { return ''; } - ActivityPub\Receiver::processActivity($ldactivity, json_encode($activity), $uid, true, false, $signer); - - Logger::notice('Activity had been fetched and processed.', ['url' => $url, 'object' => $activity['id']]); + if (($completion == Receiver::COMPLETION_RELAY) && Queue::exists($url, 'as:Create')) { + Logger::notice('Activity has already been queued.', ['url' => $url, 'object' => $activity['id']]); + } elseif (ActivityPub\Receiver::processActivity($ldactivity, json_encode($activity), 0, true, false, $signer, '', $completion)) { + Logger::notice('Activity had been fetched and processed.', ['url' => $url, 'entry' => $child['entry-id'] ?? 0, 'completion' => $completion, 'object' => $activity['id']]); + } else { + Logger::notice('Activity had been fetched and will be processed later.', ['url' => $url, 'entry' => $child['entry-id'] ?? 0, 'completion' => $completion, 'object' => $activity['id']]); + } return $activity['id']; } @@ -1354,7 +1598,7 @@ class Processor } } - return Relay::isSolicitedPost($messageTags, $body, $authorid, $id, Protocol::ACTIVITYPUB); + return Relay::isSolicitedPost($messageTags, $body, $authorid, $id, Protocol::ACTIVITYPUB, $activity['thread-completion'] ?? 0); } /** @@ -1433,9 +1677,9 @@ class Processor } if (DI::config()->get('system', 'bulk_delivery')) { Post\Delivery::add($post['uri-id'], $uid, $inbox, $post['created'], Delivery::POST, [$cid]); - Worker::add(PRIORITY_HIGH, 'APDelivery', '', 0, $inbox, 0); + Worker::add(Worker::PRIORITY_HIGH, 'APDelivery', '', 0, $inbox, 0); } else { - Worker::add(PRIORITY_HIGH, 'APDelivery', Delivery::POST, $post['id'], $inbox, $uid, [$cid], $post['uri-id']); + Worker::add(Worker::PRIORITY_HIGH, 'APDelivery', Delivery::POST, $post['id'], $inbox, $uid, [$cid], $post['uri-id']); } } } @@ -1627,7 +1871,7 @@ class Processor return; } - Item::markForDeletion(['uri' => $activity['object_id'], 'author-id' => $author_id, 'gravity' => GRAVITY_ACTIVITY]); + Item::markForDeletion(['uri' => $activity['object_id'], 'author-id' => $author_id, 'gravity' => Item::GRAVITY_ACTIVITY]); Queue::remove($activity); } @@ -1783,7 +2027,9 @@ class Processor $name = $tag['name']; } - $body = str_replace($tag['name'], $hash . '[url=' . $tag['href'] . ']' . $name . '[/url]', $body); + if (Network::isValidHttpUrl($tag['href'])) { + $body = str_replace($tag['name'], $hash . '[url=' . $tag['href'] . ']' . $name . '[/url]', $body); + } } return $body;