]> git.mxchange.org Git - friendica.git/blobdiff - src/Protocol/Feed.php
Merge pull request #12916 from nupplaphil/feat/reduce_logging_noise
[friendica.git] / src / Protocol / Feed.php
index 6428fca66277ed70150b0a63c8abaefe551e532b..a83819742e00c5e0d2edbf6ce3499d7d44982e7c 100644 (file)
@@ -1,6 +1,6 @@
 <?php
 /**
- * @copyright Copyright (C) 2010-2022, the Friendica project
+ * @copyright Copyright (C) 2010-2023, the Friendica project
  *
  * @license GNU AGPL version 3 or any later version
  *
@@ -24,19 +24,23 @@ namespace Friendica\Protocol;
 use DOMDocument;
 use DOMElement;
 use DOMXPath;
+use Friendica\App;
 use Friendica\Content\PageInfo;
 use Friendica\Content\Text\BBCode;
 use Friendica\Content\Text\HTML;
 use Friendica\Core\Cache\Enum\Duration;
 use Friendica\Core\Logger;
 use Friendica\Core\Protocol;
+use Friendica\Core\Worker;
 use Friendica\Database\DBA;
 use Friendica\DI;
 use Friendica\Model\Contact;
+use Friendica\Model\Conversation;
 use Friendica\Model\Item;
 use Friendica\Model\Post;
 use Friendica\Model\Tag;
 use Friendica\Model\User;
+use Friendica\Network\HTTPException;
 use Friendica\Util\DateTimeFormat;
 use Friendica\Util\Network;
 use Friendica\Util\ParseUrl;
@@ -98,9 +102,11 @@ class Feed
 
                $author = [];
                $entries = null;
+               $protocol = Conversation::PARCEL_UNKNOWN;
 
                // Is it RDF?
                if ($xpath->query('/rdf:RDF/rss:channel')->length > 0) {
+                       $protocol = Conversation::PARCEL_RDF;
                        $author['author-link'] = XML::getFirstNodeValue($xpath, '/rdf:RDF/rss:channel/rss:link/text()');
                        $author['author-name'] = XML::getFirstNodeValue($xpath, '/rdf:RDF/rss:channel/rss:title/text()');
 
@@ -112,6 +118,7 @@ class Feed
 
                // Is it Atom?
                if ($xpath->query('/atom:feed')->length > 0) {
+                       $protocol = Conversation::PARCEL_ATOM;
                        $alternate = XML::getFirstAttributes($xpath, "atom:link[@rel='alternate']");
                        if (is_object($alternate)) {
                                foreach ($alternate as $attribute) {
@@ -195,6 +202,7 @@ class Feed
 
                // Is it RSS?
                if ($xpath->query('/rss/channel')->length > 0) {
+                       $protocol = Conversation::PARCEL_RSS;
                        $author['author-link'] = XML::getFirstNodeValue($xpath, '/rss/channel/link/text()');
 
                        $author['author-name'] = XML::getFirstNodeValue($xpath, '/rss/channel/title/text()');
@@ -247,18 +255,21 @@ class Feed
                        $author['owner-avatar'] = $contact['thumb'];
                }
 
-               $header = [];
-               $header['uid'] = $importer['uid'] ?? 0;
-               $header['network'] = Protocol::FEED;
-               $header['wall'] = 0;
-               $header['origin'] = 0;
-               $header['gravity'] = GRAVITY_PARENT;
-               $header['private'] = Item::PUBLIC;
-               $header['verb'] = Activity::POST;
-               $header['object-type'] = Activity\ObjectType::NOTE;
-               $header['post-type'] = Item::PT_ARTICLE;
-
-               $header['contact-id'] = $contact['id'] ?? 0;
+               $header = [
+                       'uid'         => $importer['uid'] ?? 0,
+                       'network'     => Protocol::FEED,
+                       'wall'        => 0,
+                       'origin'      => 0,
+                       'gravity'     => Item::GRAVITY_PARENT,
+                       'private'     => Item::PUBLIC,
+                       'verb'        => Activity::POST,
+                       'object-type' => Activity\ObjectType::NOTE,
+                       'post-type'   => Item::PT_ARTICLE,
+                       'contact-id'  => $contact['id'] ?? 0,
+               ];
+
+               $datarray['protocol'] = $protocol;
+               $datarray['direction'] = Conversation::PULL;
 
                if (!is_object($entries)) {
                        Logger::info("There are no entries in this feed.");
@@ -313,7 +324,7 @@ class Feed
                                $item['uri'] = $guid;
 
                                // Don't use the GUID value directly but instead use it as a basis for the GUID
-                               $item['guid'] = Item::guidFromUri($guid, parse_url($guid, PHP_URL_HOST) ?? parse_url($item['plink'], PHP_URL_HOST) ?? '');
+                               $item['guid'] = Item::guidFromUri($guid, parse_url($guid, PHP_URL_HOST) ?? parse_url($item['plink'], PHP_URL_HOST));
                        }
 
                        if (empty($item['uri'])) {
@@ -365,16 +376,18 @@ class Feed
                        }
 
                        if ($published != '') {
-                               $item['created'] = $published;
+                               $item['created'] = trim($published);
                        }
 
                        if ($updated != '') {
-                               $item['edited'] = $updated;
+                               $item['edited'] = trim($updated);
                        }
 
                        if (!$dryRun) {
-                               $condition = ["`uid` = ? AND `uri` = ? AND `network` IN (?, ?)",
-                                       $importer['uid'], $item['uri'], Protocol::FEED, Protocol::DFRN];
+                               $condition = [
+                                       "`uid` = ? AND `uri` = ? AND `network` IN (?, ?)",
+                                       $importer['uid'], $item['uri'], Protocol::FEED, Protocol::DFRN
+                               ];
                                $previous = Post::selectFirst(['id', 'created'], $condition);
                                if (DBA::isResult($previous)) {
                                        // Use the creation date when the post had been stored. It can happen this date changes in the feed.
@@ -428,7 +441,7 @@ class Feed
                                }
 
                                if (!empty($href)) {
-                                       $attachment = ['type' => Post\Media::UNKNOWN, 'url' => $href, 'mimetype' => $type, 'size' => $length];
+                                       $attachment = ['uri-id' => -1, 'type' => Post\Media::UNKNOWN, 'url' => $href, 'mimetype' => $type, 'size' => $length];
 
                                        $attachment = Post\Media::fetchAdditionalData($attachment);
 
@@ -613,17 +626,19 @@ class Feed
                        Logger::info('Stored feed', ['item' => $item]);
 
                        $notify = Item::isRemoteSelf($contact, $item);
+                       $item['wall'] = (bool)$notify;
 
-                       // Distributed items should have a well formatted URI.
-                       // Additionally we have to avoid conflicts with identical URI between imported feeds and these items.
+                       // Distributed items should have a well-formatted URI.
+                       // Additionally, we have to avoid conflicts with identical URI between imported feeds and these items.
                        if ($notify) {
-                               $item['guid'] = Item::guidFromUri($orig_plink, DI::baseUrl()->getHostname());
-                               $item['uri'] = Item::newURI($item['uid'], $item['guid']);
+                               $item['guid'] = Item::guidFromUri($orig_plink, DI::baseUrl()->getHost());
+                               $item['uri']  = Item::newURI($item['guid']);
+                               unset($item['plink']);
                                unset($item['thr-parent']);
                                unset($item['parent-uri']);
 
                                // Set the delivery priority for "remote self" to "medium"
-                               $notify = PRIORITY_MEDIUM;
+                               $notify = Worker::PRIORITY_MEDIUM;
                        }
 
                        $condition = ['uid' => $item['uid'], 'uri' => $item['uri']];
@@ -631,8 +646,10 @@ class Feed
                                if (!$notify) {
                                        Post\Delayed::publish($item, $notify, $taglist, $attachments);
                                } else {
-                                       $postings[] = ['item' => $item, 'notify' => $notify,
-                                               'taglist' => $taglist, 'attachments' => $attachments];
+                                       $postings[] = [
+                                               'item' => $item, 'notify' => $notify,
+                                               'taglist' => $taglist, 'attachments' => $attachments
+                                       ];
                                }
                        } else {
                                Logger::info('Post already created or exists in the delayed posts queue', ['uid' => $item['uid'], 'uri' => $item['uri']]);
@@ -839,7 +856,7 @@ class Feed
 
                $min_poll_interval = max(1, DI::config()->get('system', 'min_poll_interval'));
 
-               $poll_intervals = [$min_poll_interval, 15, 30, 60, 120, 180, 360, 720 ,1440, 10080, 43200];
+               $poll_intervals = [$min_poll_interval, 15, 30, 60, 120, 180, 360, 7201440, 10080, 43200];
 
                //$poll_intervals = [$min_poll_interval . ' minute', '15 minute', '30 minute',
                //      '1 hour', '2 hour', '3 hour', '6 hour', '12 hour' ,'1 day', '1 week', '1 month'];
@@ -905,28 +922,23 @@ class Feed
         * Updates the provided last_update parameter if the result comes from the
         * cache or it is empty
         *
-        * @param string  $owner_nick  Nickname of the feed owner
+        * @param array   $owner       owner-view record of the feed owner
         * @param string  $last_update Date of the last update
         * @param integer $max_items   Number of maximum items to fetch
         * @param string  $filter      Feed items filter (activity, posts or comments)
         * @param boolean $nocache     Wether to bypass caching
         *
         * @return string Atom feed
-        * @throws \Friendica\Network\HTTPException\InternalServerErrorException
+        * @throws HTTPException\InternalServerErrorException
         * @throws \ImagickException
         */
-       public static function atom(string $owner_nick, string $last_update, int $max_items = 300, string $filter = 'activity', bool $nocache = false)
+       public static function atom(array $owner, string $last_update, int $max_items = 300, string $filter = 'activity', bool $nocache = false)
        {
                $stamp = microtime(true);
 
-               $owner = User::getOwnerDataByNick($owner_nick);
-               if (!$owner) {
-                       return;
-               }
-
-               $cachekey = 'feed:feed:' . $owner_nick . ':' . $filter . ':' . $last_update;
+               $cachekey = 'feed:feed:' . $owner['nickname'] . ':' . $filter . ':' . $last_update;
 
-               // Display events in the users's timezone
+               // Display events in the user's timezone
                if (strlen($owner['timezone'])) {
                        DI::app()->setTimeZone($owner['timezone']);
                }
@@ -934,10 +946,10 @@ class Feed
                $previous_created = $last_update;
 
                // Don't cache when the last item was posted less then 15 minutes ago (Cache duration)
-               if ((time() - strtotime($owner['last-item'])) < 15*60) {
+               if ((time() - strtotime($owner['last-item'])) < 15 * 60) {
                        $result = DI::cache()->get($cachekey);
                        if (!$nocache && !is_null($result)) {
-                               Logger::info('Cached feed duration', ['seconds' => number_format(microtime(true) - $stamp, 3), 'nick' => $owner_nick, 'filter' => $filter, 'created' => $previous_created]);
+                               Logger::info('Cached feed duration', ['seconds' => number_format(microtime(true) - $stamp, 3), 'nick' => $owner['nickname'], 'filter' => $filter, 'created' => $previous_created]);
                                return $result['feed'];
                        }
                }
@@ -945,15 +957,17 @@ class Feed
                $check_date = empty($last_update) ? '' : DateTimeFormat::utc($last_update);
                $authorid = Contact::getIdForURL($owner['url']);
 
-               $condition = ["`uid` = ? AND `received` > ? AND NOT `deleted` AND `gravity` IN (?, ?)
+               $condition = [
+                       "`uid` = ? AND `received` > ? AND NOT `deleted` AND `gravity` IN (?, ?)
                        AND `private` != ? AND `visible` AND `wall` AND `parent-network` IN (?, ?, ?, ?)",
-                       $owner['uid'], $check_date, GRAVITY_PARENT, GRAVITY_COMMENT,
+                       $owner['uid'], $check_date, Item::GRAVITY_PARENT, Item::GRAVITY_COMMENT,
                        Item::PRIVATE, Protocol::ACTIVITYPUB,
-                       Protocol::OSTATUS, Protocol::DFRN, Protocol::DIASPORA];
+                       Protocol::OSTATUS, Protocol::DFRN, Protocol::DIASPORA
+               ];
 
                if ($filter === 'comments') {
                        $condition[0] .= " AND `gravity` = ? ";
-                       $condition[] = GRAVITY_COMMENT;
+                       $condition[] = Item::GRAVITY_COMMENT;
                }
 
                if ($owner['account-type'] != User::ACCOUNT_TYPE_COMMUNITY) {
@@ -991,7 +1005,7 @@ class Feed
                $msg = ['feed' => $feeddata, 'last_update' => $last_update];
                DI::cache()->set($cachekey, $msg, Duration::QUARTER_HOUR);
 
-               Logger::info('Feed duration', ['seconds' => number_format(microtime(true) - $stamp, 3), 'nick' => $owner_nick, 'filter' => $filter, 'created' => $previous_created]);
+               Logger::info('Feed duration', ['seconds' => number_format(microtime(true) - $stamp, 3), 'nick' => $owner['nickname'], 'filter' => $filter, 'created' => $previous_created]);
 
                return $feeddata;
        }
@@ -1027,8 +1041,8 @@ class Feed
                                break;
                }
 
-               $attributes = ['uri' => 'https://friendi.ca', 'version' => FRIENDICA_VERSION . '-' . DB_UPDATE_VERSION];
-               XML::addElement($doc, $root, 'generator', FRIENDICA_PLATFORM, $attributes);
+               $attributes = ['uri' => 'https://friendi.ca', 'version' => App::VERSION . '-' . DB_UPDATE_VERSION];
+               XML::addElement($doc, $root, 'generator', App::PLATFORM, $attributes);
                XML::addElement($doc, $root, 'id', DI::baseUrl() . '/profile/' . $owner['nick']);
                XML::addElement($doc, $root, 'title', $title);
                XML::addElement($doc, $root, 'subtitle', sprintf("Updates from %s on %s", $owner['name'], DI::config()->get('config', 'sitename')));
@@ -1080,7 +1094,7 @@ class Feed
         */
        private static function noteEntry(DOMDocument $doc, array $item, array $owner): DOMElement
        {
-               if (($item['gravity'] != GRAVITY_PARENT) && (Strings::normaliseLink($item['author-link']) != Strings::normaliseLink($owner['url']))) {
+               if (($item['gravity'] != Item::GRAVITY_PARENT) && (Strings::normaliseLink($item['author-link']) != Strings::normaliseLink($owner['url']))) {
                        Logger::info('Feed entry author does not match feed owner', ['owner' => $owner['url'], 'author' => $item['author-link']]);
                }
 
@@ -1115,14 +1129,22 @@ class Feed
                XML::addElement($doc, $entry, 'id', $item['uri']);
                XML::addElement($doc, $entry, 'title', html_entity_decode($title, ENT_QUOTES, 'UTF-8'));
 
-               $body = OStatus::formatPicturePost($item['body'], $item['uri-id']);
+               $body = Post\Media::addAttachmentsToBody($item['uri-id'], DI::contentItem()->addSharedPost($item));
+               $body = Post\Media::addHTMLLinkToBody($item['uri-id'], $body);
 
                $body = BBCode::convertForUriId($item['uri-id'], $body, BBCode::ACTIVITYPUB);
 
                XML::addElement($doc, $entry, 'content', $body, ['type' => 'html']);
 
-               XML::addElement($doc, $entry, 'link', '', ['rel' => 'alternate', 'type' => 'text/html',
-                                                               'href' => DI::baseUrl() . '/display/' . $item['guid']]
+               XML::addElement(
+                       $doc,
+                       $entry,
+                       'link',
+                       '',
+                       [
+                               'rel' => 'alternate', 'type' => 'text/html',
+                               'href' => DI::baseUrl() . '/display/' . $item['guid']
+                       ]
                );
 
                XML::addElement($doc, $entry, 'published', DateTimeFormat::utc($item['created'] . '+00:00', DateTimeFormat::ATOM));
@@ -1144,7 +1166,7 @@ class Feed
        {
                $mentioned = [];
 
-               if ($item['gravity'] != GRAVITY_PARENT) {
+               if ($item['gravity'] != Item::GRAVITY_PARENT) {
                        $parent = Post::selectFirst(['guid', 'author-link', 'owner-link'], ['id' => $item['parent']]);
 
                        $thrparent = Post::selectFirst(['guid', 'author-link', 'owner-link', 'plink'], ['uid' => $owner['uid'], 'uri' => $item['thr-parent']]);
@@ -1164,12 +1186,14 @@ class Feed
                        if (isset($parent_plink)) {
                                $attributes = [
                                        'ref'  => $item['thr-parent'],
-                                       'href' => $parent_plink];
+                                       'href' => $parent_plink
+                               ];
                                XML::addElement($doc, $entry, 'thr:in-reply-to', '', $attributes);
 
                                $attributes = [
                                        'rel'  => 'related',
-                                       'href' => $parent_plink];
+                                       'href' => $parent_plink
+                               ];
                                XML::addElement($doc, $entry, 'link', '', $attributes);
                        }
                }
@@ -1202,16 +1226,16 @@ class Feed
                }
 
                // Fetch information about the post
-               $siteinfo = BBCode::getAttachedData($item['body']);
-               if (isset($siteinfo['title'])) {
-                       return $siteinfo['title'];
+               $media = Post\Media::getByURIId($item['uri-id'], [Post\Media::HTML]);
+               if (!empty($media) && !empty($media[0]['name']) && ($media[0]['name'] != $media[0]['url'])) {
+                       return $media[0]['name'];
                }
 
                // If no bookmark is found then take the first line
                // Remove the share element before fetching the first line
                $title = trim(preg_replace("/\[share.*?\](.*?)\[\/share\]/ism", "\n$1\n", $item['body']));
 
-               $title = BBCode::toPlaintext($title)."\n";
+               $title = BBCode::toPlaintext($title) . "\n";
                $pos = strpos($title, "\n");
                $trailer = '';
                if (($pos == 0) || ($pos > 100)) {