Merge pull request #9146 from tobiasd/2020.09-CHANGELOG

[friendica.git] / src / Protocol / Feed.php

diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php
index 8a6725d19a46bc6dc84d762ea823ea1e03cd9d67..67baf4b2aebf6412945579c7d3ca8e3cdf654bd2 100644 (file)
--- a/src/Protocol/Feed.php
+++ b/src/Protocol/Feed.php
@@ -494,6 +494,9 @@ class Feed
                        }
                        $item["body"] = HTML::toBBCode($body, $basepath);
 
+                       // Remove tracking pixels
+                       $item["body"] = preg_replace("/\[img=1x1\]([^\[\]]*)\[\/img\]/Usi", '', $item["body"]);
+
                        if (($item["body"] == '') && ($item["title"] != '')) {
                                $item["body"] = $item["title"];
                                $item["title"] = '';
@@ -533,6 +536,9 @@ class Feed
                                        $replace = true;
                                }
 
+                               $saved_body = $item["body"];
+                               $saved_title = $item["title"];
+
                                if ($replace) {
                                        $item["body"] = trim($item["title"]);
                                }
@@ -549,9 +555,24 @@ class Feed
                                        }
                                }
 
+                               $data = PageInfo::queryUrl($item["plink"], false, $preview, ($contact["fetch_further_information"] == 2), $contact["ffi_keyword_denylist"] ?? '');
+
+                               // Take the data that was provided by the feed if the query is empty
+                               if (($data['type'] == 'link') && empty($data['title']) && empty($data['text'])) {
+                                       $data['title'] = $saved_title;
+                                       $item["body"] = $saved_body;
+                               }
+
+                               $data_text = strip_tags(trim($data['text'] ?? ''));
+                               $item_body = strip_tags(trim($item['body'] ?? ''));
+
+                               if (!empty($data_text) && (($data_text == $item_body) || strstr($item_body, $data_text))) {
+                                       $data['text'] = '';
+                               }
+
                                // We always strip the title since it will be added in the page information
                                $item["title"] = "";
-                               $item["body"] = $item["body"] . "\n" . PageInfo::getFooterFromUrl($item["plink"], false, $preview, ($contact["fetch_further_information"] == 2), $contact["ffi_keyword_denylist"] ?? '');
+                               $item["body"] = $item["body"] . "\n" . PageInfo::getFooterFromData($data, false);
                                $taglist = $contact["fetch_further_information"] == 2 ? PageInfo::getTagsFromUrl($item["plink"], $preview, $contact["ffi_keyword_denylist"] ?? '') : [];
                                $item["object-type"] = Activity\ObjectType::BOOKMARK;
                                unset($item["attach"]);
@@ -618,8 +639,8 @@ class Feed
         */
        private static function adjustPollFrequency(array $contact, array $creation_dates)
        {
-               if (($contact['priority'] > 3) || ($contact['network'] != Protocol::FEED)) {
-                       Logger::info('Contact is no feed or has a low priority, skip.', ['id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url'], 'priority' => $contact['priority'], 'network' => $contact['network']]);
+               if ($contact['network'] != Protocol::FEED) {
+                       Logger::info('Contact is no feed, skip.', ['id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url'], 'network' => $contact['network']]);
                        return;
                }
 
@@ -660,13 +681,23 @@ class Feed
                                }
                        }
 
-                       if (empty($frequency)) {
-                               Logger::info('Feed had not posted for at least a week, switching to daily polling', ['newest' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]);
+                       if (count($creation_dates) == 1) {
+                               Logger::info('Feed had posted a single time, switching to daily polling', ['newest' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]);
                                $priority = 8; // Poll once a day
                        }
 
-                       if (empty($priority) && (count($creation_dates) == 1)) {
-                               Logger::info('Feed had posted a single time, switching to daily polling', ['newest' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]);
+                       if (empty($priority) && (((time() / 86400) - $newest) > 730)) {
+                               Logger::info('Feed had not posted for two years, switching to monthly polling', ['newest' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]);
+                               $priority = 10; // Poll every month
+                       }
+
+                       if (empty($priority) && (((time() / 86400) - $newest) > 365)) {
+                               Logger::info('Feed had not posted for a year, switching to weekly polling', ['newest' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]);
+                               $priority = 9; // Poll every week
+                       }
+
+                       if (empty($priority) && empty($frequency)) {
+                               Logger::info('Feed had not posted for at least a week, switching to daily polling', ['newest' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]);
                                $priority = 8; // Poll once a day
                        }
 
@@ -700,11 +731,8 @@ class Feed
                                        $priority = 5; // Poll every three hours
                                } elseif ($max > 2) {
                                        $priority = 6; // Poll every six hours
-                               } elseif ($max > 0) {
-                                       $priority = 7; // Poll twice a day
                                } else {
-                                       /// @todo In the future we could calculate the days between the posts to set even lower priorities
-                                       $priority = 8; // Poll once a day
+                                       $priority = 7; // Poll twice a day
                                }
                                Logger::info('Calculated priority by the posts per day', ['priority' => $priority, 'max' => round($max, 2), 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]);
                        }