X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;f=src%2FProtocol%2FFeed.php;h=821de8c33580556f3d8675531a65ee5f6782a5b5;hb=4fbd0a46b58cae1f04059b1ab9b0df654940cba7;hp=67baf4b2aebf6412945579c7d3ca8e3cdf654bd2;hpb=fcddf24f8808d18c2f5697994d64bddb88d4120a;p=friendica.git diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php index 67baf4b2ae..821de8c335 100644 --- a/src/Protocol/Feed.php +++ b/src/Protocol/Feed.php @@ -29,10 +29,13 @@ use Friendica\Content\Text\HTML; use Friendica\Core\Cache\Duration; use Friendica\Core\Logger; use Friendica\Core\Protocol; +use Friendica\Core\Worker; use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\Contact; +use Friendica\Model\Conversation; use Friendica\Model\Item; +use Friendica\Model\Post; use Friendica\Model\Tag; use Friendica\Model\User; use Friendica\Util\DateTimeFormat; @@ -99,7 +102,7 @@ class Feed $dfrn_importer = DFRN::getImporter($contact['id'], $importer['uid']); if (!empty($dfrn_importer)) { Logger::info('Now import the DFRN feed'); - DFRN::import($xml, $dfrn_importer, true); + DFRN::import($xml, $dfrn_importer, true, Conversation::PARCEL_LEGACY_DFRN); return; } } @@ -309,6 +312,8 @@ class Feed $total_items = $max_items; } + $postings = []; + // Importing older entries first for ($i = $total_items; $i >= 0; --$i) { $entry = $entries->item($i); @@ -353,8 +358,6 @@ class Feed $item["plink"] = DI::httpRequest()->finalUrl($item["plink"]); - $item["parent-uri"] = $item["uri"]; - $item["title"] = XML::getFirstNodeValue($xpath, 'atom:title/text()', $entry); if (empty($item["title"])) { @@ -436,28 +439,22 @@ class Feed $enclosures = $xpath->query("enclosure|atom:link[@rel='enclosure']", $entry); foreach ($enclosures AS $enclosure) { $href = ""; - $length = ""; - $type = ""; + $length = null; + $type = null; foreach ($enclosure->attributes AS $attribute) { if (in_array($attribute->name, ["url", "href"])) { $href = $attribute->textContent; } elseif ($attribute->name == "length") { - $length = $attribute->textContent; + $length = (int)$attribute->textContent; } elseif ($attribute->name == "type") { $type = $attribute->textContent; } } - if (!empty($item["attach"])) { - $item["attach"] .= ','; - } else { - $item["attach"] = ''; + if (!empty($href)) { + $attachments[] = ['type' => Post\Media::DOCUMENT, 'url' => $href, 'mimetype' => $type, 'size' => $length]; } - - $attachments[] = ["link" => $href, "type" => $type, "length" => $length]; - - $item["attach"] .= '[attach]href="' . $href . '" length="' . $length . '" type="' . $type . '"[/attach]'; } $taglist = []; @@ -506,7 +503,10 @@ class Feed $items[] = $item; break; } elseif (!Item::isValid($item)) { - Logger::info('Feed is invalid', ['created' => $item['created'], 'uid' => $item['uid'], 'uri' => $item['uri']]); + Logger::info('Feed item is invalid', ['created' => $item['created'], 'uid' => $item['uid'], 'uri' => $item['uri']]); + continue; + } elseif (Item::isTooOld($item)) { + Logger::info('Feed is too old', ['created' => $item['created'], 'uid' => $item['uid'], 'uri' => $item['uri']]); continue; } @@ -514,8 +514,8 @@ class Feed if (!empty($contact["fetch_further_information"]) && ($contact["fetch_further_information"] < 3)) { // Handle enclosures and treat them as preview picture foreach ($attachments AS $attachment) { - if ($attachment["type"] == "image/jpeg") { - $preview = $attachment["link"]; + if ($attachment["mimetype"] == "image/jpeg") { + $preview = $attachment["url"]; } } @@ -557,25 +557,27 @@ class Feed $data = PageInfo::queryUrl($item["plink"], false, $preview, ($contact["fetch_further_information"] == 2), $contact["ffi_keyword_denylist"] ?? ''); - // Take the data that was provided by the feed if the query is empty - if (($data['type'] == 'link') && empty($data['title']) && empty($data['text'])) { - $data['title'] = $saved_title; - $item["body"] = $saved_body; - } + if (!empty($data)) { + // Take the data that was provided by the feed if the query is empty + if (($data['type'] == 'link') && empty($data['title']) && empty($data['text'])) { + $data['title'] = $saved_title; + $item["body"] = $saved_body; + } - $data_text = strip_tags(trim($data['text'] ?? '')); - $item_body = strip_tags(trim($item['body'] ?? '')); + $data_text = strip_tags(trim($data['text'] ?? '')); + $item_body = strip_tags(trim($item['body'] ?? '')); - if (!empty($data_text) && (($data_text == $item_body) || strstr($item_body, $data_text))) { - $data['text'] = ''; - } + if (!empty($data_text) && (($data_text == $item_body) || strstr($item_body, $data_text))) { + $data['text'] = ''; + } - // We always strip the title since it will be added in the page information - $item["title"] = ""; - $item["body"] = $item["body"] . "\n" . PageInfo::getFooterFromData($data, false); - $taglist = $contact["fetch_further_information"] == 2 ? PageInfo::getTagsFromUrl($item["plink"], $preview, $contact["ffi_keyword_denylist"] ?? '') : []; - $item["object-type"] = Activity\ObjectType::BOOKMARK; - unset($item["attach"]); + // We always strip the title since it will be added in the page information + $item["title"] = ""; + $item["body"] = $item["body"] . "\n" . PageInfo::getFooterFromData($data, false); + $taglist = $contact["fetch_further_information"] == 2 ? PageInfo::getTagsFromUrl($item["plink"], $preview, $contact["ffi_keyword_denylist"] ?? '') : []; + $item["object-type"] = Activity\ObjectType::BOOKMARK; + $attachments = []; + } } else { if (!empty($summary)) { $item["body"] = '[abstract]' . HTML::toBBCode($summary, $basepath) . "[/abstract]\n" . $item["body"]; @@ -604,22 +606,58 @@ class Feed // Additionally we have to avoid conflicts with identical URI between imported feeds and these items. if ($notify) { $item['guid'] = Item::guidFromUri($orig_plink, DI::baseUrl()->getHostname()); - unset($item['uri']); + $item['uri'] = Item::newURI($item['uid'], $item['guid']); + unset($item['thr-parent']); unset($item['parent-uri']); // Set the delivery priority for "remote self" to "medium" $notify = PRIORITY_MEDIUM; } - $id = Item::insert($item, $notify); + if (!Post\Delayed::exists($item["uri"])) { + $postings[] = ['item' => $item, 'notify' => $notify, + 'taglist' => $taglist, 'attachments' => $attachments]; + } else { + Logger::info('Post already exists in the delayed posts queue', ['uri' => $item["uri"]]); + } + } + + if (!empty($postings)) { + $min_posting = DI::config()->get('system', 'minimum_posting_interval', 0); + $total = count($postings); + if ($total > 1) { + // Posts shouldn't be delayed more than a day + $interval = min(1440, self::getPollInterval($contact)); + $delay = max(round(($interval * 60) / $total), 60 * $min_posting); + Logger::notice('Got posting delay', ['delay' => $delay, 'interval' => $interval, 'items' => $total, 'cid' => $contact['id'], 'url' => $contact['url']]); + } else { + $delay = 0; + } - Logger::info("Feed for contact " . $contact["url"] . " stored under id " . $id); + $post_delay = 0; - if (!empty($id) && !empty($taglist)) { - $feeditem = Item::selectFirst(['uri-id'], ['id' => $id]); - foreach ($taglist as $tag) { - Tag::store($feeditem['uri-id'], Tag::HASHTAG, $tag); + foreach ($postings as $posting) { + if ($delay > 0) { + $publish_time = time() + $post_delay; + Logger::notice('Got publishing date', ['delay' => $delay, 'cid' => $contact['id'], 'url' => $contact['url']]); + $post_delay += $delay; + } else { + $publish_time = time(); + } + + $last_publish = DI::pConfig()->get($posting['item']['uid'], 'system', 'last_publish', 0, true); + $next_publish = max($last_publish + (60 * $min_posting), time()); + if ($publish_time < $next_publish) { + Logger::notice('Adapting publish time', + ['last' => date(DateTimeFormat::MYSQL, $last_publish), + 'next' => date(DateTimeFormat::MYSQL, $next_publish), + 'publish' => date(DateTimeFormat::MYSQL, $publish_time)]); + $publish_time = $next_publish; } + $publish_at = date(DateTimeFormat::MYSQL, $publish_time); + + Post\Delayed::add($publish_at, $posting['item'], $posting['notify'], $posting['taglist'], $posting['attachments']); + DI::pConfig()->set($item['uid'], 'system', 'last_publish', $next_publish); } } @@ -674,7 +712,7 @@ class Feed $oldest = $day; $oldest_date = $date; } - + if ($newest < $day) { $newest = $day; $newest_date = $date; @@ -747,6 +785,55 @@ class Feed } } + /** + * Get the poll interval for the given contact array + * + * @param array $contact + * @return int Poll interval in minutes + */ + public static function getPollInterval(array $contact) + { + if (in_array($contact['network'], [Protocol::MAIL, Protocol::FEED])) { + $ratings = [0, 3, 7, 8, 9, 10]; + if (DI::config()->get('system', 'adjust_poll_frequency') && ($contact['network'] == Protocol::FEED)) { + $rating = $contact['rating']; + } elseif (array_key_exists($contact['priority'], $ratings)) { + $rating = $ratings[$contact['priority']]; + } else { + $rating = -1; + } + } else { + // Check once a week per default for all other networks + $rating = 9; + } + + // Friendica and OStatus are checked once a day + if (in_array($contact['network'], [Protocol::DFRN, Protocol::OSTATUS])) { + $rating = 8; + } + + // Check archived contacts or contacts with unsupported protocols once a month + if ($contact['archive'] || in_array($contact['network'], [Protocol::ZOT, Protocol::PHANTOM])) { + $rating = 10; + } + + if ($rating < 0) { + return 0; + } + /* + * Based on $contact['priority'], should we poll this site now? Or later? + */ + + $min_poll_interval = max(1, DI::config()->get('system', 'min_poll_interval')); + + $poll_intervals = [$min_poll_interval, 15, 30, 60, 120, 180, 360, 720 ,1440, 10080, 43200]; + + //$poll_intervals = [$min_poll_interval . ' minute', '15 minute', '30 minute', + // '1 hour', '2 hour', '3 hour', '6 hour', '12 hour' ,'1 day', '1 week', '1 month']; + + return $poll_intervals[$rating]; + } + /** * Convert a tag array to a tag string * @@ -761,7 +848,7 @@ class Feed if ($tagstr != "") { $tagstr .= ", "; } - + $tagstr .= "#[url=" . DI::baseUrl() . "/search?tag=" . urlencode($tag) . "]" . $tag . "[/url]"; } @@ -1107,9 +1194,8 @@ class Feed if ($item['gravity'] != GRAVITY_PARENT) { $parent = Item::selectFirst(['guid', 'author-link', 'owner-link'], ['id' => $item['parent']]); - $parent_item = (($item['thr-parent']) ? $item['thr-parent'] : $item['parent-uri']); - $thrparent = Item::selectFirst(['guid', 'author-link', 'owner-link', 'plink'], ['uid' => $owner["uid"], 'uri' => $parent_item]); + $thrparent = Item::selectFirst(['guid', 'author-link', 'owner-link', 'plink'], ['uid' => $owner["uid"], 'uri' => $item['thr-parent']]); if (DBA::isResult($thrparent)) { $mentioned[$thrparent["author-link"]] = $thrparent["author-link"]; @@ -1122,7 +1208,7 @@ class Feed } $attributes = [ - "ref" => $parent_item, + "ref" => $item['thr-parent'], "href" => $parent_plink]; XML::addElement($doc, $entry, "thr:in-reply-to", "", $attributes);