X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;f=src%2FUtil%2FParseUrl.php;h=13cb55b73ee1396acd4d03714d38c2c07be59233;hb=2a68ad9b26d0a36968128be1627f5b1a427957d7;hp=06e320c40e03f29a93ce987d4f9a217deb13486f;hpb=01640a7045e146759bc936dd499ac27738b78940;p=friendica.git diff --git a/src/Util/ParseUrl.php b/src/Util/ParseUrl.php index 06e320c40e..13cb55b73e 100644 --- a/src/Util/ParseUrl.php +++ b/src/Util/ParseUrl.php @@ -26,7 +26,10 @@ use DOMXPath; use Friendica\Content\OEmbed; use Friendica\Core\Hook; use Friendica\Core\Logger; +use Friendica\Database\Database; use Friendica\Database\DBA; +use Friendica\DI; +use Friendica\Network\HTTPException; /** * Get information about a given URL @@ -35,6 +38,9 @@ use Friendica\Database\DBA; */ class ParseUrl { + const DEFAULT_EXPIRATION_FAILURE = 'now + 1 day'; + const DEFAULT_EXPIRATION_SUCCESS = 'now + 3 months'; + /** * Maximum number of characters for the description */ @@ -45,53 +51,82 @@ class ParseUrl */ const MIN_DESC_COUNT = 100; + /** + * Fetch the content type of the given url + * @param string $url URL of the page + * @return array content type + */ + public static function getContentType(string $url) + { + $curlResult = DI::httpRequest()->head($url); + if (!$curlResult->isSuccess()) { + return []; + } + + $contenttype = $curlResult->getHeader('Content-Type'); + if (empty($contenttype)) { + return []; + } + + return explode('/', current(explode(';', $contenttype))); + } + /** * Search for chached embeddable data of an url otherwise fetch it * * @param string $url The url of the page which should be scraped - * @param bool $no_guessing If true the parse doens't search for - * preview pictures * @param bool $do_oembed The false option is used by the function fetch_oembed() * to avoid endless loops * * @return array which contains needed data for embedding - * string 'url' => The url of the parsed page - * string 'type' => Content type - * string 'title' => The title of the content - * string 'text' => The description for the content - * string 'image' => A preview image of the content (only available - * if $no_geuessing = false - * array'images' = Array of preview pictures - * string 'keywords' => The tags which belong to the content + * string 'url' => The url of the parsed page + * string 'type' => Content type + * string 'title' => (optional) The title of the content + * string 'text' => (optional) The description for the content + * string 'image' => (optional) A preview image of the content + * array 'images' => (optional) Array of preview pictures + * string 'keywords' => (optional) The tags which belong to the content * - * @throws \Friendica\Network\HTTPException\InternalServerErrorException + * @throws HTTPException\InternalServerErrorException * @see ParseUrl::getSiteinfo() for more information about scraping * embeddable content */ - public static function getSiteinfoCached($url, $no_guessing = false, $do_oembed = true) + public static function getSiteinfoCached($url, $do_oembed = true): array { - if ($url == "") { - return false; + if (empty($url)) { + return [ + 'url' => '', + 'type' => 'error', + ]; } + $urlHash = hash('sha256', $url); + $parsed_url = DBA::selectFirst('parsed_url', ['content'], - ['url' => Strings::normaliseLink($url), 'guessing' => !$no_guessing, 'oembed' => $do_oembed] + ['url_hash' => $urlHash, 'oembed' => $do_oembed] ); if (!empty($parsed_url['content'])) { $data = unserialize($parsed_url['content']); return $data; } - $data = self::getSiteinfo($url, $no_guessing, $do_oembed); + $data = self::getSiteinfo($url, $do_oembed); + + $expires = $data['expires']; - DBA::insert( + unset($data['expires']); + + DI::dba()->insert( 'parsed_url', [ - 'url' => Strings::normaliseLink($url), 'guessing' => !$no_guessing, - 'oembed' => $do_oembed, 'content' => serialize($data), - 'created' => DateTimeFormat::utcNow() + 'url_hash' => $urlHash, + 'oembed' => $do_oembed, + 'url' => $url, + 'content' => serialize($data), + 'created' => DateTimeFormat::utcNow(), + 'expires' => $expires, ], - true + Database::INSERT_UPDATE ); return $data; @@ -108,21 +143,18 @@ class ParseUrl * \ * * @param string $url The url of the page which should be scraped - * @param bool $no_guessing If true the parse doens't search for - * preview pictures * @param bool $do_oembed The false option is used by the function fetch_oembed() * to avoid endless loops * @param int $count Internal counter to avoid endless loops * * @return array which contains needed data for embedding - * string 'url' => The url of the parsed page - * string 'type' => Content type - * string 'title' => The title of the content - * string 'text' => The description for the content - * string 'image' => A preview image of the content (only available - * if $no_geuessing = false - * array'images' = Array of preview pictures - * string 'keywords' => The tags which belong to the content + * string 'url' => The url of the parsed page + * string 'type' => Content type (error, link, photo, image, audio, video) + * string 'title' => (optional) The title of the content + * string 'text' => (optional) The description for the content + * string 'image' => (optional) A preview image of the content + * array 'images' => (optional) Array of preview pictures + * string 'keywords' => (optional) The tags which belong to the content * * @throws \Friendica\Network\HTTPException\InternalServerErrorException * @todo https://developers.google.com/+/plugins/snippet/ @@ -138,77 +170,129 @@ class ParseUrl * * @endverbatim */ - public static function getSiteinfo($url, $no_guessing = false, $do_oembed = true, $count = 1) + public static function getSiteinfo($url, $do_oembed = true, $count = 1) { - $siteinfo = []; + if (empty($url)) { + return [ + 'url' => '', + 'type' => 'error', + ]; + } // Check if the URL does contain a scheme $scheme = parse_url($url, PHP_URL_SCHEME); if ($scheme == '') { - $url = 'http://' . trim($url, '/'); + $url = 'http://' . ltrim($url, '/'); } + $url = trim($url, "'\""); + + $url = Network::stripTrackingQueryParams($url); + + $siteinfo = [ + 'url' => $url, + 'type' => 'link', + 'expires' => DateTimeFormat::utc(self::DEFAULT_EXPIRATION_FAILURE), + ]; + if ($count > 10) { Logger::log('Endless loop detected for ' . $url, Logger::DEBUG); return $siteinfo; } - $url = trim($url, "'"); - $url = trim($url, '"'); - - $url = Network::stripTrackingQueryParams($url); + $type = self::getContentType($url); + Logger::info('Got content-type', ['content-type' => $type, 'url' => $url]); + if (!empty($type) && in_array($type[0], ['image', 'video', 'audio'])) { + $siteinfo['type'] = $type[0]; + return $siteinfo; + } - $siteinfo['url'] = $url; - $siteinfo['type'] = 'link'; + if ((count($type) >= 2) && (($type[0] != 'text') || ($type[1] != 'html'))) { + Logger::info('Unparseable content-type, quitting here, ', ['content-type' => $type, 'url' => $url]); + return $siteinfo; + } - $curlResult = Network::curl($url); + $curlResult = DI::httpRequest()->get($url); if (!$curlResult->isSuccess()) { return $siteinfo; } + $siteinfo['expires'] = DateTimeFormat::utc(self::DEFAULT_EXPIRATION_SUCCESS); + // If the file is too large then exit if (($curlResult->getInfo()['download_content_length'] ?? 0) > 1000000) { return $siteinfo; } - // If it isn't a HTML file then exit - if (($curlResult->getContentType() != '') && !strstr(strtolower($curlResult->getContentType()), 'html')) { - return $siteinfo; + if ($cacheControlHeader = $curlResult->getHeader('Cache-Control')) { + if (preg_match('/max-age=([0-9]+)/i', $cacheControlHeader, $matches)) { + $maxAge = max(86400, (int)array_pop($matches)); + $siteinfo['expires'] = DateTimeFormat::utc("now + $maxAge seconds"); + } } $header = $curlResult->getHeader(); $body = $curlResult->getBody(); if ($do_oembed) { - $oembed_data = OEmbed::fetchURL($url); + $oembed_data = OEmbed::fetchURL($url, false, false); if (!empty($oembed_data->type)) { - if (!in_array($oembed_data->type, ['error', 'rich', ''])) { + if (!in_array($oembed_data->type, ['error', 'rich', 'image', 'video', 'audio', ''])) { $siteinfo['type'] = $oembed_data->type; } // See https://github.com/friendica/friendica/pull/5763#discussion_r217913178 if ($siteinfo['type'] != 'photo') { - if (isset($oembed_data->title)) { + if (!empty($oembed_data->title)) { $siteinfo['title'] = trim($oembed_data->title); } - if (isset($oembed_data->description)) { + if (!empty($oembed_data->description)) { $siteinfo['text'] = trim($oembed_data->description); } - if (isset($oembed_data->thumbnail_url)) { + if (!empty($oembed_data->author_name)) { + $siteinfo['author_name'] = trim($oembed_data->author_name); + } + if (!empty($oembed_data->author_url)) { + $siteinfo['author_url'] = trim($oembed_data->author_url); + } + if (!empty($oembed_data->provider_name)) { + $siteinfo['publisher_name'] = trim($oembed_data->provider_name); + } + if (!empty($oembed_data->provider_url)) { + $siteinfo['publisher_url'] = trim($oembed_data->provider_url); + } + if (!empty($oembed_data->thumbnail_url)) { $siteinfo['image'] = $oembed_data->thumbnail_url; } } } } - // Fetch the first mentioned charset. Can be in body or header $charset = ''; - if (preg_match('/charset=(.*?)[\'"\s\n]/', $header, $matches)) { + // Look for a charset, first in headers + // Expected form: Content-Type: text/html; charset=ISO-8859-4 + if (preg_match('/charset=([a-z0-9-_.\/]+)/i', $header, $matches)) { $charset = trim(trim(trim(array_pop($matches)), ';,')); } + // Then in body that gets precedence + // Expected forms: + // - + // - + // - + // - + // We escape