X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;f=src%2FUtil%2FParseUrl.php;h=04afc927ba6cec4229c8167a6bcafe66e4c317d9;hb=720a43461d67ab229de0aecfc5008f22cc4c1c54;hp=62b5d007d8b62c0238715f7e82f4ccc3f7984939;hpb=8a4a65f41874a7cf98487cf9ae11633d430967c8;p=friendica.git diff --git a/src/Util/ParseUrl.php b/src/Util/ParseUrl.php index 62b5d007d8..04afc927ba 100644 --- a/src/Util/ParseUrl.php +++ b/src/Util/ParseUrl.php @@ -1,6 +1,6 @@ head($url); + if (!$curlResult->isSuccess()) { + return []; + } + + $contenttype = $curlResult->getHeader('Content-Type')[0] ?? ''; + if (empty($contenttype)) { + return []; + } + + return explode('/', current(explode(';', $contenttype))); + } + /** * Search for chached embeddable data of an url otherwise fetch it * * @param string $url The url of the page which should be scraped - * @param bool $no_guessing If true the parse doens't search for - * preview pictures * @param bool $do_oembed The false option is used by the function fetch_oembed() * to avoid endless loops * * @return array which contains needed data for embedding - * string 'url' => The url of the parsed page - * string 'type' => Content type - * string 'title' => The title of the content - * string 'text' => The description for the content - * string 'image' => A preview image of the content (only available - * if $no_geuessing = false - * array'images' = Array of preview pictures - * string 'keywords' => The tags which belong to the content + * string 'url' => The url of the parsed page + * string 'type' => Content type + * string 'title' => (optional) The title of the content + * string 'text' => (optional) The description for the content + * string 'image' => (optional) A preview image of the content + * array 'images' => (optional) Array of preview pictures + * string 'keywords' => (optional) The tags which belong to the content * - * @throws \Friendica\Network\HTTPException\InternalServerErrorException + * @throws HTTPException\InternalServerErrorException * @see ParseUrl::getSiteinfo() for more information about scraping * embeddable content */ - public static function getSiteinfoCached($url, $no_guessing = false, $do_oembed = true) + public static function getSiteinfoCached($url, $do_oembed = true): array { - if ($url == "") { - return false; + if (empty($url)) { + return [ + 'url' => '', + 'type' => 'error', + ]; } + $urlHash = hash('sha256', $url); + $parsed_url = DBA::selectFirst('parsed_url', ['content'], - ['url' => Strings::normaliseLink($url), 'guessing' => !$no_guessing, 'oembed' => $do_oembed] + ['url_hash' => $urlHash, 'oembed' => $do_oembed] ); if (!empty($parsed_url['content'])) { $data = unserialize($parsed_url['content']); return $data; } - $data = self::getSiteinfo($url, $no_guessing, $do_oembed); + $data = self::getSiteinfo($url, $do_oembed); + + $expires = $data['expires']; + + unset($data['expires']); - DBA::insert( + DI::dba()->insert( 'parsed_url', [ - 'url' => substr(Strings::normaliseLink($url), 0, 255), 'guessing' => !$no_guessing, - 'oembed' => $do_oembed, 'content' => serialize($data), - 'created' => DateTimeFormat::utcNow() + 'url_hash' => $urlHash, + 'oembed' => $do_oembed, + 'url' => $url, + 'content' => serialize($data), + 'created' => DateTimeFormat::utcNow(), + 'expires' => $expires, ], - true + Database::INSERT_UPDATE ); return $data; @@ -108,21 +144,18 @@ class ParseUrl * \ * * @param string $url The url of the page which should be scraped - * @param bool $no_guessing If true the parse doens't search for - * preview pictures * @param bool $do_oembed The false option is used by the function fetch_oembed() * to avoid endless loops * @param int $count Internal counter to avoid endless loops * * @return array which contains needed data for embedding - * string 'url' => The url of the parsed page - * string 'type' => Content type - * string 'title' => The title of the content - * string 'text' => The description for the content - * string 'image' => A preview image of the content (only available - * if $no_geuessing = false - * array'images' = Array of preview pictures - * string 'keywords' => The tags which belong to the content + * string 'url' => The url of the parsed page + * string 'type' => Content type (error, link, photo, image, audio, video) + * string 'title' => (optional) The title of the content + * string 'text' => (optional) The description for the content + * string 'image' => (optional) A preview image of the content + * array 'images' => (optional) Array of preview pictures + * string 'keywords' => (optional) The tags which belong to the content * * @throws \Friendica\Network\HTTPException\InternalServerErrorException * @todo https://developers.google.com/+/plugins/snippet/ @@ -138,82 +171,129 @@ class ParseUrl * * @endverbatim */ - public static function getSiteinfo($url, $no_guessing = false, $do_oembed = true, $count = 1) + public static function getSiteinfo($url, $do_oembed = true, $count = 1) { - $siteinfo = []; + if (empty($url)) { + return [ + 'url' => '', + 'type' => 'error', + ]; + } // Check if the URL does contain a scheme $scheme = parse_url($url, PHP_URL_SCHEME); if ($scheme == '') { - $url = 'http://' . trim($url, '/'); + $url = 'http://' . ltrim($url, '/'); } - if ($count > 10) { - Logger::log('Endless loop detected for ' . $url, Logger::DEBUG); - return $siteinfo; - } - - $url = trim($url, "'"); - $url = trim($url, '"'); + $url = trim($url, "'\""); $url = Network::stripTrackingQueryParams($url); - $siteinfo['url'] = $url; - $siteinfo['type'] = 'link'; + $siteinfo = [ + 'url' => $url, + 'type' => 'link', + 'expires' => DateTimeFormat::utc(self::DEFAULT_EXPIRATION_FAILURE), + ]; - $curlResult = Network::curl($url); - if (!$curlResult->isSuccess()) { + if ($count > 10) { + Logger::notice('Endless loop detected', ['url' => $url]); return $siteinfo; } - // If the file is too large then exit - if (($curlResult->getInfo()['download_content_length'] ?? 0) > 1000000) { + $type = self::getContentType($url); + Logger::info('Got content-type', ['content-type' => $type, 'url' => $url]); + if (!empty($type) && in_array($type[0], ['image', 'video', 'audio'])) { + $siteinfo['type'] = $type[0]; return $siteinfo; } - // If it isn't a HTML file then exit - if (($curlResult->getContentType() != '') && !strstr(strtolower($curlResult->getContentType()), 'html')) { + if ((count($type) >= 2) && (($type[0] != 'text') || ($type[1] != 'html'))) { + Logger::info('Unparseable content-type, quitting here, ', ['content-type' => $type, 'url' => $url]); return $siteinfo; } - $header = $curlResult->getHeader(); + $curlResult = DI::httpClient()->get($url, [HttpClientOptions::CONTENT_LENGTH => 1000000]); + if (!$curlResult->isSuccess() || empty($curlResult->getBody())) { + Logger::info('Empty body or error when fetching', ['url' => $url, 'success' => $curlResult->isSuccess(), 'code' => $curlResult->getReturnCode()]); + return $siteinfo; + } + + $siteinfo['expires'] = DateTimeFormat::utc(self::DEFAULT_EXPIRATION_SUCCESS); + + if ($cacheControlHeader = $curlResult->getHeader('Cache-Control')[0] ?? '') { + if (preg_match('/max-age=([0-9]+)/i', $cacheControlHeader, $matches)) { + $maxAge = max(86400, (int)array_pop($matches)); + $siteinfo['expires'] = DateTimeFormat::utc("now + $maxAge seconds"); + } + } + $body = $curlResult->getBody(); if ($do_oembed) { - $oembed_data = OEmbed::fetchURL($url); + $oembed_data = OEmbed::fetchURL($url, false, false); if (!empty($oembed_data->type)) { - if (!in_array($oembed_data->type, ['error', 'rich', ''])) { + if (!in_array($oembed_data->type, ['error', 'rich', 'image', 'video', 'audio', ''])) { $siteinfo['type'] = $oembed_data->type; } // See https://github.com/friendica/friendica/pull/5763#discussion_r217913178 if ($siteinfo['type'] != 'photo') { - if (isset($oembed_data->title)) { + if (!empty($oembed_data->title)) { $siteinfo['title'] = trim($oembed_data->title); } - if (isset($oembed_data->description)) { + if (!empty($oembed_data->description)) { $siteinfo['text'] = trim($oembed_data->description); } - if (isset($oembed_data->thumbnail_url)) { + if (!empty($oembed_data->author_name)) { + $siteinfo['author_name'] = trim($oembed_data->author_name); + } + if (!empty($oembed_data->author_url)) { + $siteinfo['author_url'] = trim($oembed_data->author_url); + } + if (!empty($oembed_data->provider_name)) { + $siteinfo['publisher_name'] = trim($oembed_data->provider_name); + } + if (!empty($oembed_data->provider_url)) { + $siteinfo['publisher_url'] = trim($oembed_data->provider_url); + } + if (!empty($oembed_data->thumbnail_url)) { $siteinfo['image'] = $oembed_data->thumbnail_url; } } } } - // Fetch the first mentioned charset. Can be in body or header $charset = ''; - if (preg_match('/charset=(.*?)[\'"\s\n]/', $header, $matches)) { + // Look for a charset, first in headers + // Expected form: Content-Type: text/html; charset=ISO-8859-4 + if (preg_match('/charset=([a-z0-9-_.\/]+)/i', $curlResult->getContentType(), $matches)) { $charset = trim(trim(trim(array_pop($matches)), ';,')); } + // Then in body that gets precedence + // Expected forms: + // - + // - + // - + // - + // We escape