X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;f=src%2FUtil%2FParseUrl.php;h=04afc927ba6cec4229c8167a6bcafe66e4c317d9;hb=720a43461d67ab229de0aecfc5008f22cc4c1c54;hp=13cb55b73ee1396acd4d03714d38c2c07be59233;hpb=2a68ad9b26d0a36968128be1627f5b1a427957d7;p=friendica.git diff --git a/src/Util/ParseUrl.php b/src/Util/ParseUrl.php index 13cb55b73e..04afc927ba 100644 --- a/src/Util/ParseUrl.php +++ b/src/Util/ParseUrl.php @@ -1,6 +1,6 @@ head($url); + $curlResult = DI::httpClient()->head($url); if (!$curlResult->isSuccess()) { return []; } - $contenttype = $curlResult->getHeader('Content-Type'); + $contenttype = $curlResult->getHeader('Content-Type')[0] ?? ''; if (empty($contenttype)) { return []; } @@ -197,7 +198,7 @@ class ParseUrl ]; if ($count > 10) { - Logger::log('Endless loop detected for ' . $url, Logger::DEBUG); + Logger::notice('Endless loop detected', ['url' => $url]); return $siteinfo; } @@ -213,26 +214,21 @@ class ParseUrl return $siteinfo; } - $curlResult = DI::httpRequest()->get($url); - if (!$curlResult->isSuccess()) { + $curlResult = DI::httpClient()->get($url, [HttpClientOptions::CONTENT_LENGTH => 1000000]); + if (!$curlResult->isSuccess() || empty($curlResult->getBody())) { + Logger::info('Empty body or error when fetching', ['url' => $url, 'success' => $curlResult->isSuccess(), 'code' => $curlResult->getReturnCode()]); return $siteinfo; } $siteinfo['expires'] = DateTimeFormat::utc(self::DEFAULT_EXPIRATION_SUCCESS); - // If the file is too large then exit - if (($curlResult->getInfo()['download_content_length'] ?? 0) > 1000000) { - return $siteinfo; - } - - if ($cacheControlHeader = $curlResult->getHeader('Cache-Control')) { + if ($cacheControlHeader = $curlResult->getHeader('Cache-Control')[0] ?? '') { if (preg_match('/max-age=([0-9]+)/i', $cacheControlHeader, $matches)) { $maxAge = max(86400, (int)array_pop($matches)); $siteinfo['expires'] = DateTimeFormat::utc("now + $maxAge seconds"); } } - $header = $curlResult->getHeader(); $body = $curlResult->getBody(); if ($do_oembed) { @@ -273,7 +269,7 @@ class ParseUrl $charset = ''; // Look for a charset, first in headers // Expected form: Content-Type: text/html; charset=ISO-8859-4 - if (preg_match('/charset=([a-z0-9-_.\/]+)/i', $header, $matches)) { + if (preg_match('/charset=([a-z0-9-_.\/]+)/i', $curlResult->getContentType(), $matches)) { $charset = trim(trim(trim(array_pop($matches)), ';,')); } @@ -297,7 +293,7 @@ class ParseUrl // See https://github.com/friendica/friendica/issues/5470#issuecomment-418351211 $charset = str_ireplace('latin-1', 'latin1', $charset); - Logger::log('detected charset ' . $charset, Logger::DEBUG); + Logger::info('detected charset', ['charset' => $charset]); $body = iconv($charset, 'UTF-8//TRANSLIT', $body); } @@ -380,18 +376,24 @@ class ParseUrl case 'twitter:image:src': $siteinfo['image'] = $meta_tag['content']; break; - case 'twitter:card': - // Detect photo pages - if ($meta_tag['content'] == 'summary_large_image') { - $siteinfo['type'] = 'photo'; - } - break; case 'twitter:description': $siteinfo['text'] = trim($meta_tag['content']); break; case 'twitter:title': $siteinfo['title'] = trim($meta_tag['content']); break; + case 'twitter:player': + $siteinfo['player']['embed'] = trim($meta_tag['content']); + break; + case 'twitter:player:stream': + $siteinfo['player']['stream'] = trim($meta_tag['content']); + break; + case 'twitter:player:width': + $siteinfo['player']['width'] = intval($meta_tag['content']); + break; + case 'twitter:player:height': + $siteinfo['player']['height'] = intval($meta_tag['content']); + break; case 'dc.title': $siteinfo['title'] = trim($meta_tag['content']); break; @@ -450,6 +452,12 @@ class ParseUrl case 'og:site_name': $siteinfo['publisher_name'] = trim($meta_tag['content']); break; + case 'og:locale': + $siteinfo['language'] = trim($meta_tag['content']); + break; + case 'og:type': + $siteinfo['pagetype'] = trim($meta_tag['content']); + break; case 'twitter:description': $siteinfo['text'] = trim($meta_tag['content']); break; @@ -466,16 +474,27 @@ class ParseUrl $list = $xpath->query("//script[@type='application/ld+json']"); foreach ($list as $node) { if (!empty($node->nodeValue)) { - $nodevalue = html_entity_decode($node->nodeValue, ENT_COMPAT, 'UTF-8'); - if ($jsonld = json_decode($nodevalue, true)) { + if ($jsonld = json_decode($node->nodeValue, true)) { $siteinfo = self::parseParts($siteinfo, $jsonld); } } } - // Prevent to have a photo type without an image - if ((empty($siteinfo['image']) || !empty($siteinfo['text'])) && ($siteinfo['type'] == 'photo')) { - $siteinfo['type'] = 'link'; + if (!empty($siteinfo['player']['stream'])) { + // Only add player data to media arrays if there is no duplicate + $content_urls = array_merge(array_column($siteinfo['audio'] ?? [], 'content'), array_column($siteinfo['video'] ?? [], 'content')); + if (!in_array($siteinfo['player']['stream'], $content_urls)) { + $contenttype = self::getContentType($siteinfo['player']['stream']); + if (!empty($contenttype[0]) && in_array($contenttype[0], ['audio', 'video'])) { + $media = ['content' => $siteinfo['player']['stream']]; + + if (!empty($siteinfo['player']['embed'])) { + $media['embed'] = $siteinfo['player']['embed']; + } + + $siteinfo[$contenttype[0]][] = $media; + } + } } if (!empty($siteinfo['image'])) { @@ -498,6 +517,8 @@ class ParseUrl Hook::callAll('getsiteinfo', $siteinfo); + ksort($siteinfo); + return $siteinfo; } @@ -507,23 +528,27 @@ class ParseUrl * * @param string $page_url * @param array $siteinfo - * @return void + * @return array */ - private static function checkMedia(string $page_url, array $siteinfo) + private static function checkMedia(string $page_url, array $siteinfo) : array { if (!empty($siteinfo['images'])) { array_walk($siteinfo['images'], function (&$image) use ($page_url) { // According to the specifications someone could place a picture url into the content field as well. // But this doesn't seem to happen in the wild, so we don't cover it here. - $image['url'] = self::completeUrl($image['url'], $page_url); - $photodata = Images::getInfoFromURLCached($image['url']); - if (!empty($photodata) && ($photodata[0] > 50) && ($photodata[1] > 50)) { - $image['src'] = $image['url']; - $image['width'] = $photodata[0]; - $image['height'] = $photodata[1]; - $image['contenttype'] = $photodata['mime']; - unset($image['url']); - ksort($image); + if (!empty($image['url'])) { + $image['url'] = self::completeUrl($image['url'], $page_url); + $photodata = Images::getInfoFromURLCached($image['url']); + if (!empty($photodata) && ($photodata[0] > 50) && ($photodata[1] > 50)) { + $image['src'] = $image['url']; + $image['width'] = $photodata[0]; + $image['height'] = $photodata[1]; + $image['contenttype'] = $photodata['mime']; + unset($image['url']); + ksort($image); + } else { + $image = []; + } } else { $image = []; } @@ -543,7 +568,7 @@ class ParseUrl if (!empty($media[$field])) { $media[$field] = self::completeUrl($media[$field], $page_url); $type = self::getContentType($media[$field]); - if ($type[0] == 'text') { + if (($type[0] ?? '') == 'text') { if ($field == 'embed') { $embed = $media[$field]; } else { @@ -568,8 +593,8 @@ class ParseUrl } if (!empty($embed)) { $media['embed'] = $embed; - if (!empty($media['main'])) { - $siteinfo['embed'] = $embed; + if (empty($siteinfo['player']['embed'])) { + $siteinfo['player']['embed'] = $embed; } } if (!empty($content)) { @@ -686,7 +711,9 @@ class ParseUrl { if (!empty($jsonld['@graph']) && is_array($jsonld['@graph'])) { foreach ($jsonld['@graph'] as $part) { - $siteinfo = self::parseParts($siteinfo, $part); + if (!empty($part) && is_array($part)) { + $siteinfo = self::parseParts($siteinfo, $part); + } } } elseif (!empty($jsonld['@type'])) { $siteinfo = self::parseJsonLd($siteinfo, $jsonld); @@ -700,11 +727,19 @@ class ParseUrl } if ($numeric_keys) { foreach ($jsonld as $part) { - $siteinfo = self::parseParts($siteinfo, $part); - } + if (!empty($part) && is_array($part)) { + $siteinfo = self::parseParts($siteinfo, $part); + } + } } } + array_walk_recursive($siteinfo, function (&$element) { + if (is_string($element)) { + $element = trim(strip_tags(html_entity_decode($element, ENT_COMPAT, 'UTF-8'))); + } + }); + return $siteinfo; } @@ -759,7 +794,7 @@ class ParseUrl case 'QAPage': case 'RealEstateListing': case 'SearchResultsPage': - case 'MediaGallery': + case 'MediaGallery': case 'ImageGallery': case 'VideoGallery': case 'RadioEpisode': @@ -788,7 +823,7 @@ class ParseUrl case 'PerformingGroup': case 'DanceGroup'; case 'MusicGroup': - case 'TheaterGroup': + case 'TheaterGroup': return self::parseJsonLdWebPerson($siteinfo, $jsonld); case 'AudioObject': case 'Audio': @@ -820,11 +855,6 @@ class ParseUrl $jsonldinfo['publisher_name'] = trim($content); } - $content = JsonLD::fetchElement($jsonld, 'publisher', 'sameAs'); - if (!empty($content) && is_string($content)) { - $jsonldinfo['publisher_url'] = trim($content); - } - $content = JsonLD::fetchElement($jsonld, 'publisher', 'url'); if (!empty($content) && is_string($content)) { $jsonldinfo['publisher_url'] = trim($content); @@ -837,11 +867,6 @@ class ParseUrl $jsonldinfo['publisher_name'] = trim($content); } - $content = JsonLD::fetchElement($brand, 'sameAs'); - if (!empty($content) && is_string($content)) { - $jsonldinfo['publisher_url'] = trim($content); - } - $content = JsonLD::fetchElement($brand, 'url'); if (!empty($content) && is_string($content)) { $jsonldinfo['publisher_url'] = trim($content); @@ -942,13 +967,23 @@ class ParseUrl $siteinfo['keywords'][] = trim($keyword); } } - } else { + } elseif (!empty($jsonld['keywords'])) { $content = JsonLD::fetchElementArray($jsonld, 'keywords'); if (!empty($content) && is_array($content)) { $jsonldinfo['keywords'] = $content; } } + $content = JsonLD::fetchElement($jsonld, 'datePublished'); + if (!empty($content) && is_string($content)) { + $jsonldinfo['published'] = DateTimeFormat::utc($content); + } + + $content = JsonLD::fetchElement($jsonld, 'dateModified'); + if (!empty($content) && is_string($content)) { + $jsonldinfo['modified'] = DateTimeFormat::utc($content); + } + $jsonldinfo = self::parseJsonLdAuthor($jsonldinfo, $jsonld); Logger::info('Fetched article information', ['url' => $siteinfo['url'], 'fetched' => $jsonldinfo]); @@ -1055,11 +1090,6 @@ class ParseUrl $jsonldinfo['publisher_description'] = trim($content); } - $content = JsonLD::fetchElement($jsonld, 'sameAs'); - if (!empty($content) && is_string($content)) { - $jsonldinfo['publisher_url'] = trim($content); - } - $content = JsonLD::fetchElement($jsonld, 'url'); if (!empty($content)) { $jsonldinfo['publisher_url'] = trim($content); @@ -1117,7 +1147,11 @@ class ParseUrl } $content = JsonLD::fetchElement($jsonld, 'image', 'url', '@type', 'ImageObject'); - if (!empty($content)) { + if (!empty($content) && !is_string($content)) { + Logger::notice('Unexpected return value for the author image', ['content' => $content]); + } + + if (!empty($content) && is_string($content)) { $jsonldinfo['author_img'] = trim($content); }