Merge pull request #12679 from nupplaphil/feat/reduce_config_dependency

[friendica.git] / src / Util / ParseUrl.php

diff --git a/src/Util/ParseUrl.php b/src/Util/ParseUrl.php
index ee171ea047e85ccf5111e14ce3392ead59ae3686..9d19a4ebac9e02aade552216f4bbfc23f9727f21 100644 (file)
--- a/src/Util/ParseUrl.php
+++ b/src/Util/ParseUrl.php
@@ -1,6 +1,6 @@
 <?php
 /**
- * @copyright Copyright (C) 2010-2022, the Friendica project
+ * @copyright Copyright (C) 2010-2023, the Friendica project
  *
  * @license GNU AGPL version 3 or any later version
  *
@@ -24,6 +24,8 @@ namespace Friendica\Util;
 use DOMDocument;
 use DOMXPath;
 use Friendica\Content\OEmbed;
+use Friendica\Content\Text\HTML;
+use Friendica\Protocol\HTTP\MediaType;
 use Friendica\Core\Hook;
 use Friendica\Core\Logger;
 use Friendica\Database\Database;
@@ -57,15 +59,22 @@ class ParseUrl
         * Fetch the content type of the given url
         * @param string $url    URL of the page
         * @param string $accept content-type to accept
+        * @param int    $timeout
         * @return array content type
         */
-       public static function getContentType(string $url, string $accept = HttpClientAccept::DEFAULT): array
+       public static function getContentType(string $url, string $accept = HttpClientAccept::DEFAULT, int $timeout = 0): array
        {
-               $curlResult = DI::httpClient()->head($url, [HttpClientOptions::ACCEPT_CONTENT => $accept]);
+               if (!empty($timeout)) {
+                       $options = [HttpClientOptions::TIMEOUT => $timeout];
+               } else {
+                       $options = [];
+               }
+
+               $curlResult = DI::httpClient()->head($url, array_merge([HttpClientOptions::ACCEPT_CONTENT => $accept], $options));
 
-               // Workaround for systems that can't handle a HEAD request
-               if (!$curlResult->isSuccess() && ($curlResult->getReturnCode() == 405)) {
-                       $curlResult = DI::httpClient()->get($url, $accept, [HttpClientOptions::CONTENT_LENGTH => 1000000]);
+               // Workaround for systems that can't handle a HEAD request. Don't retry on timeouts.
+               if (!$curlResult->isSuccess() && ($curlResult->getReturnCode() >= 400) && !in_array($curlResult->getReturnCode(), [408, 504])) {
+                       $curlResult = DI::httpClient()->get($url, $accept, array_merge([HttpClientOptions::CONTENT_LENGTH => 1000000], $options));
                }
 
                if (!$curlResult->isSuccess()) {
@@ -180,7 +189,7 @@ class ParseUrl
         * </body>
         * @endverbatim
         */
-       public static function getSiteinfo(string $url, bool $do_oembed = true, int $count = 1)
+       public static function getSiteinfo(string $url, bool $do_oembed = true, int $count = 1): array
        {
                if (empty($url)) {
                        return [
@@ -207,7 +216,7 @@ class ParseUrl
                ];
 
                if ($count > 10) {
-                       Logger::notice('Endless loop detected', ['url' => $url]);
+                       Logger::warning('Endless loop detected', ['url' => $url]);
                        return $siteinfo;
                }
 
@@ -276,25 +285,13 @@ class ParseUrl
                }
 
                $charset = '';
-               // Look for a charset, first in headers
-               // Expected form: Content-Type: text/html; charset=ISO-8859-4
-               if (preg_match('/charset=([a-z0-9-_.\/]+)/i', $curlResult->getContentType(), $matches)) {
-                       $charset = trim(trim(trim(array_pop($matches)), ';,'));
-               }
-
-               // Then in body that gets precedence
-               // Expected forms:
-               // - <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
-               // - <meta charset="utf-8">
-               // - <meta charset=utf-8>
-               // - <meta charSet="utf-8">
-               // We escape <style> and <script> tags since they can contain irrelevant charset information
-               // (see https://github.com/friendica/friendica/issues/9251#issuecomment-698636806)
-               Strings::performWithEscapedBlocks($body, '#<(?:style|script).*?</(?:style|script)>#ism', function ($body) use (&$charset) {
-                       if (preg_match('/charset=["\']?([a-z0-9-_.\/]+)/i', $body, $matches)) {
-                               $charset = trim(trim(trim(array_pop($matches)), ';,'));
+               try {
+                       // Look for a charset, first in headers
+                       $mediaType = MediaType::fromContentType($curlResult->getContentType());
+                       if (isset($mediaType->parameters['charset'])) {
+                               $charset = $mediaType->parameters['charset'];
                        }
-               });
+               } catch(\InvalidArgumentException $e) {}
 
                $siteinfo['charset'] = $charset;
 
@@ -308,9 +305,15 @@ class ParseUrl
 
                $body = mb_convert_encoding($body, 'HTML-ENTITIES', 'UTF-8');
 
+               if (empty($body)) {
+                       return $siteinfo;
+               }
+
                $doc = new DOMDocument();
                @$doc->loadHTML($body);
 
+               $siteinfo['charset'] = HTML::extractCharset($doc) ?? $siteinfo['charset'];
+
                XML::deleteNode($doc, 'style');
                XML::deleteNode($doc, 'option');
                XML::deleteNode($doc, 'h1');
@@ -556,6 +559,7 @@ class ParseUrl
                                                $image['width'] = $photodata[0];
                                                $image['height'] = $photodata[1];
                                                $image['contenttype'] = $photodata['mime'];
+                                               $image['blurhash'] = $photodata['blurhash'] ?? null;
                                                unset($image['url']);
                                                ksort($image);
                                        } else {
@@ -634,6 +638,7 @@ class ParseUrl
         * Convert tags from CSV to an array
         *
         * @param string $string Tags
+        *
         * @return array with formatted Hashtags
         */
        public static function convertTagsToArray(string $string): array
@@ -655,6 +660,7 @@ class ParseUrl
         *
         * @param string $tag The pure tag name
         * @param int    $k   Counter for internal use
+        *
         * @return void
         */
        private static function arrAddHashes(string &$tag, int $k)
@@ -718,6 +724,7 @@ class ParseUrl
         *
         * @param array $siteinfo
         * @param array $jsonld
+        *
         * @return array siteinfo
         */
        private static function parseParts(array $siteinfo, array $jsonld): array
@@ -763,6 +770,7 @@ class ParseUrl
         *
         * @param array $siteinfo
         * @param array $jsonld
+        *
         * @return array siteinfo
         */
        private static function parseJsonLd(array $siteinfo, array $jsonld): array
@@ -856,6 +864,7 @@ class ParseUrl
         *
         * @param array $siteinfo
         * @param array $jsonld
+        *
         * @return array siteinfo
         */
        private static function parseJsonLdAuthor(array $siteinfo, array $jsonld): array
@@ -940,6 +949,7 @@ class ParseUrl
         *
         * @param array $siteinfo
         * @param array $jsonld
+        *
         * @return array siteinfo
         */
        private static function parseJsonLdArticle(array $siteinfo, array $jsonld): array
@@ -1010,6 +1020,7 @@ class ParseUrl
         *
         * @param array $siteinfo
         * @param array $jsonld
+        *
         * @return array siteinfo
         */
        private static function parseJsonLdWebPage(array $siteinfo, array $jsonld): array
@@ -1022,17 +1033,17 @@ class ParseUrl
                }
 
                $content = JsonLD::fetchElement($jsonld, 'description');
-               if (!empty($content)) {
+               if (!empty($content) && is_string($content)) {
                        $jsonldinfo['text'] = trim($content);
                }
 
                $content = JsonLD::fetchElement($jsonld, 'image');
-               if (!empty($content)) {
+               if (!empty($content) && is_string($content)) {
                        $jsonldinfo['image'] = trim($content);
                }
 
                $content = JsonLD::fetchElement($jsonld, 'thumbnailUrl');
-               if (!empty($content)) {
+               if (!empty($content) && is_string($content)) {
                        $jsonldinfo['image'] = trim($content);
                }
 
@@ -1049,6 +1060,7 @@ class ParseUrl
         *
         * @param array $siteinfo
         * @param array $jsonld
+        *
         * @return array siteinfo
         */
        private static function parseJsonLdWebSite(array $siteinfo, array $jsonld): array
@@ -1056,22 +1068,22 @@ class ParseUrl
                $jsonldinfo = [];
 
                $content = JsonLD::fetchElement($jsonld, 'name');
-               if (!empty($content)) {
+               if (!empty($content) && is_string($content)) {
                        $jsonldinfo['publisher_name'] = trim($content);
                }
 
                $content = JsonLD::fetchElement($jsonld, 'description');
-               if (!empty($content)) {
+               if (!empty($content) && is_string($content)) {
                        $jsonldinfo['publisher_description'] = trim($content);
                }
 
                $content = JsonLD::fetchElement($jsonld, 'url');
-               if (!empty($content)) {
+               if (!empty($content) && is_string($content)) {
                        $jsonldinfo['publisher_url'] = trim($content);
                }
 
                $content = JsonLD::fetchElement($jsonld, 'thumbnailUrl');
-               if (!empty($content)) {
+               if (!empty($content) && is_string($content)) {
                        $jsonldinfo['image'] = trim($content);
                }
 
@@ -1087,6 +1099,7 @@ class ParseUrl
         *
         * @param array $siteinfo
         * @param array $jsonld
+        *
         * @return array siteinfo
         */
        private static function parseJsonLdWebOrganization(array $siteinfo, array $jsonld): array
@@ -1094,32 +1107,34 @@ class ParseUrl
                $jsonldinfo = [];
 
                $content = JsonLD::fetchElement($jsonld, 'name');
-               if (!empty($content)) {
+               if (!empty($content) && is_string($content)) {
                        $jsonldinfo['publisher_name'] = trim($content);
                }
 
                $content = JsonLD::fetchElement($jsonld, 'description');
-               if (!empty($content)) {
+               if (!empty($content) && is_string($content)) {
                        $jsonldinfo['publisher_description'] = trim($content);
                }
 
                $content = JsonLD::fetchElement($jsonld, 'url');
-               if (!empty($content)) {
+               if (!empty($content) && is_string($content)) {
                        $jsonldinfo['publisher_url'] = trim($content);
                }
 
                $content = JsonLD::fetchElement($jsonld, 'logo', 'url', '@type', 'ImageObject');
-               if (!empty($content)) {
+               if (!empty($content) && is_string($content)) {
                        $jsonldinfo['publisher_img'] = trim($content);
+               } elseif (!empty($content) && is_array($content)) {
+                       $jsonldinfo['publisher_img'] = trim($content[0]);
                }
 
                $content = JsonLD::fetchElement($jsonld, 'brand', 'name', '@type', 'Organization');
-               if (!empty($content)) {
+               if (!empty($content) && is_string($content)) {
                        $jsonldinfo['publisher_name'] = trim($content);
                }
 
                $content = JsonLD::fetchElement($jsonld, 'brand', 'url', '@type', 'Organization');
-               if (!empty($content)) {
+               if (!empty($content) && is_string($content)) {
                        $jsonldinfo['publisher_url'] = trim($content);
                }
 
@@ -1133,6 +1148,7 @@ class ParseUrl
         *
         * @param array $siteinfo
         * @param array $jsonld
+        *
         * @return array siteinfo
         */
        private static function parseJsonLdWebPerson(array $siteinfo, array $jsonld): array
@@ -1140,12 +1156,12 @@ class ParseUrl
                $jsonldinfo = [];
 
                $content = JsonLD::fetchElement($jsonld, 'name');
-               if (!empty($content)) {
+               if (!empty($content) && is_string($content)) {
                        $jsonldinfo['author_name'] = trim($content);
                }
 
                $content = JsonLD::fetchElement($jsonld, 'description');
-               if (!empty($content)) {
+               if (!empty($content) && is_string($content)) {
                        $jsonldinfo['author_description'] = trim($content);
                }
 
@@ -1155,7 +1171,7 @@ class ParseUrl
                }
 
                $content = JsonLD::fetchElement($jsonld, 'url');
-               if (!empty($content)) {
+               if (!empty($content) && is_string($content)) {
                        $jsonldinfo['author_url'] = trim($content);
                }
 
@@ -1178,6 +1194,7 @@ class ParseUrl
         *
         * @param array $siteinfo
         * @param array $jsonld
+        *
         * @return array siteinfo
         */
        private static function parseJsonLdMediaObject(array $siteinfo, array $jsonld, string $name): array
@@ -1185,22 +1202,22 @@ class ParseUrl
                $media = [];
 
                $content = JsonLD::fetchElement($jsonld, 'caption');
-               if (!empty($content)) {
+               if (!empty($content) && is_string($content)) {
                        $media['caption'] = trim($content);
                }
 
                $content = JsonLD::fetchElement($jsonld, 'url');
-               if (!empty($content)) {
+               if (!empty($content) && is_string($content)) {
                        $media['url'] = trim($content);
                }
 
                $content = JsonLD::fetchElement($jsonld, 'mainEntityOfPage');
-               if (!empty($content)) {
+               if (!empty($content) && is_string($content)) {
                        $media['main'] = Strings::compareLink($content, $siteinfo['url']);
                }
 
                $content = JsonLD::fetchElement($jsonld, 'description');
-               if (!empty($content)) {
+               if (!empty($content) && is_string($content)) {
                        $media['description'] = trim($content);
                }
 
@@ -1210,27 +1227,27 @@ class ParseUrl
                }
 
                $content = JsonLD::fetchElement($jsonld, 'contentUrl');
-               if (!empty($content)) {
+               if (!empty($content) && is_string($content)) {
                        $media['content'] = trim($content);
                }
 
                $content = JsonLD::fetchElement($jsonld, 'embedUrl');
-               if (!empty($content)) {
+               if (!empty($content) && is_string($content)) {
                        $media['embed'] = trim($content);
                }
 
                $content = JsonLD::fetchElement($jsonld, 'height');
-               if (!empty($content)) {
+               if (!empty($content) && is_string($content)) {
                        $media['height'] = trim($content);
                }
 
                $content = JsonLD::fetchElement($jsonld, 'width');
-               if (!empty($content)) {
+               if (!empty($content) && is_string($content)) {
                        $media['width'] = trim($content);
                }
 
                $content = JsonLD::fetchElement($jsonld, 'image');
-               if (!empty($content)) {
+               if (!empty($content) && is_string($content)) {
                        $media['image'] = trim($content);
                }