Preparations for a moderator role

[friendica.git] / src / Util / ParseUrl.php

diff --git a/src/Util/ParseUrl.php b/src/Util/ParseUrl.php
index d0b168278563e24be39d6a79c01939846e7e1d98..79f427a654df4b5f672bc52f3470ebe01deb2b5b 100644 (file)
--- a/src/Util/ParseUrl.php
+++ b/src/Util/ParseUrl.php
@@ -1,6 +1,6 @@
 <?php
 /**
- * @copyright Copyright (C) 2010-2022, the Friendica project
+ * @copyright Copyright (C) 2010-2023, the Friendica project
  *
  * @license GNU AGPL version 3 or any later version
  *
@@ -24,6 +24,8 @@ namespace Friendica\Util;
 use DOMDocument;
 use DOMXPath;
 use Friendica\Content\OEmbed;
+use Friendica\Content\Text\HTML;
+use Friendica\Protocol\HTTP\MediaType;
 use Friendica\Core\Hook;
 use Friendica\Core\Logger;
 use Friendica\Database\Database;
@@ -68,7 +70,12 @@ class ParseUrl
                        $options = [];
                }
 
-               $curlResult = DI::httpClient()->head($url, array_merge([HttpClientOptions::ACCEPT_CONTENT => $accept], $options));
+               try {
+                       $curlResult = DI::httpClient()->head($url, array_merge([HttpClientOptions::ACCEPT_CONTENT => $accept], $options));
+               } catch (\Exception $e) {
+                       DI::logger()->debug('Got exception', ['url' => $url, 'message' => $e->getMessage()]);
+                       return [];
+               }
 
                // Workaround for systems that can't handle a HEAD request. Don't retry on timeouts.
                if (!$curlResult->isSuccess() && ($curlResult->getReturnCode() >= 400) && !in_array($curlResult->getReturnCode(), [408, 504])) {
@@ -89,7 +96,7 @@ class ParseUrl
        }
 
        /**
-        * Search for chached embeddable data of an url otherwise fetch it
+        * Search for cached embeddable data of an url otherwise fetch it
         *
         * @param string $url         The url of the page which should be scraped
         * @param bool   $do_oembed   The false option is used by the function fetch_oembed()
@@ -214,7 +221,7 @@ class ParseUrl
                ];
 
                if ($count > 10) {
-                       Logger::notice('Endless loop detected', ['url' => $url]);
+                       Logger::warning('Endless loop detected', ['url' => $url]);
                        return $siteinfo;
                }
 
@@ -283,25 +290,13 @@ class ParseUrl
                }
 
                $charset = '';
-               // Look for a charset, first in headers
-               // Expected form: Content-Type: text/html; charset=ISO-8859-4
-               if (preg_match('/charset=([a-z0-9-_.\/]+)/i', $curlResult->getContentType(), $matches)) {
-                       $charset = trim(trim(trim(array_pop($matches)), ';,'));
-               }
-
-               // Then in body that gets precedence
-               // Expected forms:
-               // - <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
-               // - <meta charset="utf-8">
-               // - <meta charset=utf-8>
-               // - <meta charSet="utf-8">
-               // We escape <style> and <script> tags since they can contain irrelevant charset information
-               // (see https://github.com/friendica/friendica/issues/9251#issuecomment-698636806)
-               Strings::performWithEscapedBlocks($body, '#<(?:style|script).*?</(?:style|script)>#ism', function ($body) use (&$charset) {
-                       if (preg_match('/charset=["\']?([a-z0-9-_.\/]+)/i', $body, $matches)) {
-                               $charset = trim(trim(trim(array_pop($matches)), ';,'));
+               try {
+                       // Look for a charset, first in headers
+                       $mediaType = MediaType::fromContentType($curlResult->getContentType());
+                       if (isset($mediaType->parameters['charset'])) {
+                               $charset = $mediaType->parameters['charset'];
                        }
-               });
+               } catch(\InvalidArgumentException $e) {}
 
                $siteinfo['charset'] = $charset;
 
@@ -322,6 +317,8 @@ class ParseUrl
                $doc = new DOMDocument();
                @$doc->loadHTML($body);
 
+               $siteinfo['charset'] = HTML::extractCharset($doc) ?? $siteinfo['charset'];
+
                XML::deleteNode($doc, 'style');
                XML::deleteNode($doc, 'option');
                XML::deleteNode($doc, 'h1');
@@ -567,6 +564,7 @@ class ParseUrl
                                                $image['width'] = $photodata[0];
                                                $image['height'] = $photodata[1];
                                                $image['contenttype'] = $photodata['mime'];
+                                               $image['blurhash'] = $photodata['blurhash'] ?? null;
                                                unset($image['url']);
                                                ksort($image);
                                        } else {
@@ -653,7 +651,7 @@ class ParseUrl
                $arr_tags = str_getcsv($string);
                if (count($arr_tags)) {
                        // add the # sign to every tag
-                       array_walk($arr_tags, ['self', 'arrAddHashes']);
+                       array_walk($arr_tags, [self::class, 'arrAddHashes']);
 
                        return $arr_tags;
                }
@@ -693,7 +691,7 @@ class ParseUrl
        {
                $urlarr = parse_url($url);
 
-               // If the url does allready have an scheme
+               // If the url does already have an scheme
                // we can stop the process here
                if (isset($urlarr['scheme'])) {
                        return $url;
@@ -1131,6 +1129,8 @@ class ParseUrl
                $content = JsonLD::fetchElement($jsonld, 'logo', 'url', '@type', 'ImageObject');
                if (!empty($content) && is_string($content)) {
                        $jsonldinfo['publisher_img'] = trim($content);
+               } elseif (!empty($content) && is_array($content)) {
+                       $jsonldinfo['publisher_img'] = trim($content[0]);
                }
 
                $content = JsonLD::fetchElement($jsonld, 'brand', 'name', '@type', 'Organization');