Replace legacy file/category handling

[friendica.git] / src / Util / ParseUrl.php

diff --git a/src/Util/ParseUrl.php b/src/Util/ParseUrl.php
index cf38ffd7bec5172989efb9f6c523a328765d05bb..15186b57374bd9246444dcf484f575d6cfec7eaf 100644 (file)
--- a/src/Util/ParseUrl.php
+++ b/src/Util/ParseUrl.php
@@ -26,7 +26,9 @@ use DOMXPath;
 use Friendica\Content\OEmbed;
 use Friendica\Core\Hook;
 use Friendica\Core\Logger;
+use Friendica\Database\Database;
 use Friendica\Database\DBA;
+use Friendica\DI;
 
 /**
  * Get information about a given URL
@@ -90,7 +92,7 @@ class ParseUrl
                                'oembed' => $do_oembed, 'content' => serialize($data),
                                'created' => DateTimeFormat::utcNow()
                        ],
-                       true
+                       Database::INSERT_UPDATE
                );
 
                return $data;
@@ -159,7 +161,7 @@ class ParseUrl
                        return $siteinfo;
                }
 
-               $curlResult = DI::httpRequest()->curl($url);
+               $curlResult = DI::httpRequest()->get($url);
                if (!$curlResult->isSuccess()) {
                        return $siteinfo;
                }
@@ -200,12 +202,29 @@ class ParseUrl
                        }
                }
 
-               // Fetch the first mentioned charset. Can be in body or header
                $charset = '';
-               if (preg_match('/charset=(.*?)[\'"\s\n]/', $header, $matches)) {
+               // Look for a charset, first in headers
+               // Expected form: Content-Type: text/html; charset=ISO-8859-4
+               if (preg_match('/charset=([a-z0-9-_.\/]+)/i', $header, $matches)) {
                        $charset = trim(trim(trim(array_pop($matches)), ';,'));
                }
 
+               // Then in body that gets precedence
+               // Expected forms:
+               // - <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+               // - <meta charset="utf-8">
+               // - <meta charset=utf-8>
+               // - <meta charSet="utf-8">
+               // We escape <style> and <script> tags since they can contain irrelevant charset information
+               // (see https://github.com/friendica/friendica/issues/9251#issuecomment-698636806)
+               Strings::performWithEscapedBlocks($body, '#<(?:style|script).*?</(?:style|script)>#ism', function ($body) use (&$charset) {
+                       if (preg_match('/charset=["\']?([a-z0-9-_.\/]+)/i', $body, $matches)) {
+                               $charset = trim(trim(trim(array_pop($matches)), ';,'));
+                       }
+               });
+
+               $siteinfo['charset'] = $charset;
+
                if ($charset && strtoupper($charset) != 'UTF-8') {
                        // See https://github.com/friendica/friendica/issues/5470#issuecomment-418351211
                        $charset = str_ireplace('latin-1', 'latin1', $charset);