Merge pull request #10967 from annando/api

[friendica.git] / src / Util / ParseUrl.php

diff --git a/src/Util/ParseUrl.php b/src/Util/ParseUrl.php
index dc5648dd7efe9d75e1cc0121ef20c3dc01e261e0..6c3c4a0259e80873e1130d0b243531097f9fa3c1 100644 (file)
--- a/src/Util/ParseUrl.php
+++ b/src/Util/ParseUrl.php
@@ -1,6 +1,6 @@
 <?php
 /**
- * @copyright Copyright (C) 2020, Friendica
+ * @copyright Copyright (C) 2010-2021, the Friendica project
  *
  * @license GNU AGPL version 3 or any later version
  *
@@ -30,6 +30,7 @@ use Friendica\Database\Database;
 use Friendica\Database\DBA;
 use Friendica\DI;
 use Friendica\Network\HTTPException;
+use Friendica\Network\HTTPClient\Client\HttpClientOptions;
 
 /**
  * Get information about a given URL
@@ -54,16 +55,16 @@ class ParseUrl
        /**
         * Fetch the content type of the given url
         * @param string $url URL of the page
-        * @return array content type 
+        * @return array content type
         */
        public static function getContentType(string $url)
        {
-               $curlResult = DI::httpRequest()->head($url);
+               $curlResult = DI::httpClient()->head($url);
                if (!$curlResult->isSuccess()) {
                        return [];
                }
 
-               $contenttype =  $curlResult->getHeader('Content-Type');
+               $contenttype =  $curlResult->getHeader('Content-Type')[0] ?? '';
                if (empty($contenttype)) {
                        return [];
                }
@@ -197,7 +198,7 @@ class ParseUrl
                ];
 
                if ($count > 10) {
-                       Logger::log('Endless loop detected for ' . $url, Logger::DEBUG);
+                       Logger::notice('Endless loop detected', ['url' => $url]);
                        return $siteinfo;
                }
 
@@ -213,26 +214,20 @@ class ParseUrl
                        return $siteinfo;
                }
 
-               $curlResult = DI::httpRequest()->get($url);
-               if (!$curlResult->isSuccess()) {
+               $curlResult = DI::httpClient()->get($url, [HttpClientOptions::CONTENT_LENGTH => 1000000]);
+               if (!$curlResult->isSuccess() || empty($curlResult->getBody())) {
                        return $siteinfo;
                }
 
                $siteinfo['expires'] = DateTimeFormat::utc(self::DEFAULT_EXPIRATION_SUCCESS);
 
-               // If the file is too large then exit
-               if (($curlResult->getInfo()['download_content_length'] ?? 0) > 1000000) {
-                       return $siteinfo;
-               }
-
-               if ($cacheControlHeader = $curlResult->getHeader('Cache-Control')) {
+               if ($cacheControlHeader = $curlResult->getHeader('Cache-Control')[0] ?? '') {
                        if (preg_match('/max-age=([0-9]+)/i', $cacheControlHeader, $matches)) {
                                $maxAge = max(86400, (int)array_pop($matches));
                                $siteinfo['expires'] = DateTimeFormat::utc("now + $maxAge seconds");
                        }
                }
 
-               $header = $curlResult->getHeader();
                $body = $curlResult->getBody();
 
                if ($do_oembed) {
@@ -273,7 +268,7 @@ class ParseUrl
                $charset = '';
                // Look for a charset, first in headers
                // Expected form: Content-Type: text/html; charset=ISO-8859-4
-               if (preg_match('/charset=([a-z0-9-_.\/]+)/i', $header, $matches)) {
+               if (preg_match('/charset=([a-z0-9-_.\/]+)/i', $curlResult->getContentType(), $matches)) {
                        $charset = trim(trim(trim(array_pop($matches)), ';,'));
                }
 
@@ -297,7 +292,7 @@ class ParseUrl
                        // See https://github.com/friendica/friendica/issues/5470#issuecomment-418351211
                        $charset = str_ireplace('latin-1', 'latin1', $charset);
 
-                       Logger::log('detected charset ' . $charset, Logger::DEBUG);
+                       Logger::info('detected charset', ['charset' => $charset]);
                        $body = iconv($charset, 'UTF-8//TRANSLIT', $body);
                }
 
@@ -349,6 +344,9 @@ class ParseUrl
                        $siteinfo['title'] = trim($list->item(0)->nodeValue);
                }
 
+               $twitter_card = false;
+               $twitter_image = false;
+
                $list = $xpath->query('//meta[@name]');
                foreach ($list as $node) {
                        $meta_tag = [];
@@ -376,6 +374,7 @@ class ParseUrl
                                        break;
                                case 'twitter:image':
                                        $siteinfo['image'] = $meta_tag['content'];
+                                       $twitter_image = true;
                                        break;
                                case 'twitter:image:src':
                                        $siteinfo['image'] = $meta_tag['content'];
@@ -383,7 +382,7 @@ class ParseUrl
                                case 'twitter:card':
                                        // Detect photo pages
                                        if ($meta_tag['content'] == 'summary_large_image') {
-                                               $siteinfo['type'] = 'photo';
+                                               $twitter_card = true;
                                        }
                                        break;
                                case 'twitter:description':
@@ -458,6 +457,7 @@ class ParseUrl
                                                break;
                                        case 'twitter:image':
                                                $siteinfo['image'] = $meta_tag['content'];
+                                               $twitter_image = true;
                                                break;
                                }
                        }
@@ -472,10 +472,11 @@ class ParseUrl
                        }
                }
 
+// Currently deactivated, see https://github.com/friendica/friendica/pull/10148#issuecomment-821512658
                // Prevent to have a photo type without an image
-               if ((empty($siteinfo['image']) || !empty($siteinfo['text'])) && ($siteinfo['type'] == 'photo')) {
-                       $siteinfo['type'] = 'link';
-               }
+//             if ($twitter_card && $twitter_image && !empty($siteinfo['image'])) {
+//                     $siteinfo['type'] = 'photo';
+//             }
 
                if (!empty($siteinfo['image'])) {
                        $siteinfo['images'] = $siteinfo['images'] ?? [];
@@ -689,7 +690,7 @@ class ParseUrl
        {
                if (!empty($jsonld['@graph']) && is_array($jsonld['@graph'])) {
                        foreach ($jsonld['@graph'] as $part) {
-                               if (!empty($part)) {
+                               if (!empty($part) && is_array($part)) {
                                        $siteinfo = self::parseParts($siteinfo, $part);
                                }
                        }
@@ -705,7 +706,7 @@ class ParseUrl
                        }
                        if ($numeric_keys) {
                                foreach ($jsonld as $part) {
-                                       if (!empty($part)) {
+                                       if (!empty($part) && is_array($part)) {
                                                $siteinfo = self::parseParts($siteinfo, $part);
                                        }
                                }
@@ -772,7 +773,7 @@ class ParseUrl
                        case 'QAPage':
                        case 'RealEstateListing':
                        case 'SearchResultsPage':
-                       case 'MediaGallery':                    
+                       case 'MediaGallery':
                        case 'ImageGallery':
                        case 'VideoGallery':
                        case 'RadioEpisode':
@@ -801,7 +802,7 @@ class ParseUrl
                        case 'PerformingGroup':
                        case 'DanceGroup';
                        case 'MusicGroup':
-                       case 'TheaterGroup':                    
+                       case 'TheaterGroup':
                                return self::parseJsonLdWebPerson($siteinfo, $jsonld);
                        case 'AudioObject':
                        case 'Audio':
@@ -1115,7 +1116,11 @@ class ParseUrl
                }
 
                $content = JsonLD::fetchElement($jsonld, 'image', 'url', '@type', 'ImageObject');
-               if (!empty($content)) {
+               if (!empty($content) && !is_string($content)) {
+                       Logger::notice('Unexpected return value for the author image', ['content' => $content]);
+               }
+
+               if (!empty($content) && is_string($content)) {
                        $jsonldinfo['author_img'] = trim($content);
                }