X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;f=src%2FUtil%2FParseUrl.php;h=4611cf8efe5f25378caa699044543af6269bab4c;hb=b5ad8c3e153976cf3e63753597377f09852d98d7;hp=2763f56abf52aefc8090779b7fca8090906478ab;hpb=52b119cba3e0ae828f73faf96ae2f358e861d621;p=friendica.git

diff --git a/src/Util/ParseUrl.php b/src/Util/ParseUrl.php
index 2763f56abf..4611cf8efe 100644
--- a/src/Util/ParseUrl.php
+++ b/src/Util/ParseUrl.php
@@ -29,6 +29,7 @@ use Friendica\Core\Logger;
 use Friendica\Database\Database;
 use Friendica\Database\DBA;
 use Friendica\DI;
+use Friendica\Network\HTTPClient\Client\HttpClientAccept;
 use Friendica\Network\HTTPException;
 use Friendica\Network\HTTPClient\Client\HttpClientOptions;
 
@@ -54,25 +55,34 @@ class ParseUrl
 
 	/**
 	 * Fetch the content type of the given url
-	 * @param string $url URL of the page
+	 * @param string $url    URL of the page
+	 * @param string $accept content-type to accept
+	 * @param int    $timeout
 	 * @return array content type
 	 */
-	public static function getContentType(string $url)
+	public static function getContentType(string $url, string $accept = HttpClientAccept::DEFAULT, int $timeout = 0): array
 	{
-		$curlResult = DI::httpClient()->head($url);
+		if (!empty($timeout)) {
+			$options = [HttpClientOptions::TIMEOUT => $timeout];
+		} else {
+			$options = [];
+		}
+
+		$curlResult = DI::httpClient()->head($url, array_merge([HttpClientOptions::ACCEPT_CONTENT => $accept], $options));
 
-		// Workaround for systems that can't handle a HEAD request
-		if (!$curlResult->isSuccess() && ($curlResult->getReturnCode() == 405)) {
-			$curlResult = DI::httpClient()->get($url, [HttpClientOptions::CONTENT_LENGTH => 1000000]);
+		// Workaround for systems that can't handle a HEAD request. Don't retry on timeouts.
+		if (!$curlResult->isSuccess() && ($curlResult->getReturnCode() >= 400) && !in_array($curlResult->getReturnCode(), [408, 504])) {
+			$curlResult = DI::httpClient()->get($url, $accept, array_merge([HttpClientOptions::CONTENT_LENGTH => 1000000], $options));
 		}
 
 		if (!$curlResult->isSuccess()) {
+			Logger::debug('Got HTTP Error', ['http error' => $curlResult->getReturnCode(), 'url' => $url]);
 			return [];
 		}
 
 		$contenttype =  $curlResult->getHeader('Content-Type')[0] ?? '';
 		if (empty($contenttype)) {
-			return [];
+			return ['application', 'octet-stream'];
 		}
 
 		return explode('/', current(explode(';', $contenttype)));
@@ -98,7 +108,7 @@ class ParseUrl
 	 * @see   ParseUrl::getSiteinfo() for more information about scraping
 	 * embeddable content
 	 */
-	public static function getSiteinfoCached($url, $do_oembed = true): array
+	public static function getSiteinfoCached(string $url, bool $do_oembed = true): array
 	{
 		if (empty($url)) {
 			return [
@@ -177,7 +187,7 @@ class ParseUrl
 	 * </body>
 	 * @endverbatim
 	 */
-	public static function getSiteinfo($url, $do_oembed = true, $count = 1)
+	public static function getSiteinfo(string $url, bool $do_oembed = true, int $count = 1): array
 	{
 		if (empty($url)) {
 			return [
@@ -204,7 +214,7 @@ class ParseUrl
 		];
 
 		if ($count > 10) {
-			Logger::notice('Endless loop detected', ['url' => $url]);
+			Logger::warning('Endless loop detected', ['url' => $url]);
 			return $siteinfo;
 		}
 
@@ -220,7 +230,7 @@ class ParseUrl
 			return $siteinfo;
 		}
 
-		$curlResult = DI::httpClient()->get($url, [HttpClientOptions::CONTENT_LENGTH => 1000000]);
+		$curlResult = DI::httpClient()->get($url, HttpClientAccept::HTML, [HttpClientOptions::CONTENT_LENGTH => 1000000]);
 		if (!$curlResult->isSuccess() || empty($curlResult->getBody())) {
 			Logger::info('Empty body or error when fetching', ['url' => $url, 'success' => $curlResult->isSuccess(), 'code' => $curlResult->getReturnCode()]);
 			return $siteinfo;
@@ -277,22 +287,22 @@ class ParseUrl
 		// Expected form: Content-Type: text/html; charset=ISO-8859-4
 		if (preg_match('/charset=([a-z0-9-_.\/]+)/i', $curlResult->getContentType(), $matches)) {
 			$charset = trim(trim(trim(array_pop($matches)), ';,'));
+		} else {
+			// Then in body that gets precedence
+			// Expected forms:
+			// - <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+			// - <meta charset="utf-8">
+			// - <meta charset=utf-8>
+			// - <meta charSet="utf-8">
+			// We escape <style> and <script> tags since they can contain irrelevant charset information
+			// (see https://github.com/friendica/friendica/issues/9251#issuecomment-698636806)
+			Strings::performWithEscapedBlocks($body, '#<(?:style|script).*?</(?:style|script)>#ism', function ($body) use (&$charset) {
+				if (preg_match('/charset=["\']?([a-z0-9-_.\/]+)/i', $body, $matches)) {
+					$charset = trim(trim(trim(array_pop($matches)), ';,'));
+				}
+			});
 		}
 
-		// Then in body that gets precedence
-		// Expected forms:
-		// - <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
-		// - <meta charset="utf-8">
-		// - <meta charset=utf-8>
-		// - <meta charSet="utf-8">
-		// We escape <style> and <script> tags since they can contain irrelevant charset information
-		// (see https://github.com/friendica/friendica/issues/9251#issuecomment-698636806)
-		Strings::performWithEscapedBlocks($body, '#<(?:style|script).*?</(?:style|script)>#ism', function ($body) use (&$charset) {
-			if (preg_match('/charset=["\']?([a-z0-9-_.\/]+)/i', $body, $matches)) {
-				$charset = trim(trim(trim(array_pop($matches)), ';,'));
-			}
-		});
-
 		$siteinfo['charset'] = $charset;
 
 		if ($charset && strtoupper($charset) != 'UTF-8') {
@@ -305,6 +315,10 @@ class ParseUrl
 
 		$body = mb_convert_encoding($body, 'HTML-ENTITIES', 'UTF-8');
 
+		if (empty($body)) {
+			return $siteinfo;
+		}
+
 		$doc = new DOMDocument();
 		@$doc->loadHTML($body);
 
@@ -540,12 +554,15 @@ class ParseUrl
 	{
 		if (!empty($siteinfo['images'])) {
 			array_walk($siteinfo['images'], function (&$image) use ($page_url) {
-				// According to the specifications someone could place a picture url into the content field as well.
-				// But this doesn't seem to happen in the wild, so we don't cover it here.
+				/*
+				 * According to the specifications someone could place a picture
+				 * URL into the content field as well. But this doesn't seem to
+				 * happen in the wild, so we don't cover it here.
+				 */
 				if (!empty($image['url'])) {
 					$image['url'] = self::completeUrl($image['url'], $page_url);
 					$photodata = Images::getInfoFromURLCached($image['url']);
-					if (!empty($photodata) && ($photodata[0] > 50) && ($photodata[1] > 50)) {
+					if (($photodata) && ($photodata[0] > 50) && ($photodata[1] > 50)) {
 						$image['src'] = $image['url'];
 						$image['width'] = $photodata[0];
 						$image['height'] = $photodata[1];
@@ -628,17 +645,19 @@ class ParseUrl
 	 * Convert tags from CSV to an array
 	 *
 	 * @param string $string Tags
+	 *
 	 * @return array with formatted Hashtags
 	 */
-	public static function convertTagsToArray($string)
+	public static function convertTagsToArray(string $string): array
 	{
 		$arr_tags = str_getcsv($string);
 		if (count($arr_tags)) {
 			// add the # sign to every tag
-			array_walk($arr_tags, ["self", "arrAddHashes"]);
+			array_walk($arr_tags, ['self', 'arrAddHashes']);
 
 			return $arr_tags;
 		}
+		return [];
 	}
 
 	/**
@@ -648,11 +667,12 @@ class ParseUrl
 	 *
 	 * @param string $tag The pure tag name
 	 * @param int    $k   Counter for internal use
+	 *
 	 * @return void
 	 */
-	private static function arrAddHashes(&$tag, $k)
+	private static function arrAddHashes(string &$tag, int $k)
 	{
-		$tag = "#" . $tag;
+		$tag = '#' . $tag;
 	}
 
 	/**
@@ -669,41 +689,41 @@ class ParseUrl
 	 *
 	 * @return string The url with a scheme
 	 */
-	private static function completeUrl($url, $scheme)
+	private static function completeUrl(string $url, string $scheme): string
 	{
 		$urlarr = parse_url($url);
 
 		// If the url does allready have an scheme
 		// we can stop the process here
-		if (isset($urlarr["scheme"])) {
-			return($url);
+		if (isset($urlarr['scheme'])) {
+			return $url;
 		}
 
 		$schemearr = parse_url($scheme);
 
-		$complete = $schemearr["scheme"]."://".$schemearr["host"];
+		$complete = $schemearr['scheme'] . '://' . $schemearr['host'];
 
-		if (!empty($schemearr["port"])) {
-			$complete .= ":".$schemearr["port"];
+		if (!empty($schemearr['port'])) {
+			$complete .= ':' . $schemearr['port'];
 		}
 
-		if (!empty($urlarr["path"])) {
-			if (strpos($urlarr["path"], "/") !== 0) {
-				$complete .= "/";
+		if (!empty($urlarr['path'])) {
+			if (strpos($urlarr['path'], '/') !== 0) {
+				$complete .= '/';
 			}
 
-			$complete .= $urlarr["path"];
+			$complete .= $urlarr['path'];
 		}
 
-		if (!empty($urlarr["query"])) {
-			$complete .= "?".$urlarr["query"];
+		if (!empty($urlarr['query'])) {
+			$complete .= '?' . $urlarr['query'];
 		}
 
-		if (!empty($urlarr["fragment"])) {
-			$complete .= "#".$urlarr["fragment"];
+		if (!empty($urlarr['fragment'])) {
+			$complete .= '#' . $urlarr['fragment'];
 		}
 
-		return($complete);
+		return $complete;
 	}
 
 	/**
@@ -711,9 +731,10 @@ class ParseUrl
 	 *
 	 * @param array $siteinfo
 	 * @param array $jsonld
+	 *
 	 * @return array siteinfo
 	 */
-	private static function parseParts(array $siteinfo, array $jsonld)
+	private static function parseParts(array $siteinfo, array $jsonld): array
 	{
 		if (!empty($jsonld['@graph']) && is_array($jsonld['@graph'])) {
 			foreach ($jsonld['@graph'] as $part) {
@@ -756,9 +777,10 @@ class ParseUrl
 	 *
 	 * @param array $siteinfo
 	 * @param array $jsonld
+	 *
 	 * @return array siteinfo
 	 */
-	private static function parseJsonLd(array $siteinfo, array $jsonld)
+	private static function parseJsonLd(array $siteinfo, array $jsonld): array
 	{
 		$type = JsonLD::fetchElement($jsonld, '@type');
 		if (empty($type)) {
@@ -849,9 +871,10 @@ class ParseUrl
 	 *
 	 * @param array $siteinfo
 	 * @param array $jsonld
+	 *
 	 * @return array siteinfo
 	 */
-	private static function parseJsonLdAuthor(array $siteinfo, array $jsonld)
+	private static function parseJsonLdAuthor(array $siteinfo, array $jsonld): array
 	{
 		$jsonldinfo = [];
 
@@ -933,9 +956,10 @@ class ParseUrl
 	 *
 	 * @param array $siteinfo
 	 * @param array $jsonld
+	 *
 	 * @return array siteinfo
 	 */
-	private static function parseJsonLdArticle(array $siteinfo, array $jsonld)
+	private static function parseJsonLdArticle(array $siteinfo, array $jsonld): array
 	{
 		$jsonldinfo = [];
 
@@ -1003,9 +1027,10 @@ class ParseUrl
 	 *
 	 * @param array $siteinfo
 	 * @param array $jsonld
+	 *
 	 * @return array siteinfo
 	 */
-	private static function parseJsonLdWebPage(array $siteinfo, array $jsonld)
+	private static function parseJsonLdWebPage(array $siteinfo, array $jsonld): array
 	{
 		$jsonldinfo = [];
 
@@ -1015,17 +1040,17 @@ class ParseUrl
 		}
 
 		$content = JsonLD::fetchElement($jsonld, 'description');
-		if (!empty($content)) {
+		if (!empty($content) && is_string($content)) {
 			$jsonldinfo['text'] = trim($content);
 		}
 
 		$content = JsonLD::fetchElement($jsonld, 'image');
-		if (!empty($content)) {
+		if (!empty($content) && is_string($content)) {
 			$jsonldinfo['image'] = trim($content);
 		}
 
 		$content = JsonLD::fetchElement($jsonld, 'thumbnailUrl');
-		if (!empty($content)) {
+		if (!empty($content) && is_string($content)) {
 			$jsonldinfo['image'] = trim($content);
 		}
 
@@ -1042,29 +1067,30 @@ class ParseUrl
 	 *
 	 * @param array $siteinfo
 	 * @param array $jsonld
+	 *
 	 * @return array siteinfo
 	 */
-	private static function parseJsonLdWebSite(array $siteinfo, array $jsonld)
+	private static function parseJsonLdWebSite(array $siteinfo, array $jsonld): array
 	{
 		$jsonldinfo = [];
 
 		$content = JsonLD::fetchElement($jsonld, 'name');
-		if (!empty($content)) {
+		if (!empty($content) && is_string($content)) {
 			$jsonldinfo['publisher_name'] = trim($content);
 		}
 
 		$content = JsonLD::fetchElement($jsonld, 'description');
-		if (!empty($content)) {
+		if (!empty($content) && is_string($content)) {
 			$jsonldinfo['publisher_description'] = trim($content);
 		}
 
 		$content = JsonLD::fetchElement($jsonld, 'url');
-		if (!empty($content)) {
+		if (!empty($content) && is_string($content)) {
 			$jsonldinfo['publisher_url'] = trim($content);
 		}
 
 		$content = JsonLD::fetchElement($jsonld, 'thumbnailUrl');
-		if (!empty($content)) {
+		if (!empty($content) && is_string($content)) {
 			$jsonldinfo['image'] = trim($content);
 		}
 
@@ -1080,39 +1106,40 @@ class ParseUrl
 	 *
 	 * @param array $siteinfo
 	 * @param array $jsonld
+	 *
 	 * @return array siteinfo
 	 */
-	private static function parseJsonLdWebOrganization(array $siteinfo, array $jsonld)
+	private static function parseJsonLdWebOrganization(array $siteinfo, array $jsonld): array
 	{
 		$jsonldinfo = [];
 
 		$content = JsonLD::fetchElement($jsonld, 'name');
-		if (!empty($content)) {
+		if (!empty($content) && is_string($content)) {
 			$jsonldinfo['publisher_name'] = trim($content);
 		}
 
 		$content = JsonLD::fetchElement($jsonld, 'description');
-		if (!empty($content)) {
+		if (!empty($content) && is_string($content)) {
 			$jsonldinfo['publisher_description'] = trim($content);
 		}
 
 		$content = JsonLD::fetchElement($jsonld, 'url');
-		if (!empty($content)) {
+		if (!empty($content) && is_string($content)) {
 			$jsonldinfo['publisher_url'] = trim($content);
 		}
 
 		$content = JsonLD::fetchElement($jsonld, 'logo', 'url', '@type', 'ImageObject');
-		if (!empty($content)) {
+		if (!empty($content) && is_string($content)) {
 			$jsonldinfo['publisher_img'] = trim($content);
 		}
 
 		$content = JsonLD::fetchElement($jsonld, 'brand', 'name', '@type', 'Organization');
-		if (!empty($content)) {
+		if (!empty($content) && is_string($content)) {
 			$jsonldinfo['publisher_name'] = trim($content);
 		}
 
 		$content = JsonLD::fetchElement($jsonld, 'brand', 'url', '@type', 'Organization');
-		if (!empty($content)) {
+		if (!empty($content) && is_string($content)) {
 			$jsonldinfo['publisher_url'] = trim($content);
 		}
 
@@ -1126,19 +1153,20 @@ class ParseUrl
 	 *
 	 * @param array $siteinfo
 	 * @param array $jsonld
+	 *
 	 * @return array siteinfo
 	 */
-	private static function parseJsonLdWebPerson(array $siteinfo, array $jsonld)
+	private static function parseJsonLdWebPerson(array $siteinfo, array $jsonld): array
 	{
 		$jsonldinfo = [];
 
 		$content = JsonLD::fetchElement($jsonld, 'name');
-		if (!empty($content)) {
+		if (!empty($content) && is_string($content)) {
 			$jsonldinfo['author_name'] = trim($content);
 		}
 
 		$content = JsonLD::fetchElement($jsonld, 'description');
-		if (!empty($content)) {
+		if (!empty($content) && is_string($content)) {
 			$jsonldinfo['author_description'] = trim($content);
 		}
 
@@ -1148,7 +1176,7 @@ class ParseUrl
 		}
 
 		$content = JsonLD::fetchElement($jsonld, 'url');
-		if (!empty($content)) {
+		if (!empty($content) && is_string($content)) {
 			$jsonldinfo['author_url'] = trim($content);
 		}
 
@@ -1171,29 +1199,30 @@ class ParseUrl
 	 *
 	 * @param array $siteinfo
 	 * @param array $jsonld
+	 *
 	 * @return array siteinfo
 	 */
-	private static function parseJsonLdMediaObject(array $siteinfo, array $jsonld, string $name)
+	private static function parseJsonLdMediaObject(array $siteinfo, array $jsonld, string $name): array
 	{
 		$media = [];
 
 		$content = JsonLD::fetchElement($jsonld, 'caption');
-		if (!empty($content)) {
+		if (!empty($content) && is_string($content)) {
 			$media['caption'] = trim($content);
 		}
 
 		$content = JsonLD::fetchElement($jsonld, 'url');
-		if (!empty($content)) {
+		if (!empty($content) && is_string($content)) {
 			$media['url'] = trim($content);
 		}
 
 		$content = JsonLD::fetchElement($jsonld, 'mainEntityOfPage');
-		if (!empty($content)) {
+		if (!empty($content) && is_string($content)) {
 			$media['main'] = Strings::compareLink($content, $siteinfo['url']);
 		}
 
 		$content = JsonLD::fetchElement($jsonld, 'description');
-		if (!empty($content)) {
+		if (!empty($content) && is_string($content)) {
 			$media['description'] = trim($content);
 		}
 
@@ -1203,27 +1232,27 @@ class ParseUrl
 		}
 
 		$content = JsonLD::fetchElement($jsonld, 'contentUrl');
-		if (!empty($content)) {
+		if (!empty($content) && is_string($content)) {
 			$media['content'] = trim($content);
 		}
 
 		$content = JsonLD::fetchElement($jsonld, 'embedUrl');
-		if (!empty($content)) {
+		if (!empty($content) && is_string($content)) {
 			$media['embed'] = trim($content);
 		}
 
 		$content = JsonLD::fetchElement($jsonld, 'height');
-		if (!empty($content)) {
+		if (!empty($content) && is_string($content)) {
 			$media['height'] = trim($content);
 		}
 
 		$content = JsonLD::fetchElement($jsonld, 'width');
-		if (!empty($content)) {
+		if (!empty($content) && is_string($content)) {
 			$media['width'] = trim($content);
 		}
 
 		$content = JsonLD::fetchElement($jsonld, 'image');
-		if (!empty($content)) {
+		if (!empty($content) && is_string($content)) {
 			$media['image'] = trim($content);
 		}