<?php
/**
- * @copyright Copyright (C) 2020, Friendica
+ * @copyright Copyright (C) 2010-2022, the Friendica project
*
* @license GNU AGPL version 3 or any later version
*
use Friendica\Database\Database;
use Friendica\Database\DBA;
use Friendica\DI;
+use Friendica\Network\HTTPClient\Client\HttpClientAccept;
use Friendica\Network\HTTPException;
+use Friendica\Network\HTTPClient\Client\HttpClientOptions;
/**
* Get information about a given URL
/**
* Fetch the content type of the given url
- * @param string $url URL of the page
- * @return array content type
+ * @param string $url URL of the page
+ * @param string $accept content-type to accept
+ * @return array content type
*/
- public static function getContentType(string $url)
+ public static function getContentType(string $url, string $accept = HttpClientAccept::DEFAULT): array
{
- $curlResult = DI::httpRequest()->head($url);
+ $curlResult = DI::httpClient()->head($url, [HttpClientOptions::ACCEPT_CONTENT => $accept]);
+
+ // Workaround for systems that can't handle a HEAD request
+ if (!$curlResult->isSuccess() && ($curlResult->getReturnCode() == 405)) {
+ $curlResult = DI::httpClient()->get($url, $accept, [HttpClientOptions::CONTENT_LENGTH => 1000000]);
+ }
+
if (!$curlResult->isSuccess()) {
+ Logger::debug('Got HTTP Error', ['http error' => $curlResult->getReturnCode(), 'url' => $url]);
return [];
}
- $contenttype = $curlResult->getHeader('Content-Type');
+ $contenttype = $curlResult->getHeader('Content-Type')[0] ?? '';
if (empty($contenttype)) {
- return [];
+ return ['application', 'octet-stream'];
}
return explode('/', current(explode(';', $contenttype)));
* @see ParseUrl::getSiteinfo() for more information about scraping
* embeddable content
*/
- public static function getSiteinfoCached($url, $do_oembed = true): array
+ public static function getSiteinfoCached(string $url, bool $do_oembed = true): array
{
if (empty($url)) {
return [
* </body>
* @endverbatim
*/
- public static function getSiteinfo($url, $do_oembed = true, $count = 1)
+ public static function getSiteinfo(string $url, bool $do_oembed = true, int $count = 1)
{
if (empty($url)) {
return [
];
if ($count > 10) {
- Logger::log('Endless loop detected for ' . $url, Logger::DEBUG);
+ Logger::notice('Endless loop detected', ['url' => $url]);
return $siteinfo;
}
return $siteinfo;
}
- $curlResult = DI::httpRequest()->get($url);
- if (!$curlResult->isSuccess()) {
+ $curlResult = DI::httpClient()->get($url, HttpClientAccept::HTML, [HttpClientOptions::CONTENT_LENGTH => 1000000]);
+ if (!$curlResult->isSuccess() || empty($curlResult->getBody())) {
+ Logger::info('Empty body or error when fetching', ['url' => $url, 'success' => $curlResult->isSuccess(), 'code' => $curlResult->getReturnCode()]);
return $siteinfo;
}
$siteinfo['expires'] = DateTimeFormat::utc(self::DEFAULT_EXPIRATION_SUCCESS);
- // If the file is too large then exit
- if (($curlResult->getInfo()['download_content_length'] ?? 0) > 1000000) {
- return $siteinfo;
- }
-
- if ($cacheControlHeader = $curlResult->getHeader('Cache-Control')) {
+ if ($cacheControlHeader = $curlResult->getHeader('Cache-Control')[0] ?? '') {
if (preg_match('/max-age=([0-9]+)/i', $cacheControlHeader, $matches)) {
$maxAge = max(86400, (int)array_pop($matches));
$siteinfo['expires'] = DateTimeFormat::utc("now + $maxAge seconds");
}
}
- $header = $curlResult->getHeader();
$body = $curlResult->getBody();
if ($do_oembed) {
$charset = '';
// Look for a charset, first in headers
// Expected form: Content-Type: text/html; charset=ISO-8859-4
- if (preg_match('/charset=([a-z0-9-_.\/]+)/i', $header, $matches)) {
+ if (preg_match('/charset=([a-z0-9-_.\/]+)/i', $curlResult->getContentType(), $matches)) {
$charset = trim(trim(trim(array_pop($matches)), ';,'));
}
// See https://github.com/friendica/friendica/issues/5470#issuecomment-418351211
$charset = str_ireplace('latin-1', 'latin1', $charset);
- Logger::log('detected charset ' . $charset, Logger::DEBUG);
+ Logger::info('detected charset', ['charset' => $charset]);
$body = iconv($charset, 'UTF-8//TRANSLIT', $body);
}
case 'twitter:image:src':
$siteinfo['image'] = $meta_tag['content'];
break;
- case 'twitter:card':
- // Detect photo pages
- if ($meta_tag['content'] == 'summary_large_image') {
- $siteinfo['type'] = 'photo';
- }
- break;
case 'twitter:description':
$siteinfo['text'] = trim($meta_tag['content']);
break;
case 'twitter:title':
$siteinfo['title'] = trim($meta_tag['content']);
break;
+ case 'twitter:player':
+ $siteinfo['player']['embed'] = trim($meta_tag['content']);
+ break;
+ case 'twitter:player:stream':
+ $siteinfo['player']['stream'] = trim($meta_tag['content']);
+ break;
+ case 'twitter:player:width':
+ $siteinfo['player']['width'] = intval($meta_tag['content']);
+ break;
+ case 'twitter:player:height':
+ $siteinfo['player']['height'] = intval($meta_tag['content']);
+ break;
case 'dc.title':
$siteinfo['title'] = trim($meta_tag['content']);
break;
case 'og:site_name':
$siteinfo['publisher_name'] = trim($meta_tag['content']);
break;
+ case 'og:locale':
+ $siteinfo['language'] = trim($meta_tag['content']);
+ break;
+ case 'og:type':
+ $siteinfo['pagetype'] = trim($meta_tag['content']);
+ break;
case 'twitter:description':
$siteinfo['text'] = trim($meta_tag['content']);
break;
}
}
- // Prevent to have a photo type without an image
- if ((empty($siteinfo['image']) || !empty($siteinfo['text'])) && ($siteinfo['type'] == 'photo')) {
- $siteinfo['type'] = 'link';
+ if (!empty($siteinfo['player']['stream'])) {
+ // Only add player data to media arrays if there is no duplicate
+ $content_urls = array_merge(array_column($siteinfo['audio'] ?? [], 'content'), array_column($siteinfo['video'] ?? [], 'content'));
+ if (!in_array($siteinfo['player']['stream'], $content_urls)) {
+ $contenttype = self::getContentType($siteinfo['player']['stream']);
+ if (!empty($contenttype[0]) && in_array($contenttype[0], ['audio', 'video'])) {
+ $media = ['content' => $siteinfo['player']['stream']];
+
+ if (!empty($siteinfo['player']['embed'])) {
+ $media['embed'] = $siteinfo['player']['embed'];
+ }
+
+ $siteinfo[$contenttype[0]][] = $media;
+ }
+ }
}
if (!empty($siteinfo['image'])) {
Hook::callAll('getsiteinfo', $siteinfo);
+ ksort($siteinfo);
+
return $siteinfo;
}
*
* @param string $page_url
* @param array $siteinfo
- * @return void
+ * @return array
*/
- private static function checkMedia(string $page_url, array $siteinfo)
+ private static function checkMedia(string $page_url, array $siteinfo) : array
{
if (!empty($siteinfo['images'])) {
array_walk($siteinfo['images'], function (&$image) use ($page_url) {
- // According to the specifications someone could place a picture url into the content field as well.
- // But this doesn't seem to happen in the wild, so we don't cover it here.
- $image['url'] = self::completeUrl($image['url'], $page_url);
- $photodata = Images::getInfoFromURLCached($image['url']);
- if (!empty($photodata) && ($photodata[0] > 50) && ($photodata[1] > 50)) {
- $image['src'] = $image['url'];
- $image['width'] = $photodata[0];
- $image['height'] = $photodata[1];
- $image['contenttype'] = $photodata['mime'];
- unset($image['url']);
- ksort($image);
+ /*
+ * According to the specifications someone could place a picture
+ * URL into the content field as well. But this doesn't seem to
+ * happen in the wild, so we don't cover it here.
+ */
+ if (!empty($image['url'])) {
+ $image['url'] = self::completeUrl($image['url'], $page_url);
+ $photodata = Images::getInfoFromURLCached($image['url']);
+ if (!empty($photodata) && ($photodata[0] > 50) && ($photodata[1] > 50)) {
+ $image['src'] = $image['url'];
+ $image['width'] = $photodata[0];
+ $image['height'] = $photodata[1];
+ $image['contenttype'] = $photodata['mime'];
+ unset($image['url']);
+ ksort($image);
+ } else {
+ $image = [];
+ }
} else {
$image = [];
}
if (!empty($media[$field])) {
$media[$field] = self::completeUrl($media[$field], $page_url);
$type = self::getContentType($media[$field]);
- if ($type[0] == 'text') {
+ if (($type[0] ?? '') == 'text') {
if ($field == 'embed') {
$embed = $media[$field];
} else {
}
if (!empty($embed)) {
$media['embed'] = $embed;
- if (!empty($media['main'])) {
- $siteinfo['embed'] = $embed;
+ if (empty($siteinfo['player']['embed'])) {
+ $siteinfo['player']['embed'] = $embed;
}
}
if (!empty($content)) {
* @param string $string Tags
* @return array with formatted Hashtags
*/
- public static function convertTagsToArray($string)
+ public static function convertTagsToArray(string $string): array
{
$arr_tags = str_getcsv($string);
if (count($arr_tags)) {
// add the # sign to every tag
- array_walk($arr_tags, ["self", "arrAddHashes"]);
+ array_walk($arr_tags, ['self', 'arrAddHashes']);
return $arr_tags;
}
+ return [];
}
/**
* @param int $k Counter for internal use
* @return void
*/
- private static function arrAddHashes(&$tag, $k)
+ private static function arrAddHashes(string &$tag, int $k)
{
- $tag = "#" . $tag;
+ $tag = '#' . $tag;
}
/**
*
* @return string The url with a scheme
*/
- private static function completeUrl($url, $scheme)
+ private static function completeUrl(string $url, string $scheme): string
{
$urlarr = parse_url($url);
// If the url does allready have an scheme
// we can stop the process here
- if (isset($urlarr["scheme"])) {
- return($url);
+ if (isset($urlarr['scheme'])) {
+ return $url;
}
$schemearr = parse_url($scheme);
- $complete = $schemearr["scheme"]."://".$schemearr["host"];
+ $complete = $schemearr['scheme'] . '://' . $schemearr['host'];
- if (!empty($schemearr["port"])) {
- $complete .= ":".$schemearr["port"];
+ if (!empty($schemearr['port'])) {
+ $complete .= ':' . $schemearr['port'];
}
- if (!empty($urlarr["path"])) {
- if (strpos($urlarr["path"], "/") !== 0) {
- $complete .= "/";
+ if (!empty($urlarr['path'])) {
+ if (strpos($urlarr['path'], '/') !== 0) {
+ $complete .= '/';
}
- $complete .= $urlarr["path"];
+ $complete .= $urlarr['path'];
}
- if (!empty($urlarr["query"])) {
- $complete .= "?".$urlarr["query"];
+ if (!empty($urlarr['query'])) {
+ $complete .= '?' . $urlarr['query'];
}
- if (!empty($urlarr["fragment"])) {
- $complete .= "#".$urlarr["fragment"];
+ if (!empty($urlarr['fragment'])) {
+ $complete .= '#' . $urlarr['fragment'];
}
- return($complete);
+ return $complete;
}
/**
* @param array $jsonld
* @return array siteinfo
*/
- private static function parseParts(array $siteinfo, array $jsonld)
+ private static function parseParts(array $siteinfo, array $jsonld): array
{
if (!empty($jsonld['@graph']) && is_array($jsonld['@graph'])) {
foreach ($jsonld['@graph'] as $part) {
- $siteinfo = self::parseParts($siteinfo, $part);
+ if (!empty($part) && is_array($part)) {
+ $siteinfo = self::parseParts($siteinfo, $part);
+ }
}
} elseif (!empty($jsonld['@type'])) {
$siteinfo = self::parseJsonLd($siteinfo, $jsonld);
}
if ($numeric_keys) {
foreach ($jsonld as $part) {
- $siteinfo = self::parseParts($siteinfo, $part);
+ if (!empty($part) && is_array($part)) {
+ $siteinfo = self::parseParts($siteinfo, $part);
+ }
}
}
}
array_walk_recursive($siteinfo, function (&$element) {
if (is_string($element)) {
- $element = html_entity_decode($element, ENT_COMPAT, 'UTF-8');
+ $element = trim(strip_tags(html_entity_decode($element, ENT_COMPAT, 'UTF-8')));
}
});
* @param array $jsonld
* @return array siteinfo
*/
- private static function parseJsonLd(array $siteinfo, array $jsonld)
+ private static function parseJsonLd(array $siteinfo, array $jsonld): array
{
$type = JsonLD::fetchElement($jsonld, '@type');
if (empty($type)) {
case 'QAPage':
case 'RealEstateListing':
case 'SearchResultsPage':
- case 'MediaGallery':
+ case 'MediaGallery':
case 'ImageGallery':
case 'VideoGallery':
case 'RadioEpisode':
case 'PerformingGroup':
case 'DanceGroup';
case 'MusicGroup':
- case 'TheaterGroup':
+ case 'TheaterGroup':
return self::parseJsonLdWebPerson($siteinfo, $jsonld);
case 'AudioObject':
case 'Audio':
* @param array $jsonld
* @return array siteinfo
*/
- private static function parseJsonLdAuthor(array $siteinfo, array $jsonld)
+ private static function parseJsonLdAuthor(array $siteinfo, array $jsonld): array
{
$jsonldinfo = [];
* @param array $jsonld
* @return array siteinfo
*/
- private static function parseJsonLdArticle(array $siteinfo, array $jsonld)
+ private static function parseJsonLdArticle(array $siteinfo, array $jsonld): array
{
$jsonldinfo = [];
$siteinfo['keywords'][] = trim($keyword);
}
}
- } else {
+ } elseif (!empty($jsonld['keywords'])) {
$content = JsonLD::fetchElementArray($jsonld, 'keywords');
if (!empty($content) && is_array($content)) {
$jsonldinfo['keywords'] = $content;
}
}
+ $content = JsonLD::fetchElement($jsonld, 'datePublished');
+ if (!empty($content) && is_string($content)) {
+ $jsonldinfo['published'] = DateTimeFormat::utc($content);
+ }
+
+ $content = JsonLD::fetchElement($jsonld, 'dateModified');
+ if (!empty($content) && is_string($content)) {
+ $jsonldinfo['modified'] = DateTimeFormat::utc($content);
+ }
+
$jsonldinfo = self::parseJsonLdAuthor($jsonldinfo, $jsonld);
Logger::info('Fetched article information', ['url' => $siteinfo['url'], 'fetched' => $jsonldinfo]);
* @param array $jsonld
* @return array siteinfo
*/
- private static function parseJsonLdWebPage(array $siteinfo, array $jsonld)
+ private static function parseJsonLdWebPage(array $siteinfo, array $jsonld): array
{
$jsonldinfo = [];
* @param array $jsonld
* @return array siteinfo
*/
- private static function parseJsonLdWebSite(array $siteinfo, array $jsonld)
+ private static function parseJsonLdWebSite(array $siteinfo, array $jsonld): array
{
$jsonldinfo = [];
* @param array $jsonld
* @return array siteinfo
*/
- private static function parseJsonLdWebOrganization(array $siteinfo, array $jsonld)
+ private static function parseJsonLdWebOrganization(array $siteinfo, array $jsonld): array
{
$jsonldinfo = [];
* @param array $jsonld
* @return array siteinfo
*/
- private static function parseJsonLdWebPerson(array $siteinfo, array $jsonld)
+ private static function parseJsonLdWebPerson(array $siteinfo, array $jsonld): array
{
$jsonldinfo = [];
}
$content = JsonLD::fetchElement($jsonld, 'image', 'url', '@type', 'ImageObject');
- if (!empty($content)) {
+ if (!empty($content) && !is_string($content)) {
+ Logger::notice('Unexpected return value for the author image', ['content' => $content]);
+ }
+
+ if (!empty($content) && is_string($content)) {
$jsonldinfo['author_img'] = trim($content);
}
* @param array $jsonld
* @return array siteinfo
*/
- private static function parseJsonLdMediaObject(array $siteinfo, array $jsonld, string $name)
+ private static function parseJsonLdMediaObject(array $siteinfo, array $jsonld, string $name): array
{
$media = [];