<?php
/**
- * @copyright Copyright (C) 2020, Friendica
+ * @copyright Copyright (C) 2010-2021, the Friendica project
*
* @license GNU AGPL version 3 or any later version
*
/**
* Fetch the content type of the given url
* @param string $url URL of the page
- * @return array content type
+ * @return array content type
*/
public static function getContentType(string $url)
{
];
if ($count > 10) {
- Logger::log('Endless loop detected for ' . $url, Logger::DEBUG);
+ Logger::notice('Endless loop detected', ['url' => $url]);
return $siteinfo;
}
}
$curlResult = DI::httpRequest()->get($url);
- if (!$curlResult->isSuccess()) {
+ if (!$curlResult->isSuccess() || empty($curlResult->getBody())) {
return $siteinfo;
}
// See https://github.com/friendica/friendica/issues/5470#issuecomment-418351211
$charset = str_ireplace('latin-1', 'latin1', $charset);
- Logger::log('detected charset ' . $charset, Logger::DEBUG);
+ Logger::info('detected charset', ['charset' => $charset]);
$body = iconv($charset, 'UTF-8//TRANSLIT', $body);
}
$siteinfo['title'] = trim($list->item(0)->nodeValue);
}
+ $twitter_card = false;
+ $twitter_image = false;
+
$list = $xpath->query('//meta[@name]');
foreach ($list as $node) {
$meta_tag = [];
break;
case 'twitter:image':
$siteinfo['image'] = $meta_tag['content'];
+ $twitter_image = true;
break;
case 'twitter:image:src':
$siteinfo['image'] = $meta_tag['content'];
case 'twitter:card':
// Detect photo pages
if ($meta_tag['content'] == 'summary_large_image') {
- $siteinfo['type'] = 'photo';
+ $twitter_card = true;
}
break;
case 'twitter:description':
break;
case 'twitter:image':
$siteinfo['image'] = $meta_tag['content'];
+ $twitter_image = true;
break;
}
}
$list = $xpath->query("//script[@type='application/ld+json']");
foreach ($list as $node) {
if (!empty($node->nodeValue)) {
- $nodevalue = html_entity_decode($node->nodeValue, ENT_COMPAT, 'UTF-8');
- if ($jsonld = json_decode($nodevalue, true)) {
+ if ($jsonld = json_decode($node->nodeValue, true)) {
$siteinfo = self::parseParts($siteinfo, $jsonld);
}
}
}
+// Currently deactivated, see https://github.com/friendica/friendica/pull/10148#issuecomment-821512658
// Prevent to have a photo type without an image
- if ((empty($siteinfo['image']) || !empty($siteinfo['text'])) && ($siteinfo['type'] == 'photo')) {
- $siteinfo['type'] = 'link';
- }
-
- if (!empty($siteinfo['image']) && empty($siteinfo['images'])) {
- $src = self::completeUrl($siteinfo['image'], $url);
+// if ($twitter_card && $twitter_image && !empty($siteinfo['image'])) {
+// $siteinfo['type'] = 'photo';
+// }
+ if (!empty($siteinfo['image'])) {
+ $siteinfo['images'] = $siteinfo['images'] ?? [];
+ array_unshift($siteinfo['images'], ['url' => $siteinfo['image']]);
unset($siteinfo['image']);
-
- $photodata = Images::getInfoFromURLCached($src);
-
- if (($photodata) && ($photodata[0] > 10) && ($photodata[1] > 10)) {
- $siteinfo['images'][] = ['src' => $src,
- 'width' => $photodata[0],
- 'height' => $photodata[1]];
- }
}
+ $siteinfo = self::checkMedia($url, $siteinfo);
+
if (!empty($siteinfo['text']) && mb_strlen($siteinfo['text']) > self::MAX_DESC_COUNT) {
$siteinfo['text'] = mb_substr($siteinfo['text'], 0, self::MAX_DESC_COUNT) . '…';
$pos = mb_strrpos($siteinfo['text'], '.');
return $siteinfo;
}
+ /**
+ * Check the attached media elements.
+ * Fix existing data and add missing data.
+ *
+ * @param string $page_url
+ * @param array $siteinfo
+ * @return void
+ */
+ private static function checkMedia(string $page_url, array $siteinfo)
+ {
+ if (!empty($siteinfo['images'])) {
+ array_walk($siteinfo['images'], function (&$image) use ($page_url) {
+ // According to the specifications someone could place a picture url into the content field as well.
+ // But this doesn't seem to happen in the wild, so we don't cover it here.
+ if (!empty($image['url'])) {
+ $image['url'] = self::completeUrl($image['url'], $page_url);
+ $photodata = Images::getInfoFromURLCached($image['url']);
+ if (!empty($photodata) && ($photodata[0] > 50) && ($photodata[1] > 50)) {
+ $image['src'] = $image['url'];
+ $image['width'] = $photodata[0];
+ $image['height'] = $photodata[1];
+ $image['contenttype'] = $photodata['mime'];
+ unset($image['url']);
+ ksort($image);
+ } else {
+ $image = [];
+ }
+ } else {
+ $image = [];
+ }
+ });
+
+ $siteinfo['images'] = array_values(array_filter($siteinfo['images']));
+ }
+
+ foreach (['audio', 'video'] as $element) {
+ if (!empty($siteinfo[$element])) {
+ array_walk($siteinfo[$element], function (&$media) use ($page_url, &$siteinfo) {
+ $url = '';
+ $embed = '';
+ $content = '';
+ $contenttype = '';
+ foreach (['embed', 'content', 'url'] as $field) {
+ if (!empty($media[$field])) {
+ $media[$field] = self::completeUrl($media[$field], $page_url);
+ $type = self::getContentType($media[$field]);
+ if (($type[0] ?? '') == 'text') {
+ if ($field == 'embed') {
+ $embed = $media[$field];
+ } else {
+ $url = $media[$field];
+ }
+ } elseif (!empty($type[0])) {
+ $content = $media[$field];
+ $contenttype = implode('/', $type);
+ }
+ }
+ unset($media[$field]);
+ }
+
+ foreach (['image', 'preview'] as $field) {
+ if (!empty($media[$field])) {
+ $media[$field] = self::completeUrl($media[$field], $page_url);
+ }
+ }
+
+ if (!empty($url)) {
+ $media['url'] = $url;
+ }
+ if (!empty($embed)) {
+ $media['embed'] = $embed;
+ if (!empty($media['main'])) {
+ $siteinfo['embed'] = $embed;
+ }
+ }
+ if (!empty($content)) {
+ $media['src'] = $content;
+ }
+ if (!empty($contenttype)) {
+ $media['contenttype'] = $contenttype;
+ }
+ if (empty($url) && empty($content) && empty($embed)) {
+ $media = [];
+ }
+ ksort($media);
+ });
+
+ $siteinfo[$element] = array_values(array_filter($siteinfo[$element]));
+ }
+ if (empty($siteinfo[$element])) {
+ unset($siteinfo[$element]);
+ }
+ }
+ return $siteinfo;
+ }
+
/**
* Convert tags from CSV to an array
*
{
if (!empty($jsonld['@graph']) && is_array($jsonld['@graph'])) {
foreach ($jsonld['@graph'] as $part) {
- $siteinfo = self::parseJsonLd($siteinfo, $part);
+ if (!empty($part) && is_array($part)) {
+ $siteinfo = self::parseParts($siteinfo, $part);
+ }
}
} elseif (!empty($jsonld['@type'])) {
$siteinfo = self::parseJsonLd($siteinfo, $jsonld);
}
if ($numeric_keys) {
foreach ($jsonld as $part) {
- $siteinfo = self::parseParts($siteinfo, $part);
- }
+ if (!empty($part) && is_array($part)) {
+ $siteinfo = self::parseParts($siteinfo, $part);
+ }
+ }
}
}
+ array_walk_recursive($siteinfo, function (&$element) {
+ if (is_string($element)) {
+ $element = trim(strip_tags(html_entity_decode($element, ENT_COMPAT, 'UTF-8')));
+ }
+ });
+
return $siteinfo;
}
// Silently ignore some types that aren't processed
if (in_array($type, ['SiteNavigationElement', 'JobPosting', 'CreativeWork', 'MusicAlbum',
- 'WPHeader', 'WPSideBar', 'WPFooter', 'LegalService',
+ 'WPHeader', 'WPSideBar', 'WPFooter', 'LegalService', 'MusicRecording',
'ItemList', 'BreadcrumbList', 'Blog', 'Dataset', 'Product'])) {
return $siteinfo;
}
case 'QAPage':
case 'RealEstateListing':
case 'SearchResultsPage':
- case 'MediaGallery':
+ case 'MediaGallery':
case 'ImageGallery':
case 'VideoGallery':
case 'RadioEpisode':
case 'PerformingGroup':
case 'DanceGroup';
case 'MusicGroup':
- case 'TheaterGroup':
+ case 'TheaterGroup':
return self::parseJsonLdWebPerson($siteinfo, $jsonld);
case 'AudioObject':
case 'Audio':
$jsonldinfo['publisher_name'] = trim($content);
}
- $content = JsonLD::fetchElement($jsonld, 'publisher', 'sameAs');
- if (!empty($content) && is_string($content)) {
- $jsonldinfo['publisher_url'] = trim($content);
- }
-
$content = JsonLD::fetchElement($jsonld, 'publisher', 'url');
if (!empty($content) && is_string($content)) {
$jsonldinfo['publisher_url'] = trim($content);
$brand = JsonLD::fetchElement($jsonld, 'publisher', 'brand', '@type', 'Organization');
if (!empty($brand) && is_array($brand)) {
- $content = JsonLD::fetchElement($brand, 'name', '@type', 'brand');
+ $content = JsonLD::fetchElement($brand, 'name');
if (!empty($content) && is_string($content)) {
$jsonldinfo['publisher_name'] = trim($content);
}
- $content = JsonLD::fetchElement($brand, 'url', '@type', 'brand');
+
+ $content = JsonLD::fetchElement($brand, 'url');
if (!empty($content) && is_string($content)) {
$jsonldinfo['publisher_url'] = trim($content);
}
+
+ $content = JsonLD::fetchElement($brand, 'logo', 'url');
+ if (!empty($content) && is_string($content)) {
+ $jsonldinfo['publisher_img'] = trim($content);
+ }
+ }
+
+ $logo = JsonLD::fetchElement($jsonld, 'publisher', 'logo');
+ if (!empty($logo) && is_array($logo)) {
+ $content = JsonLD::fetchElement($logo, 'url');
+ if (!empty($content) && is_string($content)) {
+ $jsonldinfo['publisher_img'] = trim($content);
+ }
}
} elseif (!empty($jsonld['publisher']) && is_string($jsonld['publisher'])) {
$jsonldinfo['publisher_name'] = trim($jsonld['publisher']);
if (!empty($content) && is_string($content)) {
$jsonldinfo['author_url'] = trim($content);
}
+
+ $logo = JsonLD::fetchElement($jsonld, 'author', 'logo');
+ if (!empty($logo) && is_array($logo)) {
+ $content = JsonLD::fetchElement($logo, 'url');
+ if (!empty($content) && is_string($content)) {
+ $jsonldinfo['author_img'] = trim($content);
+ }
+ }
} elseif (!empty($jsonld['author']) && is_string($jsonld['author'])) {
$jsonldinfo['author_name'] = trim($jsonld['author']);
}
$siteinfo['keywords'][] = trim($keyword);
}
}
- } else {
+ } elseif (!empty($jsonld['keywords'])) {
$content = JsonLD::fetchElementArray($jsonld, 'keywords');
if (!empty($content) && is_array($content)) {
$jsonldinfo['keywords'] = $content;
$jsonldinfo['publisher_description'] = trim($content);
}
- $content = JsonLD::fetchElement($jsonld, 'sameAs');
- if (!empty($content) && is_string($content)) {
- $jsonldinfo['publisher_url'] = trim($content);
- }
-
$content = JsonLD::fetchElement($jsonld, 'url');
if (!empty($content)) {
$jsonldinfo['publisher_url'] = trim($content);
$content = JsonLD::fetchElement($jsonld, 'url');
if (!empty($content)) {
- $media['src'] = trim($content);
+ $media['url'] = trim($content);
+ }
+
+ $content = JsonLD::fetchElement($jsonld, 'mainEntityOfPage');
+ if (!empty($content)) {
+ $media['main'] = Strings::compareLink($content, $siteinfo['url']);
}
$content = JsonLD::fetchElement($jsonld, 'description');
$media['width'] = trim($content);
}
- $content = JsonLD::fetchElement($jsonld, 'thumbnailUrl');
- if (!empty($content)) {
- $media['preview'] = trim($content);
- }
-
$content = JsonLD::fetchElement($jsonld, 'image');
if (!empty($content)) {
$media['image'] = trim($content);
}
+ $content = JsonLD::fetchElement($jsonld, 'thumbnailUrl');
+ if (!empty($content) && (($media['image'] ?? '') != trim($content))) {
+ if (!empty($media['image'])) {
+ $media['preview'] = trim($content);
+ } else {
+ $media['image'] = trim($content);
+ }
+ }
+
Logger::info('Fetched Media information', ['url' => $siteinfo['url'], 'fetched' => $media]);
$siteinfo[$name][] = $media;
return $siteinfo;
projects / friendica.git / blobdiff