* Fetch the content type of the given url
* @param string $url URL of the page
* @param string $accept content-type to accept
+ * @param int $timeout
* @return array content type
*/
- public static function getContentType(string $url, string $accept = HttpClientAccept::DEFAULT)
+ public static function getContentType(string $url, string $accept = HttpClientAccept::DEFAULT, int $timeout = 0): array
{
- $curlResult = DI::httpClient()->head($url, [HttpClientOptions::ACCEPT_CONTENT => $accept]);
+ if (!empty($timeout)) {
+ $options = [HttpClientOptions::TIMEOUT => $timeout];
+ } else {
+ $options = [];
+ }
+
+ $curlResult = DI::httpClient()->head($url, array_merge([HttpClientOptions::ACCEPT_CONTENT => $accept], $options));
- // Workaround for systems that can't handle a HEAD request
- if (!$curlResult->isSuccess() && ($curlResult->getReturnCode() == 405)) {
- $curlResult = DI::httpClient()->get($url, $accept, [HttpClientOptions::CONTENT_LENGTH => 1000000]);
+ // Workaround for systems that can't handle a HEAD request. Don't retry on timeouts.
+ if (!$curlResult->isSuccess() && ($curlResult->getReturnCode() >= 400) && !in_array($curlResult->getReturnCode(), [408, 504])) {
+ $curlResult = DI::httpClient()->get($url, $accept, array_merge([HttpClientOptions::CONTENT_LENGTH => 1000000], $options));
}
if (!$curlResult->isSuccess()) {
* @see ParseUrl::getSiteinfo() for more information about scraping
* embeddable content
*/
- public static function getSiteinfoCached($url, $do_oembed = true): array
+ public static function getSiteinfoCached(string $url, bool $do_oembed = true): array
{
if (empty($url)) {
return [
* </body>
* @endverbatim
*/
- public static function getSiteinfo($url, $do_oembed = true, $count = 1)
+ public static function getSiteinfo(string $url, bool $do_oembed = true, int $count = 1): array
{
if (empty($url)) {
return [
];
if ($count > 10) {
- Logger::notice('Endless loop detected', ['url' => $url]);
+ Logger::warning('Endless loop detected', ['url' => $url]);
return $siteinfo;
}
// Expected form: Content-Type: text/html; charset=ISO-8859-4
if (preg_match('/charset=([a-z0-9-_.\/]+)/i', $curlResult->getContentType(), $matches)) {
$charset = trim(trim(trim(array_pop($matches)), ';,'));
+ } else {
+ // Then in body that gets precedence
+ // Expected forms:
+ // - <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+ // - <meta charset="utf-8">
+ // - <meta charset=utf-8>
+ // - <meta charSet="utf-8">
+ // We escape <style> and <script> tags since they can contain irrelevant charset information
+ // (see https://github.com/friendica/friendica/issues/9251#issuecomment-698636806)
+ Strings::performWithEscapedBlocks($body, '#<(?:style|script).*?</(?:style|script)>#ism', function ($body) use (&$charset) {
+ if (preg_match('/charset=["\']?([a-z0-9-_.\/]+)/i', $body, $matches)) {
+ $charset = trim(trim(trim(array_pop($matches)), ';,'));
+ }
+ });
}
- // Then in body that gets precedence
- // Expected forms:
- // - <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
- // - <meta charset="utf-8">
- // - <meta charset=utf-8>
- // - <meta charSet="utf-8">
- // We escape <style> and <script> tags since they can contain irrelevant charset information
- // (see https://github.com/friendica/friendica/issues/9251#issuecomment-698636806)
- Strings::performWithEscapedBlocks($body, '#<(?:style|script).*?</(?:style|script)>#ism', function ($body) use (&$charset) {
- if (preg_match('/charset=["\']?([a-z0-9-_.\/]+)/i', $body, $matches)) {
- $charset = trim(trim(trim(array_pop($matches)), ';,'));
- }
- });
-
$siteinfo['charset'] = $charset;
if ($charset && strtoupper($charset) != 'UTF-8') {
$body = mb_convert_encoding($body, 'HTML-ENTITIES', 'UTF-8');
+ if (empty($body)) {
+ return $siteinfo;
+ }
+
$doc = new DOMDocument();
@$doc->loadHTML($body);
{
if (!empty($siteinfo['images'])) {
array_walk($siteinfo['images'], function (&$image) use ($page_url) {
- // According to the specifications someone could place a picture url into the content field as well.
- // But this doesn't seem to happen in the wild, so we don't cover it here.
+ /*
+ * According to the specifications someone could place a picture
+ * URL into the content field as well. But this doesn't seem to
+ * happen in the wild, so we don't cover it here.
+ */
if (!empty($image['url'])) {
$image['url'] = self::completeUrl($image['url'], $page_url);
$photodata = Images::getInfoFromURLCached($image['url']);
- if (!empty($photodata) && ($photodata[0] > 50) && ($photodata[1] > 50)) {
+ if (($photodata) && ($photodata[0] > 50) && ($photodata[1] > 50)) {
$image['src'] = $image['url'];
$image['width'] = $photodata[0];
$image['height'] = $photodata[1];
* Convert tags from CSV to an array
*
* @param string $string Tags
+ *
* @return array with formatted Hashtags
*/
- public static function convertTagsToArray($string)
+ public static function convertTagsToArray(string $string): array
{
$arr_tags = str_getcsv($string);
if (count($arr_tags)) {
// add the # sign to every tag
- array_walk($arr_tags, ["self", "arrAddHashes"]);
+ array_walk($arr_tags, ['self', 'arrAddHashes']);
return $arr_tags;
}
+ return [];
}
/**
*
* @param string $tag The pure tag name
* @param int $k Counter for internal use
+ *
* @return void
*/
- private static function arrAddHashes(&$tag, $k)
+ private static function arrAddHashes(string &$tag, int $k)
{
- $tag = "#" . $tag;
+ $tag = '#' . $tag;
}
/**
*
* @return string The url with a scheme
*/
- private static function completeUrl($url, $scheme)
+ private static function completeUrl(string $url, string $scheme): string
{
$urlarr = parse_url($url);
// If the url does allready have an scheme
// we can stop the process here
- if (isset($urlarr["scheme"])) {
- return($url);
+ if (isset($urlarr['scheme'])) {
+ return $url;
}
$schemearr = parse_url($scheme);
- $complete = $schemearr["scheme"]."://".$schemearr["host"];
+ $complete = $schemearr['scheme'] . '://' . $schemearr['host'];
- if (!empty($schemearr["port"])) {
- $complete .= ":".$schemearr["port"];
+ if (!empty($schemearr['port'])) {
+ $complete .= ':' . $schemearr['port'];
}
- if (!empty($urlarr["path"])) {
- if (strpos($urlarr["path"], "/") !== 0) {
- $complete .= "/";
+ if (!empty($urlarr['path'])) {
+ if (strpos($urlarr['path'], '/') !== 0) {
+ $complete .= '/';
}
- $complete .= $urlarr["path"];
+ $complete .= $urlarr['path'];
}
- if (!empty($urlarr["query"])) {
- $complete .= "?".$urlarr["query"];
+ if (!empty($urlarr['query'])) {
+ $complete .= '?' . $urlarr['query'];
}
- if (!empty($urlarr["fragment"])) {
- $complete .= "#".$urlarr["fragment"];
+ if (!empty($urlarr['fragment'])) {
+ $complete .= '#' . $urlarr['fragment'];
}
- return($complete);
+ return $complete;
}
/**
*
* @param array $siteinfo
* @param array $jsonld
+ *
* @return array siteinfo
*/
- private static function parseParts(array $siteinfo, array $jsonld)
+ private static function parseParts(array $siteinfo, array $jsonld): array
{
if (!empty($jsonld['@graph']) && is_array($jsonld['@graph'])) {
foreach ($jsonld['@graph'] as $part) {
*
* @param array $siteinfo
* @param array $jsonld
+ *
* @return array siteinfo
*/
- private static function parseJsonLd(array $siteinfo, array $jsonld)
+ private static function parseJsonLd(array $siteinfo, array $jsonld): array
{
$type = JsonLD::fetchElement($jsonld, '@type');
if (empty($type)) {
*
* @param array $siteinfo
* @param array $jsonld
+ *
* @return array siteinfo
*/
- private static function parseJsonLdAuthor(array $siteinfo, array $jsonld)
+ private static function parseJsonLdAuthor(array $siteinfo, array $jsonld): array
{
$jsonldinfo = [];
*
* @param array $siteinfo
* @param array $jsonld
+ *
* @return array siteinfo
*/
- private static function parseJsonLdArticle(array $siteinfo, array $jsonld)
+ private static function parseJsonLdArticle(array $siteinfo, array $jsonld): array
{
$jsonldinfo = [];
*
* @param array $siteinfo
* @param array $jsonld
+ *
* @return array siteinfo
*/
- private static function parseJsonLdWebPage(array $siteinfo, array $jsonld)
+ private static function parseJsonLdWebPage(array $siteinfo, array $jsonld): array
{
$jsonldinfo = [];
}
$content = JsonLD::fetchElement($jsonld, 'description');
- if (!empty($content)) {
+ if (!empty($content) && is_string($content)) {
$jsonldinfo['text'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'image');
- if (!empty($content)) {
+ if (!empty($content) && is_string($content)) {
$jsonldinfo['image'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'thumbnailUrl');
- if (!empty($content)) {
+ if (!empty($content) && is_string($content)) {
$jsonldinfo['image'] = trim($content);
}
*
* @param array $siteinfo
* @param array $jsonld
+ *
* @return array siteinfo
*/
- private static function parseJsonLdWebSite(array $siteinfo, array $jsonld)
+ private static function parseJsonLdWebSite(array $siteinfo, array $jsonld): array
{
$jsonldinfo = [];
$content = JsonLD::fetchElement($jsonld, 'name');
- if (!empty($content)) {
+ if (!empty($content) && is_string($content)) {
$jsonldinfo['publisher_name'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'description');
- if (!empty($content)) {
+ if (!empty($content) && is_string($content)) {
$jsonldinfo['publisher_description'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'url');
- if (!empty($content)) {
+ if (!empty($content) && is_string($content)) {
$jsonldinfo['publisher_url'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'thumbnailUrl');
- if (!empty($content)) {
+ if (!empty($content) && is_string($content)) {
$jsonldinfo['image'] = trim($content);
}
*
* @param array $siteinfo
* @param array $jsonld
+ *
* @return array siteinfo
*/
- private static function parseJsonLdWebOrganization(array $siteinfo, array $jsonld)
+ private static function parseJsonLdWebOrganization(array $siteinfo, array $jsonld): array
{
$jsonldinfo = [];
$content = JsonLD::fetchElement($jsonld, 'name');
- if (!empty($content)) {
+ if (!empty($content) && is_string($content)) {
$jsonldinfo['publisher_name'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'description');
- if (!empty($content)) {
+ if (!empty($content) && is_string($content)) {
$jsonldinfo['publisher_description'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'url');
- if (!empty($content)) {
+ if (!empty($content) && is_string($content)) {
$jsonldinfo['publisher_url'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'logo', 'url', '@type', 'ImageObject');
- if (!empty($content)) {
+ if (!empty($content) && is_string($content)) {
$jsonldinfo['publisher_img'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'brand', 'name', '@type', 'Organization');
- if (!empty($content)) {
+ if (!empty($content) && is_string($content)) {
$jsonldinfo['publisher_name'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'brand', 'url', '@type', 'Organization');
- if (!empty($content)) {
+ if (!empty($content) && is_string($content)) {
$jsonldinfo['publisher_url'] = trim($content);
}
*
* @param array $siteinfo
* @param array $jsonld
+ *
* @return array siteinfo
*/
- private static function parseJsonLdWebPerson(array $siteinfo, array $jsonld)
+ private static function parseJsonLdWebPerson(array $siteinfo, array $jsonld): array
{
$jsonldinfo = [];
$content = JsonLD::fetchElement($jsonld, 'name');
- if (!empty($content)) {
+ if (!empty($content) && is_string($content)) {
$jsonldinfo['author_name'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'description');
- if (!empty($content)) {
+ if (!empty($content) && is_string($content)) {
$jsonldinfo['author_description'] = trim($content);
}
}
$content = JsonLD::fetchElement($jsonld, 'url');
- if (!empty($content)) {
+ if (!empty($content) && is_string($content)) {
$jsonldinfo['author_url'] = trim($content);
}
*
* @param array $siteinfo
* @param array $jsonld
+ *
* @return array siteinfo
*/
- private static function parseJsonLdMediaObject(array $siteinfo, array $jsonld, string $name)
+ private static function parseJsonLdMediaObject(array $siteinfo, array $jsonld, string $name): array
{
$media = [];
$content = JsonLD::fetchElement($jsonld, 'caption');
- if (!empty($content)) {
+ if (!empty($content) && is_string($content)) {
$media['caption'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'url');
- if (!empty($content)) {
+ if (!empty($content) && is_string($content)) {
$media['url'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'mainEntityOfPage');
- if (!empty($content)) {
+ if (!empty($content) && is_string($content)) {
$media['main'] = Strings::compareLink($content, $siteinfo['url']);
}
$content = JsonLD::fetchElement($jsonld, 'description');
- if (!empty($content)) {
+ if (!empty($content) && is_string($content)) {
$media['description'] = trim($content);
}
}
$content = JsonLD::fetchElement($jsonld, 'contentUrl');
- if (!empty($content)) {
+ if (!empty($content) && is_string($content)) {
$media['content'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'embedUrl');
- if (!empty($content)) {
+ if (!empty($content) && is_string($content)) {
$media['embed'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'height');
- if (!empty($content)) {
+ if (!empty($content) && is_string($content)) {
$media['height'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'width');
- if (!empty($content)) {
+ if (!empty($content) && is_string($content)) {
$media['width'] = trim($content);
}
$content = JsonLD::fetchElement($jsonld, 'image');
- if (!empty($content)) {
+ if (!empty($content) && is_string($content)) {
$media['image'] = trim($content);
}