use DOMDocument;
use DOMXPath;
use Friendica\Content\OEmbed;
+use Friendica\Content\Text\HTML;
+use Friendica\Protocol\HTTP\MediaType;
use Friendica\Core\Hook;
use Friendica\Core\Logger;
use Friendica\Database\Database;
$options = [];
}
- $curlResult = DI::httpClient()->head($url, array_merge([HttpClientOptions::ACCEPT_CONTENT => $accept], $options));
+ try {
+ $curlResult = DI::httpClient()->head($url, array_merge([HttpClientOptions::ACCEPT_CONTENT => $accept], $options));
+ } catch (\Exception $e) {
+ DI::logger()->debug('Got exception', ['url' => $url, 'message' => $e->getMessage()]);
+ return [];
+ }
// Workaround for systems that can't handle a HEAD request. Don't retry on timeouts.
if (!$curlResult->isSuccess() && ($curlResult->getReturnCode() >= 400) && !in_array($curlResult->getReturnCode(), [408, 504])) {
}
/**
- * Search for chached embeddable data of an url otherwise fetch it
+ * Search for cached embeddable data of an url otherwise fetch it
*
* @param string $url The url of the page which should be scraped
* @param bool $do_oembed The false option is used by the function fetch_oembed()
}
$charset = '';
- // Look for a charset, first in headers
- // Expected form: Content-Type: text/html; charset=ISO-8859-4
- if (preg_match('/charset=([a-z0-9-_.\/]+)/i', $curlResult->getContentType(), $matches)) {
- $charset = trim(trim(trim(array_pop($matches)), ';,'));
- } else {
- // Then in body that gets precedence
- // Expected forms:
- // - <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
- // - <meta charset="utf-8">
- // - <meta charset=utf-8>
- // - <meta charSet="utf-8">
- // We escape <style> and <script> tags since they can contain irrelevant charset information
- // (see https://github.com/friendica/friendica/issues/9251#issuecomment-698636806)
- Strings::performWithEscapedBlocks($body, '#<(?:style|script).*?</(?:style|script)>#ism', function ($body) use (&$charset) {
- if (preg_match('/charset=["\']?([a-z0-9-_.\/]+)/i', $body, $matches)) {
- $charset = trim(trim(trim(array_pop($matches)), ';,'));
- }
- });
- }
+ try {
+ // Look for a charset, first in headers
+ $mediaType = MediaType::fromContentType($curlResult->getContentType());
+ if (isset($mediaType->parameters['charset'])) {
+ $charset = $mediaType->parameters['charset'];
+ }
+ } catch(\InvalidArgumentException $e) {}
$siteinfo['charset'] = $charset;
$doc = new DOMDocument();
@$doc->loadHTML($body);
+ $siteinfo['charset'] = HTML::extractCharset($doc) ?? $siteinfo['charset'];
+
XML::deleteNode($doc, 'style');
XML::deleteNode($doc, 'option');
XML::deleteNode($doc, 'h1');
$arr_tags = str_getcsv($string);
if (count($arr_tags)) {
// add the # sign to every tag
- array_walk($arr_tags, ['self', 'arrAddHashes']);
+ array_walk($arr_tags, [self::class, 'arrAddHashes']);
return $arr_tags;
}
{
$urlarr = parse_url($url);
- // If the url does allready have an scheme
+ // If the url does already have an scheme
// we can stop the process here
if (isset($urlarr['scheme'])) {
return $url;