]> git.mxchange.org Git - friendica.git/blob - src/Util/ParseUrl.php
add self for unfollowed contact
[friendica.git] / src / Util / ParseUrl.php
1 <?php
2 /**
3  * @copyright Copyright (C) 2010-2023, the Friendica project
4  *
5  * @license GNU AGPL version 3 or any later version
6  *
7  * This program is free software: you can redistribute it and/or modify
8  * it under the terms of the GNU Affero General Public License as
9  * published by the Free Software Foundation, either version 3 of the
10  * License, or (at your option) any later version.
11  *
12  * This program is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15  * GNU Affero General Public License for more details.
16  *
17  * You should have received a copy of the GNU Affero General Public License
18  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
19  *
20  */
21
22 namespace Friendica\Util;
23
24 use DOMDocument;
25 use DOMXPath;
26 use Friendica\Content\OEmbed;
27 use Friendica\Content\Text\HTML;
28 use Friendica\Protocol\HTTP\MediaType;
29 use Friendica\Core\Hook;
30 use Friendica\Core\Logger;
31 use Friendica\Database\Database;
32 use Friendica\Database\DBA;
33 use Friendica\DI;
34 use Friendica\Network\HTTPClient\Client\HttpClientAccept;
35 use Friendica\Network\HTTPException;
36 use Friendica\Network\HTTPClient\Client\HttpClientOptions;
37
38 /**
39  * Get information about a given URL
40  *
41  * Class with methods for extracting certain content from an url
42  */
43 class ParseUrl
44 {
45         const DEFAULT_EXPIRATION_FAILURE = 'now + 1 day';
46         const DEFAULT_EXPIRATION_SUCCESS = 'now + 3 months';
47
48         /**
49          * Maximum number of characters for the description
50          */
51         const MAX_DESC_COUNT = 250;
52
53         /**
54          * Minimum number of characters for the description
55          */
56         const MIN_DESC_COUNT = 100;
57
58         /**
59          * Fetch the content type of the given url
60          * @param string $url    URL of the page
61          * @param string $accept content-type to accept
62          * @param int    $timeout
63          * @return array content type
64          */
65         public static function getContentType(string $url, string $accept = HttpClientAccept::DEFAULT, int $timeout = 0): array
66         {
67                 if (!empty($timeout)) {
68                         $options = [HttpClientOptions::TIMEOUT => $timeout];
69                 } else {
70                         $options = [];
71                 }
72
73                 try {
74                         $curlResult = DI::httpClient()->head($url, array_merge([HttpClientOptions::ACCEPT_CONTENT => $accept], $options));
75                 } catch (\Exception $e) {
76                         DI::logger()->debug('Got exception', ['url' => $url, 'message' => $e->getMessage()]);
77                         return [];
78                 }
79
80                 // Workaround for systems that can't handle a HEAD request. Don't retry on timeouts.
81                 if (!$curlResult->isSuccess() && ($curlResult->getReturnCode() >= 400) && !in_array($curlResult->getReturnCode(), [408, 504])) {
82                         $curlResult = DI::httpClient()->get($url, $accept, array_merge([HttpClientOptions::CONTENT_LENGTH => 1000000], $options));
83                 }
84
85                 if (!$curlResult->isSuccess()) {
86                         Logger::debug('Got HTTP Error', ['http error' => $curlResult->getReturnCode(), 'url' => $url]);
87                         return [];
88                 }
89
90                 $contenttype =  $curlResult->getHeader('Content-Type')[0] ?? '';
91                 if (empty($contenttype)) {
92                         return ['application', 'octet-stream'];
93                 }
94
95                 return explode('/', current(explode(';', $contenttype)));
96         }
97
98         /**
99          * Search for cached embeddable data of an url otherwise fetch it
100          *
101          * @param string $url         The url of the page which should be scraped
102          * @param bool   $do_oembed   The false option is used by the function fetch_oembed()
103          *                            to avoid endless loops
104          *
105          * @return array which contains needed data for embedding
106          *    string 'url'      => The url of the parsed page
107          *    string 'type'     => Content type
108          *    string 'title'    => (optional) The title of the content
109          *    string 'text'     => (optional) The description for the content
110          *    string 'image'    => (optional) A preview image of the content
111          *    array  'images'   => (optional) Array of preview pictures
112          *    string 'keywords' => (optional) The tags which belong to the content
113          *
114          * @throws HTTPException\InternalServerErrorException
115          * @see   ParseUrl::getSiteinfo() for more information about scraping
116          * embeddable content
117          */
118         public static function getSiteinfoCached(string $url, bool $do_oembed = true): array
119         {
120                 if (empty($url)) {
121                         return [
122                                 'url' => '',
123                                 'type' => 'error',
124                         ];
125                 }
126
127                 $urlHash = hash('sha256', $url);
128
129                 $parsed_url = DBA::selectFirst('parsed_url', ['content'],
130                         ['url_hash' => $urlHash, 'oembed' => $do_oembed]
131                 );
132                 if (!empty($parsed_url['content'])) {
133                         $data = unserialize($parsed_url['content']);
134                         return $data;
135                 }
136
137                 $data = self::getSiteinfo($url, $do_oembed);
138
139                 $expires = $data['expires'];
140
141                 unset($data['expires']);
142
143                 DI::dba()->insert(
144                         'parsed_url',
145                         [
146                                 'url_hash' => $urlHash,
147                                 'oembed'   => $do_oembed,
148                                 'url'      => $url,
149                                 'content'  => serialize($data),
150                                 'created'  => DateTimeFormat::utcNow(),
151                                 'expires'  => $expires,
152                         ],
153                         Database::INSERT_UPDATE
154                 );
155
156                 return $data;
157         }
158
159         /**
160          * Parse a page for embeddable content information
161          *
162          * This method parses to url for meta data which can be used to embed
163          * the content. If available it prioritizes Open Graph meta tags.
164          * If this is not available it uses the twitter cards meta tags.
165          * As fallback it uses standard html elements with meta informations
166          * like \<title\>Awesome Title\</title\> or
167          * \<meta name="description" content="An awesome description"\>
168          *
169          * @param string $url         The url of the page which should be scraped
170          * @param bool   $do_oembed   The false option is used by the function fetch_oembed()
171          *                            to avoid endless loops
172          * @param int    $count       Internal counter to avoid endless loops
173          *
174          * @return array which contains needed data for embedding
175          *    string 'url'      => The url of the parsed page
176          *    string 'type'     => Content type (error, link, photo, image, audio, video)
177          *    string 'title'    => (optional) The title of the content
178          *    string 'text'     => (optional) The description for the content
179          *    string 'image'    => (optional) A preview image of the content
180          *    array  'images'   => (optional) Array of preview pictures
181          *    string 'keywords' => (optional) The tags which belong to the content
182          *
183          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
184          * @todo  https://developers.google.com/+/plugins/snippet/
185          * @verbatim
186          * <meta itemprop="name" content="Awesome title">
187          * <meta itemprop="description" content="An awesome description">
188          * <meta itemprop="image" content="http://maple.libertreeproject.org/images/tree-icon.png">
189          *
190          * <body itemscope itemtype="http://schema.org/Product">
191          *   <h1 itemprop="name">Shiny Trinket</h1>
192          *   <img itemprop="image" src="{image-url}" />
193          *   <p itemprop="description">Shiny trinkets are shiny.</p>
194          * </body>
195          * @endverbatim
196          */
197         public static function getSiteinfo(string $url, bool $do_oembed = true, int $count = 1): array
198         {
199                 if (empty($url)) {
200                         return [
201                                 'url' => '',
202                                 'type' => 'error',
203                         ];
204                 }
205
206                 // Check if the URL does contain a scheme
207                 $scheme = parse_url($url, PHP_URL_SCHEME);
208
209                 if ($scheme == '') {
210                         $url = 'http://' . ltrim($url, '/');
211                 }
212
213                 $url = trim($url, "'\"");
214
215                 $url = Network::stripTrackingQueryParams($url);
216
217                 $siteinfo = [
218                         'url' => $url,
219                         'type' => 'link',
220                         'expires' => DateTimeFormat::utc(self::DEFAULT_EXPIRATION_FAILURE),
221                 ];
222
223                 if ($count > 10) {
224                         Logger::warning('Endless loop detected', ['url' => $url]);
225                         return $siteinfo;
226                 }
227
228                 $type = self::getContentType($url);
229                 Logger::info('Got content-type', ['content-type' => $type, 'url' => $url]);
230                 if (!empty($type) && in_array($type[0], ['image', 'video', 'audio'])) {
231                         $siteinfo['type'] = $type[0];
232                         return $siteinfo;
233                 }
234
235                 if ((count($type) >= 2) && (($type[0] != 'text') || ($type[1] != 'html'))) {
236                         Logger::info('Unparseable content-type, quitting here, ', ['content-type' => $type, 'url' => $url]);
237                         return $siteinfo;
238                 }
239
240                 $curlResult = DI::httpClient()->get($url, HttpClientAccept::HTML, [HttpClientOptions::CONTENT_LENGTH => 1000000]);
241                 if (!$curlResult->isSuccess() || empty($curlResult->getBody())) {
242                         Logger::info('Empty body or error when fetching', ['url' => $url, 'success' => $curlResult->isSuccess(), 'code' => $curlResult->getReturnCode()]);
243                         return $siteinfo;
244                 }
245
246                 $siteinfo['expires'] = DateTimeFormat::utc(self::DEFAULT_EXPIRATION_SUCCESS);
247
248                 if ($cacheControlHeader = $curlResult->getHeader('Cache-Control')[0] ?? '') {
249                         if (preg_match('/max-age=([0-9]+)/i', $cacheControlHeader, $matches)) {
250                                 $maxAge = max(86400, (int)array_pop($matches));
251                                 $siteinfo['expires'] = DateTimeFormat::utc("now + $maxAge seconds");
252                         }
253                 }
254
255                 $body = $curlResult->getBody();
256
257                 if ($do_oembed) {
258                         $oembed_data = OEmbed::fetchURL($url, false, false);
259
260                         if (!empty($oembed_data->type)) {
261                                 if (!in_array($oembed_data->type, ['error', 'rich', 'image', 'video', 'audio', ''])) {
262                                         $siteinfo['type'] = $oembed_data->type;
263                                 }
264
265                                 // See https://github.com/friendica/friendica/pull/5763#discussion_r217913178
266                                 if ($siteinfo['type'] != 'photo') {
267                                         if (!empty($oembed_data->title)) {
268                                                 $siteinfo['title'] = trim($oembed_data->title);
269                                         }
270                                         if (!empty($oembed_data->description)) {
271                                                 $siteinfo['text'] = trim($oembed_data->description);
272                                         }
273                                         if (!empty($oembed_data->author_name)) {
274                                                 $siteinfo['author_name'] = trim($oembed_data->author_name);
275                                         }
276                                         if (!empty($oembed_data->author_url)) {
277                                                 $siteinfo['author_url'] = trim($oembed_data->author_url);
278                                         }
279                                         if (!empty($oembed_data->provider_name)) {
280                                                 $siteinfo['publisher_name'] = trim($oembed_data->provider_name);
281                                         }
282                                         if (!empty($oembed_data->provider_url)) {
283                                                 $siteinfo['publisher_url'] = trim($oembed_data->provider_url);
284                                         }
285                                         if (!empty($oembed_data->thumbnail_url)) {
286                                                 $siteinfo['image'] = $oembed_data->thumbnail_url;
287                                         }
288                                 }
289                         }
290                 }
291
292                 $charset = '';
293                 try {
294                         // Look for a charset, first in headers
295                         $mediaType = MediaType::fromContentType($curlResult->getContentType());
296                         if (isset($mediaType->parameters['charset'])) {
297                                 $charset = $mediaType->parameters['charset'];
298                         }
299                 } catch(\InvalidArgumentException $e) {}
300
301                 $siteinfo['charset'] = $charset;
302
303                 if ($charset && strtoupper($charset) != 'UTF-8') {
304                         // See https://github.com/friendica/friendica/issues/5470#issuecomment-418351211
305                         $charset = str_ireplace('latin-1', 'latin1', $charset);
306
307                         Logger::info('detected charset', ['charset' => $charset]);
308                         $body = iconv($charset, 'UTF-8//TRANSLIT', $body);
309                 }
310
311                 $body = mb_convert_encoding($body, 'HTML-ENTITIES', 'UTF-8');
312
313                 if (empty($body)) {
314                         return $siteinfo;
315                 }
316
317                 $doc = new DOMDocument();
318                 @$doc->loadHTML($body);
319
320                 $siteinfo['charset'] = HTML::extractCharset($doc) ?? $siteinfo['charset'];
321
322                 XML::deleteNode($doc, 'style');
323                 XML::deleteNode($doc, 'option');
324                 XML::deleteNode($doc, 'h1');
325                 XML::deleteNode($doc, 'h2');
326                 XML::deleteNode($doc, 'h3');
327                 XML::deleteNode($doc, 'h4');
328                 XML::deleteNode($doc, 'h5');
329                 XML::deleteNode($doc, 'h6');
330                 XML::deleteNode($doc, 'ol');
331                 XML::deleteNode($doc, 'ul');
332
333                 $xpath = new DOMXPath($doc);
334
335                 $list = $xpath->query('//meta[@content]');
336                 foreach ($list as $node) {
337                         $meta_tag = [];
338                         if ($node->attributes->length) {
339                                 foreach ($node->attributes as $attribute) {
340                                         $meta_tag[$attribute->name] = $attribute->value;
341                                 }
342                         }
343
344                         if (@$meta_tag['http-equiv'] == 'refresh') {
345                                 $path = $meta_tag['content'];
346                                 $pathinfo = explode(';', $path);
347                                 $content = '';
348                                 foreach ($pathinfo as $value) {
349                                         if (substr(strtolower($value), 0, 4) == 'url=') {
350                                                 $content = substr($value, 4);
351                                         }
352                                 }
353                                 if ($content != '') {
354                                         $siteinfo = self::getSiteinfo($content, $do_oembed, ++$count);
355                                         return $siteinfo;
356                                 }
357                         }
358                 }
359
360                 $list = $xpath->query('//title');
361                 if ($list->length > 0) {
362                         $siteinfo['title'] = trim($list->item(0)->nodeValue);
363                 }
364
365                 $list = $xpath->query('//meta[@name]');
366                 foreach ($list as $node) {
367                         $meta_tag = [];
368                         if ($node->attributes->length) {
369                                 foreach ($node->attributes as $attribute) {
370                                         $meta_tag[$attribute->name] = $attribute->value;
371                                 }
372                         }
373
374                         if (empty($meta_tag['content'])) {
375                                 continue;
376                         }
377
378                         $meta_tag['content'] = trim(html_entity_decode($meta_tag['content'], ENT_QUOTES, 'UTF-8'));
379
380                         switch (strtolower($meta_tag['name'])) {
381                                 case 'fulltitle':
382                                         $siteinfo['title'] = trim($meta_tag['content']);
383                                         break;
384                                 case 'description':
385                                         $siteinfo['text'] = trim($meta_tag['content']);
386                                         break;
387                                 case 'thumbnail':
388                                         $siteinfo['image'] = $meta_tag['content'];
389                                         break;
390                                 case 'twitter:image':
391                                         $siteinfo['image'] = $meta_tag['content'];
392                                         break;
393                                 case 'twitter:image:src':
394                                         $siteinfo['image'] = $meta_tag['content'];
395                                         break;
396                                 case 'twitter:description':
397                                         $siteinfo['text'] = trim($meta_tag['content']);
398                                         break;
399                                 case 'twitter:title':
400                                         $siteinfo['title'] = trim($meta_tag['content']);
401                                         break;
402                                 case 'twitter:player':
403                                         $siteinfo['player']['embed'] = trim($meta_tag['content']);
404                                         break;
405                                 case 'twitter:player:stream':
406                                         $siteinfo['player']['stream'] = trim($meta_tag['content']);
407                                         break;
408                                 case 'twitter:player:width':
409                                         $siteinfo['player']['width'] = intval($meta_tag['content']);
410                                         break;
411                                 case 'twitter:player:height':
412                                         $siteinfo['player']['height'] = intval($meta_tag['content']);
413                                         break;
414                                 case 'dc.title':
415                                         $siteinfo['title'] = trim($meta_tag['content']);
416                                         break;
417                                 case 'dc.description':
418                                         $siteinfo['text'] = trim($meta_tag['content']);
419                                         break;
420                                 case 'dc.creator':
421                                         $siteinfo['publisher_name'] = trim($meta_tag['content']);
422                                         break;
423                                 case 'keywords':
424                                         $keywords = explode(',', $meta_tag['content']);
425                                         break;
426                                 case 'news_keywords':
427                                         $keywords = explode(',', $meta_tag['content']);
428                                         break;
429                         }
430                 }
431
432                 if (isset($keywords)) {
433                         $siteinfo['keywords'] = [];
434                         foreach ($keywords as $keyword) {
435                                 if (!in_array(trim($keyword), $siteinfo['keywords'])) {
436                                         $siteinfo['keywords'][] = trim($keyword);
437                                 }
438                         }
439                 }
440
441                 $list = $xpath->query('//meta[@property]');
442                 foreach ($list as $node) {
443                         $meta_tag = [];
444                         if ($node->attributes->length) {
445                                 foreach ($node->attributes as $attribute) {
446                                         $meta_tag[$attribute->name] = $attribute->value;
447                                 }
448                         }
449
450                         if (!empty($meta_tag['content'])) {
451                                 $meta_tag['content'] = trim(html_entity_decode($meta_tag['content'], ENT_QUOTES, 'UTF-8'));
452
453                                 switch (strtolower($meta_tag['property'])) {
454                                         case 'og:image':
455                                                 $siteinfo['image'] = $meta_tag['content'];
456                                                 break;
457                                         case 'og:image:url':
458                                                 $siteinfo['image'] = $meta_tag['content'];
459                                                 break;
460                                         case 'og:image:secure_url':
461                                                 $siteinfo['image'] = $meta_tag['content'];
462                                                 break;
463                                         case 'og:title':
464                                                 $siteinfo['title'] = trim($meta_tag['content']);
465                                                 break;
466                                         case 'og:description':
467                                                 $siteinfo['text'] = trim($meta_tag['content']);
468                                                 break;
469                                         case 'og:site_name':
470                                                 $siteinfo['publisher_name'] = trim($meta_tag['content']);
471                                                 break;
472                                         case 'og:locale':
473                                                 $siteinfo['language'] = trim($meta_tag['content']);
474                                                 break;
475                                         case 'og:type':
476                                                 $siteinfo['pagetype'] = trim($meta_tag['content']);
477                                                 break;
478                                         case 'twitter:description':
479                                                 $siteinfo['text'] = trim($meta_tag['content']);
480                                                 break;
481                                         case 'twitter:title':
482                                                 $siteinfo['title'] = trim($meta_tag['content']);
483                                                 break;
484                                         case 'twitter:image':
485                                                 $siteinfo['image'] = $meta_tag['content'];
486                                                 break;
487                                 }
488                         }
489                 }
490
491                 $list = $xpath->query("//script[@type='application/ld+json']");
492                 foreach ($list as $node) {
493                         if (!empty($node->nodeValue)) {
494                                 if ($jsonld = json_decode($node->nodeValue, true)) {
495                                         $siteinfo = self::parseParts($siteinfo, $jsonld);
496                                 }
497                         }
498                 }
499
500                 if (!empty($siteinfo['player']['stream'])) {
501                         // Only add player data to media arrays if there is no duplicate
502                         $content_urls = array_merge(array_column($siteinfo['audio'] ?? [], 'content'), array_column($siteinfo['video'] ?? [], 'content'));
503                         if (!in_array($siteinfo['player']['stream'], $content_urls)) {
504                                 $contenttype = self::getContentType($siteinfo['player']['stream']);
505                                 if (!empty($contenttype[0]) && in_array($contenttype[0], ['audio', 'video'])) {
506                                         $media = ['content' => $siteinfo['player']['stream']];
507
508                                         if (!empty($siteinfo['player']['embed'])) {
509                                                 $media['embed'] = $siteinfo['player']['embed'];
510                                         }
511
512                                         $siteinfo[$contenttype[0]][] = $media;
513                                 }
514                         }
515                 }
516
517                 if (!empty($siteinfo['image'])) {
518                         $siteinfo['images'] = $siteinfo['images'] ?? [];
519                         array_unshift($siteinfo['images'], ['url' => $siteinfo['image']]);
520                         unset($siteinfo['image']);
521                 }
522
523                 $siteinfo = self::checkMedia($url, $siteinfo);
524
525                 if (!empty($siteinfo['text']) && mb_strlen($siteinfo['text']) > self::MAX_DESC_COUNT) {
526                         $siteinfo['text'] = mb_substr($siteinfo['text'], 0, self::MAX_DESC_COUNT) . '…';
527                         $pos = mb_strrpos($siteinfo['text'], '.');
528                         if ($pos > self::MIN_DESC_COUNT) {
529                                 $siteinfo['text'] = mb_substr($siteinfo['text'], 0, $pos + 1);
530                         }
531                 }
532
533                 Logger::info('Siteinfo fetched', ['url' => $url, 'siteinfo' => $siteinfo]);
534
535                 Hook::callAll('getsiteinfo', $siteinfo);
536
537                 ksort($siteinfo);
538
539                 return $siteinfo;
540         }
541
542         /**
543          * Check the attached media elements.
544          * Fix existing data and add missing data.
545          *
546          * @param string $page_url
547          * @param array $siteinfo
548          * @return array
549          */
550         private static function checkMedia(string $page_url, array $siteinfo) : array
551         {
552                 if (!empty($siteinfo['images'])) {
553                         array_walk($siteinfo['images'], function (&$image) use ($page_url) {
554                                 /*
555                                  * According to the specifications someone could place a picture
556                                  * URL into the content field as well. But this doesn't seem to
557                                  * happen in the wild, so we don't cover it here.
558                                  */
559                                 if (!empty($image['url'])) {
560                                         $image['url'] = self::completeUrl($image['url'], $page_url);
561                                         $photodata = Images::getInfoFromURLCached($image['url']);
562                                         if (($photodata) && ($photodata[0] > 50) && ($photodata[1] > 50)) {
563                                                 $image['src'] = $image['url'];
564                                                 $image['width'] = $photodata[0];
565                                                 $image['height'] = $photodata[1];
566                                                 $image['contenttype'] = $photodata['mime'];
567                                                 $image['blurhash'] = $photodata['blurhash'] ?? null;
568                                                 unset($image['url']);
569                                                 ksort($image);
570                                         } else {
571                                                 $image = [];
572                                         }
573                                 } else {
574                                         $image = [];
575                                 }
576                         });
577
578                         $siteinfo['images'] = array_values(array_filter($siteinfo['images']));
579                 }
580
581                 foreach (['audio', 'video'] as $element) {
582                         if (!empty($siteinfo[$element])) {
583                                 array_walk($siteinfo[$element], function (&$media) use ($page_url, &$siteinfo) {
584                                         $url = '';
585                                         $embed = '';
586                                         $content = '';
587                                         $contenttype = '';
588                                         foreach (['embed', 'content', 'url'] as $field) {
589                                                 if (!empty($media[$field])) {
590                                                         $media[$field] = self::completeUrl($media[$field], $page_url);
591                                                         $type = self::getContentType($media[$field]);
592                                                         if (($type[0] ?? '') == 'text') {
593                                                                 if ($field == 'embed') {
594                                                                         $embed = $media[$field];
595                                                                 } else {
596                                                                         $url = $media[$field];
597                                                                 }
598                                                         } elseif (!empty($type[0])) {
599                                                                 $content = $media[$field];
600                                                                 $contenttype = implode('/', $type);
601                                                         }
602                                                 }
603                                                 unset($media[$field]);
604                                         }
605
606                                         foreach (['image', 'preview'] as $field) {
607                                                 if (!empty($media[$field])) {
608                                                         $media[$field] = self::completeUrl($media[$field], $page_url);
609                                                 }
610                                         }
611
612                                         if (!empty($url)) {
613                                                 $media['url'] = $url;
614                                         }
615                                         if (!empty($embed)) {
616                                                 $media['embed'] = $embed;
617                                                 if (empty($siteinfo['player']['embed'])) {
618                                                         $siteinfo['player']['embed'] = $embed;
619                                                 }
620                                         }
621                                         if (!empty($content)) {
622                                                 $media['src'] = $content;
623                                         }
624                                         if (!empty($contenttype)) {
625                                                 $media['contenttype'] = $contenttype;
626                                         }
627                                         if (empty($url) && empty($content) && empty($embed)) {
628                                                 $media = [];
629                                         }
630                                         ksort($media);
631                                 });
632
633                                 $siteinfo[$element] = array_values(array_filter($siteinfo[$element]));
634                         }
635                         if (empty($siteinfo[$element])) {
636                                 unset($siteinfo[$element]);
637                         }
638                 }
639                 return $siteinfo;
640         }
641
642         /**
643          * Convert tags from CSV to an array
644          *
645          * @param string $string Tags
646          *
647          * @return array with formatted Hashtags
648          */
649         public static function convertTagsToArray(string $string): array
650         {
651                 $arr_tags = str_getcsv($string);
652                 if (count($arr_tags)) {
653                         // add the # sign to every tag
654                         array_walk($arr_tags, [self::class, 'arrAddHashes']);
655
656                         return $arr_tags;
657                 }
658                 return [];
659         }
660
661         /**
662          * Add a hasht sign to a string
663          *
664          * This method is used as callback function
665          *
666          * @param string $tag The pure tag name
667          * @param int    $k   Counter for internal use
668          *
669          * @return void
670          */
671         private static function arrAddHashes(string &$tag, int $k)
672         {
673                 $tag = '#' . $tag;
674         }
675
676         /**
677          * Add a scheme to an url
678          *
679          * The src attribute of some html elements (e.g. images)
680          * can miss the scheme so we need to add the correct
681          * scheme
682          *
683          * @param string $url    The url which possibly does have
684          *                       a missing scheme (a link to an image)
685          * @param string $scheme The url with a correct scheme
686          *                       (e.g. the url from the webpage which does contain the image)
687          *
688          * @return string The url with a scheme
689          */
690         private static function completeUrl(string $url, string $scheme): string
691         {
692                 $urlarr = parse_url($url);
693
694                 // If the url does already have an scheme
695                 // we can stop the process here
696                 if (isset($urlarr['scheme'])) {
697                         return $url;
698                 }
699
700                 $schemearr = parse_url($scheme);
701
702                 $complete = $schemearr['scheme'] . '://' . $schemearr['host'];
703
704                 if (!empty($schemearr['port'])) {
705                         $complete .= ':' . $schemearr['port'];
706                 }
707
708                 if (!empty($urlarr['path'])) {
709                         if (strpos($urlarr['path'], '/') !== 0) {
710                                 $complete .= '/';
711                         }
712
713                         $complete .= $urlarr['path'];
714                 }
715
716                 if (!empty($urlarr['query'])) {
717                         $complete .= '?' . $urlarr['query'];
718                 }
719
720                 if (!empty($urlarr['fragment'])) {
721                         $complete .= '#' . $urlarr['fragment'];
722                 }
723
724                 return $complete;
725         }
726
727         /**
728          * Parse the Json-Ld parts of a web page
729          *
730          * @param array $siteinfo
731          * @param array $jsonld
732          *
733          * @return array siteinfo
734          */
735         private static function parseParts(array $siteinfo, array $jsonld): array
736         {
737                 if (!empty($jsonld['@graph']) && is_array($jsonld['@graph'])) {
738                         foreach ($jsonld['@graph'] as $part) {
739                                 if (!empty($part) && is_array($part)) {
740                                         $siteinfo = self::parseParts($siteinfo, $part);
741                                 }
742                         }
743                 } elseif (!empty($jsonld['@type'])) {
744                         $siteinfo = self::parseJsonLd($siteinfo, $jsonld);
745                 } elseif (!empty($jsonld)) {
746                         $keys = array_keys($jsonld);
747                         $numeric_keys = true;
748                         foreach ($keys as $key) {
749                                 if (!is_int($key)) {
750                                         $numeric_keys = false;
751                                 }
752                         }
753                         if ($numeric_keys) {
754                                 foreach ($jsonld as $part) {
755                                         if (!empty($part) && is_array($part)) {
756                                                 $siteinfo = self::parseParts($siteinfo, $part);
757                                         }
758                                 }
759                         }
760                 }
761
762                 array_walk_recursive($siteinfo, function (&$element) {
763                         if (is_string($element)) {
764                                 $element = trim(strip_tags(html_entity_decode($element, ENT_COMPAT, 'UTF-8')));
765                         }
766                 });
767
768                 return $siteinfo;
769         }
770
771         /**
772          * Improve the siteinfo with information from the provided JSON-LD information
773          * @see https://jsonld.com/
774          * @see https://schema.org/
775          *
776          * @param array $siteinfo
777          * @param array $jsonld
778          *
779          * @return array siteinfo
780          */
781         private static function parseJsonLd(array $siteinfo, array $jsonld): array
782         {
783                 $type = JsonLD::fetchElement($jsonld, '@type');
784                 if (empty($type)) {
785                         Logger::info('Empty type', ['url' => $siteinfo['url']]);
786                         return $siteinfo;
787                 }
788
789                 // Silently ignore some types that aren't processed
790                 if (in_array($type, ['SiteNavigationElement', 'JobPosting', 'CreativeWork', 'MusicAlbum',
791                         'WPHeader', 'WPSideBar', 'WPFooter', 'LegalService', 'MusicRecording',
792                         'ItemList', 'BreadcrumbList', 'Blog', 'Dataset', 'Product'])) {
793                         return $siteinfo;
794                 }
795
796                 switch ($type) {
797                         case 'Article':
798                         case 'AdvertiserContentArticle':
799                         case 'NewsArticle':
800                         case 'Report':
801                         case 'SatiricalArticle':
802                         case 'ScholarlyArticle':
803                         case 'SocialMediaPosting':
804                         case 'TechArticle':
805                         case 'ReportageNewsArticle':
806                         case 'SocialMediaPosting':
807                         case 'BlogPosting':
808                         case 'LiveBlogPosting':
809                         case 'DiscussionForumPosting':
810                                 return self::parseJsonLdArticle($siteinfo, $jsonld);
811                         case 'WebPage':
812                         case 'AboutPage':
813                         case 'CheckoutPage':
814                         case 'CollectionPage':
815                         case 'ContactPage':
816                         case 'FAQPage':
817                         case 'ItemPage':
818                         case 'MedicalWebPage':
819                         case 'ProfilePage':
820                         case 'QAPage':
821                         case 'RealEstateListing':
822                         case 'SearchResultsPage':
823                         case 'MediaGallery':
824                         case 'ImageGallery':
825                         case 'VideoGallery':
826                         case 'RadioEpisode':
827                         case 'Event':
828                                 return self::parseJsonLdWebPage($siteinfo, $jsonld);
829                         case 'WebSite':
830                                 return self::parseJsonLdWebSite($siteinfo, $jsonld);
831                         case 'Organization':
832                         case 'Airline':
833                         case 'Consortium':
834                         case 'Corporation':
835                         case 'EducationalOrganization':
836                         case 'FundingScheme':
837                         case 'GovernmentOrganization':
838                         case 'LibrarySystem':
839                         case 'LocalBusiness':
840                         case 'MedicalOrganization':
841                         case 'NGO':
842                         case 'NewsMediaOrganization':
843                         case 'Project':
844                         case 'SportsOrganization':
845                         case 'WorkersUnion':
846                                 return self::parseJsonLdWebOrganization($siteinfo, $jsonld);
847                         case 'Person':
848                         case 'Patient':
849                         case 'PerformingGroup':
850                         case 'DanceGroup';
851                         case 'MusicGroup':
852                         case 'TheaterGroup':
853                                 return self::parseJsonLdWebPerson($siteinfo, $jsonld);
854                         case 'AudioObject':
855                         case 'Audio':
856                                 return self::parseJsonLdMediaObject($siteinfo, $jsonld, 'audio');
857                         case 'VideoObject':
858                                 return self::parseJsonLdMediaObject($siteinfo, $jsonld, 'video');
859                         case 'ImageObject':
860                                 return self::parseJsonLdMediaObject($siteinfo, $jsonld, 'images');
861                         default:
862                                 Logger::info('Unknown type', ['type' => $type, 'url' => $siteinfo['url']]);
863                                 return $siteinfo;
864                 }
865         }
866
867         /**
868          * Fetch author and publisher data
869          *
870          * @param array $siteinfo
871          * @param array $jsonld
872          *
873          * @return array siteinfo
874          */
875         private static function parseJsonLdAuthor(array $siteinfo, array $jsonld): array
876         {
877                 $jsonldinfo = [];
878
879                 if (!empty($jsonld['publisher']) && is_array($jsonld['publisher'])) {
880                         $content = JsonLD::fetchElement($jsonld, 'publisher', 'name');
881                         if (!empty($content) && is_string($content)) {
882                                 $jsonldinfo['publisher_name'] = trim($content);
883                         }
884
885                         $content = JsonLD::fetchElement($jsonld, 'publisher', 'url');
886                         if (!empty($content) && is_string($content)) {
887                                 $jsonldinfo['publisher_url'] = trim($content);
888                         }
889
890                         $brand = JsonLD::fetchElement($jsonld, 'publisher', 'brand', '@type', 'Organization');
891                         if (!empty($brand) && is_array($brand)) {
892                                 $content = JsonLD::fetchElement($brand, 'name');
893                                 if (!empty($content) && is_string($content)) {
894                                         $jsonldinfo['publisher_name'] = trim($content);
895                                 }
896
897                                 $content = JsonLD::fetchElement($brand, 'url');
898                                 if (!empty($content) && is_string($content)) {
899                                         $jsonldinfo['publisher_url'] = trim($content);
900                                 }
901
902                                 $content = JsonLD::fetchElement($brand, 'logo', 'url');
903                                 if (!empty($content) && is_string($content)) {
904                                         $jsonldinfo['publisher_img'] = trim($content);
905                                 }
906                         }
907
908                         $logo = JsonLD::fetchElement($jsonld, 'publisher', 'logo');
909                         if (!empty($logo) && is_array($logo)) {
910                                 $content = JsonLD::fetchElement($logo, 'url');
911                                 if (!empty($content) && is_string($content)) {
912                                         $jsonldinfo['publisher_img'] = trim($content);
913                                 }
914                         }
915                 } elseif (!empty($jsonld['publisher']) && is_string($jsonld['publisher'])) {
916                         $jsonldinfo['publisher_name'] = trim($jsonld['publisher']);
917                 }
918
919                 if (!empty($jsonld['author']) && is_array($jsonld['author'])) {
920                         $content = JsonLD::fetchElement($jsonld, 'author', 'name');
921                         if (!empty($content) && is_string($content)) {
922                                 $jsonldinfo['author_name'] = trim($content);
923                         }
924
925                         $content = JsonLD::fetchElement($jsonld, 'author', 'sameAs');
926                         if (!empty($content) && is_string($content)) {
927                                 $jsonldinfo['author_url'] = trim($content);
928                         }
929
930                         $content = JsonLD::fetchElement($jsonld, 'author', 'url');
931                         if (!empty($content) && is_string($content)) {
932                                 $jsonldinfo['author_url'] = trim($content);
933                         }
934
935                         $logo = JsonLD::fetchElement($jsonld, 'author', 'logo');
936                         if (!empty($logo) && is_array($logo)) {
937                                 $content = JsonLD::fetchElement($logo, 'url');
938                                 if (!empty($content) && is_string($content)) {
939                                         $jsonldinfo['author_img'] = trim($content);
940                                 }
941                         }
942                 } elseif (!empty($jsonld['author']) && is_string($jsonld['author'])) {
943                         $jsonldinfo['author_name'] = trim($jsonld['author']);
944                 }
945
946                 Logger::info('Fetched Author information', ['fetched' => $jsonldinfo]);
947
948                 return array_merge($siteinfo, $jsonldinfo);
949         }
950
951         /**
952          * Fetch data from the provided JSON-LD Article type
953          * @see https://schema.org/Article
954          *
955          * @param array $siteinfo
956          * @param array $jsonld
957          *
958          * @return array siteinfo
959          */
960         private static function parseJsonLdArticle(array $siteinfo, array $jsonld): array
961         {
962                 $jsonldinfo = [];
963
964                 $content = JsonLD::fetchElement($jsonld, 'headline');
965                 if (!empty($content) && is_string($content)) {
966                         $jsonldinfo['title'] = trim($content);
967                 }
968
969                 $content = JsonLD::fetchElement($jsonld, 'alternativeHeadline');
970                 if (!empty($content) && is_string($content) && (($jsonldinfo['title'] ?? '') != trim($content))) {
971                         $jsonldinfo['alternative_title'] = trim($content);
972                 }
973
974                 $content = JsonLD::fetchElement($jsonld, 'description');
975                 if (!empty($content) && is_string($content)) {
976                         $jsonldinfo['text'] = trim($content);
977                 }
978
979                 $content = JsonLD::fetchElement($jsonld, 'thumbnailUrl');
980                 if (!empty($content)) {
981                         $jsonldinfo['image'] = trim($content);
982                 }
983
984                 $content = JsonLD::fetchElement($jsonld, 'image', 'url', '@type', 'ImageObject');
985                 if (!empty($content) && is_string($content)) {
986                         $jsonldinfo['image'] = trim($content);
987                 }
988
989                 if (!empty($jsonld['keywords']) && !is_array($jsonld['keywords'])) {
990                         $content = JsonLD::fetchElement($jsonld, 'keywords');
991                         if (!empty($content)) {
992                                 $siteinfo['keywords'] = [];
993                                 $keywords = explode(',', $content);
994                                 foreach ($keywords as $keyword) {
995                                         $siteinfo['keywords'][] = trim($keyword);
996                                 }
997                         }
998                 } elseif (!empty($jsonld['keywords'])) {
999                         $content = JsonLD::fetchElementArray($jsonld, 'keywords');
1000                         if (!empty($content) && is_array($content)) {
1001                                 $jsonldinfo['keywords'] = $content;
1002                         }
1003                 }
1004
1005                 $content = JsonLD::fetchElement($jsonld, 'datePublished');
1006                 if (!empty($content) && is_string($content)) {
1007                         $jsonldinfo['published'] = DateTimeFormat::utc($content);
1008                 }
1009
1010                 $content = JsonLD::fetchElement($jsonld, 'dateModified');
1011                 if (!empty($content) && is_string($content)) {
1012                         $jsonldinfo['modified'] = DateTimeFormat::utc($content);
1013                 }
1014
1015                 $jsonldinfo = self::parseJsonLdAuthor($jsonldinfo, $jsonld);
1016
1017                 Logger::info('Fetched article information', ['url' => $siteinfo['url'], 'fetched' => $jsonldinfo]);
1018
1019                 return array_merge($siteinfo, $jsonldinfo);
1020         }
1021
1022         /**
1023          * Fetch data from the provided JSON-LD WebPage type
1024          * @see https://schema.org/WebPage
1025          *
1026          * @param array $siteinfo
1027          * @param array $jsonld
1028          *
1029          * @return array siteinfo
1030          */
1031         private static function parseJsonLdWebPage(array $siteinfo, array $jsonld): array
1032         {
1033                 $jsonldinfo = [];
1034
1035                 $content = JsonLD::fetchElement($jsonld, 'name');
1036                 if (!empty($content)) {
1037                         $jsonldinfo['title'] = trim($content);
1038                 }
1039
1040                 $content = JsonLD::fetchElement($jsonld, 'description');
1041                 if (!empty($content) && is_string($content)) {
1042                         $jsonldinfo['text'] = trim($content);
1043                 }
1044
1045                 $content = JsonLD::fetchElement($jsonld, 'image');
1046                 if (!empty($content) && is_string($content)) {
1047                         $jsonldinfo['image'] = trim($content);
1048                 }
1049
1050                 $content = JsonLD::fetchElement($jsonld, 'thumbnailUrl');
1051                 if (!empty($content) && is_string($content)) {
1052                         $jsonldinfo['image'] = trim($content);
1053                 }
1054
1055                 $jsonldinfo = self::parseJsonLdAuthor($jsonldinfo, $jsonld);
1056
1057                 Logger::info('Fetched WebPage information', ['url' => $siteinfo['url'], 'fetched' => $jsonldinfo]);
1058
1059                 return array_merge($siteinfo, $jsonldinfo);
1060         }
1061
1062         /**
1063          * Fetch data from the provided JSON-LD WebSite type
1064          * @see https://schema.org/WebSite
1065          *
1066          * @param array $siteinfo
1067          * @param array $jsonld
1068          *
1069          * @return array siteinfo
1070          */
1071         private static function parseJsonLdWebSite(array $siteinfo, array $jsonld): array
1072         {
1073                 $jsonldinfo = [];
1074
1075                 $content = JsonLD::fetchElement($jsonld, 'name');
1076                 if (!empty($content) && is_string($content)) {
1077                         $jsonldinfo['publisher_name'] = trim($content);
1078                 }
1079
1080                 $content = JsonLD::fetchElement($jsonld, 'description');
1081                 if (!empty($content) && is_string($content)) {
1082                         $jsonldinfo['publisher_description'] = trim($content);
1083                 }
1084
1085                 $content = JsonLD::fetchElement($jsonld, 'url');
1086                 if (!empty($content) && is_string($content)) {
1087                         $jsonldinfo['publisher_url'] = trim($content);
1088                 }
1089
1090                 $content = JsonLD::fetchElement($jsonld, 'thumbnailUrl');
1091                 if (!empty($content) && is_string($content)) {
1092                         $jsonldinfo['image'] = trim($content);
1093                 }
1094
1095                 $jsonldinfo = self::parseJsonLdAuthor($jsonldinfo, $jsonld);
1096
1097                 Logger::info('Fetched WebSite information', ['url' => $siteinfo['url'], 'fetched' => $jsonldinfo]);
1098                 return array_merge($siteinfo, $jsonldinfo);
1099         }
1100
1101         /**
1102          * Fetch data from the provided JSON-LD Organization type
1103          * @see https://schema.org/Organization
1104          *
1105          * @param array $siteinfo
1106          * @param array $jsonld
1107          *
1108          * @return array siteinfo
1109          */
1110         private static function parseJsonLdWebOrganization(array $siteinfo, array $jsonld): array
1111         {
1112                 $jsonldinfo = [];
1113
1114                 $content = JsonLD::fetchElement($jsonld, 'name');
1115                 if (!empty($content) && is_string($content)) {
1116                         $jsonldinfo['publisher_name'] = trim($content);
1117                 }
1118
1119                 $content = JsonLD::fetchElement($jsonld, 'description');
1120                 if (!empty($content) && is_string($content)) {
1121                         $jsonldinfo['publisher_description'] = trim($content);
1122                 }
1123
1124                 $content = JsonLD::fetchElement($jsonld, 'url');
1125                 if (!empty($content) && is_string($content)) {
1126                         $jsonldinfo['publisher_url'] = trim($content);
1127                 }
1128
1129                 $content = JsonLD::fetchElement($jsonld, 'logo', 'url', '@type', 'ImageObject');
1130                 if (!empty($content) && is_string($content)) {
1131                         $jsonldinfo['publisher_img'] = trim($content);
1132                 } elseif (!empty($content) && is_array($content)) {
1133                         $jsonldinfo['publisher_img'] = trim($content[0]);
1134                 }
1135
1136                 $content = JsonLD::fetchElement($jsonld, 'brand', 'name', '@type', 'Organization');
1137                 if (!empty($content) && is_string($content)) {
1138                         $jsonldinfo['publisher_name'] = trim($content);
1139                 }
1140
1141                 $content = JsonLD::fetchElement($jsonld, 'brand', 'url', '@type', 'Organization');
1142                 if (!empty($content) && is_string($content)) {
1143                         $jsonldinfo['publisher_url'] = trim($content);
1144                 }
1145
1146                 Logger::info('Fetched Organization information', ['url' => $siteinfo['url'], 'fetched' => $jsonldinfo]);
1147                 return array_merge($siteinfo, $jsonldinfo);
1148         }
1149
1150         /**
1151          * Fetch data from the provided JSON-LD Person type
1152          * @see https://schema.org/Person
1153          *
1154          * @param array $siteinfo
1155          * @param array $jsonld
1156          *
1157          * @return array siteinfo
1158          */
1159         private static function parseJsonLdWebPerson(array $siteinfo, array $jsonld): array
1160         {
1161                 $jsonldinfo = [];
1162
1163                 $content = JsonLD::fetchElement($jsonld, 'name');
1164                 if (!empty($content) && is_string($content)) {
1165                         $jsonldinfo['author_name'] = trim($content);
1166                 }
1167
1168                 $content = JsonLD::fetchElement($jsonld, 'description');
1169                 if (!empty($content) && is_string($content)) {
1170                         $jsonldinfo['author_description'] = trim($content);
1171                 }
1172
1173                 $content = JsonLD::fetchElement($jsonld, 'sameAs');
1174                 if (!empty($content) && is_string($content)) {
1175                         $jsonldinfo['author_url'] = trim($content);
1176                 }
1177
1178                 $content = JsonLD::fetchElement($jsonld, 'url');
1179                 if (!empty($content) && is_string($content)) {
1180                         $jsonldinfo['author_url'] = trim($content);
1181                 }
1182
1183                 $content = JsonLD::fetchElement($jsonld, 'image', 'url', '@type', 'ImageObject');
1184                 if (!empty($content) && !is_string($content)) {
1185                         Logger::notice('Unexpected return value for the author image', ['content' => $content]);
1186                 }
1187
1188                 if (!empty($content) && is_string($content)) {
1189                         $jsonldinfo['author_img'] = trim($content);
1190                 }
1191
1192                 Logger::info('Fetched Person information', ['url' => $siteinfo['url'], 'fetched' => $jsonldinfo]);
1193                 return array_merge($siteinfo, $jsonldinfo);
1194         }
1195
1196         /**
1197          * Fetch data from the provided JSON-LD MediaObject type
1198          * @see https://schema.org/MediaObject
1199          *
1200          * @param array $siteinfo
1201          * @param array $jsonld
1202          *
1203          * @return array siteinfo
1204          */
1205         private static function parseJsonLdMediaObject(array $siteinfo, array $jsonld, string $name): array
1206         {
1207                 $media = [];
1208
1209                 $content = JsonLD::fetchElement($jsonld, 'caption');
1210                 if (!empty($content) && is_string($content)) {
1211                         $media['caption'] = trim($content);
1212                 }
1213
1214                 $content = JsonLD::fetchElement($jsonld, 'url');
1215                 if (!empty($content) && is_string($content)) {
1216                         $media['url'] = trim($content);
1217                 }
1218
1219                 $content = JsonLD::fetchElement($jsonld, 'mainEntityOfPage');
1220                 if (!empty($content) && is_string($content)) {
1221                         $media['main'] = Strings::compareLink($content, $siteinfo['url']);
1222                 }
1223
1224                 $content = JsonLD::fetchElement($jsonld, 'description');
1225                 if (!empty($content) && is_string($content)) {
1226                         $media['description'] = trim($content);
1227                 }
1228
1229                 $content = JsonLD::fetchElement($jsonld, 'name');
1230                 if (!empty($content) && (($media['description'] ?? '') != trim($content))) {
1231                         $media['name'] = trim($content);
1232                 }
1233
1234                 $content = JsonLD::fetchElement($jsonld, 'contentUrl');
1235                 if (!empty($content) && is_string($content)) {
1236                         $media['content'] = trim($content);
1237                 }
1238
1239                 $content = JsonLD::fetchElement($jsonld, 'embedUrl');
1240                 if (!empty($content) && is_string($content)) {
1241                         $media['embed'] = trim($content);
1242                 }
1243
1244                 $content = JsonLD::fetchElement($jsonld, 'height');
1245                 if (!empty($content) && is_string($content)) {
1246                         $media['height'] = trim($content);
1247                 }
1248
1249                 $content = JsonLD::fetchElement($jsonld, 'width');
1250                 if (!empty($content) && is_string($content)) {
1251                         $media['width'] = trim($content);
1252                 }
1253
1254                 $content = JsonLD::fetchElement($jsonld, 'image');
1255                 if (!empty($content) && is_string($content)) {
1256                         $media['image'] = trim($content);
1257                 }
1258
1259                 $content = JsonLD::fetchElement($jsonld, 'thumbnailUrl');
1260                 if (!empty($content) && (($media['image'] ?? '') != trim($content))) {
1261                         if (!empty($media['image'])) {
1262                                 $media['preview'] = trim($content);
1263                         } else {
1264                                 $media['image'] = trim($content);
1265                         }
1266                 }
1267
1268                 Logger::info('Fetched Media information', ['url' => $siteinfo['url'], 'fetched' => $media]);
1269                 $siteinfo[$name][] = $media;
1270                 return $siteinfo;
1271         }
1272 }