]> git.mxchange.org Git - friendica.git/blob - src/ParseUrl.php
Commenting to (nearly) every Friendica contact is now possible
[friendica.git] / src / ParseUrl.php
1 <?php
2 /**
3  * @file include/ParseUrl.php
4  * @brief Get informations about a given URL
5  */
6 namespace Friendica;
7
8 use Friendica\Content\OEmbed;
9 use Friendica\Object\Image;
10 use Friendica\Util\XML;
11
12 use dba;
13 use DOMXPath;
14 use DOMDocument;
15
16 require_once 'include/dba.php';
17 require_once "include/network.php";
18
19 /**
20  * @brief Class with methods for extracting certain content from an url
21  */
22 class ParseUrl
23 {
24         /**
25          * @brief Search for chached embeddable data of an url otherwise fetch it
26          *
27          * @param string $url         The url of the page which should be scraped
28          * @param bool $no_guessing If true the parse doens't search for
29          *                          preview pictures
30          * @param bool $do_oembed   The false option is used by the function fetch_oembed()
31          *                          to avoid endless loops
32          *
33          * @return array which contains needed data for embedding
34          *    string 'url' => The url of the parsed page
35          *    string 'type' => Content type
36          *    string 'title' => The title of the content
37          *    string 'text' => The description for the content
38          *    string 'image' => A preview image of the content (only available
39          *                if $no_geuessing = false
40          *    array'images' = Array of preview pictures
41          *    string 'keywords' => The tags which belong to the content
42          *
43          * @see ParseUrl::getSiteinfo() for more information about scraping
44          * embeddable content
45          */
46         public static function getSiteinfoCached($url, $no_guessing = false, $do_oembed = true)
47         {
48                 if ($url == "") {
49                         return false;
50                 }
51
52                 $r = q(
53                         "SELECT * FROM `parsed_url` WHERE `url` = '%s' AND `guessing` = %d AND `oembed` = %d",
54                         dbesc(normalise_link($url)),
55                         intval(!$no_guessing),
56                         intval($do_oembed)
57                 );
58
59                 if ($r) {
60                         $data = $r[0]["content"];
61                 }
62
63                 if (!is_null($data)) {
64                         $data = unserialize($data);
65                         return $data;
66                 }
67
68                 $data = self::getSiteinfo($url, $no_guessing, $do_oembed);
69
70                 dba::insert(
71                         'parsed_url',
72                         array(
73                                 'url' => normalise_link($url), 'guessing' => !$no_guessing,
74                                 'oembed' => $do_oembed, 'content' => serialize($data),
75                                 'created' => datetime_convert()),
76                         true
77                 );
78
79                 return $data;
80         }
81         /**
82          * @brief Parse a page for embeddable content information
83          *
84          * This method parses to url for meta data which can be used to embed
85          * the content. If available it prioritizes Open Graph meta tags.
86          * If this is not available it uses the twitter cards meta tags.
87          * As fallback it uses standard html elements with meta informations
88          * like \<title\>Awesome Title\</title\> or
89          * \<meta name="description" content="An awesome description"\>
90          *
91          * @param string $url         The url of the page which should be scraped
92          * @param bool $no_guessing If true the parse doens't search for
93          *                          preview pictures
94          * @param bool $do_oembed   The false option is used by the function fetch_oembed()
95          *                          to avoid endless loops
96          * @param int $count       Internal counter to avoid endless loops
97          *
98          * @return array which contains needed data for embedding
99          *    string 'url' => The url of the parsed page
100          *    string 'type' => Content type
101          *    string 'title' => The title of the content
102          *    string 'text' => The description for the content
103          *    string 'image' => A preview image of the content (only available
104          *                if $no_geuessing = false
105          *    array'images' = Array of preview pictures
106          *    string 'keywords' => The tags which belong to the content
107          *
108          * @todo https://developers.google.com/+/plugins/snippet/
109          * @verbatim
110          * <meta itemprop="name" content="Awesome title">
111          * <meta itemprop="description" content="An awesome description">
112          * <meta itemprop="image" content="http://maple.libertreeproject.org/images/tree-icon.png">
113          *
114          * <body itemscope itemtype="http://schema.org/Product">
115          *   <h1 itemprop="name">Shiny Trinket</h1>
116          *   <img itemprop="image" src="{image-url}" />
117          *   <p itemprop="description">Shiny trinkets are shiny.</p>
118          * </body>
119          * @endverbatim
120          */
121         public static function getSiteinfo($url, $no_guessing = false, $do_oembed = true, $count = 1)
122         {
123                 $a = get_app();
124
125                 $siteinfo = array();
126
127                 // Check if the URL does contain a scheme
128                 $scheme = parse_url($url, PHP_URL_SCHEME);
129
130                 if ($scheme == "") {
131                         $url = "http://".trim($url, "/");
132                 }
133
134                 if ($count > 10) {
135                         logger("parseurl_getsiteinfo: Endless loop detected for ".$url, LOGGER_DEBUG);
136                         return($siteinfo);
137                 }
138
139                 $url = trim($url, "'");
140                 $url = trim($url, '"');
141
142                 $url = strip_tracking_query_params($url);
143
144                 $siteinfo["url"] = $url;
145                 $siteinfo["type"] = "link";
146
147                 $data = z_fetch_url($url);
148                 if (!$data['success']) {
149                         return($siteinfo);
150                 }
151
152                 // If the file is too large then exit
153                 if ($data["info"]["download_content_length"] > 1000000) {
154                         return($siteinfo);
155                 }
156
157                 // If it isn't a HTML file then exit
158                 if (($data["info"]["content_type"] != "") && !strstr(strtolower($data["info"]["content_type"]), "html")) {
159                         return($siteinfo);
160                 }
161
162                 $header = $data["header"];
163                 $body = $data["body"];
164
165                 if ($do_oembed) {
166                         $oembed_data = OEmbed::fetchURL($url);
167
168                         if (!in_array($oembed_data->type, array("error", "rich", ""))) {
169                                 $siteinfo["type"] = $oembed_data->type;
170                         }
171
172                         if (($oembed_data->type == "link") && ($siteinfo["type"] != "photo")) {
173                                 if (isset($oembed_data->title)) {
174                                         $siteinfo["title"] = trim($oembed_data->title);
175                                 }
176                                 if (isset($oembed_data->description)) {
177                                         $siteinfo["text"] = trim($oembed_data->description);
178                                 }
179                                 if (isset($oembed_data->thumbnail_url)) {
180                                         $siteinfo["image"] = $oembed_data->thumbnail_url;
181                                 }
182                         }
183                 }
184
185                 // Fetch the first mentioned charset. Can be in body or header
186                 $charset = "";
187                 if (preg_match('/charset=(.*?)['."'".'"\s\n]/', $header, $matches)) {
188                         $charset = trim(trim(trim(array_pop($matches)), ';,'));
189                 }
190
191                 if ($charset == "") {
192                         $charset = "utf-8";
193                 }
194
195                 if (($charset != "") && (strtoupper($charset) != "UTF-8")) {
196                         logger("parseurl_getsiteinfo: detected charset ".$charset, LOGGER_DEBUG);
197                         //$body = mb_convert_encoding($body, "UTF-8", $charset);
198                         $body = iconv($charset, "UTF-8//TRANSLIT", $body);
199                 }
200
201                 $body = mb_convert_encoding($body, 'HTML-ENTITIES', "UTF-8");
202
203                 $doc = new DOMDocument();
204                 @$doc->loadHTML($body);
205
206                 XML::deleteNode($doc, "style");
207                 XML::deleteNode($doc, "script");
208                 XML::deleteNode($doc, "option");
209                 XML::deleteNode($doc, "h1");
210                 XML::deleteNode($doc, "h2");
211                 XML::deleteNode($doc, "h3");
212                 XML::deleteNode($doc, "h4");
213                 XML::deleteNode($doc, "h5");
214                 XML::deleteNode($doc, "h6");
215                 XML::deleteNode($doc, "ol");
216                 XML::deleteNode($doc, "ul");
217
218                 $xpath = new DOMXPath($doc);
219
220                 $list = $xpath->query("//meta[@content]");
221                 foreach ($list as $node) {
222                         $attr = array();
223                         if ($node->attributes->length) {
224                                 foreach ($node->attributes as $attribute) {
225                                         $attr[$attribute->name] = $attribute->value;
226                                 }
227                         }
228
229                         if (@$attr["http-equiv"] == "refresh") {
230                                 $path = $attr["content"];
231                                 $pathinfo = explode(";", $path);
232                                 $content = "";
233                                 foreach ($pathinfo as $value) {
234                                         if (substr(strtolower($value), 0, 4) == "url=") {
235                                                 $content = substr($value, 4);
236                                         }
237                                 }
238                                 if ($content != "") {
239                                         $siteinfo = self::getSiteinfo($content, $no_guessing, $do_oembed, ++$count);
240                                         return($siteinfo);
241                                 }
242                         }
243                 }
244
245                 $list = $xpath->query("//title");
246                 if ($list->length > 0) {
247                         $siteinfo["title"] = trim($list->item(0)->nodeValue);
248                 }
249
250                 //$list = $xpath->query("head/meta[@name]");
251                 $list = $xpath->query("//meta[@name]");
252                 foreach ($list as $node) {
253                         $attr = array();
254                         if ($node->attributes->length) {
255                                 foreach ($node->attributes as $attribute) {
256                                         $attr[$attribute->name] = $attribute->value;
257                                 }
258                         }
259
260                         $attr["content"] = trim(html_entity_decode($attr["content"], ENT_QUOTES, "UTF-8"));
261
262                         if ($attr["content"] != "") {
263                                 switch (strtolower($attr["name"])) {
264                                         case "fulltitle":
265                                                 $siteinfo["title"] = trim($attr["content"]);
266                                                 break;
267                                         case "description":
268                                                 $siteinfo["text"] = trim($attr["content"]);
269                                                 break;
270                                         case "thumbnail":
271                                                 $siteinfo["image"] = $attr["content"];
272                                                 break;
273                                         case "twitter:image":
274                                                 $siteinfo["image"] = $attr["content"];
275                                                 break;
276                                         case "twitter:image:src":
277                                                 $siteinfo["image"] = $attr["content"];
278                                                 break;
279                                         case "twitter:card":
280                                                 if (($siteinfo["type"] == "") || ($attr["content"] == "photo")) {
281                                                         $siteinfo["type"] = $attr["content"];
282                                                 }
283                                                 break;
284                                         case "twitter:description":
285                                                 $siteinfo["text"] = trim($attr["content"]);
286                                                 break;
287                                         case "twitter:title":
288                                                 $siteinfo["title"] = trim($attr["content"]);
289                                                 break;
290                                         case "dc.title":
291                                                 $siteinfo["title"] = trim($attr["content"]);
292                                                 break;
293                                         case "dc.description":
294                                                 $siteinfo["text"] = trim($attr["content"]);
295                                                 break;
296                                         case "keywords":
297                                                 $keywords = explode(",", $attr["content"]);
298                                                 break;
299                                         case "news_keywords":
300                                                 $keywords = explode(",", $attr["content"]);
301                                                 break;
302                                 }
303                         }
304                         if ($siteinfo["type"] == "summary") {
305                                 $siteinfo["type"] = "link";
306                         }
307                 }
308
309                 if (isset($keywords)) {
310                         $siteinfo["keywords"] = array();
311                         foreach ($keywords as $keyword) {
312                                 if (!in_array(trim($keyword), $siteinfo["keywords"])) {
313                                         $siteinfo["keywords"][] = trim($keyword);
314                                 }
315                         }
316                 }
317
318                 //$list = $xpath->query("head/meta[@property]");
319                 $list = $xpath->query("//meta[@property]");
320                 foreach ($list as $node) {
321                         $attr = array();
322                         if ($node->attributes->length) {
323                                 foreach ($node->attributes as $attribute) {
324                                         $attr[$attribute->name] = $attribute->value;
325                                 }
326                         }
327
328                         $attr["content"] = trim(html_entity_decode($attr["content"], ENT_QUOTES, "UTF-8"));
329
330                         if ($attr["content"] != "") {
331                                 switch (strtolower($attr["property"])) {
332                                         case "og:image":
333                                                 $siteinfo["image"] = $attr["content"];
334                                                 break;
335                                         case "og:title":
336                                                 $siteinfo["title"] = trim($attr["content"]);
337                                                 break;
338                                         case "og:description":
339                                                 $siteinfo["text"] = trim($attr["content"]);
340                                                 break;
341                                 }
342                         }
343                 }
344
345                 if ((@$siteinfo["image"] == "") && !$no_guessing) {
346                         $list = $xpath->query("//img[@src]");
347                         foreach ($list as $node) {
348                                 $attr = array();
349                                 if ($node->attributes->length) {
350                                         foreach ($node->attributes as $attribute) {
351                                                 $attr[$attribute->name] = $attribute->value;
352                                         }
353                                 }
354
355                                 $src = self::completeUrl($attr["src"], $url);
356                                 $photodata = Image::getInfoFromURL($src);
357
358                                 if (($photodata) && ($photodata[0] > 150) && ($photodata[1] > 150)) {
359                                         if ($photodata[0] > 300) {
360                                                 $photodata[1] = round($photodata[1] * (300 / $photodata[0]));
361                                                 $photodata[0] = 300;
362                                         }
363                                         if ($photodata[1] > 300) {
364                                                 $photodata[0] = round($photodata[0] * (300 / $photodata[1]));
365                                                 $photodata[1] = 300;
366                                         }
367                                         $siteinfo["images"][] = array("src" => $src,
368                                                                         "width" => $photodata[0],
369                                                                         "height" => $photodata[1]);
370                                 }
371                         }
372                 } elseif ($siteinfo["image"] != "") {
373                         $src = self::completeUrl($siteinfo["image"], $url);
374
375                         unset($siteinfo["image"]);
376
377                         $photodata = Image::getInfoFromURL($src);
378
379                         if (($photodata) && ($photodata[0] > 10) && ($photodata[1] > 10)) {
380                                 $siteinfo["images"][] = array("src" => $src,
381                                                                 "width" => $photodata[0],
382                                                                 "height" => $photodata[1]);
383                         }
384                 }
385
386                 if ((@$siteinfo["text"] == "") && (@$siteinfo["title"] != "") && !$no_guessing) {
387                         $text = "";
388
389                         $list = $xpath->query("//div[@class='article']");
390                         foreach ($list as $node) {
391                                 if (strlen($node->nodeValue) > 40) {
392                                         $text .= " ".trim($node->nodeValue);
393                                 }
394                         }
395
396                         if ($text == "") {
397                                 $list = $xpath->query("//div[@class='content']");
398                                 foreach ($list as $node) {
399                                         if (strlen($node->nodeValue) > 40) {
400                                                 $text .= " ".trim($node->nodeValue);
401                                         }
402                                 }
403                         }
404
405                         // If none text was found then take the paragraph content
406                         if ($text == "") {
407                                 $list = $xpath->query("//p");
408                                 foreach ($list as $node) {
409                                         if (strlen($node->nodeValue) > 40) {
410                                                 $text .= " ".trim($node->nodeValue);
411                                         }
412                                 }
413                         }
414
415                         if ($text != "") {
416                                 $text = trim(str_replace(array("\n", "\r"), array(" ", " "), $text));
417
418                                 while (strpos($text, "  ")) {
419                                         $text = trim(str_replace("  ", " ", $text));
420                                 }
421
422                                 $siteinfo["text"] = trim(html_entity_decode(substr($text, 0, 350), ENT_QUOTES, "UTF-8").'...');
423                         }
424                 }
425
426                 logger("parseurl_getsiteinfo: Siteinfo for ".$url." ".print_r($siteinfo, true), LOGGER_DEBUG);
427
428                 call_hooks("getsiteinfo", $siteinfo);
429
430                 return($siteinfo);
431         }
432
433         /**
434          * @brief Convert tags from CSV to an array
435          *
436          * @param string $string Tags
437          * @return array with formatted Hashtags
438          */
439         public static function convertTagsToArray($string)
440         {
441                 $arr_tags = str_getcsv($string);
442                 if (count($arr_tags)) {
443                         // add the # sign to every tag
444                         array_walk($arr_tags, array("self", "arrAddHashes"));
445
446                         return $arr_tags;
447                 }
448         }
449
450         /**
451          * @brief Add a hasht sign to a string
452          *
453          *  This method is used as callback function
454          *
455          * @param string $tag The pure tag name
456          * @param int    $k   Counter for internal use
457          * @return void
458          */
459         private static function arrAddHashes(&$tag, $k)
460         {
461                 $tag = "#" . $tag;
462         }
463
464         /**
465          * @brief Add a scheme to an url
466          *
467          * The src attribute of some html elements (e.g. images)
468          * can miss the scheme so we need to add the correct
469          * scheme
470          *
471          * @param string $url    The url which possibly does have
472          *                       a missing scheme (a link to an image)
473          * @param string $scheme The url with a correct scheme
474          *                       (e.g. the url from the webpage which does contain the image)
475          *
476          * @return string The url with a scheme
477          */
478         private static function completeUrl($url, $scheme)
479         {
480                 $urlarr = parse_url($url);
481
482                 // If the url does allready have an scheme
483                 // we can stop the process here
484                 if (isset($urlarr["scheme"])) {
485                         return($url);
486                 }
487
488                 $schemearr = parse_url($scheme);
489
490                 $complete = $schemearr["scheme"]."://".$schemearr["host"];
491
492                 if (@$schemearr["port"] != "") {
493                         $complete .= ":".$schemearr["port"];
494                 }
495
496                 if (strpos($urlarr["path"], "/") !== 0) {
497                         $complete .= "/";
498                 }
499
500                 $complete .= $urlarr["path"];
501
502                 if (@$urlarr["query"] != "") {
503                         $complete .= "?".$urlarr["query"];
504                 }
505
506                 if (@$urlarr["fragment"] != "") {
507                         $complete .= "#".$urlarr["fragment"];
508                 }
509
510                 return($complete);
511         }
512 }