mod/parse_url.php

   1 <?php
   2 /**
   3  * @file mod/parse_url.php
   4  *
   5  * @todo https://developers.google.com/+/plugins/snippet/
   6  *
   7  * @verbatim
   8  * <meta itemprop="name" content="Toller Titel">
   9  * <meta itemprop="description" content="Eine tolle Beschreibung">
  10  * <meta itemprop="image" content="http://maple.libertreeproject.org/images/tree-icon.png">
  11  *
  12  * <body itemscope itemtype="http://schema.org/Product">
  13  *   <h1 itemprop="name">Shiny Trinket</h1>
  14  *   <img itemprop="image" src="{image-url}" />
  15  *   <p itemprop="description">Shiny trinkets are shiny.</p>
  16  * </body>
  17  * @endverbatim
  18 */
  19
  20 if(!function_exists('deletenode')) {
  21         function deletenode(&$doc, $node)
  22         {
  23                 $xpath = new DomXPath($doc);
  24                 $list = $xpath->query("//".$node);
  25                 foreach ($list as $child)
  26                         $child->parentNode->removeChild($child);
  27         }
  28 }
  29
  30 function completeurl($url, $scheme) {
  31         $urlarr = parse_url($url);
  32
  33         if (isset($urlarr["scheme"]))
  34                 return($url);
  35
  36         $schemearr = parse_url($scheme);
  37
  38         $complete = $schemearr["scheme"]."://".$schemearr["host"];
  39
  40         if (@$schemearr["port"] != "")
  41                 $complete .= ":".$schemearr["port"];
  42
  43                 if(strpos($urlarr['path'],'/') !== 0)
  44                         $complete .= '/';
  45
  46         $complete .= $urlarr["path"];
  47
  48         if (@$urlarr["query"] != "")
  49                 $complete .= "?".$urlarr["query"];
  50
  51         if (@$urlarr["fragment"] != "")
  52                 $complete .= "#".$urlarr["fragment"];
  53
  54         return($complete);
  55 }
  56
  57 function parseurl_getsiteinfo_cached($url, $no_guessing = false, $do_oembed = true) {
  58
  59         if ($url == "")
  60                 return false;
  61
  62         $r = q("SELECT * FROM `parsed_url` WHERE `url` = '%s' AND `guessing` = %d AND `oembed` = %d",
  63                 dbesc(normalise_link($url)), intval(!$no_guessing), intval($do_oembed));
  64
  65         if ($r)
  66                 $data = $r[0]["content"];
  67
  68         if (!is_null($data)) {
  69                 $data = unserialize($data);
  70                 return $data;
  71         }
  72
  73         $data = parseurl_getsiteinfo($url, $no_guessing, $do_oembed);
  74
  75         q("INSERT INTO `parsed_url` (`url`, `guessing`, `oembed`, `content`, `created`) VALUES ('%s', %d, %d, '%s', '%s')
  76                  ON DUPLICATE KEY UPDATE `content` = '%s', `created` = '%s'",
  77                 dbesc(normalise_link($url)), intval(!$no_guessing), intval($do_oembed),
  78                 dbesc(serialize($data)), dbesc(datetime_convert()),
  79                 dbesc(serialize($data)), dbesc(datetime_convert()));
  80
  81         return $data;
  82 }
  83
  84 function parseurl_getsiteinfo($url, $no_guessing = false, $do_oembed = true, $count = 1) {
  85         require_once("include/network.php");
  86         require_once("include/Photo.php");
  87
  88         $a = get_app();
  89
  90         $siteinfo = array();
  91
  92         // Check if the URL does contain a scheme
  93         $scheme = parse_url($url, PHP_URL_SCHEME);
  94
  95         if ($scheme == "") {
  96                 $url = "http://".trim($url, "/");
  97         }
  98
  99         if ($count > 10) {
 100                 logger("parseurl_getsiteinfo: Endless loop detected for ".$url, LOGGER_DEBUG);
 101                 return($siteinfo);
 102         }
 103
 104         $url = trim($url, "'");
 105         $url = trim($url, '"');
 106
 107         $url = original_url($url);
 108
 109         $siteinfo["url"] = $url;
 110         $siteinfo["type"] = "link";
 111
 112         $check_cert = get_config('system','verifyssl');
 113
 114         $stamp1 = microtime(true);
 115
 116         $ch = curl_init();
 117         curl_setopt($ch, CURLOPT_URL, $url);
 118         curl_setopt($ch, CURLOPT_HEADER, 1);
 119         curl_setopt($ch, CURLOPT_NOBODY, 1);
 120         curl_setopt($ch, CURLOPT_TIMEOUT, 3);
 121         curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
 122         curl_setopt($ch, CURLOPT_USERAGENT, $a->get_useragent());
 123         curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, (($check_cert) ? true : false));
 124         curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, (($check_cert) ? 2 : false));
 125
 126         $header = curl_exec($ch);
 127         $curl_info = @curl_getinfo($ch);
 128         $http_code = $curl_info['http_code'];
 129         curl_close($ch);
 130
 131         $a->save_timestamp($stamp1, "network");
 132
 133         if ((($curl_info['http_code'] == "301") OR ($curl_info['http_code'] == "302") OR ($curl_info['http_code'] == "303") OR ($curl_info['http_code'] == "307"))
 134                 AND (($curl_info['redirect_url'] != "") OR ($curl_info['location'] != ""))) {
 135                 if ($curl_info['redirect_url'] != "")
 136                         $siteinfo = parseurl_getsiteinfo($curl_info['redirect_url'], $no_guessing, $do_oembed, ++$count);
 137                 else
 138                         $siteinfo = parseurl_getsiteinfo($curl_info['location'], $no_guessing, $do_oembed, ++$count);
 139                 return($siteinfo);
 140         }
 141
 142         // if the file is too large then exit
 143         if ($curl_info["download_content_length"] > 1000000)
 144                 return($siteinfo);
 145
 146         // if it isn't a HTML file then exit
 147         if (($curl_info["content_type"] != "") AND !strstr(strtolower($curl_info["content_type"]),"html"))
 148                 return($siteinfo);
 149
 150         if ($do_oembed) {
 151                 require_once("include/oembed.php");
 152
 153                 $oembed_data = oembed_fetch_url($url);
 154
 155                 if (!in_array($oembed_data->type, array("error", "rich"))) {
 156                         $siteinfo["type"] = $oembed_data->type;
 157                 }
 158
 159                 if (($oembed_data->type == "link") AND ($siteinfo["type"] != "photo")) {
 160                         if (isset($oembed_data->title))
 161                                 $siteinfo["title"] = $oembed_data->title;
 162                         if (isset($oembed_data->description))
 163                                 $siteinfo["text"] = trim($oembed_data->description);
 164                         if (isset($oembed_data->thumbnail_url))
 165                                 $siteinfo["image"] = $oembed_data->thumbnail_url;
 166                 }
 167         }
 168
 169         $stamp1 = microtime(true);
 170
 171         // Now fetch the body as well
 172         $ch = curl_init();
 173         curl_setopt($ch, CURLOPT_URL, $url);
 174         curl_setopt($ch, CURLOPT_HEADER, 1);
 175         curl_setopt($ch, CURLOPT_NOBODY, 0);
 176         curl_setopt($ch, CURLOPT_TIMEOUT, 10);
 177         curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
 178         curl_setopt($ch, CURLOPT_USERAGENT, $a->get_useragent());
 179         curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, (($check_cert) ? true : false));
 180         curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, (($check_cert) ? 2 : false));
 181
 182         $header = curl_exec($ch);
 183         $curl_info = @curl_getinfo($ch);
 184         $http_code = $curl_info['http_code'];
 185         curl_close($ch);
 186
 187         $a->save_timestamp($stamp1, "network");
 188
 189         // Fetch the first mentioned charset. Can be in body or header
 190         $charset = "";
 191         if (preg_match('/charset=(.*?)['."'".'"\s\n]/', $header, $matches))
 192                 $charset = trim(trim(trim(array_pop($matches)), ';,'));
 193
 194         if ($charset == "")
 195                 $charset = "utf-8";
 196
 197         $pos = strpos($header, "\r\n\r\n");
 198
 199         if ($pos)
 200                 $body = trim(substr($header, $pos));
 201         else
 202                 $body = $header;
 203
 204         if (($charset != '') AND (strtoupper($charset) != "UTF-8")) {
 205                 logger("parseurl_getsiteinfo: detected charset ".$charset, LOGGER_DEBUG);
 206                 //$body = mb_convert_encoding($body, "UTF-8", $charset);
 207                 $body = iconv($charset, "UTF-8//TRANSLIT", $body);
 208         }
 209
 210         $body = mb_convert_encoding($body, 'HTML-ENTITIES', "UTF-8");
 211
 212         $doc = new DOMDocument();
 213         @$doc->loadHTML($body);
 214
 215         deletenode($doc, 'style');
 216         deletenode($doc, 'script');
 217         deletenode($doc, 'option');
 218         deletenode($doc, 'h1');
 219         deletenode($doc, 'h2');
 220         deletenode($doc, 'h3');
 221         deletenode($doc, 'h4');
 222         deletenode($doc, 'h5');
 223         deletenode($doc, 'h6');
 224         deletenode($doc, 'ol');
 225         deletenode($doc, 'ul');
 226
 227         $xpath = new DomXPath($doc);
 228
 229         $list = $xpath->query("//meta[@content]");
 230         foreach ($list as $node) {
 231                 $attr = array();
 232                 if ($node->attributes->length)
 233                         foreach ($node->attributes as $attribute)
 234                                 $attr[$attribute->name] = $attribute->value;
 235
 236                 if (@$attr["http-equiv"] == 'refresh') {
 237                         $path = $attr["content"];
 238                         $pathinfo = explode(";", $path);
 239                         $content = "";
 240                         foreach ($pathinfo AS $value) {
 241                                 if (substr(strtolower($value), 0, 4) == "url=")
 242                                         $content = substr($value, 4);
 243                         }
 244                         if ($content != "") {
 245                                 $siteinfo = parseurl_getsiteinfo($content, $no_guessing, $do_oembed, ++$count);
 246                                 return($siteinfo);
 247                         }
 248                 }
 249         }
 250
 251         $list = $xpath->query("//title");
 252         if ($list->length > 0)
 253                 $siteinfo["title"] = $list->item(0)->nodeValue;
 254
 255         //$list = $xpath->query("head/meta[@name]");
 256         $list = $xpath->query("//meta[@name]");
 257         foreach ($list as $node) {
 258                 $attr = array();
 259                 if ($node->attributes->length)
 260                         foreach ($node->attributes as $attribute)
 261                                 $attr[$attribute->name] = $attribute->value;
 262
 263                 $attr["content"] = trim(html_entity_decode($attr["content"], ENT_QUOTES, "UTF-8"));
 264
 265                 if ($attr["content"] != "")
 266                         switch (strtolower($attr["name"])) {
 267                                 case "fulltitle":
 268                                         $siteinfo["title"] = $attr["content"];
 269                                         break;
 270                                 case "description":
 271                                         $siteinfo["text"] = $attr["content"];
 272                                         break;
 273                                 case "thumbnail":
 274                                         $siteinfo["image"] = $attr["content"];
 275                                         break;
 276                                 case "twitter:image":
 277                                         $siteinfo["image"] = $attr["content"];
 278                                         break;
 279                                 case "twitter:image:src":
 280                                         $siteinfo["image"] = $attr["content"];
 281                                         break;
 282                                 case "twitter:card":
 283                                         if (($siteinfo["type"] == "") OR ($attr["content"] == "photo"))
 284                                                 $siteinfo["type"] = $attr["content"];
 285                                         break;
 286                                 case "twitter:description":
 287                                         $siteinfo["text"] = $attr["content"];
 288                                         break;
 289                                 case "twitter:title":
 290                                         $siteinfo["title"] = $attr["content"];
 291                                         break;
 292                                 case "dc.title":
 293                                         $siteinfo["title"] = $attr["content"];
 294                                         break;
 295                                 case "dc.description":
 296                                         $siteinfo["text"] = $attr["content"];
 297                                         break;
 298                                 case "keywords":
 299                                         $keywords = explode(",", $attr["content"]);
 300                                         break;
 301                                 case "news_keywords":
 302                                         $keywords = explode(",", $attr["content"]);
 303                                         break;
 304                         }
 305                 if ($siteinfo["type"] == "summary")
 306                         $siteinfo["type"] = "link";
 307         }
 308
 309         if (isset($keywords)) {
 310                 $siteinfo["keywords"] = array();
 311                 foreach ($keywords as $keyword)
 312                         if (!in_array(trim($keyword), $siteinfo["keywords"]))
 313                                 $siteinfo["keywords"][] = trim($keyword);
 314         }
 315
 316         //$list = $xpath->query("head/meta[@property]");
 317         $list = $xpath->query("//meta[@property]");
 318         foreach ($list as $node) {
 319                 $attr = array();
 320                 if ($node->attributes->length)
 321                         foreach ($node->attributes as $attribute)
 322                                 $attr[$attribute->name] = $attribute->value;
 323
 324                 $attr["content"] = trim(html_entity_decode($attr["content"], ENT_QUOTES, "UTF-8"));
 325
 326                 if ($attr["content"] != "")
 327                         switch (strtolower($attr["property"])) {
 328                                 case "og:image":
 329                                         $siteinfo["image"] = $attr["content"];
 330                                         break;
 331                                 case "og:title":
 332                                         $siteinfo["title"] = $attr["content"];
 333                                         break;
 334                                 case "og:description":
 335                                         $siteinfo["text"] = $attr["content"];
 336                                         break;
 337                         }
 338         }
 339
 340         if ((@$siteinfo["image"] == "") AND !$no_guessing) {
 341             $list = $xpath->query("//img[@src]");
 342             foreach ($list as $node) {
 343                 $attr = array();
 344                 if ($node->attributes->length)
 345                     foreach ($node->attributes as $attribute)
 346                         $attr[$attribute->name] = $attribute->value;
 347
 348                         $src = completeurl($attr["src"], $url);
 349                         $photodata = get_photo_info($src);
 350
 351                         if (($photodata) && ($photodata[0] > 150) and ($photodata[1] > 150)) {
 352                                 if ($photodata[0] > 300) {
 353                                         $photodata[1] = round($photodata[1] * (300 / $photodata[0]));
 354                                         $photodata[0] = 300;
 355                                 }
 356                                 if ($photodata[1] > 300) {
 357                                         $photodata[0] = round($photodata[0] * (300 / $photodata[1]));
 358                                         $photodata[1] = 300;
 359                                 }
 360                                 $siteinfo["images"][] = array("src"=>$src,
 361                                                                 "width"=>$photodata[0],
 362                                                                 "height"=>$photodata[1]);
 363                         }
 364
 365                 }
 366     } elseif ($siteinfo["image"] != "") {
 367                 $src = completeurl($siteinfo["image"], $url);
 368
 369                 unset($siteinfo["image"]);
 370
 371                 $photodata = get_photo_info($src);
 372
 373                 if (($photodata) && ($photodata[0] > 10) and ($photodata[1] > 10))
 374                         $siteinfo["images"][] = array("src"=>$src,
 375                                                         "width"=>$photodata[0],
 376                                                         "height"=>$photodata[1]);
 377         }
 378
 379         if ((@$siteinfo["text"] == "") AND (@$siteinfo["title"] != "") AND !$no_guessing) {
 380                 $text = "";
 381
 382                 $list = $xpath->query("//div[@class='article']");
 383                 foreach ($list as $node)
 384                         if (strlen($node->nodeValue) > 40)
 385                                 $text .= " ".trim($node->nodeValue);
 386
 387                 if ($text == "") {
 388                         $list = $xpath->query("//div[@class='content']");
 389                         foreach ($list as $node)
 390                                 if (strlen($node->nodeValue) > 40)
 391                                         $text .= " ".trim($node->nodeValue);
 392                 }
 393
 394                 // If none text was found then take the paragraph content
 395                 if ($text == "") {
 396                         $list = $xpath->query("//p");
 397                         foreach ($list as $node)
 398                                 if (strlen($node->nodeValue) > 40)
 399                                         $text .= " ".trim($node->nodeValue);
 400                 }
 401
 402                 if ($text != "") {
 403                         $text = trim(str_replace(array("\n", "\r"), array(" ", " "), $text));
 404
 405                         while (strpos($text, "  "))
 406                                 $text = trim(str_replace("  ", " ", $text));
 407
 408                         $siteinfo["text"] = trim(html_entity_decode(substr($text,0,350), ENT_QUOTES, "UTF-8").'...');
 409                 }
 410         }
 411
 412         logger("parseurl_getsiteinfo: Siteinfo for ".$url." ".print_r($siteinfo, true), LOGGER_DEBUG);
 413
 414         call_hooks('getsiteinfo', $siteinfo);
 415
 416         return($siteinfo);
 417 }
 418
 419 function arr_add_hashes(&$item,$k) {
 420         $item = '#' . $item;
 421 }
 422
 423 function parse_url_content(&$a) {
 424
 425         require_once("include/items.php");
 426
 427         $text = null;
 428         $str_tags = '';
 429
 430         $textmode = false;
 431
 432         if(local_user() && (! feature_enabled(local_user(),'richtext')))
 433                 $textmode = true;
 434
 435         //if($textmode)
 436         $br = (($textmode) ? "\n" : '<br />');
 437
 438         if(x($_GET,'binurl'))
 439                 $url = trim(hex2bin($_GET['binurl']));
 440         else
 441                 $url = trim($_GET['url']);
 442
 443         if($_GET['title'])
 444                 $title = strip_tags(trim($_GET['title']));
 445
 446         if($_GET['description'])
 447                 $text = strip_tags(trim($_GET['description']));
 448
 449         if($_GET['tags']) {
 450                 $arr_tags = str_getcsv($_GET['tags']);
 451                 if(count($arr_tags)) {
 452                         array_walk($arr_tags,'arr_add_hashes');
 453                         $str_tags = $br . implode(' ',$arr_tags) . $br;
 454                 }
 455         }
 456
 457         // add url scheme if missing
 458         $arrurl = parse_url($url);
 459         if (!x($arrurl, 'scheme')) {
 460                 if (x($arrurl, 'host'))
 461                         $url = "http:".$url;
 462                 else
 463                         $url = "http://".$url;
 464         }
 465
 466         logger('parse_url: ' . $url);
 467
 468         if($textmode)
 469                 $template = '[bookmark=%s]%s[/bookmark]%s';
 470         else
 471                 $template = "<a class=\"bookmark\" href=\"%s\" >%s</a>%s";
 472
 473         $arr = array('url' => $url, 'text' => '');
 474
 475         call_hooks('parse_link', $arr);
 476
 477         if(strlen($arr['text'])) {
 478                 echo $arr['text'];
 479                 killme();
 480         }
 481
 482
 483         if($url && $title && $text) {
 484
 485                 $title = str_replace(array("\r","\n"),array('',''),$title);
 486
 487                 if($textmode)
 488                         $text = '[quote]' . trim($text) . '[/quote]' . $br;
 489                 else {
 490                         $text = '<blockquote>' . htmlspecialchars(trim($text)) . '</blockquote><br />';
 491                         $title = htmlspecialchars($title);
 492                 }
 493
 494                 $result = sprintf($template,$url,($title) ? $title : $url,$text) . $str_tags;
 495
 496                 logger('parse_url (unparsed): returns: ' . $result);
 497
 498                 echo $result;
 499                 killme();
 500         }
 501
 502         $siteinfo = parseurl_getsiteinfo($url);
 503
 504         unset($siteinfo["keywords"]);
 505
 506         $info = add_page_info_data($siteinfo);
 507
 508         if (!$textmode)
 509                 // Replace ' with ’ - not perfect - but the richtext editor has problems otherwise
 510                 $info = str_replace(array("&#039;"), array("&#8217;"), $info);
 511
 512         echo $info;
 513
 514         killme();
 515 }
 516 ?>