]> git.mxchange.org Git - friendica.git/commitdiff
Merge remote branch 'upstream/master'
authorMichael Vogel <icarus@dabo.de>
Fri, 13 Jul 2012 21:39:51 +0000 (23:39 +0200)
committerMichael Vogel <icarus@dabo.de>
Fri, 13 Jul 2012 21:39:51 +0000 (23:39 +0200)
Conflicts:
include/bb2diaspora.php

include/api.php
include/bb2diaspora.php
include/bbcode.php
include/dba.php
include/items.php
include/network.php
mod/parse_url.php

index d790b4b875516bc39f661b99041723b8998d1c10..7d230629b0dab4445e41a0d9de140c46f71fef3e 100644 (file)
                                //echo "<pre>"; var_dump($r); die();
                        }
                }
+               header("HTTP/1.1 404 Not Found");
                logger('API call not implemented: '.$a->query_string." - ".print_r($_REQUEST,true));
                $r = '<status><error>not implemented</error></status>';
                switch($type){
                 $_REQUEST['type'] = 'wall';
                 $_REQUEST['profile_uid'] = local_user();
                 $_REQUEST['api_source'] = true;
-                $txt = urldecode(requestdata('status'));
+                $txt = requestdata('status');
+                //$txt = urldecode(requestdata('status'));
 
                 require_once('library/HTMLPurifier.auto.php');
                 require_once('include/html2bbcode.php');
 
                }
                else
-                       $_REQUEST['body'] = urldecode(requestdata('status'));
+                       $_REQUEST['body'] = requestdata('status');
+                       //$_REQUEST['body'] = urldecode(requestdata('status'));
 
                $parent = requestdata('in_reply_to_status_id');
                if(ctype_digit($parent))
@@ -1725,4 +1728,6 @@ notifications/follow
 notifications/leave
 blocks/exists
 blocks/blocking
+lists
 */
+
index b5feacea8b4ae5a70fbd54dff2729cc554de0b45..436412dbd487bc700465314ab812d5670f417df8 100644 (file)
@@ -192,61 +192,28 @@ function diaspora_ol($s) {
 
 function bb2diaspora($Text,$preserve_nl = false) {
 
-//////////////////////
-// An attempt was made to convert bbcode to html and then to markdown
-// consisting of the following lines.
-// I'm undoing this as we have a lot of bbcode constructs which
-// were simply getting lost, for instance bookmark, vimeo, video, youtube, events, etc.
-// We can try this again, but need a very good test sequence to verify
-// all the major bbcode constructs that we use are getting through.
-//////////////////////
-/*
-       // bbcode() will convert "[*]" into "<li>" with no closing "</li>"
-       // Markdownify() is unable to handle these, as it makes each new
-       // "<li>" into a deeper nested element until it crashes. So pre-format
-       // the lists as Diaspora lists before sending the $Text to bbcode()
-       //
-       // Note that to get nested lists to work for Diaspora, we would need
-       // to define the closing tag for the list elements. So nested lists
-       // are going to be flattened out in Diaspora for now
-
-       $endlessloop = 0;
-       while ((((strpos($Text, "[/list]") !== false) && (strpos($Text, "[list") !== false)) ||
-              ((strpos($Text, "[/ol]") !== false) && (strpos($Text, "[ol]") !== false)) || 
-              ((strpos($Text, "[/ul]") !== false) && (strpos($Text, "[ul]") !== false))) && (++$endlessloop < 20)) {
-               $Text = preg_replace_callback("/\[list\](.*?)\[\/list\]/is", 'diaspora_ul', $Text);
-               $Text = preg_replace_callback("/\[list=1\](.*?)\[\/list\]/is", 'diaspora_ol', $Text);
-               $Text = preg_replace_callback("/\[list=i\](.*?)\[\/list\]/s",'diaspora_ol', $Text);
-               $Text = preg_replace_callback("/\[list=I\](.*?)\[\/list\]/s", 'diaspora_ol', $Text);
-               $Text = preg_replace_callback("/\[list=a\](.*?)\[\/list\]/s", 'diaspora_ol', $Text);
-               $Text = preg_replace_callback("/\[list=A\](.*?)\[\/list\]/s", 'diaspora_ol', $Text);
-               $Text = preg_replace_callback("/\[ul\](.*?)\[\/ul\]/is", 'diaspora_ul', $Text);
-               $Text = preg_replace_callback("/\[ol\](.*?)\[\/ol\]/is", 'diaspora_ol', $Text);
-       }
-
-*/
+       // Re-enabling the converter again.
+       // The bbcode parser now handles youtube-links (and the other stuff) correctly.
+       // Additionally the html code is now fixed so that lists are now working.
 
        // Convert it to HTML - don't try oembed
-//     $Text = bbcode($Text, $preserve_nl, false);
+       $Text = bbcode($Text, $preserve_nl, false);
 
        // Now convert HTML to Markdown
-//     $md = new Markdownify(false, false, false);
-//     $Text = $md->parseString($Text);
+       $md = new Markdownify(false, false, false);
+       $Text = $md->parseString($Text);
 
        // If the text going into bbcode() has a plain URL in it, i.e.
        // with no [url] tags around it, it will come out of parseString()
        // looking like: <http://url.com>, which gets removed by strip_tags().
        // So take off the angle brackets of any such URL
-//     $Text = preg_replace("/<http(.*?)>/is", "http$1", $Text);
+       $Text = preg_replace("/<http(.*?)>/is", "http$1", $Text);
 
        // Remove all unconverted tags
-//     $Text = strip_tags($Text);
-
-////// 
-// end of bb->html->md conversion attempt
-//////
+       $Text = strip_tags($Text);
 
 
+/* Old routine
 
        $ev = bbtoevent($Text);
 
@@ -422,6 +389,7 @@ function bb2diaspora($Text,$preserve_nl = false) {
 
        $Text = preg_replace_callback('/\[(.*?)\]\((.*?)\)/ism','unescape_underscores_in_links',$Text);
 
+*/
 
        // Remove any leading or trailing whitespace, as this will mess up
        // the Diaspora signature verification and cause the item to disappear
index 63dd9695e76f1bad88a19d922a9098eec53360b3..4aac33f112e9badf10953d260baabae11ebece90 100644 (file)
@@ -313,21 +313,25 @@ function bbcode($Text,$preserve_nl = false, $tryoembed = true) {
        $Text = preg_replace("/\[img\](.*?)\[\/img\]/ism", '<img src="$1" alt="' . t('Image/photo') . '" />', $Text);
 
 
-       $Text = preg_replace("/\[video\](.*?\.(ogg|ogv|oga|ogm|webm|mp4))\[\/video\]/ism", '<video src="$1" controls="controls" width="425" height="350"><a href="$1">$1</a></video>', $Text);
-
-       $Text = preg_replace("/\[audio\](.*?\.(ogg|ogv|oga|ogm|webm|mp4|mp3))\[\/audio\]/ism", '<audio src="$1" controls="controls"><a href="$1">$1</a></audio>', $Text);
-
        // Try to Oembed
        if ($tryoembed) {
+               $Text = preg_replace("/\[video\](.*?\.(ogg|ogv|oga|ogm|webm|mp4))\[\/video\]/ism", '<video src="$1" controls="controls" width="425" height="350"><a href="$1">$1</a></video>', $Text);
+               $Text = preg_replace("/\[audio\](.*?\.(ogg|ogv|oga|ogm|webm|mp4|mp3))\[\/audio\]/ism", '<audio src="$1" controls="controls"><a href="$1">$1</a></audio>', $Text);
+
                $Text = preg_replace_callback("/\[video\](.*?)\[\/video\]/ism", 'tryoembed', $Text);
                $Text = preg_replace_callback("/\[audio\](.*?)\[\/audio\]/ism", 'tryoembed', $Text);
+       } else {
+               $Text = preg_replace("/\[video\](.*?)\[\/video\]/", '$1', $Text);
+               $Text = preg_replace("/\[audio\](.*?)\[\/audio\]/", '$1', $Text);
        }
 
        // html5 video and audio
 
 
-       $Text = preg_replace("/\[iframe\](.*?)\[\/iframe\]/ism", '<iframe src="$1" width="425" height="350"><a href="$1">$1</a></iframe>', $Text);
-
+       if ($tryoembed)
+               $Text = preg_replace("/\[iframe\](.*?)\[\/iframe\]/ism", '<iframe src="$1" width="425" height="350"><a href="$1">$1</a></iframe>', $Text);
+       else
+               $Text = preg_replace("/\[iframe\](.*?)\[\/iframe\]/ism", '<a href="$1">$1</a>', $Text);
 
        // Youtube extensions
        if ($tryoembed) {
@@ -340,7 +344,10 @@ function bbcode($Text,$preserve_nl = false, $tryoembed = true) {
        $Text = preg_replace("/\[youtube\]https?:\/\/www.youtube.com\/embed\/(.*?)\[\/youtube\]/ism",'[youtube]$1[/youtube]',$Text); 
        $Text = preg_replace("/\[youtube\]https?:\/\/youtu.be\/(.*?)\[\/youtube\]/ism",'[youtube]$1[/youtube]',$Text);
 
-       $Text = preg_replace("/\[youtube\]([A-Za-z0-9\-_=]+)(.*?)\[\/youtube\]/ism", '<iframe width="425" height="350" src="http://www.youtube.com/embed/$1" frameborder="0" ></iframe>', $Text);
+       if ($tryoembed)
+               $Text = preg_replace("/\[youtube\]([A-Za-z0-9\-_=]+)(.*?)\[\/youtube\]/ism", '<iframe width="425" height="350" src="http://www.youtube.com/embed/$1" frameborder="0" ></iframe>', $Text);
+       else
+               $Text = preg_replace("/\[youtube\]([A-Za-z0-9\-_=]+)(.*?)\[\/youtube\]/ism", "http://www.youtube.com/watch?v=$1", $Text);
 
 
        if ($tryoembed) {
@@ -349,8 +356,12 @@ function bbcode($Text,$preserve_nl = false, $tryoembed = true) {
        }
 
        $Text = preg_replace("/\[vimeo\]https?:\/\/player.vimeo.com\/video\/([0-9]+)(.*?)\[\/vimeo\]/ism",'[vimeo]$1[/vimeo]',$Text); 
-       $Text = preg_replace("/\[vimeo\]https?:\/\/vimeo.com\/([0-9]+)(.*?)\[\/vimeo\]/ism",'[vimeo]$1[/vimeo]',$Text); 
-       $Text = preg_replace("/\[vimeo\]([0-9]+)(.*?)\[\/vimeo\]/ism", '<iframe width="425" height="350" src="http://player.vimeo.com/video/$1" frameborder="0" ></iframe>', $Text);
+       $Text = preg_replace("/\[vimeo\]https?:\/\/vimeo.com\/([0-9]+)(.*?)\[\/vimeo\]/ism",'[vimeo]$1[/vimeo]',$Text);
+
+       if ($tryoembed)
+               $Text = preg_replace("/\[vimeo\]([0-9]+)(.*?)\[\/vimeo\]/ism", '<iframe width="425" height="350" src="http://player.vimeo.com/video/$1" frameborder="0" ></iframe>', $Text);
+       else
+               $Text = preg_replace("/\[vimeo\]([0-9]+)(.*?)\[\/vimeo\]/ism", "http://vimeo.com/$1", $Text);
 
 //     $Text = preg_replace("/\[youtube\](.*?)\[\/youtube\]/", '<object width="425" height="350" type="application/x-shockwave-flash" data="http://www.youtube.com/v/$1" ><param name="movie" value="http://www.youtube.com/v/$1"></param><!--[if IE]><embed src="http://www.youtube.com/v/$1" type="application/x-shockwave-flash" width="425" height="350" /><![endif]--></object>', $Text);
 
@@ -390,6 +401,25 @@ function bbcode($Text,$preserve_nl = false, $tryoembed = true) {
        if($saved_image)
                $Text = bb_replace_images($Text, $saved_image);
 
+       // Clean up the HTML by loading and saving the HTML with the DOM
+       // Only do it when it has to be done - for performance reasons
+       if (!$tryoembed) {
+               $doc = new DOMDocument();
+               $doc->preserveWhiteSpace = false;
+
+               $Text = mb_convert_encoding($Text, 'HTML-ENTITIES', "UTF-8");
+
+               $doctype = '<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/loose.dtd">';
+               @$doc->loadHTML($doctype."<html><body>".$Text."</body></html>");
+
+               $Text = $doc->saveHTML();
+               $Text = str_replace(array("<html><body>", "</body></html>", $doctype), array("", "", ""), $Text);
+
+               $Text = str_replace('<br></li>','</li>', $Text);
+
+               $Text = mb_convert_encoding($Text, "UTF-8", 'HTML-ENTITIES');
+       }
+
        call_hooks('bbcode',$Text);
 
        return $Text;
index 71c1ba859152da2aeb621a38cfff9a47fefc3ca9..879d7e67e12d33c1c997faa6edcd4a57797dba20 100644 (file)
@@ -71,22 +71,32 @@ class dba {
        }
 
        public function q($sql) {
+               global $a;
 
                if((! $this->db) || (! $this->connected))
                        return false;
 
                $this->error = '';
 
-               //if (get_config("system", "db_log") != "")
-               //      @file_put_contents(get_config("system", "db_log"), datetime_convert().':'.session_id(). ' Start '.$sql."\n", FILE_APPEND);
+               if ($a->config["system"]["db_log"] != "")
+                       $stamp1 = microtime(true);
 
                if($this->mysqli)
                        $result = @$this->db->query($sql);
                else
                        $result = @mysql_query($sql,$this->db);
 
-               //if (get_config("system", "db_log") != "")
-               //      @file_put_contents(get_config("system", "db_log"), datetime_convert().':'.session_id(). ' Stop '."\n", FILE_APPEND);
+               if ($a->config["system"]["db_log"] != "") {
+                       $stamp2 = microtime(true);
+                       $duration = round($stamp2-$stamp1, 3);
+                       if ($duration > $a->config["system"]["db_loglimit"]) {
+                               $backtrace = debug_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS);
+                               @file_put_contents($a->config["system"]["db_log"], $duration."\t".
+                                               basename($backtrace[1]["file"])."\t".
+                                               $backtrace[1]["line"]."\t".$backtrace[2]["function"]."\t".
+                                               substr($sql, 0, 2000)."\n", FILE_APPEND);
+                       }
+               }
 
                if($this->mysqli) {
                        if($this->db->errno)
index 6d58bd182aab30a91939f7bd4fa1faf141773fab..b81208f3666bda612614011667979fb2fa119084 100755 (executable)
@@ -374,6 +374,29 @@ function limit_body_size($body) {
                return $body;
 }}
 
+function title_is_body($title, $body) {
+
+       $title = strip_tags($title);
+       $title = trim($title);
+       $title = str_replace(array("\n", "\r", "\t", " "), array("","","",""), $title);
+
+       $body = strip_tags($body);
+       $body = trim($body);
+       $body = str_replace(array("\n", "\r", "\t", " "), array("","","",""), $body);
+
+       if (strlen($title) < strlen($body))
+               $body = substr($body, 0, strlen($title));
+
+       if (($title != $body) and (substr($title, -3) == "...")) {
+               $pos = strrpos($title, "...");
+               if ($pos > 0) {
+                       $title = substr($title, 0, $pos);
+                       $body = substr($body, 0, $pos);
+               }
+       }
+
+       return($title == $body);
+}
 
 
 
@@ -400,6 +423,11 @@ function get_atom_elements($feed,$item) {
        $res['body'] = unxmlify($item->get_content());
        $res['plink'] = unxmlify($item->get_link(0));
 
+       // removing the content of the title if its identically to the body
+       // This helps with auto generated titles e.g. from tumblr
+       if (title_is_body($res["title"], $res["body"]))
+               $res['title'] = "";
+
        if($res['plink'])
                $base_url = implode('/', array_slice(explode('/',$res['plink']),0,3));
        else
@@ -418,7 +446,7 @@ function get_atom_elements($feed,$item) {
                                        $res['author-avatar'] = unxmlify($link['attribs']['']['href']);
                        }
                }
-       }                       
+       }
 
        $rawactor = $item->get_item_tags(NAMESPACE_ACTIVITY, 'actor');
 
@@ -450,7 +478,7 @@ function get_atom_elements($feed,$item) {
                                                $res['author-avatar'] = unxmlify($link['attribs']['']['href']);
                                }
                        }
-               }                       
+               }
 
                $rawactor = $feed->get_feed_tags(NAMESPACE_ACTIVITY, 'subject');
 
@@ -475,7 +503,7 @@ function get_atom_elements($feed,$item) {
                $res['app'] = strip_tags(unxmlify($apps[0]['attribs']['']['source']));
                if($res['app'] === 'web')
                        $res['app'] = 'OStatus';
-       }                  
+       }
 
        // base64 encoded json structure representing Diaspora signature
 
@@ -618,7 +646,7 @@ function get_atom_elements($feed,$item) {
 
                foreach($base as $link) {
                        if(!x($res, 'owner-avatar') || !$res['owner-avatar']) {
-                               if($link['attribs']['']['rel'] === 'photo' || $link['attribs']['']['rel'] === 'avatar')                 
+                               if($link['attribs']['']['rel'] === 'photo' || $link['attribs']['']['rel'] === 'avatar')
                                        $res['owner-avatar'] = unxmlify($link['attribs']['']['href']);
                        }
                }
@@ -762,6 +790,12 @@ function get_atom_elements($feed,$item) {
 
        call_hooks('parse_atom', $arr);
 
+       //if (($res["title"] != "") or (strpos($res["body"], "RT @") > 0)) {
+       if (strpos($res["body"], "RT @") !== false) {
+               $debugfile = tempnam("/home/ike/log", "item-res2-");
+               file_put_contents($debugfile, serialize($arr));
+       }
+
        return $res;
 }
 
@@ -1715,7 +1749,7 @@ function consume_feed($xml,$importer,&$contact, &$hub, $datedir = 0, $pass = 0)
 
        // Now process the feed
 
-       if($feed->get_item_quantity()) {                
+       if($feed->get_item_quantity()) {
 
                logger('consume_feed: feed item count = ' . $feed->get_item_quantity());
 
@@ -1728,7 +1762,7 @@ function consume_feed($xml,$importer,&$contact, &$hub, $datedir = 0, $pass = 0)
 
                foreach($items as $item) {
 
-                       $is_reply = false;              
+                       $is_reply = false;
                        $item_id = $item->get_id();
                        $rawthread = $item->get_item_tags( NAMESPACE_THREAD,'in-reply-to');
                        if(isset($rawthread[0]['attribs']['']['ref'])) {
@@ -1742,11 +1776,10 @@ function consume_feed($xml,$importer,&$contact, &$hub, $datedir = 0, $pass = 0)
                                        continue;
 
                                // Have we seen it? If not, import it.
-       
+
                                $item_id  = $item->get_id();
                                $datarray = get_atom_elements($feed,$item);
 
-
                                if((! x($datarray,'author-name')) && ($contact['network'] != NETWORK_DFRN))
                                        $datarray['author-name'] = $contact['name'];
                                if((! x($datarray,'author-link')) && ($contact['network'] != NETWORK_DFRN))
@@ -3663,7 +3696,6 @@ function posted_date_widget($url,$uid,$wall) {
        return $o;
 }
 
-
 function store_diaspora_retract_sig($item, $user, $baseurl) {
        // Note that we can't add a target_author_signature
        // if the comment was deleted by a remote user. That should be ok, because if a remote user is deleting
index 9e6f8355b75d5b1168f02da9e8bd8ea0f11e7ad2..a95dde535c9fa967156b28806f9c508b9ed1b064 100644 (file)
@@ -14,15 +14,16 @@ function fetch_url($url,$binary = false, &$redirects = 0, $timeout = 0, $accept_
                return false;
 
        @curl_setopt($ch, CURLOPT_HEADER, true);
-       
+
        if (!is_null($accept_content)){
                curl_setopt($ch,CURLOPT_HTTPHEADER, array (
                        "Accept: " . $accept_content
                ));
        }
-       
+
        @curl_setopt($ch, CURLOPT_RETURNTRANSFER,true);
-       @curl_setopt($ch, CURLOPT_USERAGENT, "Friendica");
+       //@curl_setopt($ch, CURLOPT_USERAGENT, "Friendica");
+       @curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/5.0 (compatible; Friendica)");
 
 
        if(intval($timeout)) {
@@ -59,7 +60,6 @@ function fetch_url($url,$binary = false, &$redirects = 0, $timeout = 0, $accept_
        $base = $s;
        $curl_info = @curl_getinfo($ch);
        $http_code = $curl_info['http_code'];
-
 //     logger('fetch_url:' . $http_code . ' data: ' . $s);
        $header = '';
 
@@ -73,24 +73,22 @@ function fetch_url($url,$binary = false, &$redirects = 0, $timeout = 0, $accept_
        }
 
        if($http_code == 301 || $http_code == 302 || $http_code == 303 || $http_code == 307) {
-        $matches = array();
-        preg_match('/(Location:|URI:)(.*?)\n/', $header, $matches);
-        $newurl = trim(array_pop($matches));
+               $matches = array();
+               preg_match('/(Location:|URI:)(.*?)\n/', $header, $matches);
+               $newurl = trim(array_pop($matches));
                if(strpos($newurl,'/') === 0)
                        $newurl = $url . $newurl;
-        $url_parsed = @parse_url($newurl);
-        if (isset($url_parsed)) {
-            $redirects++;
-            return fetch_url($newurl,$binary,$redirects,$timeout);
-        }
-    }
+               $url_parsed = @parse_url($newurl);
+               if (isset($url_parsed)) {
+                       $redirects++;
+                       return fetch_url($newurl,$binary,$redirects,$timeout);
+               }
+       }
 
        $a->set_curl_code($http_code);
 
        $body = substr($s,strlen($header));
-
        $a->set_curl_headers($header);
-
        @curl_close($ch);
        return($body);
 }}
@@ -800,6 +798,9 @@ function scale_external_images($s, $include_link = true, $scale_replace = false)
 
        $a = get_app();
 
+       // Picture addresses can contain special characters
+       $s = htmlspecialchars_decode($s);
+
        $matches = null;
        $c = preg_match_all('/\[img\](.*?)\[\/img\]/ism',$s,$matches,PREG_SET_ORDER);
        if($c) {
index a38f7e27027ea2923bd2cae777f9bdd26793e214..32f28b7b7bce3c6f67d6260e97e7f74147c6d1e1 100644 (file)
@@ -1,7 +1,210 @@
 <?php
+if(!function_exists('deletenode')) {
+       function deletenode(&$doc, $node)
+       {
+               $xpath = new DomXPath($doc);
+               $list = $xpath->query("//".$node);
+               foreach ($list as $child)
+                       $child->parentNode->removeChild($child);
+       }
+}
+
+function completeurl($url, $scheme) {
+        $urlarr = parse_url($url);
+
+        if (isset($urlarr["scheme"]))
+                return($url);
+
+        $schemearr = parse_url($scheme);
+
+        $complete = $schemearr["scheme"]."://".$schemearr["host"];
+
+        if ($schemearr["port"] != "")
+                $complete .= ":".$schemearr["port"];
+
+        $complete .= $urlarr["path"];
+
+        if ($urlarr["query"] != "")
+                $complete .= "?".$urlarr["query"];
+
+        if ($urlarr["fragment"] != "")
+                $complete .= "#".$urlarr["fragment"];
+
+        return($complete);
+}
 
-require_once('library/HTML5/Parser.php');
-require_once('library/HTMLPurifier.auto.php');
+function parseurl_getsiteinfo($url) {
+       $siteinfo = array();
+
+       $ch = curl_init();
+       curl_setopt($ch, CURLOPT_URL, $url);
+       curl_setopt($ch, CURLOPT_HEADER, 1);
+       curl_setopt($ch, CURLOPT_NOBODY, 0);
+       curl_setopt($ch, CURLOPT_TIMEOUT, 3);
+       curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
+       curl_setopt($ch,CURLOPT_USERAGENT,'Opera/9.64(Windows NT 5.1; U; de) Presto/2.1.1');
+
+       $header = curl_exec($ch);
+       curl_close($ch);
+
+       // Fetch the first mentioned charset. Can be in body or header
+       if (preg_match('/charset=(.*?)['."'".'"\s\n]/', $header, $matches))
+               $charset = trim(array_pop($matches));
+       else
+               $charset = "utf-8";
+
+       $pos = strpos($header, "\r\n\r\n");
+
+       if ($pos)
+               $body = trim(substr($header, $pos));
+       else
+               $body = $header;
+
+       $body = mb_convert_encoding($body, "UTF-8", $charset);
+       $body = mb_convert_encoding($body, 'HTML-ENTITIES', "UTF-8");
+
+       $doc = new DOMDocument();
+       @$doc->loadHTML($body);
+
+       deletenode($doc, 'style');
+       deletenode($doc, 'script');
+       deletenode($doc, 'option');
+       deletenode($doc, 'h1');
+       deletenode($doc, 'h2');
+       deletenode($doc, 'h3');
+       deletenode($doc, 'h4');
+       deletenode($doc, 'h5');
+       deletenode($doc, 'h6');
+       deletenode($doc, 'ol');
+       deletenode($doc, 'ul');
+
+       $xpath = new DomXPath($doc);
+
+       //$list = $xpath->query("head/title");
+       $list = $xpath->query("//title");
+       foreach ($list as $node)
+               $siteinfo["title"] =  html_entity_decode($node->nodeValue, ENT_QUOTES, "UTF-8");
+
+       //$list = $xpath->query("head/meta[@name]");
+       $list = $xpath->query("//meta[@name]");
+       foreach ($list as $node) {
+               $attr = array();
+               if ($node->attributes->length)
+                        foreach ($node->attributes as $attribute)
+                                $attr[$attribute->name] = $attribute->value;
+
+               $attr["content"] = html_entity_decode($attr["content"], ENT_QUOTES, "UTF-8");
+
+               switch (strtolower($attr["name"])) {
+                       case "fulltitle":
+                               $siteinfo["title"] = $attr["content"];
+                               break;
+                       case "description":
+                               $siteinfo["text"] = $attr["content"];
+                               break;
+                       case "dc.title":
+                               $siteinfo["title"] = $attr["content"];
+                               break;
+                       case "dc.description":
+                               $siteinfo["text"] = $attr["content"];
+                               break;
+               }
+       }
+
+       //$list = $xpath->query("head/meta[@property]");
+       $list = $xpath->query("//meta[@property]");
+       foreach ($list as $node) {
+               $attr = array();
+               if ($node->attributes->length)
+                        foreach ($node->attributes as $attribute)
+                                $attr[$attribute->name] = $attribute->value;
+
+               $attr["content"] = html_entity_decode($attr["content"], ENT_QUOTES, "UTF-8");
+
+               switch (strtolower($attr["property"])) {
+                       case "og:image":
+                               $siteinfo["image"] = $attr["content"];
+                               break;
+                       case "og:title":
+                               $siteinfo["title"] = $attr["content"];
+                               break;
+                       case "og:description":
+                               $siteinfo["text"] = $attr["content"];
+                               break;
+               }
+       }
+
+       if ($siteinfo["image"] == "") {
+                $list = $xpath->query("//img[@src]");
+                foreach ($list as $node) {
+                        $attr = array();
+                        if ($node->attributes->length)
+                                foreach ($node->attributes as $attribute)
+                                        $attr[$attribute->name] = $attribute->value;
+
+                       $src = completeurl($attr["src"], $url);
+                       $photodata = getimagesize($src);
+
+                       if (($photodata[0] > 150) and ($photodata[1] > 150)) {
+                               if ($photodata[0] > 300) {
+                                       $photodata[1] = $photodata[1] * (300 / $photodata[0]);
+                                       $photodata[0] = 300;
+                               }
+                               if ($photodata[1] > 300) {
+                                       $photodata[0] = $photodata[0] * (300 / $photodata[1]);
+                                       $photodata[1] = 300;
+                               }
+                               $siteinfo["images"][] = array("src"=>$src,
+                                                               "width"=>$photodata[0],
+                                                               "height"=>$photodata[1]);
+                       }
+
+                }
+        } else {
+               $src = completeurl($siteinfo["image"], $url);
+               $photodata = getimagesize($src);
+
+               if (($photodata[0] > 10) and ($photodata[1] > 10))
+                       $siteinfo["images"][] = array("src"=>$src,
+                                                       "width"=>$photodata[0],
+                                                       "height"=>$photodata[1]);
+       }
+
+       if ($siteinfo["text"] == "") {
+               $text = "";
+
+               $list = $xpath->query("//div[@class='article']");
+               foreach ($list as $node)
+                       if (strlen($node->nodeValue) > 40)
+                               $text .= " ".trim($node->nodeValue);
+
+               if ($text == "") {
+                       $list = $xpath->query("//div[@class='content']");
+                       foreach ($list as $node)
+                               if (strlen($node->nodeValue) > 40)
+                                       $text .= " ".trim($node->nodeValue);
+               }
+
+               // If none text was found then take the paragraph content
+               if ($text == "") {
+                       $list = $xpath->query("//p");
+                       foreach ($list as $node)
+                               if (strlen($node->nodeValue) > 40)
+                                       $text .= " ".trim($node->nodeValue);
+               }
+
+               if ($text != "") {
+                       $text = trim(str_replace(array("\n", "\r"), array(" ", " "), $text));
+
+                       while (strpos($text, "  "))
+                               $text = trim(str_replace("  ", " ", $text));
+
+                       $siteinfo["text"] = html_entity_decode(substr($text,0,350), ENT_QUOTES, "UTF-8").'...';
+               }
+       }
+
+       return($siteinfo);
+}
 
 function arr_add_hashes(&$item,$k) {
        $item = '#' . $item;
@@ -16,8 +219,8 @@ function parse_url_content(&$a) {
        if(local_user() && intval(get_pconfig(local_user(),'system','plaintext')))
                $textmode = true;
 
-       if($textmode)
-       $br = (($textmode) ? "\n" : '<br /?');
+       //if($textmode)
+       $br = (($textmode) ? "\n" : '<br />');
 
        if(x($_GET,'binurl'))
                $url = trim(hex2bin($_GET['binurl']));
@@ -40,13 +243,11 @@ function parse_url_content(&$a) {
 
        logger('parse_url: ' . $url);
 
-
        if($textmode)
                $template = $br . '[bookmark=%s]%s[/bookmark]%s' . $br;
        else
                $template = "<br /><a class=\"bookmark\" href=\"%s\" >%s</a>%s<br />";
 
-
        $arr = array('url' => $url, 'text' => '');
 
        call_hooks('parse_link', $arr);
@@ -60,208 +261,55 @@ function parse_url_content(&$a) {
        if($url && $title && $text) {
 
                if($textmode)
-                       $text = $br . $br . '[quote]' . $text . '[/quote]' . $br;
+                       $text = $br . '[quote]' . trim($text) . '[/quote]' . $br;
                else
-                       $text = '<br /><br /><blockquote>' . $text . '</blockquote><br />';
+                       $text = '<br /><blockquote>' . trim($text) . '</blockquote><br />';
 
                $title = str_replace(array("\r","\n"),array('',''),$title);
 
                $result = sprintf($template,$url,($title) ? $title : $url,$text) . $str_tags;
 
-               logger('parse_url (unparsed): returns: ' . $result); 
+               logger('parse_url (unparsed): returns: ' . $result);
 
                echo $result;
                killme();
        }
 
+       $siteinfo = parseurl_getsiteinfo($url);
 
-       if($url) {
-               $s = fetch_url($url);
-       } else {
-               echo '';
-               killme();
-       }
-
-//     logger('parse_url: data: ' . $s, LOGGER_DATA);
-
-       if(! $s) {
-               echo sprintf($template,$url,$url,'') . $str_tags;
-               killme();
-       }
-
-       $matches = '';
-       $c = preg_match('/\<head(.*?)\>(.*?)\<\/head\>/ism',$s,$matches);
-       if($c) {
-//             logger('parse_url: header: ' . $matches[2], LOGGER_DATA);
-               try {
-                       $domhead = HTML5_Parser::parse($matches[2]);
-               } catch (DOMException $e) {
-                       logger('scrape_dfrn: parse error: ' . $e);
-               }
-               if($domhead)
-                       logger('parsed header');
-       }
-
-       if(! $title) {
-               if(strpos($s,'<title>')) {
-                       $title = substr($s,strpos($s,'<title>')+7,64);
-                       if(strpos($title,'<') !== false)
-                               $title = strip_tags(substr($title,0,strpos($title,'<')));
-               }
-       }
-
-       $config = HTMLPurifier_Config::createDefault();
-       $config->set('Cache.DefinitionImpl', null);
-       $purifier = new HTMLPurifier($config);
-       $s = $purifier->purify($s);
-
-//     logger('purify_output: ' . $s);
-
-       try {
-               $dom = HTML5_Parser::parse($s);
-       } catch (DOMException $e) {
-               logger('scrape_dfrn: parse error: ' . $e);
-       }
-
-       if(! $dom) {
-               echo sprintf($template,$url,$url,'') . $str_tags;
+       if($siteinfo["title"] == "") {
+               echo print_r($siteinfo, true);
+               //echo sprintf($template,$url,$url,'') . $str_tags;
                killme();
+       } else {
+               $text = $siteinfo["text"];
+               $title = $siteinfo["title"];
        }
 
-       $items = $dom->getElementsByTagName('title');
-
-       if($items) {
-               foreach($items as $item) {
-                       $title = trim($item->textContent);
-                       break;
-               }
-       }
-
-
-       if(! $text) {
-               $divs = $dom->getElementsByTagName('div');
-               if($divs) {
-                       foreach($divs as $div) {
-                               $class = $div->getAttribute('class');
-                               if($class && (stristr($class,'article') || stristr($class,'content'))) {
-                                       $items = $div->getElementsByTagName('p');
-                                       if($items) {
-                                               foreach($items as $item) {
-                                                       $text = $item->textContent;
-                                                       if(stristr($text,'<script')) {
-                                                               $text = '';
-                                                               continue;
-                                                       }
-                                                       $text = strip_tags($text);
-                                                       if(strlen($text) < 100) {
-                                                               $text = '';
-                                                               continue;
-                                                       }
-                                                       $text = substr($text,0,250) . '...' ;
-                                                       break;
-                                               }
-                                       }
-                               }
-                               if($text)
-                                       break;
-                       }
-               }
-
-               if(! $text) {
-                       $items = $dom->getElementsByTagName('p');
-                       if($items) {
-                               foreach($items as $item) {
-                                       $text = $item->textContent;
-                                       if(stristr($text,'<script'))
-                                               continue;
-                                       $text = strip_tags($text);
-                                       if(strlen($text) < 100) {
-                                               $text = '';
-                                               continue;
-                                       }
-                                       $text = substr($text,0,250) . '...' ;
-                                       break;
-                               }
-                       }
-               }
-       }
-
-       if(! $text) {
-               logger('parsing meta');
-               $items = (isset($domhead) && is_object($domhead) ? $domhead->getElementsByTagName('meta') : null);
-               if($items) {
-                       foreach($items as $item) {
-                               $property = $item->getAttribute('property');
-                               if($property && (stristr($property,':description'))) {
-
-                                       $text = $item->getAttribute('content');
-                                       if(stristr($text,'<script')) {
-                                               $text = '';
-                                               continue;
-                                       }
-                                       $text = strip_tags($text);
+       $image = "";
 
-
-                                       $text = substr($text,0,250) . '...' ;
-                               }
-                               if($property && (stristr($property,':image'))) {
-
-                                       $image = $item->getAttribute('content');
-                                       if(stristr($text,'<script')) {
-                                               $image = '';
-                                               continue;
-                                       }
-                                       $image = strip_tags($image);
-                                       
-                                       $i = fetch_url($image);
-                                       if($i) {
-                                               require_once('include/Photo.php');
-                                               // guess mimetype from headers or filename
-                                               $type = guess_image_type($image,true);
-
-                                               $ph = new Photo($i, $type);
-                                               if($ph->is_valid()) {
-                                                       if($ph->getWidth() > 300 || $ph->getHeight() > 300) {
-                                                               $ph->scaleImage(300);
-                                                               $new_width = $ph->getWidth();
-                                                               $new_height = $ph->getHeight();
-                                                               if($textmode)
-                                                                       $image = $br . $br . '[img=' . $new_width . 'x' . $new_height . ']' . $image . '[/img]';
-                                                               else
-                                                                       $image = '<br /><br /><img height="' . $new_height . '" width="' . $new_width . '" src="' .$image . '" alt="photo" />';
-                                                       }
-                                                       else {
-                                                               if($textmode) 
-                                                                       $image = $br . $br . '[img]' . $image . '[/img]';
-                                                               else
-                                                                       $image = '<br /><br /><img src="' . $image . '" alt="photo" />';
-                                                       }
-                                               }
-                                               else
-                                                       $image = '';
-                                       
-                                       }
-                               }
-                       }
-               }
-       }
+       foreach ($siteinfo["images"] as $imagedata)
+               if($textmode)
+                       $image .= '[img='.$imagedata["width"].'x'.$imagedata["height"].']'.$imagedata["src"].'[/img]';
+               else
+                       $image .= '<img height="'.$imagedata["height"].'" width="'.$imagedata["width"].'" src="'.$imagedata["src"].'" alt="photo" />';
 
        if(strlen($text)) {
                if($textmode)
-                       $text = $br .$br . '[quote]' . $text . '[/quote]' . $br ;
+                       $text = $br.'[quote]'.trim($text).'[/quote]'.$br ;
                else
-                       $text = '<br /><br /><blockquote>' . $text . '</blockquote><br />';
+                       $text = '<br /><blockquote>'.trim($text).'</blockquote><br />';
        }
 
        if($image) {
-               $text = $image . $br . $text;
+               $text = $br.$br.$image.$text;
        }
        $title = str_replace(array("\r","\n"),array('',''),$title);
 
        $result = sprintf($template,$url,($title) ? $title : $url,$text) . $str_tags;
 
-       logger('parse_url: returns: ' . $result); 
+       logger('parse_url: returns: ' . $result);
 
-       echo $result;
+       echo trim($result);
        killme();
 }