]> git.mxchange.org Git - friendica.git/blobdiff - include/Scrape.php
fix setting array values to config items
[friendica.git] / include / Scrape.php
index bfe795e19dac51bfd04d74f0734442035ac021bb..22725260091cac26f848183c4ac5f5d12e46e938 100644 (file)
@@ -30,8 +30,11 @@ function scrape_dfrn($url) {
                }
        }
 
-
-       $dom = HTML5_Parser::parse($s);
+       try {
+               $dom = HTML5_Parser::parse($s);
+       } catch (DOMException $e) {
+               logger('scrape_dfrn: parse error: ' . $e);
+       }
 
        if(! $dom)
                return $ret;
@@ -44,8 +47,9 @@ function scrape_dfrn($url) {
                $x = $item->getAttribute('rel');
                if(($x === 'alternate') && ($item->getAttribute('type') === 'application/atom+xml'))
                        $ret['feed_atom'] = $item->getAttribute('href');
-               if(substr($x,0,5) == "dfrn-")
+               if(substr($x,0,5) == "dfrn-") {
                        $ret[$x] = $item->getAttribute('href');
+               }
                if($x === 'lrdd') {
                        $decoded = urldecode($item->getAttribute('href'));
                        if(preg_match('/acct:([^@]*)@/',$decoded,$matches))
@@ -55,17 +59,28 @@ function scrape_dfrn($url) {
 
        // Pull out hCard profile elements
 
+       $largest_photo = 0;
+
        $items = $dom->getElementsByTagName('*');
        foreach($items as $item) {
                if(attribute_contains($item->getAttribute('class'), 'vcard')) {
                        $level2 = $item->getElementsByTagName('*');
                        foreach($level2 as $x) {
-                               if(attribute_contains($x->getAttribute('class'),'fn'))
+                               if(attribute_contains($x->getAttribute('class'),'fn')) {
                                        $ret['fn'] = $x->textContent;
-                               if(attribute_contains($x->getAttribute('class'),'photo'))
-                                       $ret['photo'] = $x->getAttribute('src');
-                               if(attribute_contains($x->getAttribute('class'),'key'))
+                               }
+                               if((attribute_contains($x->getAttribute('class'),'photo'))
+                                       || (attribute_contains($x->getAttribute('class'),'avatar'))) {
+                                       $size = intval($x->getAttribute('width'));
+                                       // dfrn prefers 175, so if we find this, we set largest_size so it can't be topped.
+                                       if(($size > $largest_photo) || ($size == 175) || (! $largest_photo)) {
+                                               $ret['photo'] = $x->getAttribute('src');
+                                               $largest_photo = (($size == 175) ? 9999 : $size);
+                                       }
+                               }
+                               if(attribute_contains($x->getAttribute('class'),'key')) {
                                        $ret['key'] = $x->textContent;
+                               }
                        }
                }
        }
@@ -120,9 +135,11 @@ function scrape_meta($url) {
                }
        }
 
-
-
-       $dom = HTML5_Parser::parse($s);
+       try {
+               $dom = HTML5_Parser::parse($s);
+       } catch (DOMException $e) {
+               logger('scrape_meta: parse error: ' . $e);
+       }
 
        if(! $dom)
                return $ret;
@@ -165,7 +182,11 @@ function scrape_vcard($url) {
                }
        }
 
-       $dom = HTML5_Parser::parse($s);
+       try {
+               $dom = HTML5_Parser::parse($s);
+       } catch (DOMException $e) {
+               logger('scrape_vcard: parse error: ' . $e);
+       }
 
        if(! $dom)
                return $ret;
@@ -190,8 +211,9 @@ function scrape_vcard($url) {
                                        }
                                }
                                if((attribute_contains($x->getAttribute('class'),'nickname'))
-                                       || (attribute_contains($x->getAttribute('class'),'uid')))
+                                       || (attribute_contains($x->getAttribute('class'),'uid'))) {
                                        $ret['nick'] = $x->textContent;
+                               }
                        }
                }
        }
@@ -208,11 +230,16 @@ function scrape_feed($url) {
        $ret = array();
        $s = fetch_url($url);
 
-       if(! $s) 
+       $headers = $a->get_curl_headers();
+       $code = $a->get_curl_code();
+
+       logger('scrape_feed: returns: ' . $code . ' headers=' . $headers, LOGGER_DEBUG);
+
+       if(! $s) {
+               logger('scrape_feed: no data returned for ' . $url); 
                return $ret;
+       }
 
-       $headers = $a->get_curl_headers();
-       logger('scrape_feed: headers=' . $headers, LOGGER_DEBUG);
 
        $lines = explode("\n",$headers);
        if(count($lines)) {
@@ -228,25 +255,22 @@ function scrape_feed($url) {
                                }
                        }
                }
+               // perhaps an RSS version 1 feed with a generic or incorrect content-type?
+               if(stristr($s,'</item>')) {
+                       $ret['feed_rss'] = $url;
+                       return $ret;
+               }
        }
 
-       $dom = HTML5_Parser::parse($s);
+       try {
+               $dom = HTML5_Parser::parse($s);
+       } catch (DOMException $e) {
+               logger('scrape_feed: parse error: ' . $e);
+       }
 
-       if(! $dom)
+       if(! $dom) {
+               logger('scrape_feed: failed to parse.');
                return $ret;
-
-
-       $items = $dom->getElementsByTagName('img');
-
-       // get img elements (twitter)
-
-       if($items) {
-               foreach($items as $item) {
-                       $x = $item->getAttribute('id');
-                       if($x === 'profile-image') {
-                               $ret['photo'] = $item->getAttribute('src');
-                       }
-               }
        }
 
 
@@ -258,7 +282,7 @@ function scrape_feed($url) {
                }
        }
        if(! $basename)
-               $basename = substr($url,0,strrpos($url,'/')) . '/';
+               $basename = implode('/', array_slice(explode('/',$url),0,3)) . '/';
 
        $items = $dom->getElementsByTagName('link');
 
@@ -289,7 +313,29 @@ function scrape_feed($url) {
 }}
 
 
-function probe_url($url) {
+/**
+ *
+ * Probe a network address to discover what kind of protocols we need to communicate with it.
+ *
+ * Warning: this function is a bit touchy and there are some subtle dependencies within the logic flow.
+ * Edit with care.
+ *
+ */
+
+/**
+ *
+ * PROBE_DIASPORA has a bias towards returning Diaspora information
+ * while PROBE_NORMAL has a bias towards dfrn/zot - in the case where
+ * an address (such as a Friendica address) supports more than one type
+ * of network. 
+ *
+ */
+
+
+define ( 'PROBE_NORMAL',   0);
+define ( 'PROBE_DIASPORA', 1);
+
+function probe_url($url, $mode = PROBE_NORMAL) {
        require_once('include/email.php');
 
        $result = array();
@@ -297,10 +343,12 @@ function probe_url($url) {
        if(! $url)
                return $result;
 
+       $network = null;
        $diaspora = false;
        $diaspora_base = '';
        $diaspora_guid = '';    
        $diaspora_key = '';
+       $has_lrdd = false;
        $email_conversant = false;
 
        $twitter = ((strpos($url,'twitter.com') !== false) ? true : false);
@@ -317,6 +365,8 @@ function probe_url($url) {
                        $links = lrdd($url);
 
                if(count($links)) {
+                       $has_lrdd = true;
+
                        logger('probe_url: found lrdd links: ' . print_r($links,true), LOGGER_DATA);
                        foreach($links as $link) {
                                if($link['@attributes']['rel'] === NAMESPACE_ZOT)
@@ -331,6 +381,8 @@ function probe_url($url) {
                                        $hcard = unamp($link['@attributes']['href']);
                                if($link['@attributes']['rel'] === 'http://webfinger.net/rel/profile-page')
                                        $profile = unamp($link['@attributes']['href']);
+                               if($link['@attributes']['rel'] === 'http://portablecontacts.net/spec/1.0')
+                                       $poco = unamp($link['@attributes']['href']);
                                if($link['@attributes']['rel'] === 'http://joindiaspora.com/seed_location') {
                                        $diaspora_base = unamp($link['@attributes']['href']);
                                        $diaspora = true;
@@ -366,7 +418,7 @@ function probe_url($url) {
                                }
                        }
                }
-               else {
+               elseif($mode == PROBE_NORMAL) {
 
                        // Check email
 
@@ -391,9 +443,10 @@ function probe_url($url) {
                                                $addr = $orig_url;
                                                $network = NETWORK_MAIL;
                                                $name = substr($url,0,strpos($url,'@'));
-                                               $profile = 'http://' . substr($url,strpos($url,'@')+1);
+                                               $phost = substr($url,strpos($url,'@')+1);
+                                               $profile = 'http://' . $phost;
                                                // fix nick character range
-                                               $vcard = array('fn' => $name, 'nick' => $name, 'photo' => gravatar_img($url));
+                                               $vcard = array('fn' => $name, 'nick' => $name, 'photo' => avatar_img($url));
                                                $notify = 'smtp ' . random_string();
                                                $poll = 'email ' . random_string();
                                                $priority = 0;
@@ -402,8 +455,24 @@ function probe_url($url) {
                                                        $adr = imap_rfc822_parse_adrlist($x->from,'');
                                                elseif(stristr($x->to,$orig_url))
                                                        $adr = imap_rfc822_parse_adrlist($x->to,'');
-                                               if(isset($adr) && strlen($adr[0]->personal))
-                                                       $vcard['fn'] = notags($adr[0]->personal);
+                                               if(isset($adr)) {
+                                                       foreach($adr as $feadr) {
+                                                               if((strcasecmp($feadr->mailbox,$name) == 0)
+                                                                       &&(strcasecmp($feadr->host,$phost) == 0)
+                                                                       && (strlen($feadr->personal))) {
+
+                                                                       $personal = imap_mime_header_decode($feadr->personal);
+                                                                       $vcard['fn'] = "";
+                                                                       foreach($personal as $perspart)
+                                                                               if ($perspart->charset != "default")
+                                                                                       $vcard['fn'] .= iconv($perspart->charset, 'UTF-8//IGNORE', $perspart->text);
+                                                                               else
+                                                                                       $vcard['fn'] .= $perspart->text;
+
+                                                                       $vcard['fn'] = notags($vcard['fn']);
+                                                               }
+                                                       }
+                                               }
                                        }
                                        imap_close($mbox);
                                }
@@ -411,38 +480,48 @@ function probe_url($url) {
                }
        }       
 
-       if(strlen($zot)) {
-               $s = fetch_url($zot);
-               if($s) {
-                       $j = json_decode($s);
-                       if($j) {
-                               $network = NETWORK_ZOT;
-                               $vcard   = array(
-                                       'fn'    => $j->fullname, 
-                                       'nick'  => $j->nickname, 
-                                       'photo' => $j->photo
-                               );
-                               $profile  = $j->url;
-                               $notify   = $j->post;
-                               $pubkey   = $j->pubkey;
-                               $poll     = 'N/A';
+       if($mode == PROBE_NORMAL) {
+               if(strlen($zot)) {
+                       $s = fetch_url($zot);
+                       if($s) {
+                               $j = json_decode($s);
+                               if($j) {
+                                       $network = NETWORK_ZOT;
+                                       $vcard   = array(
+                                               'fn'    => $j->fullname, 
+                                               'nick'  => $j->nickname, 
+                                               'photo' => $j->photo
+                                       );
+                                       $profile  = $j->url;
+                                       $notify   = $j->post;
+                                       $pubkey   = $j->pubkey;
+                                       $poll     = 'N/A';
+                               }
                        }
                }
-       }
 
-       if(strlen($dfrn)) {
-               $ret = scrape_dfrn($dfrn);
-               if(is_array($ret) && x($ret,'dfrn-request')) {
-                       $network = NETWORK_DFRN;
-                       $request = $ret['dfrn-request'];
-                       $confirm = $ret['dfrn-confirm'];
-                       $notify  = $ret['dfrn-notify'];
-                       $poll    = $ret['dfrn-poll'];
+               if(strlen($dfrn)) {
+                       $ret = scrape_dfrn(($hcard) ? $hcard : $dfrn);
+                       if(is_array($ret) && x($ret,'dfrn-request')) {
+                               $network = NETWORK_DFRN;
+                               $request = $ret['dfrn-request'];
+                               $confirm = $ret['dfrn-confirm'];
+                               $notify  = $ret['dfrn-notify'];
+                               $poll    = $ret['dfrn-poll'];
+
+                               $vcard = array();
+                               $vcard['fn'] = $ret['fn'];
+                               $vcard['nick'] = $ret['nick'];
+                               $vcard['photo'] = $ret['photo'];
+                       }
                }
        }
 
        if($diaspora && $diaspora_base && $diaspora_guid) {
-               $notify = $diaspora_base . 'receive/post/' . $diaspora_guid;
+               if($mode == PROBE_DIASPORA || ! $notify) {
+                       $notify = $diaspora_base . 'receive/users/' . $diaspora_guid;
+                       $batch  = $diaspora_base . 'receive/public' ;
+               }
                if(strpos($url,'@'))
                        $addr = str_replace('acct:', '', $url);
        }                       
@@ -450,11 +529,11 @@ function probe_url($url) {
        if($network !== NETWORK_ZOT && $network !== NETWORK_DFRN && $network !== NETWORK_MAIL) {
                if($diaspora)
                        $network = NETWORK_DIASPORA;
-               else
+               elseif($has_lrdd)
                        $network  = NETWORK_OSTATUS;
                $priority = 0;
 
-               if($hcard) {
+               if($hcard && ! $vcard) {
                        $vcard = scrape_vcard($hcard);
 
                        // Google doesn't use absolute url in profile photos
@@ -477,6 +556,9 @@ function probe_url($url) {
                        else
                                $poll = $tapi . '?screen_name=' . $tid;
                        $profile = 'http://twitter.com/#!/' . $tid;
+                       $vcard['photo'] = 'https://api.twitter.com/1/users/profile_image/' . $tid;
+                       $vcard['nick'] = $tid;
+                       $vcard['fn'] = $tid . '@twitter';
                }
 
                if(! x($vcard,'fn'))
@@ -487,7 +569,7 @@ function probe_url($url) {
 
                if($twitter || ! $poll)
                        $check_feed = true;
-               if((! isset($vcard)) || (! $profile))
+               if((! isset($vcard)) || (! x($vcard,'fn')) || (! $profile))
                        $check_feed = true;
                if(($at_addr) && (! count($links)))
                        $check_feed = false;
@@ -495,13 +577,14 @@ function probe_url($url) {
                if($check_feed) {
 
                        $feedret = scrape_feed(($poll) ? $poll : $url);
-                       logger('probe_url: scrape_feed returns: ' . print_r($feedret,true), LOGGER_DATA);
+                       logger('probe_url: scrape_feed ' . (($poll)? $poll : $url) . ' returns: ' . print_r($feedret,true), LOGGER_DATA);
                        if(count($feedret) && ($feedret['feed_atom'] || $feedret['feed_rss'])) {
                                $poll = ((x($feedret,'feed_atom')) ? unamp($feedret['feed_atom']) : unamp($feedret['feed_rss']));
-                               $vcard = array();
+                               if(! x($vcard)) 
+                                       $vcard = array();
                        }
 
-                       if(x($feedret,'photo'))
+                       if(x($feedret,'photo') && (! x($vcard,'photo')))
                                $vcard['photo'] = $feedret['photo'];
                        require_once('library/simplepie/simplepie.inc');
                    $feed = new SimplePie();
@@ -510,7 +593,7 @@ function probe_url($url) {
                        logger('probe_url: fetch feed: ' . $poll . ' returns: ' . $xml, LOGGER_DATA);
                        $a = get_app();
 
-                       logger('probe_url: scrape_feed: headers: ' . $a->get_curl_headers(), $LOGGER_DATA);
+                       logger('probe_url: scrape_feed: headers: ' . $a->get_curl_headers(), LOGGER_DATA);
 
                        $feed->set_raw_data($xml);
 
@@ -518,9 +601,11 @@ function probe_url($url) {
                        if($feed->error())
                                logger('probe_url: scrape_feed: Error parsing XML: ' . $feed->error());
 
+
                        if(! x($vcard,'photo'))
                                $vcard['photo'] = $feed->get_image_url();
                        $author = $feed->get_author();
+
                        if($author) {                   
                                $vcard['fn'] = unxmlify(trim($author->get_name()));
                                if(! $vcard['fn'])
@@ -568,8 +653,9 @@ function probe_url($url) {
                                        }
                                }
                        }
+
                        if((! $vcard['photo']) && strlen($email))
-                               $vcard['photo'] = gravatar_img($email);
+                               $vcard['photo'] = avatar_img($email);
                        if($poll === $profile)
                                $lnk = $feed->get_permalink();
                        if(isset($lnk) && strlen($lnk))
@@ -590,7 +676,7 @@ function probe_url($url) {
                                        $vcard['nick'] = trim(substr($vcard['nick'],0,strpos($vcard['nick'],' ')));
                        }
                        if(! $network)
-                               $network = 'feed';
+                               $network = NETWORK_FEED;
                        if(! $priority)
                                $priority = 2;
                }
@@ -598,24 +684,30 @@ function probe_url($url) {
 
        if(! x($vcard,'photo')) {
                $a = get_app();
-               $vcard['photo'] = $a->get_baseurl() . '/images/default-profile.jpg' ; 
+               $vcard['photo'] = $a->get_baseurl() . '/images/person-175.jpg' ; 
        }
 
        if(! $profile)
                $profile = $url;
 
-       $vcard['fn'] = notags($vcard['fn']);
-       $vcard['nick'] = str_replace(' ','',notags($vcard['nick']));
+       // No human could be associated with this link, use the URL as the contact name
 
+       if(($network === NETWORK_FEED) && ($poll) && (! x($vcard,'fn')))
+               $vcard['fn'] = $url;
 
+       $vcard['fn'] = notags($vcard['fn']);
+       $vcard['nick'] = str_replace(' ','',notags($vcard['nick']));
+               
        $result['name'] = $vcard['fn'];
        $result['nick'] = $vcard['nick'];
        $result['url'] = $profile;
        $result['addr'] = $addr;
+       $result['batch'] = $batch;
        $result['notify'] = $notify;
        $result['poll'] = $poll;
        $result['request'] = $request;
        $result['confirm'] = $confirm;
+       $result['poco'] = $poco;
        $result['photo'] = $vcard['photo'];
        $result['priority'] = $priority;
        $result['network'] = $network;