X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;f=include%2FScrape.php;h=a585047ebdcdc2d98d0cf9890ae5e6de6b9e2a9e;hb=85fede486e6ff57daeb3bc3472eb9bf0bde990e3;hp=4c4ad3cdb935233d35bc820ff2b607c5c06553da;hpb=b71a3af4570ccf349804b9ffa151fc1d35287d88;p=friendica.git diff --git a/include/Scrape.php b/include/Scrape.php old mode 100755 new mode 100644 index 4c4ad3cdb9..a585047ebd --- a/include/Scrape.php +++ b/include/Scrape.php @@ -14,7 +14,7 @@ function scrape_dfrn($url) { $s = fetch_url($url); - if(! $s) + if(! $s) return $ret; $headers = $a->get_curl_headers(); @@ -23,7 +23,7 @@ function scrape_dfrn($url) { $lines = explode("\n",$headers); if(count($lines)) { - foreach($lines as $line) { + foreach($lines as $line) { // don't try and run feeds through the html5 parser if(stristr($line,'content-type:') && ((stristr($line,'application/atom+xml')) || (stristr($line,'application/rss+xml')))) return ret; @@ -255,6 +255,11 @@ function scrape_feed($url) { } } } + // perhaps an RSS version 1 feed with a generic or incorrect content-type? + if(stristr($s,'')) { + $ret['feed_rss'] = $url; + return $ret; + } } try { @@ -277,7 +282,7 @@ function scrape_feed($url) { } } if(! $basename) - $basename = substr($url,0,strrpos($url,'/')) . '/'; + $basename = implode('/', array_slice(explode('/',$url),0,3)) . '/'; $items = $dom->getElementsByTagName('link'); @@ -341,16 +346,18 @@ function probe_url($url, $mode = PROBE_NORMAL) { $network = null; $diaspora = false; $diaspora_base = ''; - $diaspora_guid = ''; + $diaspora_guid = ''; $diaspora_key = ''; $has_lrdd = false; $email_conversant = false; - $twitter = ((strpos($url,'twitter.com') !== false) ? true : false); + // Twitter is deactivated since twitter closed its old API + //$twitter = ((strpos($url,'twitter.com') !== false) ? true : false); + $lastfm = ((strpos($url,'last.fm/user') !== false) ? true : false); $at_addr = ((strpos($url,'@') !== false) ? true : false); - if(! $twitter) { + if((! $twitter) && (! $lastfm)) { if(strpos($url,'mailto:') !== false && $at_addr) { $url = str_replace('mailto:','',$url); @@ -388,7 +395,10 @@ function probe_url($url, $mode = PROBE_NORMAL) { } if($link['@attributes']['rel'] === 'diaspora-public-key') { $diaspora_key = base64_decode(unamp($link['@attributes']['href'])); - $pubkey = rsatopem($diaspora_key); + if(strstr($diaspora_key,'RSA ')) + $pubkey = rsatopem($diaspora_key); + else + $pubkey = $diaspora_key; $diaspora = true; } } @@ -426,14 +436,17 @@ function probe_url($url, $mode = PROBE_NORMAL) { intval(local_user()) ); if(count($x) && count($r)) { - $mailbox = construct_mailbox_name($r[0]); + $mailbox = construct_mailbox_name($r[0]); $password = ''; openssl_private_decrypt(hex2bin($r[0]['pass']),$password,$x[0]['prvkey']); $mbox = email_connect($mailbox,$r[0]['user'],$password); + if(! $mbox) + logger('probe_url: email_connect failed.'); unset($password); } if($mbox) { $msgs = email_poll($mbox,$orig_url); + logger('probe_url: searching ' . $orig_url . ', ' . count($msgs) . ' messages found.', LOGGER_DEBUG); if(count($msgs)) { $addr = $orig_url; $network = NETWORK_MAIL; @@ -441,15 +454,15 @@ function probe_url($url, $mode = PROBE_NORMAL) { $phost = substr($url,strpos($url,'@')+1); $profile = 'http://' . $phost; // fix nick character range - $vcard = array('fn' => $name, 'nick' => $name, 'photo' => gravatar_img($url)); + $vcard = array('fn' => $name, 'nick' => $name, 'photo' => avatar_img($url)); $notify = 'smtp ' . random_string(); $poll = 'email ' . random_string(); $priority = 0; $x = email_msg_meta($mbox,$msgs[0]); - if(stristr($x->from,$orig_url)) - $adr = imap_rfc822_parse_adrlist($x->from,''); - elseif(stristr($x->to,$orig_url)) - $adr = imap_rfc822_parse_adrlist($x->to,''); + if(stristr($x[0]->from,$orig_url)) + $adr = imap_rfc822_parse_adrlist($x[0]->from,''); + elseif(stristr($x[0]->to,$orig_url)) + $adr = imap_rfc822_parse_adrlist($x[0]->to,''); if(isset($adr)) { foreach($adr as $feadr) { if((strcasecmp($feadr->mailbox,$name) == 0) @@ -473,7 +486,7 @@ function probe_url($url, $mode = PROBE_NORMAL) { } } } - } + } if($mode == PROBE_NORMAL) { if(strlen($zot)) { @@ -519,7 +532,7 @@ function probe_url($url, $mode = PROBE_NORMAL) { } if(strpos($url,'@')) $addr = str_replace('acct:', '', $url); - } + } if($network !== NETWORK_ZOT && $network !== NETWORK_DFRN && $network !== NETWORK_MAIL) { if($diaspora) @@ -532,17 +545,24 @@ function probe_url($url, $mode = PROBE_NORMAL) { $vcard = scrape_vcard($hcard); // Google doesn't use absolute url in profile photos - + if((x($vcard,'photo')) && substr($vcard['photo'],0,1) == '/') { $h = @parse_url($hcard); if($h) $vcard['photo'] = $h['scheme'] . '://' . $h['host'] . $vcard['photo']; } - + logger('probe_url: scrape_vcard: ' . print_r($vcard,true), LOGGER_DATA); } - if($twitter) { + if($diaspora && $addr) { + // Diaspora returns the name as the nick. As the nick will never be updated, + // let's use the Diaspora nickname (the first part of the handle) as the nick instead + $addr_parts = explode('@', $addr); + $vcard['nick'] = $addr_parts[0]; + } + + /* if($twitter) { logger('twitter: setup'); $tid = basename($url); $tapi = 'https://api.twitter.com/1/statuses/user_timeline.rss'; @@ -551,9 +571,18 @@ function probe_url($url, $mode = PROBE_NORMAL) { else $poll = $tapi . '?screen_name=' . $tid; $profile = 'http://twitter.com/#!/' . $tid; - $vcard['photo'] = 'https://api.twitter.com/1/users/profile_image/' . $tid; + //$vcard['photo'] = 'https://api.twitter.com/1/users/profile_image/' . $tid; + $vcard['photo'] = 'https://api.twitter.com/1/users/profile_image?screen_name=' . $tid . '&size=bigger'; $vcard['nick'] = $tid; - $vcard['fn'] = $tid . '@twitter'; + $vcard['fn'] = $tid; + } */ + + if($lastfm) { + $profile = $url; + $poll = str_replace(array('www.','last.fm/'),array('','ws.audioscrobbler.com/1.0/'),$url) . '/recenttracks.rss'; + $vcard['nick'] = basename($url); + $vcard['fn'] = $vcard['nick'] . t(' on Last.fm'); + $network = NETWORK_FEED; } if(! x($vcard,'fn')) @@ -562,6 +591,12 @@ function probe_url($url, $mode = PROBE_NORMAL) { $check_feed = false; + if(stristr($url,'tumblr.com') && (! stristr($url,'/rss'))) { + $poll = $url . '/rss'; + $check_feed = true; + // Will leave it to others to figure out how to grab the avatar, which is on the $url page in the open graph meta links + } + if($twitter || ! $poll) $check_feed = true; if((! isset($vcard)) || (! x($vcard,'fn')) || (! $profile)) @@ -575,7 +610,7 @@ function probe_url($url, $mode = PROBE_NORMAL) { logger('probe_url: scrape_feed ' . (($poll)? $poll : $url) . ' returns: ' . print_r($feedret,true), LOGGER_DATA); if(count($feedret) && ($feedret['feed_atom'] || $feedret['feed_rss'])) { $poll = ((x($feedret,'feed_atom')) ? unamp($feedret['feed_atom']) : unamp($feedret['feed_rss'])); - if(! x($vcard)) + if(! x($vcard)) $vcard = array(); } @@ -588,9 +623,10 @@ function probe_url($url, $mode = PROBE_NORMAL) { logger('probe_url: fetch feed: ' . $poll . ' returns: ' . $xml, LOGGER_DATA); $a = get_app(); - logger('probe_url: scrape_feed: headers: ' . $a->get_curl_headers(), $LOGGER_DATA); + logger('probe_url: scrape_feed: headers: ' . $a->get_curl_headers(), LOGGER_DATA); - $feed->set_raw_data($xml); + // Don't try and parse an empty string + $feed->set_raw_data(($xml) ? $xml : ''); $feed->init(); if($feed->error()) @@ -650,7 +686,7 @@ function probe_url($url, $mode = PROBE_NORMAL) { } if((! $vcard['photo']) && strlen($email)) - $vcard['photo'] = gravatar_img($email); + $vcard['photo'] = avatar_img($email); if($poll === $profile) $lnk = $feed->get_permalink(); if(isset($lnk) && strlen($lnk)) @@ -679,7 +715,7 @@ function probe_url($url, $mode = PROBE_NORMAL) { if(! x($vcard,'photo')) { $a = get_app(); - $vcard['photo'] = $a->get_baseurl() . '/images/default-profile.jpg' ; + $vcard['photo'] = $a->get_baseurl() . '/images/person-175.jpg' ; } if(! $profile)