OStatus: The gcontact table will now be updated from the conversation as well

[friendica.git] / include / Scrape.php

diff --git a/include/Scrape.php b/include/Scrape.php
index 93d68be22b892af7f95ae34bab506617b443204d..bc6aebbdcfdee7b388c57f6905e2f008dc258647 100644 (file)
--- a/include/Scrape.php
+++ b/include/Scrape.php
@@ -4,7 +4,7 @@ require_once('library/HTML5/Parser.php');
 require_once('include/crypto.php');
 
 if(! function_exists('scrape_dfrn')) {
-function scrape_dfrn($url) {
+function scrape_dfrn($url, $dont_probe = false) {
 
        $a = get_app();
 
@@ -12,11 +12,30 @@ function scrape_dfrn($url) {
 
        logger('scrape_dfrn: url=' . $url);
 
+       // Try to fetch the data from noscrape. This is faster than parsing the HTML
+       $noscrape = str_replace("/hcard/", "/noscrape/", $url);
+       $noscrapejson = fetch_url($noscrape);
+       $noscrapedata = array();
+       if ($noscrapejson) {
+               $noscrapedata = json_decode($noscrapejson, true);
+
+               if (is_array($noscrapedata))
+                       if ($noscrapedata["nick"] != "")
+                               return($noscrapedata);
+       }
+
        $s = fetch_url($url);
 
        if(! $s)
                return $ret;
 
+       if (!$dont_probe) {
+               $probe = probe_url($url);
+
+               if (isset($probe["addr"]))
+                       $ret["addr"] = $probe["addr"];
+       }
+
        $headers = $a->get_curl_headers();
        logger('scrape_dfrn: headers=' . $headers, LOGGER_DEBUG);
 
@@ -84,8 +103,7 @@ function scrape_dfrn($url) {
                        }
                }
        }
-
-       return $ret;
+       return array_merge($ret, $noscrapedata);
 }}
 
 
@@ -228,7 +246,9 @@ function scrape_feed($url) {
        $a = get_app();
 
        $ret = array();
-       $s = fetch_url($url);
+       $cookiejar = tempnam(get_temppath(), 'cookiejar-scrape-feed-');
+       $s = fetch_url($url, false, $redirects, 0, Null, $cookiejar);
+       unlink($cookiejar);
 
        $headers = $a->get_curl_headers();
        $code = $a->get_curl_code();
@@ -320,7 +340,7 @@ function scrape_feed($url) {
  * PROBE_DIASPORA has a bias towards returning Diaspora information
  * while PROBE_NORMAL has a bias towards dfrn/zot - in the case where
  * an address (such as a Friendica address) supports more than one type
- * of network. 
+ * of network.
  *
  */
 
@@ -407,7 +427,7 @@ function probe_url($url, $mode = PROBE_NORMAL, $level = 1) {
                                                $pubkey = $diaspora_key;
                                        $diaspora = true;
                                }
-                               if($link['@attributes']['rel'] === 'http://ostatus.org/schema/1.0/subscribe') {
+                               if(($link['@attributes']['rel'] === 'http://ostatus.org/schema/1.0/subscribe') AND ($mode == PROBE_NORMAL)) {
                                        $diaspora = false;
                                }
                        }
@@ -524,7 +544,7 @@ function probe_url($url, $mode = PROBE_NORMAL, $level = 1) {
 
 
                if(strlen($dfrn)) {
-                       $ret = scrape_dfrn(($hcard) ? $hcard : $dfrn);
+                       $ret = scrape_dfrn(($hcard) ? $hcard : $dfrn, true);
                        if(is_array($ret) && x($ret,'dfrn-request')) {
                                $network = NETWORK_DFRN;
                                $request = $ret['dfrn-request'];
@@ -540,9 +560,23 @@ function probe_url($url, $mode = PROBE_NORMAL, $level = 1) {
                }
        }
 
+       // Scrape the public key from the hcard.
+       // Diaspora will remove it from the webfinger somewhere in the future.
+       if (($hcard != "") AND ($pubkey == "")) {
+               $ret = scrape_dfrn(($hcard) ? $hcard : $dfrn, true);
+               if (isset($ret["key"])) {
+                       $hcard_key = $ret["key"];
+                       if(strstr($hcard_key,'RSA '))
+                               $pubkey = rsatopem($hcard_key);
+                       else
+                               $pubkey = $hcard_key;
+               }
+       }
        if($diaspora && $diaspora_base && $diaspora_guid) {
-               if($mode == PROBE_DIASPORA || ! $notify) {
-                       $notify = $diaspora_base . 'receive/users/' . $diaspora_guid;
+               $diaspora_notify = $diaspora_base.'receive/users/'.$diaspora_guid;
+
+               if($mode == PROBE_DIASPORA || ! $notify || ($notify == $diaspora_notify)) {
+                       $notify = $diaspora_notify;
                        $batch  = $diaspora_base . 'receive/public' ;
                }
                if(strpos($url,'@'))
@@ -552,7 +586,7 @@ function probe_url($url, $mode = PROBE_NORMAL, $level = 1) {
        if($network !== NETWORK_ZOT && $network !== NETWORK_DFRN && $network !== NETWORK_MAIL) {
                if($diaspora)
                        $network = NETWORK_DIASPORA;
-               elseif($has_lrdd)
+               elseif($has_lrdd AND ($notify))
                        $network  = NETWORK_OSTATUS;
 
                if(strpos($url,'@'))
@@ -641,7 +675,9 @@ function probe_url($url, $mode = PROBE_NORMAL, $level = 1) {
                                $vcard['photo'] = $feedret['photo'];
                        require_once('library/simplepie/simplepie.inc');
                        $feed = new SimplePie();
-                       $xml = fetch_url($poll);
+                       $cookiejar = tempnam(get_temppath(), 'cookiejar-scrape-feed-');
+                       $xml = fetch_url($poll, false, $redirects, 0, Null, $cookiejar);
+                       unlink($cookiejar);
 
                        logger('probe_url: fetch feed: ' . $poll . ' returns: ' . $xml, LOGGER_DATA);
                        $a = get_app();
@@ -652,9 +688,10 @@ function probe_url($url, $mode = PROBE_NORMAL, $level = 1) {
                        $feed->set_raw_data(($xml) ? $xml : '<?xml version="1.0" encoding="utf-8" ?><xml></xml>');
 
                        $feed->init();
-                       if($feed->error())
+                       if($feed->error()) {
                                logger('probe_url: scrape_feed: Error parsing XML: ' . $feed->error());
-
+                               $network = NETWORK_PHANTOM;
+                       }
 
                        if(! x($vcard,'photo'))
                                $vcard['photo'] = $feed->get_image_url();
@@ -718,6 +755,45 @@ function probe_url($url, $mode = PROBE_NORMAL, $level = 1) {
                                }
                        }
 
+                       // Workaround for misconfigured Friendica servers
+                       if (($network == "") AND (strstr($url, "/profile/"))) {
+                               $noscrape = str_replace("/profile/", "/noscrape/", $url);
+                               $noscrapejson = fetch_url($noscrape);
+                               if ($noscrapejson) {
+
+                                       $network = NETWORK_DFRN;
+
+                                       $poco = str_replace("/profile/", "/poco/", $url);
+
+                                       $noscrapedata = json_decode($noscrapejson, true);
+
+                                       if (isset($noscrapedata["addr"]))
+                                               $addr = $noscrapedata["addr"];
+
+                                       if (isset($noscrapedata["fn"]))
+                                               $vcard["fn"] = $noscrapedata["fn"];
+
+                                       if (isset($noscrapedata["key"]))
+                                               $pubkey = $noscrapedata["key"];
+
+                                       if (isset($noscrapedata["photo"]))
+                                               $vcard["photo"] = $noscrapedata["photo"];
+
+                                       if (isset($noscrapedata["dfrn-request"]))
+                                               $request = $noscrapedata["dfrn-request"];
+
+                                       if (isset($noscrapedata["dfrn-confirm"]))
+                                               $confirm = $noscrapedata["dfrn-confirm"];
+
+                                       if (isset($noscrapedata["dfrn-notify"]))
+                                               $notify = $noscrapedata["dfrn-notify"];
+
+                                       if (isset($noscrapedata["dfrn-poll"]))
+                                               $poll = $noscrapedata["dfrn-poll"];
+
+                               }
+                       }
+
                        if((! $vcard['photo']) && strlen($email))
                                $vcard['photo'] = avatar_img($email);
                        if($poll === $profile)
@@ -778,6 +854,9 @@ function probe_url($url, $mode = PROBE_NORMAL, $level = 1) {
 
        $baseurl = rtrim($baseurl, "/");
 
+       if(strpos($url,'@') AND ($addr == "") AND ($network == NETWORK_DFRN))
+               $addr = str_replace('acct:', '', $url);
+
        $vcard['fn'] = notags($vcard['fn']);
        $vcard['nick'] = str_replace(' ','',notags($vcard['nick']));
 
@@ -820,7 +899,7 @@ function probe_url($url, $mode = PROBE_NORMAL, $level = 1) {
        }
 
        // Only store into the cache if the value seems to be valid
-       if ($result['network'] != NETWORK_FEED)
+       if ($result['network'] != NETWORK_PHANTOM)
                Cache::set("probe_url:".$mode.":".$url,serialize($result), CACHE_DAY);
 
        return $result;