]> git.mxchange.org Git - friendica.git/blobdiff - src/Network/Probe.php
Fix: The "extid" field wasn't updated
[friendica.git] / src / Network / Probe.php
index b1f4f233a63153e3b5f4701f02641393916a8703..5f3c516820f544f10fe2bfd6e71fcd68797fd822 100644 (file)
@@ -1,6 +1,6 @@
 <?php
 /**
 <?php
 /**
- * @copyright Copyright (C) 2020, Friendica
+ * @copyright Copyright (C) 2010-2021, the Friendica project
  *
  * @license GNU AGPL version 3 or any later version
  *
  *
  * @license GNU AGPL version 3 or any later version
  *
@@ -23,7 +23,6 @@ namespace Friendica\Network;
 
 use DOMDocument;
 use DomXPath;
 
 use DOMDocument;
 use DomXPath;
-use Friendica\Core\Cache\Duration;
 use Friendica\Core\Hook;
 use Friendica\Core\Logger;
 use Friendica\Core\Protocol;
 use Friendica\Core\Hook;
 use Friendica\Core\Logger;
 use Friendica\Core\Protocol;
@@ -33,11 +32,13 @@ use Friendica\DI;
 use Friendica\Model\Contact;
 use Friendica\Model\GServer;
 use Friendica\Model\Profile;
 use Friendica\Model\Contact;
 use Friendica\Model\GServer;
 use Friendica\Model\Profile;
+use Friendica\Model\User;
 use Friendica\Protocol\ActivityNamespace;
 use Friendica\Protocol\ActivityPub;
 use Friendica\Protocol\Email;
 use Friendica\Protocol\Feed;
 use Friendica\Util\Crypto;
 use Friendica\Protocol\ActivityNamespace;
 use Friendica\Protocol\ActivityPub;
 use Friendica\Protocol\Email;
 use Friendica\Protocol\Feed;
 use Friendica\Util\Crypto;
+use Friendica\Util\DateTimeFormat;
 use Friendica\Util\Network;
 use Friendica\Util\Strings;
 use Friendica\Util\XML;
 use Friendica\Util\Network;
 use Friendica\Util\Strings;
 use Friendica\Util\XML;
@@ -90,17 +91,19 @@ class Probe
                                "community", "keywords", "location", "about", "hide",
                                "batch", "notify", "poll", "request", "confirm", "subscribe", "poco",
                                "following", "followers", "inbox", "outbox", "sharedinbox",
                                "community", "keywords", "location", "about", "hide",
                                "batch", "notify", "poll", "request", "confirm", "subscribe", "poco",
                                "following", "followers", "inbox", "outbox", "sharedinbox",
-                               "priority", "network", "pubkey", "baseurl", "gsid"];
+                               "priority", "network", "pubkey", "manually-approve", "baseurl", "gsid"];
+
+               $numeric_fields = ["gsid", "hide", "account-type", "manually-approve"];
 
                $newdata = [];
                foreach ($fields as $field) {
                        if (isset($data[$field])) {
 
                $newdata = [];
                foreach ($fields as $field) {
                        if (isset($data[$field])) {
-                               if (in_array($field, ["gsid", "hide", "account-type"])) {
+                               if (in_array($field, $numeric_fields)) {
                                        $newdata[$field] = (int)$data[$field];
                                } else {        
                                        $newdata[$field] = $data[$field];
                                }
                                        $newdata[$field] = (int)$data[$field];
                                } else {        
                                        $newdata[$field] = $data[$field];
                                }
-                       } elseif ($field != "gsid") {
+                       } elseif (!in_array($field, $numeric_fields)) {
                                $newdata[$field] = "";
                        } else {
                                $newdata[$field] = null;
                                $newdata[$field] = "";
                        } else {
                                $newdata[$field] = null;
@@ -166,7 +169,7 @@ class Probe
                Logger::info('Probing', ['host' => $host, 'ssl_url' => $ssl_url, 'url' => $url, 'callstack' => System::callstack(20)]);
                $xrd = null;
 
                Logger::info('Probing', ['host' => $host, 'ssl_url' => $ssl_url, 'url' => $url, 'callstack' => System::callstack(20)]);
                $xrd = null;
 
-               $curlResult = Network::curl($ssl_url, false, ['timeout' => $xrd_timeout, 'accept_content' => 'application/xrd+xml']);
+               $curlResult = DI::httpRequest()->get($ssl_url, ['timeout' => $xrd_timeout, 'accept_content' => 'application/xrd+xml']);
                $ssl_connection_error = ($curlResult->getErrorNumber() == CURLE_COULDNT_CONNECT) || ($curlResult->getReturnCode() == 0);
                if ($curlResult->isSuccess()) {
                        $xml = $curlResult->getBody();
                $ssl_connection_error = ($curlResult->getErrorNumber() == CURLE_COULDNT_CONNECT) || ($curlResult->getReturnCode() == 0);
                if ($curlResult->isSuccess()) {
                        $xml = $curlResult->getBody();
@@ -177,16 +180,16 @@ class Probe
                                $host_url = $host;
                        }
                } elseif ($curlResult->isTimeout()) {
                                $host_url = $host;
                        }
                } elseif ($curlResult->isTimeout()) {
-                       Logger::info('Probing timeout', ['url' => $ssl_url], Logger::DEBUG);
+                       Logger::info('Probing timeout', ['url' => $ssl_url]);
                        self::$istimeout = true;
                        return [];
                }
 
                if (!is_object($xrd) && !empty($url)) {
                        self::$istimeout = true;
                        return [];
                }
 
                if (!is_object($xrd) && !empty($url)) {
-                       $curlResult = Network::curl($url, false, ['timeout' => $xrd_timeout, 'accept_content' => 'application/xrd+xml']);
+                       $curlResult = DI::httpRequest()->get($url, ['timeout' => $xrd_timeout, 'accept_content' => 'application/xrd+xml']);
                        $connection_error = ($curlResult->getErrorNumber() == CURLE_COULDNT_CONNECT) || ($curlResult->getReturnCode() == 0);
                        if ($curlResult->isTimeout()) {
                        $connection_error = ($curlResult->getErrorNumber() == CURLE_COULDNT_CONNECT) || ($curlResult->getReturnCode() == 0);
                        if ($curlResult->isTimeout()) {
-                               Logger::info('Probing timeout', ['url' => $url], Logger::DEBUG);
+                               Logger::info('Probing timeout', ['url' => $url]);
                                self::$istimeout = true;
                                return [];
                        } elseif ($connection_error && $ssl_connection_error) {
                                self::$istimeout = true;
                                return [];
                        } elseif ($connection_error && $ssl_connection_error) {
@@ -199,13 +202,13 @@ class Probe
                        $host_url = 'http://'.$host;
                }
                if (!is_object($xrd)) {
                        $host_url = 'http://'.$host;
                }
                if (!is_object($xrd)) {
-                       Logger::log("No xrd object found for ".$host, Logger::DEBUG);
+                       Logger::info('No xrd object found', ['host' => $host]);
                        return [];
                }
 
                $links = XML::elementToArray($xrd);
                if (!isset($links["xrd"]["link"])) {
                        return [];
                }
 
                $links = XML::elementToArray($xrd);
                if (!isset($links["xrd"]["link"])) {
-                       Logger::log("No xrd data found for ".$host, Logger::DEBUG);
+                       Logger::info('No xrd data found', ['host' => $host]);
                        return [];
                }
 
                        return [];
                }
 
@@ -229,7 +232,7 @@ class Probe
 
                self::$baseurl = $host_url;
 
 
                self::$baseurl = $host_url;
 
-               Logger::log("Probing successful for ".$host, Logger::DEBUG);
+               Logger::info('Probing successful', ['host' => $host]);
 
                return $lrdd;
        }
 
                return $lrdd;
        }
@@ -255,12 +258,12 @@ class Probe
         * @return string profile link
         * @throws HTTPException\InternalServerErrorException
         */
         * @return string profile link
         * @throws HTTPException\InternalServerErrorException
         */
-       public static function webfingerDfrn($webbie, &$hcard_url)
+       public static function webfingerDfrn(string $webbie, string &$hcard_url)
        {
                $profile_link = '';
 
                $links = self::lrdd($webbie);
        {
                $profile_link = '';
 
                $links = self::lrdd($webbie);
-               Logger::log('webfingerDfrn: '.$webbie.':'.print_r($links, true), Logger::DATA);
+               Logger::debug('Result', ['url' => $webbie, 'links' => $links]);
                if (!empty($links) && is_array($links)) {
                        foreach ($links as $link) {
                                if ($link['@attributes']['rel'] === ActivityNamespace::DFRN) {
                if (!empty($links) && is_array($links)) {
                        foreach ($links as $link) {
                                if ($link['@attributes']['rel'] === ActivityNamespace::DFRN) {
@@ -294,7 +297,7 @@ class Probe
                $webfinger = $data['webfinger'];
 
                if (empty($webfinger["links"])) {
                $webfinger = $data['webfinger'];
 
                if (empty($webfinger["links"])) {
-                       Logger::log("No webfinger links found for ".$uri, Logger::DEBUG);
+                       Logger::info('No webfinger links found', ['uri' => $uri]);
                        return [];
                }
 
                        return [];
                }
 
@@ -327,13 +330,13 @@ class Probe
         * @throws HTTPException\InternalServerErrorException
         * @throws \ImagickException
         */
         * @throws HTTPException\InternalServerErrorException
         * @throws \ImagickException
         */
-       public static function uri($uri, $network = '', $uid = -1, $cache = true)
+       public static function uri($uri, $network = '', $uid = -1)
        {
        {
-               $cachekey = 'Probe::uri:' . $network . ':' . $uri;
-               if ($cache) {
-                       $result = DI::cache()->get($cachekey);
-                       if (!is_null($result)) {
-                               return $result;
+               // Local profiles aren't probed via network
+               if (empty($network) && strpos($uri, DI::baseUrl()->getHostname())) {
+                       $data = self::localProbe($uri);
+                       if (!empty($data)) {
+                               return $data;
                        }
                }
 
                        }
                }
 
@@ -341,28 +344,27 @@ class Probe
                        $uid = local_user();
                }
 
                        $uid = local_user();
                }
 
+               if (empty($network) || ($network == Protocol::ACTIVITYPUB)) {
+                       $ap_profile = ActivityPub::probeProfile($uri);
+               } else {
+                       $ap_profile = [];
+               }
+
                self::$istimeout = false;
 
                if ($network != Protocol::ACTIVITYPUB) {
                self::$istimeout = false;
 
                if ($network != Protocol::ACTIVITYPUB) {
-                       $data = self::detect($uri, $network, $uid);
+                       $data = self::detect($uri, $network, $uid, $ap_profile);
                        if (!is_array($data)) {
                                $data = [];
                        }
                        if (!is_array($data)) {
                                $data = [];
                        }
-               } else {
-                       $data = [];
-               }
-
-               // When the previous detection process had got a time out
-               // we could falsely detect a Friendica profile as AP profile.
-               if (!self::$istimeout && (empty($network) || $network == Protocol::ACTIVITYPUB)) {
-                       $ap_profile = ActivityPub::probeProfile($uri, !$cache);
-
                        if (empty($data) || (!empty($ap_profile) && empty($network) && (($data['network'] ?? '') != Protocol::DFRN))) {
                                $data = $ap_profile;
                        } elseif (!empty($ap_profile)) {
                                $ap_profile['batch'] = '';
                                $data = array_merge($ap_profile, $data);
                        }
                        if (empty($data) || (!empty($ap_profile) && empty($network) && (($data['network'] ?? '') != Protocol::DFRN))) {
                                $data = $ap_profile;
                        } elseif (!empty($ap_profile)) {
                                $ap_profile['batch'] = '';
                                $data = array_merge($ap_profile, $data);
                        }
+               } else {
+                       $data = $ap_profile;
                }
 
                if (!isset($data['url'])) {
                }
 
                if (!isset($data['url'])) {
@@ -370,7 +372,7 @@ class Probe
                }
 
                if (empty($data['photo'])) {
                }
 
                if (empty($data['photo'])) {
-                       $data['photo'] = DI::baseUrl() . '/images/person-300.jpg';
+                       $data['photo'] = DI::baseUrl() . Contact::DEFAULT_AVATAR_PHOTO;
                }
 
                if (empty($data['name'])) {
                }
 
                if (empty($data['name'])) {
@@ -408,14 +410,7 @@ class Probe
                        $data['hide'] = self::getHideStatus($data['url']);
                }
 
                        $data['hide'] = self::getHideStatus($data['url']);
                }
 
-               $data = self::rearrangeData($data);
-
-               // Only store into the cache if the value seems to be valid
-               if (!in_array($data['network'], [Protocol::PHANTOM, Protocol::MAIL])) {
-                       DI::cache()->set($cachekey, $data, Duration::DAY);
-               }
-
-               return $data;
+               return self::rearrangeData($data);
        }
 
 
        }
 
 
@@ -428,7 +423,7 @@ class Probe
         */
        private static function getHideStatus($url)
        {
         */
        private static function getHideStatus($url)
        {
-               $curlResult = Network::curl($url);
+               $curlResult = DI::httpRequest()->get($url);
                if (!$curlResult->isSuccess()) {
                        return false;
                }
                if (!$curlResult->isSuccess()) {
                        return false;
                }
@@ -444,6 +439,9 @@ class Probe
                }
 
                $body = $curlResult->getBody();
                }
 
                $body = $curlResult->getBody();
+               if (empty($body)) {
+                       return false;
+               }
 
                $doc = new DOMDocument();
                @$doc->loadHTML($body);
 
                $doc = new DOMDocument();
                @$doc->loadHTML($body);
@@ -663,14 +661,15 @@ class Probe
         *
         * This function is only called by the "uri" function that adds caching and rearranging of data.
         *
         *
         * This function is only called by the "uri" function that adds caching and rearranging of data.
         *
-        * @param string  $uri     Address that should be probed
-        * @param string  $network Test for this specific network
-        * @param integer $uid     User ID for the probe (only used for mails)
+        * @param string  $uri        Address that should be probed
+        * @param string  $network    Test for this specific network
+        * @param integer $uid        User ID for the probe (only used for mails)
+        * @param array   $ap_profile Previously probed AP profile
         *
         * @return array uri data
         * @throws HTTPException\InternalServerErrorException
         */
         *
         * @return array uri data
         * @throws HTTPException\InternalServerErrorException
         */
-       private static function detect($uri, $network, $uid)
+       private static function detect(string $uri, string $network, int $uid, array $ap_profile)
        {
                $hookData = [
                        'uri'     => $uri,
        {
                $hookData = [
                        'uri'     => $uri,
@@ -692,7 +691,7 @@ class Probe
                $parts = parse_url($uri);
 
                if (!empty($parts['scheme']) && !empty($parts['host'])) {
                $parts = parse_url($uri);
 
                if (!empty($parts['scheme']) && !empty($parts['host'])) {
-                       if ($parts['host'] == 'twitter.com') {
+                       if (in_array($parts['host'], ['twitter.com', 'mobile.twitter.com'])) {
                                return self::twitter($uri);
                        }
                } elseif (strstr($uri, '@')) {
                                return self::twitter($uri);
                        }
                } elseif (strstr($uri, '@')) {
@@ -706,7 +705,9 @@ class Probe
                                return self::mail($uri, $uid);
                        }
 
                                return self::mail($uri, $uid);
                        }
 
-                       if (strpos($uri, '@twitter.com')) {
+                       if (Strings::endsWith($uri, '@twitter.com')
+                               || Strings::endsWith($uri, '@mobile.twitter.com')
+                       ) {
                                return self::twitter($uri);
                        }
                } else {
                                return self::twitter($uri);
                        }
                } else {
@@ -716,7 +717,14 @@ class Probe
 
                Logger::info('Probing start', ['uri' => $uri]);
 
 
                Logger::info('Probing start', ['uri' => $uri]);
 
-               $data = self::getWebfingerArray($uri);
+               if (!empty($ap_profile['addr']) && ($ap_profile['addr'] != $uri)) {
+                       $data = self::getWebfingerArray($ap_profile['addr']);
+               }
+
+               if (empty($data)) {
+                       $data = self::getWebfingerArray($uri);
+               }
+
                if (empty($data)) {
                        if (!empty($parts['scheme'])) {
                                return self::feed($uri);
                if (empty($data)) {
                        if (!empty($parts['scheme'])) {
                                return self::feed($uri);
@@ -749,7 +757,7 @@ class Probe
                if ((!$result && ($network == "")) || ($network == Protocol::PUMPIO)) {
                        $result = self::pumpio($webfinger, $addr);
                }
                if ((!$result && ($network == "")) || ($network == Protocol::PUMPIO)) {
                        $result = self::pumpio($webfinger, $addr);
                }
-               if ((!$result && ($network == "")) || ($network == Protocol::FEED)) {
+               if (empty($result['network']) && empty($ap_profile['network']) || ($network == Protocol::FEED)) {
                        $result = self::feed($uri);
                } else {
                        // We overwrite the detected nick with our try if the previois routines hadn't detected it.
                        $result = self::feed($uri);
                } else {
                        // We overwrite the detected nick with our try if the previois routines hadn't detected it.
@@ -839,7 +847,7 @@ class Probe
 
        public static function pollZot($url, $data)
        {
 
        public static function pollZot($url, $data)
        {
-               $curlResult = Network::curl($url);
+               $curlResult = DI::httpRequest()->get($url);
                if ($curlResult->isTimeout()) {
                        return $data;
                }
                if ($curlResult->isTimeout()) {
                        return $data;
                }
@@ -890,7 +898,7 @@ class Probe
                }
                if (!empty($json['public_forum'])) {
                        $data['community'] = $json['public_forum'];
                }
                if (!empty($json['public_forum'])) {
                        $data['community'] = $json['public_forum'];
-                       $data['account-type'] = Contact::PAGE_COMMUNITY;
+                       $data['account-type'] = User::PAGE_FLAGS_COMMUNITY;
                }
 
                if (!empty($json['profile'])) {
                }
 
                if (!empty($json['profile'])) {
@@ -936,7 +944,7 @@ class Probe
        {
                $xrd_timeout = DI::config()->get('system', 'xrd_timeout', 20);
 
        {
                $xrd_timeout = DI::config()->get('system', 'xrd_timeout', 20);
 
-               $curlResult = Network::curl($url, false, ['timeout' => $xrd_timeout, 'accept_content' => $type]);
+               $curlResult = DI::httpRequest()->get($url, ['timeout' => $xrd_timeout, 'accept_content' => $type]);
                if ($curlResult->isTimeout()) {
                        self::$istimeout = true;
                        return [];
                if ($curlResult->isTimeout()) {
                        self::$istimeout = true;
                        return [];
@@ -946,7 +954,7 @@ class Probe
                $webfinger = json_decode($data, true);
                if (!empty($webfinger)) {
                        if (!isset($webfinger["links"])) {
                $webfinger = json_decode($data, true);
                if (!empty($webfinger)) {
                        if (!isset($webfinger["links"])) {
-                               Logger::log("No json webfinger links for ".$url, Logger::DEBUG);
+                               Logger::info('No json webfinger links', ['url' => $url]);
                                return [];
                        }
                        return $webfinger;
                                return [];
                        }
                        return $webfinger;
@@ -955,13 +963,13 @@ class Probe
                // If it is not JSON, maybe it is XML
                $xrd = XML::parseString($data, true);
                if (!is_object($xrd)) {
                // If it is not JSON, maybe it is XML
                $xrd = XML::parseString($data, true);
                if (!is_object($xrd)) {
-                       Logger::log("No webfinger data retrievable for ".$url, Logger::DEBUG);
+                       Logger::info('No webfinger data retrievable', ['url' => $url]);
                        return [];
                }
 
                $xrd_arr = XML::elementToArray($xrd);
                if (!isset($xrd_arr["xrd"]["link"])) {
                        return [];
                }
 
                $xrd_arr = XML::elementToArray($xrd);
                if (!isset($xrd_arr["xrd"]["link"])) {
-                       Logger::log("No XML webfinger links for ".$url, Logger::DEBUG);
+                       Logger::info('No XML webfinger links', ['url' => $url]);
                        return [];
                }
 
                        return [];
                }
 
@@ -1005,20 +1013,20 @@ class Probe
         */
        private static function pollNoscrape($noscrape_url, $data)
        {
         */
        private static function pollNoscrape($noscrape_url, $data)
        {
-               $curlResult = Network::curl($noscrape_url);
+               $curlResult = DI::httpRequest()->get($noscrape_url);
                if ($curlResult->isTimeout()) {
                        self::$istimeout = true;
                        return [];
                }
                $content = $curlResult->getBody();
                if (!$content) {
                if ($curlResult->isTimeout()) {
                        self::$istimeout = true;
                        return [];
                }
                $content = $curlResult->getBody();
                if (!$content) {
-                       Logger::log("Empty body for ".$noscrape_url, Logger::DEBUG);
+                       Logger::info('Empty body', ['url' => $noscrape_url]);
                        return [];
                }
 
                $json = json_decode($content, true);
                if (!is_array($json)) {
                        return [];
                }
 
                $json = json_decode($content, true);
                if (!is_array($json)) {
-                       Logger::log("No json data for ".$noscrape_url, Logger::DEBUG);
+                       Logger::info('No json data', ['url' => $noscrape_url]);
                        return [];
                }
 
                        return [];
                }
 
@@ -1132,7 +1140,7 @@ class Probe
        {
                $data = [];
 
        {
                $data = [];
 
-               Logger::log("Check profile ".$profile_link, Logger::DEBUG);
+               Logger::info('Check profile', ['link' => $profile_link]);
 
                // Fetch data via noscrape - this is faster
                $noscrape_url = str_replace(["/hcard/", "/profile/"], "/noscrape/", $profile_link);
 
                // Fetch data via noscrape - this is faster
                $noscrape_url = str_replace(["/hcard/", "/profile/"], "/noscrape/", $profile_link);
@@ -1166,7 +1174,7 @@ class Probe
                $prof_data["fn"]           = $data['name']    ?? null;
                $prof_data["key"]          = $data['pubkey']  ?? null;
 
                $prof_data["fn"]           = $data['name']    ?? null;
                $prof_data["key"]          = $data['pubkey']  ?? null;
 
-               Logger::log("Result for profile ".$profile_link.": ".print_r($prof_data, true), Logger::DEBUG);
+               Logger::debug('Result', ['link' => $profile_link, 'data' => $prof_data]);
 
                return $prof_data;
        }
 
                return $prof_data;
        }
@@ -1263,13 +1271,13 @@ class Probe
         */
        private static function pollHcard($hcard_url, $data, $dfrn = false)
        {
         */
        private static function pollHcard($hcard_url, $data, $dfrn = false)
        {
-               $curlResult = Network::curl($hcard_url);
+               $curlResult = DI::httpRequest()->get($hcard_url);
                if ($curlResult->isTimeout()) {
                        self::$istimeout = true;
                        return [];
                }
                $content = $curlResult->getBody();
                if ($curlResult->isTimeout()) {
                        self::$istimeout = true;
                        return [];
                }
                $content = $curlResult->getBody();
-               if (!$content) {
+               if (empty($content)) {
                        return [];
                }
 
                        return [];
                }
 
@@ -1451,6 +1459,7 @@ class Probe
                        && !empty($hcard_url)
                ) {
                        $data["network"] = Protocol::DIASPORA;
                        && !empty($hcard_url)
                ) {
                        $data["network"] = Protocol::DIASPORA;
+                       $data["manually-approve"] = false;
 
                        // The Diaspora handle must always be lowercase
                        if (!empty($data["addr"])) {
 
                        // The Diaspora handle must always be lowercase
                        if (!empty($data["addr"])) {
@@ -1517,7 +1526,7 @@ class Probe
                                                        $pubkey = substr($pubkey, 5);
                                                }
                                        } elseif (Strings::normaliseLink($pubkey) == 'http://') {
                                                        $pubkey = substr($pubkey, 5);
                                                }
                                        } elseif (Strings::normaliseLink($pubkey) == 'http://') {
-                                               $curlResult = Network::curl($pubkey);
+                                               $curlResult = DI::httpRequest()->get($pubkey);
                                                if ($curlResult->isTimeout()) {
                                                        self::$istimeout = true;
                                                        return $short ? false : [];
                                                if ($curlResult->isTimeout()) {
                                                        self::$istimeout = true;
                                                        return $short ? false : [];
@@ -1541,6 +1550,7 @@ class Probe
                        && isset($data["url"])
                ) {
                        $data["network"] = Protocol::OSTATUS;
                        && isset($data["url"])
                ) {
                        $data["network"] = Protocol::OSTATUS;
+                       $data["manually-approve"] = false;
                } else {
                        return $short ? false : [];
                }
                } else {
                        return $short ? false : [];
                }
@@ -1550,7 +1560,7 @@ class Probe
                }
 
                // Fetch all additional data from the feed
                }
 
                // Fetch all additional data from the feed
-               $curlResult = Network::curl($data["poll"]);
+               $curlResult = DI::httpRequest()->get($data["poll"]);
                if ($curlResult->isTimeout()) {
                        self::$istimeout = true;
                        return [];
                if ($curlResult->isTimeout()) {
                        self::$istimeout = true;
                        return [];
@@ -1602,8 +1612,8 @@ class Probe
         */
        private static function pumpioProfileData($profile_link)
        {
         */
        private static function pumpioProfileData($profile_link)
        {
-               $curlResult = Network::curl($profile_link);
-               if (!$curlResult->isSuccess()) {
+               $curlResult = DI::httpRequest()->get($profile_link);
+               if (!$curlResult->isSuccess() || empty($curlResult->getBody())) {
                        return [];
                }
 
                        return [];
                }
 
@@ -1720,9 +1730,9 @@ class Probe
         */
        private static function twitter($uri)
        {
         */
        private static function twitter($uri)
        {
-               if (preg_match('=(.*)@twitter.com=i', $uri, $matches)) {
+               if (preg_match('=([^@]+)@(?:mobile\.)?twitter\.com$=i', $uri, $matches)) {
                        $nick = $matches[1];
                        $nick = $matches[1];
-               } elseif (preg_match('=https?://twitter.com/(.*)=i', $uri, $matches)) {
+               } elseif (preg_match('=^https?://(?:mobile\.)?twitter\.com/(.+)=i', $uri, $matches)) {
                        $nick = $matches[1];
                } else {
                        return [];
                        $nick = $matches[1];
                } else {
                        return [];
@@ -1747,6 +1757,10 @@ class Probe
         */
        public static function getFeedLink(string $url, string $body)
        {
         */
        public static function getFeedLink(string $url, string $body)
        {
+               if (empty($body)) {
+                       return '';
+               }
+
                $doc = new DOMDocument();
                if (!@$doc->loadHTML($body)) {
                        return false;
                $doc = new DOMDocument();
                if (!@$doc->loadHTML($body)) {
                        return false;
@@ -1782,6 +1796,9 @@ class Probe
                $base = $xpath->evaluate('string(/html/head/base/@href)') ?: $base;
 
                $baseParts = parse_url($base);
                $base = $xpath->evaluate('string(/html/head/base/@href)') ?: $base;
 
                $baseParts = parse_url($base);
+               if (empty($baseParts['host'])) {
+                       return $href;
+               }
 
                // Naked domain case (scheme://basehost)
                $path = $baseParts['path'] ?? '/';
 
                // Naked domain case (scheme://basehost)
                $path = $baseParts['path'] ?? '/';
@@ -1791,26 +1808,28 @@ class Probe
 
                $hrefParts = parse_url($href);
 
 
                $hrefParts = parse_url($href);
 
-               // Root path case (/path) including relative scheme case (//host/path)
-               if ($hrefParts['path'] && $hrefParts['path'][0] == '/') {
-                       $path = $hrefParts['path'];
-               } else {
-                       $path = $path . '/' . $hrefParts['path'];
-
-                       // Resolve arbitrary relative path
-                       // Lifted from https://www.php.net/manual/en/function.realpath.php#84012
-                       $parts = array_filter(explode('/', $path), 'strlen');
-                       $absolutes = array();
-                       foreach ($parts as $part) {
-                               if ('.' == $part) continue;
-                               if ('..' == $part) {
-                                       array_pop($absolutes);
-                               } else {
-                                       $absolutes[] = $part;
+               if (!empty($hrefParts['path'])) {
+                       // Root path case (/path) including relative scheme case (//host/path)
+                       if ($hrefParts['path'] && $hrefParts['path'][0] == '/') {
+                               $path = $hrefParts['path'];
+                       } else {
+                               $path = $path . '/' . $hrefParts['path'];
+
+                               // Resolve arbitrary relative path
+                               // Lifted from https://www.php.net/manual/en/function.realpath.php#84012
+                               $parts = array_filter(explode('/', $path), 'strlen');
+                               $absolutes = array();
+                               foreach ($parts as $part) {
+                                       if ('.' == $part) continue;
+                                       if ('..' == $part) {
+                                               array_pop($absolutes);
+                                       } else {
+                                               $absolutes[] = $part;
+                                       }
                                }
                                }
-                       }
 
 
-                       $path = '/' . implode('/', $absolutes);
+                               $path = '/' . implode('/', $absolutes);
+                       }
                }
 
                // Relative scheme case (//host/path)
                }
 
                // Relative scheme case (//host/path)
@@ -1833,7 +1852,7 @@ class Probe
         */
        private static function feed($url, $probe = true)
        {
         */
        private static function feed($url, $probe = true)
        {
-               $curlResult = Network::curl($url);
+               $curlResult = DI::httpRequest()->get($url);
                if ($curlResult->isTimeout()) {
                        self::$istimeout = true;
                        return [];
                if ($curlResult->isTimeout()) {
                        self::$istimeout = true;
                        return [];
@@ -1874,12 +1893,6 @@ class Probe
                $data["url"] = $url;
                $data["poll"] = $url;
 
                $data["url"] = $url;
                $data["poll"] = $url;
 
-               if (!empty($feed_data["header"]["author-link"])) {
-                       $data["baseurl"] = $feed_data["header"]["author-link"];
-               } else {
-                       $data["baseurl"] = $data["url"];
-               }
-
                $data["network"] = Protocol::FEED;
 
                return $data;
                $data["network"] = Protocol::FEED;
 
                return $data;
@@ -1923,7 +1936,7 @@ class Probe
                }
 
                $msgs = Email::poll($mbox, $uri);
                }
 
                $msgs = Email::poll($mbox, $uri);
-               Logger::log('searching '.$uri.', '.count($msgs).' messages found.', Logger::DEBUG);
+               Logger::info('Messages found', ['uri' => $uri, 'count' => count($msgs)]);
 
                if (!count($msgs)) {
                        return [];
 
                if (!count($msgs)) {
                        return [];
@@ -2006,8 +2019,220 @@ class Probe
 
                $fixed = $scheme.$host.$port.$path.$query.$fragment;
 
 
                $fixed = $scheme.$host.$port.$path.$query.$fragment;
 
-               Logger::log('Base: '.$base.' - Avatar: '.$avatar.' - Fixed: '.$fixed, Logger::DATA);
+               Logger::debug('Avatar fixed', ['base' => $base, 'avatar' => $avatar, 'fixed' => $fixed]);
 
                return $fixed;
        }
 
                return $fixed;
        }
+
+       /**
+        * Fetch the last date that the contact had posted something (publically)
+        *
+        * @param string $data  probing result
+        * @return string last activity
+        */
+       public static function getLastUpdate(array $data)
+       {
+               $uid = User::getIdForURL($data['url']);
+               if (!empty($uid)) {
+                       $contact = Contact::selectFirst(['url', 'last-item'], ['self' => true, 'uid' => $uid]);
+                       if (!empty($contact['last-item'])) {
+                               return $contact['last-item'];
+                       }
+               }
+
+               if ($lastUpdate = self::updateFromNoScrape($data)) {
+                       return $lastUpdate;
+               }
+
+               if (!empty($data['outbox'])) {
+                       return self::updateFromOutbox($data['outbox'], $data);
+               } elseif (!empty($data['poll']) && ($data['network'] == Protocol::ACTIVITYPUB)) {
+                       return self::updateFromOutbox($data['poll'], $data);
+               } elseif (!empty($data['poll'])) {
+                       return self::updateFromFeed($data);
+               }
+
+               return '';
+       }
+
+       /**
+        * Fetch the last activity date from the "noscrape" endpoint
+        *
+        * @param array $data Probing result
+        * @return string last activity
+        *
+        * @return bool 'true' if update was successful or the server was unreachable
+        */
+       private static function updateFromNoScrape(array $data)
+       {
+               if (empty($data['baseurl'])) {
+                       return '';
+               }
+
+               // Check the 'noscrape' endpoint when it is a Friendica server
+               $gserver = DBA::selectFirst('gserver', ['noscrape'], ["`nurl` = ? AND `noscrape` != ''",
+                       Strings::normaliseLink($data['baseurl'])]);
+               if (!DBA::isResult($gserver)) {
+                       return '';
+               }
+
+               $curlResult = DI::httpRequest()->get($gserver['noscrape'] . '/' . $data['nick']);
+
+               if ($curlResult->isSuccess() && !empty($curlResult->getBody())) {
+                       $noscrape = json_decode($curlResult->getBody(), true);
+                       if (!empty($noscrape) && !empty($noscrape['updated'])) {
+                               return DateTimeFormat::utc($noscrape['updated'], DateTimeFormat::MYSQL);
+                       }
+               }
+
+               return '';
+       }
+
+       /**
+        * Fetch the last activity date from an ActivityPub Outbox
+        *
+        * @param string $feed
+        * @param array  $data Probing result
+        * @return string last activity
+        * @throws \Friendica\Network\HTTPException\InternalServerErrorException
+        */
+       private static function updateFromOutbox(string $feed, array $data)
+       {
+               $outbox = ActivityPub::fetchContent($feed);
+               if (empty($outbox)) {
+                       return '';
+               }
+
+               if (!empty($outbox['orderedItems'])) {
+                       $items = $outbox['orderedItems'];
+               } elseif (!empty($outbox['first']['orderedItems'])) {
+                       $items = $outbox['first']['orderedItems'];
+               } elseif (!empty($outbox['first']['href']) && ($outbox['first']['href'] != $feed)) {
+                       return self::updateFromOutbox($outbox['first']['href'], $data);
+               } elseif (!empty($outbox['first'])) {
+                       if (is_string($outbox['first']) && ($outbox['first'] != $feed)) {
+                               return self::updateFromOutbox($outbox['first'], $data);
+                       } else {
+                               Logger::warning('Unexpected data', ['outbox' => $outbox]);
+                       }
+                       return '';
+               } else {
+                       $items = [];
+               }
+
+               $last_updated = '';
+               foreach ($items as $activity) {
+                       if (!empty($activity['published'])) {
+                               $published =  DateTimeFormat::utc($activity['published']);
+                       } elseif (!empty($activity['object']['published'])) {
+                               $published =  DateTimeFormat::utc($activity['object']['published']);
+                       } else {
+                               continue;
+                       }
+
+                       if ($last_updated < $published) {
+                               $last_updated = $published;
+                       }
+               }
+
+               if (!empty($last_updated)) {
+                       return $last_updated;
+               }
+
+               return '';
+       }
+
+       /**
+        * Fetch the last activity date from an XML feed
+        *
+        * @param array $data Probing result
+        * @return string last activity
+        */
+       private static function updateFromFeed(array $data)
+       {
+               // Search for the newest entry in the feed
+               $curlResult = DI::httpRequest()->get($data['poll']);
+               if (!$curlResult->isSuccess()) {
+                       return '';
+               }
+
+               $doc = new DOMDocument();
+               @$doc->loadXML($curlResult->getBody());
+
+               $xpath = new DOMXPath($doc);
+               $xpath->registerNamespace('atom', 'http://www.w3.org/2005/Atom');
+
+               $entries = $xpath->query('/atom:feed/atom:entry');
+
+               $last_updated = '';
+
+               foreach ($entries as $entry) {
+                       $published_item = $xpath->query('atom:published/text()', $entry)->item(0);
+                       $updated_item   = $xpath->query('atom:updated/text()'  , $entry)->item(0);
+                       $published      = !empty($published_item->nodeValue) ? DateTimeFormat::utc($published_item->nodeValue) : null;
+                       $updated        = !empty($updated_item->nodeValue) ? DateTimeFormat::utc($updated_item->nodeValue) : null;
+
+                       if (empty($published) || empty($updated)) {
+                               Logger::notice('Invalid entry for XPath.', ['entry' => $entry, 'url' => $data['url']]);
+                               continue;
+                       }
+
+                       if ($last_updated < $published) {
+                               $last_updated = $published;
+                       }
+
+                       if ($last_updated < $updated) {
+                               $last_updated = $updated;
+                       }
+               }
+
+               if (!empty($last_updated)) {
+                       return $last_updated;
+               }
+
+               return '';
+       }
+
+       /**
+        * Probe data from local profiles without network traffic
+        *
+        * @param string $url
+        * @return array probed data
+        */
+       private static function localProbe(string $url)
+       {
+               $uid = User::getIdForURL($url);
+               if (empty($uid)) {
+                       return [];
+               }
+
+               $profile = User::getOwnerDataById($uid);
+               if (empty($profile)) {
+                       return [];
+               }
+
+               $approfile = ActivityPub\Transmitter::getProfile($uid);
+               if (empty($approfile)) {
+                       return [];
+               }
+
+               if (empty($profile['gsid'])) {
+                       $profile['gsid'] = GServer::getID($approfile['generator']['url']);
+               }
+
+               $data = ['name' => $profile['name'], 'nick' => $profile['nick'], 'guid' => $approfile['diaspora:guid'] ?? '',
+                       'url' => $profile['url'], 'addr' => $profile['addr'], 'alias' => $profile['alias'],
+                       'photo' => $profile['photo'], 'account-type' => $profile['contact-type'],
+                       'community' => ($profile['contact-type'] == User::ACCOUNT_TYPE_COMMUNITY),
+                       'keywords' => $profile['keywords'], 'location' => $profile['location'], 'about' => $profile['about'], 
+                       'hide' => !$profile['net-publish'], 'batch' => '', 'notify' => $profile['notify'],
+                       'poll' => $profile['poll'], 'request' => $profile['request'], 'confirm' => $profile['confirm'],
+                       'subscribe' => $approfile['generator']['url'] . '/follow?url={uri}', 'poco' => $profile['poco'], 
+                       'following' => $approfile['following'], 'followers' => $approfile['followers'],
+                       'inbox' => $approfile['inbox'], 'outbox' => $approfile['outbox'],
+                       'sharedinbox' => $approfile['endpoints']['sharedInbox'], 'network' => Protocol::DFRN, 
+                       'pubkey' => $profile['upubkey'], 'baseurl' => $approfile['generator']['url'], 'gsid' => $profile['gsid'],
+                       'manually-approve' => in_array($profile['page-flags'], [User::PAGE_FLAGS_NORMAL, User::PAGE_FLAGS_PRVGROUP])];
+               return self::rearrangeData($data);              
+       }
 }
 }