X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;f=src%2FNetwork%2FProbe.php;h=cfd03684397c9afb023dfdd55c139696c65310a9;hb=f3cd973cbe9d41be6e5bb7c36dad96c716f2f4b3;hp=3c43fb28ea5fd033b5dcb983ae676fd66befe363;hpb=eaf159fc207ced77dcf2e280f6dff2563e826826;p=friendica.git diff --git a/src/Network/Probe.php b/src/Network/Probe.php index 3c43fb28ea..cfd0368439 100644 --- a/src/Network/Probe.php +++ b/src/Network/Probe.php @@ -23,13 +23,13 @@ namespace Friendica\Network; use DOMDocument; use DomXPath; -use Friendica\Core\Cache\Duration; use Friendica\Core\Hook; use Friendica\Core\Logger; use Friendica\Core\Protocol; use Friendica\Core\System; use Friendica\Database\DBA; use Friendica\DI; +use Friendica\Model\Contact; use Friendica\Model\GServer; use Friendica\Model\Profile; use Friendica\Model\User; @@ -38,6 +38,7 @@ use Friendica\Protocol\ActivityPub; use Friendica\Protocol\Email; use Friendica\Protocol\Feed; use Friendica\Util\Crypto; +use Friendica\Util\DateTimeFormat; use Friendica\Util\Network; use Friendica\Util\Strings; use Friendica\Util\XML; @@ -90,17 +91,19 @@ class Probe "community", "keywords", "location", "about", "hide", "batch", "notify", "poll", "request", "confirm", "subscribe", "poco", "following", "followers", "inbox", "outbox", "sharedinbox", - "priority", "network", "pubkey", "baseurl", "gsid"]; + "priority", "network", "pubkey", "manually-approve", "baseurl", "gsid"]; + + $numeric_fields = ["gsid", "hide", "account-type", "manually-approve"]; $newdata = []; foreach ($fields as $field) { if (isset($data[$field])) { - if (in_array($field, ["gsid", "hide", "account-type"])) { + if (in_array($field, $numeric_fields)) { $newdata[$field] = (int)$data[$field]; } else { $newdata[$field] = $data[$field]; } - } elseif ($field != "gsid") { + } elseif (!in_array($field, $numeric_fields)) { $newdata[$field] = ""; } else { $newdata[$field] = null; @@ -327,13 +330,13 @@ class Probe * @throws HTTPException\InternalServerErrorException * @throws \ImagickException */ - public static function uri($uri, $network = '', $uid = -1, $cache = true) + public static function uri($uri, $network = '', $uid = -1) { - $cachekey = 'Probe::uri:' . $network . ':' . $uri; - if ($cache) { - $result = DI::cache()->get($cachekey); - if (!is_null($result)) { - return $result; + // Local profiles aren't probed via network + if (empty($network) && strpos($uri, DI::baseUrl()->getHostname())) { + $data = self::localProbe($uri); + if (!empty($data)) { + return $data; } } @@ -369,7 +372,7 @@ class Probe } if (empty($data['photo'])) { - $data['photo'] = DI::baseUrl() . '/images/person-300.jpg'; + $data['photo'] = DI::baseUrl() . Contact::DEFAULT_AVATAR_PHOTO; } if (empty($data['name'])) { @@ -407,14 +410,7 @@ class Probe $data['hide'] = self::getHideStatus($data['url']); } - $data = self::rearrangeData($data); - - // Only store into the cache if the value seems to be valid - if (!in_array($data['network'], [Protocol::PHANTOM, Protocol::MAIL])) { - DI::cache()->set($cachekey, $data, Duration::DAY); - } - - return $data; + return self::rearrangeData($data); } @@ -427,16 +423,11 @@ class Probe */ private static function getHideStatus($url) { - $curlResult = DI::httpRequest()->get($url); + $curlResult = DI::httpRequest()->get($url, false, ['content_length' => 1000000]); if (!$curlResult->isSuccess()) { return false; } - // If the file is too large then exit - if (($curlResult->getInfo()['download_content_length'] ?? 0) > 1000000) { - return false; - } - // If it isn't a HTML file then exit if (($curlResult->getContentType() != '') && !strstr(strtolower($curlResult->getContentType()), 'html')) { return false; @@ -718,7 +709,14 @@ class Probe Logger::info('Probing start', ['uri' => $uri]); - $data = self::getWebfingerArray($uri); + if (!empty($ap_profile['addr']) && ($ap_profile['addr'] != $uri)) { + $data = self::getWebfingerArray($ap_profile['addr']); + } + + if (empty($data)) { + $data = self::getWebfingerArray($uri); + } + if (empty($data)) { if (!empty($parts['scheme'])) { return self::feed($uri); @@ -1453,6 +1451,7 @@ class Probe && !empty($hcard_url) ) { $data["network"] = Protocol::DIASPORA; + $data["manually-approve"] = false; // The Diaspora handle must always be lowercase if (!empty($data["addr"])) { @@ -1543,6 +1542,7 @@ class Probe && isset($data["url"]) ) { $data["network"] = Protocol::OSTATUS; + $data["manually-approve"] = false; } else { return $short ? false : []; } @@ -2009,4 +2009,216 @@ class Probe return $fixed; } + + /** + * Fetch the last date that the contact had posted something (publically) + * + * @param string $data probing result + * @return string last activity + */ + public static function getLastUpdate(array $data) + { + $uid = User::getIdForURL($data['url']); + if (!empty($uid)) { + $contact = Contact::selectFirst(['url', 'last-item'], ['self' => true, 'uid' => $uid]); + if (!empty($contact['last-item'])) { + return $contact['last-item']; + } + } + + if ($lastUpdate = self::updateFromNoScrape($data)) { + return $lastUpdate; + } + + if (!empty($data['outbox'])) { + return self::updateFromOutbox($data['outbox'], $data); + } elseif (!empty($data['poll']) && ($data['network'] == Protocol::ACTIVITYPUB)) { + return self::updateFromOutbox($data['poll'], $data); + } elseif (!empty($data['poll'])) { + return self::updateFromFeed($data); + } + + return ''; + } + + /** + * Fetch the last activity date from the "noscrape" endpoint + * + * @param array $data Probing result + * @return string last activity + * + * @return bool 'true' if update was successful or the server was unreachable + */ + private static function updateFromNoScrape(array $data) + { + if (empty($data['baseurl'])) { + return ''; + } + + // Check the 'noscrape' endpoint when it is a Friendica server + $gserver = DBA::selectFirst('gserver', ['noscrape'], ["`nurl` = ? AND `noscrape` != ''", + Strings::normaliseLink($data['baseurl'])]); + if (!DBA::isResult($gserver)) { + return ''; + } + + $curlResult = DI::httpRequest()->get($gserver['noscrape'] . '/' . $data['nick']); + + if ($curlResult->isSuccess() && !empty($curlResult->getBody())) { + $noscrape = json_decode($curlResult->getBody(), true); + if (!empty($noscrape) && !empty($noscrape['updated'])) { + return DateTimeFormat::utc($noscrape['updated'], DateTimeFormat::MYSQL); + } + } + + return ''; + } + + /** + * Fetch the last activity date from an ActivityPub Outbox + * + * @param string $feed + * @param array $data Probing result + * @return string last activity + * @throws \Friendica\Network\HTTPException\InternalServerErrorException + */ + private static function updateFromOutbox(string $feed, array $data) + { + $outbox = ActivityPub::fetchContent($feed); + if (empty($outbox)) { + return ''; + } + + if (!empty($outbox['orderedItems'])) { + $items = $outbox['orderedItems']; + } elseif (!empty($outbox['first']['orderedItems'])) { + $items = $outbox['first']['orderedItems']; + } elseif (!empty($outbox['first']['href']) && ($outbox['first']['href'] != $feed)) { + return self::updateFromOutbox($outbox['first']['href'], $data); + } elseif (!empty($outbox['first'])) { + if (is_string($outbox['first']) && ($outbox['first'] != $feed)) { + return self::updateFromOutbox($outbox['first'], $data); + } else { + Logger::warning('Unexpected data', ['outbox' => $outbox]); + } + return ''; + } else { + $items = []; + } + + $last_updated = ''; + foreach ($items as $activity) { + if (!empty($activity['published'])) { + $published = DateTimeFormat::utc($activity['published']); + } elseif (!empty($activity['object']['published'])) { + $published = DateTimeFormat::utc($activity['object']['published']); + } else { + continue; + } + + if ($last_updated < $published) { + $last_updated = $published; + } + } + + if (!empty($last_updated)) { + return $last_updated; + } + + return ''; + } + + /** + * Fetch the last activity date from an XML feed + * + * @param array $data Probing result + * @return string last activity + */ + private static function updateFromFeed(array $data) + { + // Search for the newest entry in the feed + $curlResult = DI::httpRequest()->get($data['poll']); + if (!$curlResult->isSuccess()) { + return ''; + } + + $doc = new DOMDocument(); + @$doc->loadXML($curlResult->getBody()); + + $xpath = new DOMXPath($doc); + $xpath->registerNamespace('atom', 'http://www.w3.org/2005/Atom'); + + $entries = $xpath->query('/atom:feed/atom:entry'); + + $last_updated = ''; + + foreach ($entries as $entry) { + $published_item = $xpath->query('atom:published/text()', $entry)->item(0); + $updated_item = $xpath->query('atom:updated/text()' , $entry)->item(0); + $published = !empty($published_item->nodeValue) ? DateTimeFormat::utc($published_item->nodeValue) : null; + $updated = !empty($updated_item->nodeValue) ? DateTimeFormat::utc($updated_item->nodeValue) : null; + + if (empty($published) || empty($updated)) { + Logger::notice('Invalid entry for XPath.', ['entry' => $entry, 'url' => $data['url']]); + continue; + } + + if ($last_updated < $published) { + $last_updated = $published; + } + + if ($last_updated < $updated) { + $last_updated = $updated; + } + } + + if (!empty($last_updated)) { + return $last_updated; + } + + return ''; + } + + /** + * Probe data from local profiles without network traffic + * + * @param string $url + * @return array probed data + */ + private static function localProbe(string $url) + { + $uid = User::getIdForURL($url); + if (empty($uid)) { + return []; + } + + $profile = User::getOwnerDataById($uid); + if (empty($profile)) { + return []; + } + + $approfile = ActivityPub\Transmitter::getProfile($uid); + if (empty($approfile)) { + return []; + } + + if (empty($profile['gsid'])) { + $profile['gsid'] = GServer::getID($approfile['generator']['url']); + } + + $data = ['name' => $profile['name'], 'nick' => $profile['nick'], 'guid' => $approfile['diaspora:guid'] ?? '', + 'url' => $profile['url'], 'addr' => $profile['addr'], 'alias' => $profile['alias'], + 'photo' => $profile['photo'], 'account-type' => $profile['contact-type'], + 'community' => ($profile['contact-type'] == User::ACCOUNT_TYPE_COMMUNITY), + 'keywords' => $profile['keywords'], 'location' => $profile['location'], 'about' => $profile['about'], + 'hide' => !$profile['net-publish'], 'batch' => '', 'notify' => $profile['notify'], + 'poll' => $profile['poll'], 'request' => $profile['request'], 'confirm' => $profile['confirm'], + 'subscribe' => $approfile['generator']['url'] . '/follow?url={uri}', 'poco' => $profile['poco'], + 'following' => $approfile['following'], 'followers' => $approfile['followers'], + 'inbox' => $approfile['inbox'], 'outbox' => $approfile['outbox'], + 'sharedinbox' => $approfile['endpoints']['sharedInbox'], 'network' => Protocol::DFRN, + 'pubkey' => $profile['upubkey'], 'baseurl' => $approfile['generator']['url'], 'gsid' => $profile['gsid'], + 'manually-approve' => in_array($profile['page-flags'], [User::PAGE_FLAGS_NORMAL, User::PAGE_FLAGS_PRVGROUP])]; + return self::rearrangeData($data); + } }