X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;f=src%2FModel%2FGServer.php;h=0f5f55ab671a19979fd4b1a02659d199a889fb24;hb=7b7132971a64a92a685a5fd860fe4709dce1765a;hp=4eaee9dc4fc0ed4c4db64c56173edcb0cdb6e5c0;hpb=25f93cb03ae44b4d4498a035c923b482c8ad10db;p=friendica.git diff --git a/src/Model/GServer.php b/src/Model/GServer.php index 4eaee9dc4f..0f5f55ab67 100644 --- a/src/Model/GServer.php +++ b/src/Model/GServer.php @@ -1,55 +1,213 @@ . + * */ + namespace Friendica\Model; use DOMDocument; use DOMXPath; -use Friendica\Core\Config; use Friendica\Core\Protocol; +use Friendica\Core\Worker; use Friendica\Database\DBA; +use Friendica\DI; use Friendica\Module\Register; +use Friendica\Network\CurlResult; use Friendica\Util\Network; use Friendica\Util\DateTimeFormat; use Friendica\Util\Strings; use Friendica\Util\XML; use Friendica\Core\Logger; - -/* -use Exception; -use Friendica\Core\System; -use Friendica\Core\Worker; -use Friendica\Network\Probe; use Friendica\Protocol\PortableContact; -*/ +use Friendica\Protocol\Diaspora; +use Friendica\Network\Probe; + /** - * @brief This class handles GServer related functions + * This class handles GServer related functions */ class GServer { + // Directory types + const DT_NONE = 0; + const DT_POCO = 1; + const DT_MASTODON = 2; /** - * Detect server type + * Checks if the given server is reachable * - * @param string $url Server url - * @param boolean $force Force update + * @param string $profile URL of the given profile + * @param string $server URL of the given server (If empty, taken from profile) + * @param string $network Network value that is used, when detection failed + * @param boolean $force Force an update. * - * @return boolean 'true' if server could be detected + * @return boolean 'true' if server seems vital + */ + public static function reachable(string $profile, string $server = '', string $network = '', bool $force = false) + { + if ($server == '') { + $server = GContact::getBasepath($profile); + } + + if ($server == '') { + return true; + } + + return self::check($server, $network, $force); + } + + /** + * Decides if a server needs to be updated, based upon several date fields + * + * @param date $created Creation date of that server entry + * @param date $updated When had the server entry be updated + * @param date $last_failure Last failure when contacting that server + * @param date $last_contact Last time the server had been contacted + * + * @return boolean Does the server record needs an update? + */ + public static function updateNeeded($created, $updated, $last_failure, $last_contact) + { + $now = strtotime(DateTimeFormat::utcNow()); + + if ($updated > $last_contact) { + $contact_time = strtotime($updated); + } else { + $contact_time = strtotime($last_contact); + } + + $failure_time = strtotime($last_failure); + $created_time = strtotime($created); + + // If there is no "created" time then use the current time + if ($created_time <= 0) { + $created_time = $now; + } + + // If the last contact was less than 24 hours then don't update + if (($now - $contact_time) < (60 * 60 * 24)) { + return false; + } + + // If the last failure was less than 24 hours then don't update + if (($now - $failure_time) < (60 * 60 * 24)) { + return false; + } + + // If the last contact was less than a week ago and the last failure is older than a week then don't update + //if ((($now - $contact_time) < (60 * 60 * 24 * 7)) && ($contact_time > $failure_time)) + // return false; + + // If the last contact time was more than a week ago and the contact was created more than a week ago, then only try once a week + if ((($now - $contact_time) > (60 * 60 * 24 * 7)) && (($now - $created_time) > (60 * 60 * 24 * 7)) && (($now - $failure_time) < (60 * 60 * 24 * 7))) { + return false; + } + + // If the last contact time was more than a month ago and the contact was created more than a month ago, then only try once a month + if ((($now - $contact_time) > (60 * 60 * 24 * 30)) && (($now - $created_time) > (60 * 60 * 24 * 30)) && (($now - $failure_time) < (60 * 60 * 24 * 30))) { + return false; + } + + return true; + } + + /** + * Checks the state of the given server. + * + * @param string $server_url URL of the given server + * @param string $network Network value that is used, when detection failed + * @param boolean $force Force an update. + * + * @return boolean 'true' if server seems vital */ - public static function detect($url, $force = false) + public static function check(string $server_url, string $network = '', bool $force = false) { - /// @Todo: - // - Update Check - // - poco - // - Pleroma version number + // Unify the server address + $server_url = trim($server_url, '/'); + $server_url = str_replace('/index.php', '', $server_url); + + if ($server_url == '') { + return false; + } + + $gserver = DBA::selectFirst('gserver', [], ['nurl' => Strings::normaliseLink($server_url)]); + if (DBA::isResult($gserver)) { + if ($gserver['created'] <= DBA::NULL_DATETIME) { + $fields = ['created' => DateTimeFormat::utcNow()]; + $condition = ['nurl' => Strings::normaliseLink($server_url)]; + DBA::update('gserver', $fields, $condition); + } + + $last_contact = $gserver['last_contact']; + $last_failure = $gserver['last_failure']; -// $gserver = DBA::selectFirst('gserver', [], ['nurl' => Strings::normaliseLink($url)]); + // See discussion under https://forum.friendi.ca/display/0b6b25a8135aabc37a5a0f5684081633 + // It can happen that a zero date is in the database, but storing it again is forbidden. + if ($last_contact < DBA::NULL_DATETIME) { + $last_contact = DBA::NULL_DATETIME; + } + + if ($last_failure < DBA::NULL_DATETIME) { + $last_failure = DBA::NULL_DATETIME; + } + + if (!$force && !self::updateNeeded($gserver['created'], '', $last_failure, $last_contact)) { + Logger::info('No update needed', ['server' => $server_url]); + return ($last_contact >= $last_failure); + } + Logger::info('Server is outdated. Start discovery.', ['Server' => $server_url, 'Force' => $force, 'Created' => $gserver['created'], 'Failure' => $last_failure, 'Contact' => $last_contact]); + } else { + Logger::info('Server is unknown. Start discovery.', ['Server' => $server_url]); + } + + return self::detect($server_url, $network); + } + + /** + * Detect server data (type, protocol, version number, ...) + * The detected data is then updated or inserted in the gserver table. + * + * @param string $url URL of the given server + * @param string $network Network value that is used, when detection failed + * + * @return boolean 'true' if server could be detected + */ + public static function detect(string $url, string $network = '') + { + Logger::info('Detect server type', ['server' => $url]); $serverdata = []; + $original_url = $url; + + // Remove URL content that is not supposed to exist for a server url + $urlparts = parse_url($url); + unset($urlparts['user']); + unset($urlparts['pass']); + unset($urlparts['query']); + unset($urlparts['fragment']); + $url = Network::unparseURL($urlparts); + + // If the URL missmatches, then we mark the old entry as failure + if ($url != $original_url) { + DBA::update('gserver', ['last_failure' => DateTimeFormat::utcNow()], ['nurl' => Strings::normaliseLink($original_url)]); + } + // When a nodeinfo is present, we don't need to dig further - $xrd_timeout = Config::get('system', 'xrd_timeout'); + $xrd_timeout = DI::config()->get('system', 'xrd_timeout'); $curlResult = Network::curl($url . '/.well-known/nodeinfo', false, ['timeout' => $xrd_timeout]); if ($curlResult->isTimeout()) { DBA::update('gserver', ['last_failure' => DateTimeFormat::utcNow()], ['nurl' => Strings::normaliseLink($url)]); @@ -58,32 +216,54 @@ class GServer $nodeinfo = self::fetchNodeinfo($url, $curlResult); - if (empty($nodeinfo) || ($nodeinfo['network'] == Protocol::DFRN)) { + // When nodeinfo isn't present, we use the older 'statistics.json' endpoint + if (empty($nodeinfo)) { + $nodeinfo = self::fetchStatistics($url); + } + + // If that didn't work out well, we use some protocol specific endpoints + // For Friendica and Zot based networks we have to dive deeper to reveal more details + if (empty($nodeinfo['network']) || in_array($nodeinfo['network'], [Protocol::DFRN, Protocol::ZOT])) { // Fetch the landing page, possibly it reveals some data - $curlResult = Network::curl($url, false, ['timeout' => $xrd_timeout]); - if ($curlResult->isSuccess()) { - $serverdata = self::analyseRootHeader($curlResult, $serverdata); - $serverdata = self::analyseRootBody($curlResult, $serverdata); + if (empty($nodeinfo['network'])) { + $curlResult = Network::curl($url, false, ['timeout' => $xrd_timeout]); + if ($curlResult->isSuccess()) { + $serverdata = self::analyseRootHeader($curlResult, $serverdata); + $serverdata = self::analyseRootBody($curlResult, $serverdata, $url); + } + + if (!$curlResult->isSuccess() || empty($curlResult->getBody()) || self::invalidBody($curlResult->getBody())) { + DBA::update('gserver', ['last_failure' => DateTimeFormat::utcNow()], ['nurl' => Strings::normaliseLink($url)]); + return false; + } } - if (!$curlResult->isSuccess() || empty($curlResult->getBody())) { + if (empty($serverdata['network']) || ($serverdata['network'] == Protocol::ACTIVITYPUB)) { + $serverdata = self::detectMastodonAlikes($url, $serverdata); + } + + // All following checks are done for systems that always have got a "host-meta" endpoint. + // With this check we don't have to waste time and ressources for dead systems. + // Also this hopefully prevents us from receiving abuse messages. + if (empty($serverdata['network']) && !self::validHostMeta($url)) { DBA::update('gserver', ['last_failure' => DateTimeFormat::utcNow()], ['nurl' => Strings::normaliseLink($url)]); return false; } - if (empty($serverdata['network']) || ($serverdata['network'] == Protocol::DFRN)) { + if (empty($serverdata['network']) || in_array($serverdata['network'], [Protocol::DFRN, Protocol::ACTIVITYPUB])) { $serverdata = self::detectFriendica($url, $serverdata); } - if (empty($serverdata['network']) || ($serverdata['network'] == Protocol::ACTIVITYPUB)) { - $serverdata = self::detectMastodonAlikes($url, $serverdata); - } - - // the "siteinfo.json" is some specific endpoint of Hubzilla and Red + // the 'siteinfo.json' is some specific endpoint of Hubzilla and Red if (empty($serverdata['network']) || ($serverdata['network'] == Protocol::ZOT)) { $serverdata = self::fetchSiteinfo($url, $serverdata); } + // The 'siteinfo.json' doesn't seem to be present on older Hubzilla installations + if (empty($serverdata['network'])) { + $serverdata = self::detectHubzilla($url, $serverdata); + } + if (empty($serverdata['network'])) { $serverdata = self::detectNextcloud($url, $serverdata); } @@ -95,48 +275,206 @@ class GServer $serverdata = $nodeinfo; } + // Detect the directory type + $serverdata['directory-type'] = self::DT_NONE; + $serverdata = self::checkPoCo($url, $serverdata); + $serverdata = self::checkMastodonDirectory($url, $serverdata); + // We can't detect the network type. Possibly it is some system that we don't know yet if (empty($serverdata['network'])) { $serverdata['network'] = Protocol::PHANTOM; } + // When we hadn't been able to detect the network type, we use the hint from the parameter + if (($serverdata['network'] == Protocol::PHANTOM) && !empty($network)) { + $serverdata['network'] = $network; + } + $serverdata['url'] = $url; $serverdata['nurl'] = Strings::normaliseLink($url); - // When we don't have the registered users, we simply count what we know - if (empty($serverdata['registered-users'])) { + // We take the highest number that we do find + $registeredUsers = $serverdata['registered-users'] ?? 0; + + // On an active server there has to be at least a single user + if (($serverdata['network'] != Protocol::PHANTOM) && ($registeredUsers == 0)) { + $registeredUsers = 1; + } + + if ($serverdata['network'] != Protocol::PHANTOM) { $gcontacts = DBA::count('gcontact', ['server_url' => [$url, $serverdata['nurl']]]); $apcontacts = DBA::count('apcontact', ['baseurl' => [$url, $serverdata['nurl']]]); $contacts = DBA::count('contact', ['uid' => 0, 'baseurl' => [$url, $serverdata['nurl']]]); - $serverdata['registered-users'] = max($gcontacts, $apcontacts, $contacts); + $serverdata['registered-users'] = max($gcontacts, $apcontacts, $contacts, $registeredUsers); + } else { + $serverdata['registered-users'] = $registeredUsers; + $serverdata = self::detectNetworkViaContacts($url, $serverdata); } - $fields = array_keys($serverdata); - $old_data = DBA::selectFirst('gserver', $fields, ['nurl' => Strings::normaliseLink($url)]); - if (!DBA::isResult($old_data)) { -die('Möööp'); + $serverdata['last_contact'] = DateTimeFormat::utcNow(); + + $gserver = DBA::selectFirst('gserver', ['network'], ['nurl' => Strings::normaliseLink($url)]); + if (!DBA::isResult($gserver)) { $serverdata['created'] = DateTimeFormat::utcNow(); - DBA::insert('gserver', $serverdata); + $ret = DBA::insert('gserver', $serverdata); } else { - $serverdata['last_contact'] = DateTimeFormat::utcNow(); - DBA::update('gserver', $serverdata, ['nurl' => $serverdata['nurl']], $old_data); + // Don't override the network with 'unknown' when there had been a valid entry before + if (($serverdata['network'] == Protocol::PHANTOM) && !empty($gserver['network'])) { + unset($serverdata['network']); + } + + $ret = DBA::update('gserver', $serverdata, ['nurl' => $serverdata['nurl']]); + } + + if (!empty($serverdata['network']) && in_array($serverdata['network'], [Protocol::DFRN, Protocol::DIASPORA])) { + self::discoverRelay($url); + } + + return $ret; + } + + /** + * Fetch relay data from a given server url + * + * @param string $server_url address of the server + * @throws \Friendica\Network\HTTPException\InternalServerErrorException + */ + private static function discoverRelay(string $server_url) + { + Logger::info('Discover relay data', ['server' => $server_url]); + + $curlResult = Network::curl($server_url . '/.well-known/x-social-relay'); + if (!$curlResult->isSuccess()) { + return; + } + + $data = json_decode($curlResult->getBody(), true); + if (!is_array($data)) { + return; + } + + $gserver = DBA::selectFirst('gserver', ['id', 'relay-subscribe', 'relay-scope'], ['nurl' => Strings::normaliseLink($server_url)]); + if (!DBA::isResult($gserver)) { + return; + } + + if (($gserver['relay-subscribe'] != $data['subscribe']) || ($gserver['relay-scope'] != $data['scope'])) { + $fields = ['relay-subscribe' => $data['subscribe'], 'relay-scope' => $data['scope']]; + DBA::update('gserver', $fields, ['id' => $gserver['id']]); + } + + DBA::delete('gserver-tag', ['gserver-id' => $gserver['id']]); + + if ($data['scope'] == 'tags') { + // Avoid duplicates + $tags = []; + foreach ($data['tags'] as $tag) { + $tag = mb_strtolower($tag); + if (strlen($tag) < 100) { + $tags[$tag] = $tag; + } + } + + foreach ($tags as $tag) { + DBA::insert('gserver-tag', ['gserver-id' => $gserver['id'], 'tag' => $tag], true); + } + } + + // Create or update the relay contact + $fields = []; + if (isset($data['protocols'])) { + if (isset($data['protocols']['diaspora'])) { + $fields['network'] = Protocol::DIASPORA; + + if (isset($data['protocols']['diaspora']['receive'])) { + $fields['batch'] = $data['protocols']['diaspora']['receive']; + } elseif (is_string($data['protocols']['diaspora'])) { + $fields['batch'] = $data['protocols']['diaspora']; + } + } + + if (isset($data['protocols']['dfrn'])) { + $fields['network'] = Protocol::DFRN; + + if (isset($data['protocols']['dfrn']['receive'])) { + $fields['batch'] = $data['protocols']['dfrn']['receive']; + } elseif (is_string($data['protocols']['dfrn'])) { + $fields['batch'] = $data['protocols']['dfrn']; + } + } + } + Diaspora::setRelayContact($server_url, $fields); + } + + /** + * Fetch server data from '/statistics.json' on the given server + * + * @param string $url URL of the given server + * + * @return array server data + */ + private static function fetchStatistics(string $url) + { + $curlResult = Network::curl($url . '/statistics.json'); + if (!$curlResult->isSuccess()) { + return []; + } + + $data = json_decode($curlResult->getBody(), true); + if (empty($data)) { + return []; + } + + $serverdata = []; + + if (!empty($data['version'])) { + $serverdata['version'] = $data['version']; + // Version numbers on statistics.json are presented with additional info, e.g.: + // 0.6.3.0-p1702cc1c, 0.6.99.0-p1b9ab160 or 3.4.3-2-1191. + $serverdata['version'] = preg_replace('=(.+)-(.{4,})=ism', '$1', $serverdata['version']); + } + + if (!empty($data['name'])) { + $serverdata['site_name'] = $data['name']; + } + + if (!empty($data['network'])) { + $serverdata['platform'] = strtolower($data['network']); + + if ($serverdata['platform'] == 'diaspora') { + $serverdata['network'] = Protocol::DIASPORA; + } elseif ($serverdata['platform'] == 'friendica') { + $serverdata['network'] = Protocol::DFRN; + } elseif ($serverdata['platform'] == 'hubzilla') { + $serverdata['network'] = Protocol::ZOT; + } elseif ($serverdata['platform'] == 'redmatrix') { + $serverdata['network'] = Protocol::ZOT; + } + } + + + if (!empty($data['registrations_open'])) { + $serverdata['register_policy'] = Register::OPEN; + } else { + $serverdata['register_policy'] = Register::CLOSED; } return $serverdata; } /** - * @brief Detect server type by using the nodeinfo data + * Detect server type by using the nodeinfo data * - * @param string $url address of the server + * @param string $url address of the server + * @param CurlResult $curlResult * @return array Server data * @throws \Friendica\Network\HTTPException\InternalServerErrorException */ - private static function fetchNodeinfo($url, $curlResult) + private static function fetchNodeinfo(string $url, CurlResult $curlResult) { $nodeinfo = json_decode($curlResult->getBody(), true); - if (!is_array($nodeinfo) || !isset($nodeinfo['links'])) { + if (!is_array($nodeinfo) || empty($nodeinfo['links'])) { return []; } @@ -175,24 +513,24 @@ die('Möööp'); } /** - * @brief Parses Nodeinfo 1 + * Parses Nodeinfo 1 * * @param string $nodeinfo_url address of the nodeinfo path * @return array Server data * @throws \Friendica\Network\HTTPException\InternalServerErrorException */ - private static function parseNodeinfo1($nodeinfo_url) + private static function parseNodeinfo1(string $nodeinfo_url) { $curlResult = Network::curl($nodeinfo_url); if (!$curlResult->isSuccess()) { - return false; + return []; } $nodeinfo = json_decode($curlResult->getBody(), true); if (!is_array($nodeinfo)) { - return false; + return []; } $server = []; @@ -204,11 +542,11 @@ die('Möööp'); } if (is_array($nodeinfo['software'])) { - if (isset($nodeinfo['software']['name'])) { - $server['platform'] = $nodeinfo['software']['name']; + if (!empty($nodeinfo['software']['name'])) { + $server['platform'] = strtolower($nodeinfo['software']['name']); } - if (isset($nodeinfo['software']['version'])) { + if (!empty($nodeinfo['software']['version'])) { $server['version'] = $nodeinfo['software']['version']; // Version numbers on Nodeinfo are presented with additional info, e.g.: // 0.6.3.0-p1702cc1c, 0.6.99.0-p1b9ab160 or 3.4.3-2-1191. @@ -216,7 +554,7 @@ die('Möööp'); } } - if (isset($nodeinfo['metadata']['nodeName'])) { + if (!empty($nodeinfo['metadata']['nodeName'])) { $server['site_name'] = $nodeinfo['metadata']['nodeName']; } @@ -240,37 +578,36 @@ die('Möööp'); $server['network'] = Protocol::OSTATUS; } elseif (!empty($protocols['gnusocial'])) { $server['network'] = Protocol::OSTATUS; - } else { - print_r($protocols); - die('Protocol 1'); + } elseif (!empty($protocols['zot'])) { + $server['network'] = Protocol::ZOT; } } - if (!$server) { - return false; + if (empty($server)) { + return []; } return $server; } /** - * @brief Parses Nodeinfo 2 + * Parses Nodeinfo 2 * * @param string $nodeinfo_url address of the nodeinfo path * @return array Server data * @throws \Friendica\Network\HTTPException\InternalServerErrorException */ - private static function parseNodeinfo2($nodeinfo_url) + private static function parseNodeinfo2(string $nodeinfo_url) { $curlResult = Network::curl($nodeinfo_url); if (!$curlResult->isSuccess()) { - return false; + return []; } $nodeinfo = json_decode($curlResult->getBody(), true); if (!is_array($nodeinfo)) { - return false; + return []; } $server = []; @@ -282,11 +619,11 @@ die('Möööp'); } if (is_array($nodeinfo['software'])) { - if (isset($nodeinfo['software']['name'])) { - $server['platform'] = $nodeinfo['software']['name']; + if (!empty($nodeinfo['software']['name'])) { + $server['platform'] = strtolower($nodeinfo['software']['name']); } - if (isset($nodeinfo['software']['version'])) { + if (!empty($nodeinfo['software']['version'])) { $server['version'] = $nodeinfo['software']['version']; // Version numbers on Nodeinfo are presented with additional info, e.g.: // 0.6.3.0-p1702cc1c, 0.6.99.0-p1b9ab160 or 3.4.3-2-1191. @@ -294,7 +631,7 @@ die('Möööp'); } } - if (isset($nodeinfo['metadata']['nodeName'])) { + if (!empty($nodeinfo['metadata']['nodeName'])) { $server['site_name'] = $nodeinfo['metadata']['nodeName']; } @@ -308,7 +645,7 @@ die('Möööp'); $protocols[$protocol] = true; } - if (!empty($protocols['friendica'])) { + if (!empty($protocols['dfrn'])) { $server['network'] = Protocol::DFRN; } elseif (!empty($protocols['activitypub'])) { $server['network'] = Protocol::ACTIVITYPUB; @@ -318,20 +655,27 @@ die('Möööp'); $server['network'] = Protocol::OSTATUS; } elseif (!empty($protocols['gnusocial'])) { $server['network'] = Protocol::OSTATUS; - } else { - print_r($protocols); - die('Protocol 2'); + } elseif (!empty($protocols['zot'])) { + $server['network'] = Protocol::ZOT; } } if (empty($server)) { - return false; + return []; } return $server; } - private static function fetchSiteinfo($url, $serverdata) + /** + * Fetch server information from a 'siteinfo.json' file on the given server + * + * @param string $url URL of the given server + * @param array $serverdata array with server data + * + * @return array server data + */ + private static function fetchSiteinfo(string $url, array $serverdata) { $curlResult = Network::curl($url . '/siteinfo.json'); if (!$curlResult->isSuccess()) { @@ -343,8 +687,8 @@ die('Möööp'); return $serverdata; } - if (isset($data['url'])) { - $serverdata['platform'] = $data['platform']; + if (!empty($data['url'])) { + $serverdata['platform'] = strtolower($data['platform']); $serverdata['version'] = $data['version']; } @@ -370,15 +714,15 @@ die('Möööp'); if (!empty($data['register_policy'])) { switch ($data['register_policy']) { - case "REGISTER_OPEN": + case 'REGISTER_OPEN': $serverdata['register_policy'] = Register::OPEN; break; - case "REGISTER_APPROVE": + case 'REGISTER_APPROVE': $serverdata['register_policy'] = Register::APPROVE; break; - case "REGISTER_CLOSED": + case 'REGISTER_CLOSED': default: $serverdata['register_policy'] = Register::CLOSED; break; @@ -388,7 +732,168 @@ die('Möööp'); return $serverdata; } - private static function detectNextcloud($url, $serverdata) + /** + * Checks if the server contains a valid host meta file + * + * @param string $url URL of the given server + * + * @return boolean 'true' if the server seems to be vital + */ + private static function validHostMeta(string $url) + { + $xrd_timeout = DI::config()->get('system', 'xrd_timeout'); + $curlResult = Network::curl($url . '/.well-known/host-meta', false, ['timeout' => $xrd_timeout]); + if (!$curlResult->isSuccess()) { + return false; + } + + $xrd = XML::parseString($curlResult->getBody(), false); + if (!is_object($xrd)) { + return false; + } + + $elements = XML::elementToArray($xrd); + if (empty($elements) || empty($elements['xrd']) || empty($elements['xrd']['link'])) { + return false; + } + + $valid = false; + foreach ($elements['xrd']['link'] as $link) { + // When there is more than a single "link" element, the array looks slightly different + if (!empty($link['@attributes'])) { + $link = $link['@attributes']; + } + + if (empty($link['rel']) || empty($link['template'])) { + continue; + } + + if ($link['rel'] == 'lrdd') { + // When the webfinger host is the same like the system host, it should be ok. + $valid = (parse_url($url, PHP_URL_HOST) == parse_url($link['template'], PHP_URL_HOST)); + } + } + + return $valid; + } + + /** + * Detect the network of the given server via their known contacts + * + * @param string $url URL of the given server + * @param array $serverdata array with server data + * + * @return array server data + */ + private static function detectNetworkViaContacts(string $url, array $serverdata) + { + $contacts = []; + + $gcontacts = DBA::select('gcontact', ['url', 'nurl'], ['server_url' => [$url, $serverdata['nurl']]]); + while ($gcontact = DBA::fetch($gcontacts)) { + $contacts[$gcontact['nurl']] = $gcontact['url']; + } + DBA::close($gcontacts); + + $apcontacts = DBA::select('apcontact', ['url'], ['baseurl' => [$url, $serverdata['nurl']]]); + while ($gcontact = DBA::fetch($gcontacts)) { + $contacts[Strings::normaliseLink($apcontact['url'])] = $apcontact['url']; + } + DBA::close($apcontacts); + + $pcontacts = DBA::select('contact', ['url', 'nurl'], ['uid' => 0, 'baseurl' => [$url, $serverdata['nurl']]]); + while ($gcontact = DBA::fetch($gcontacts)) { + $contacts[$pcontact['nurl']] = $pcontact['url']; + } + DBA::close($pcontacts); + + if (empty($contacts)) { + return $serverdata; + } + + foreach ($contacts as $contact) { + $probed = Probe::uri($contact); + if (in_array($probed['network'], Protocol::FEDERATED)) { + $serverdata['network'] = $probed['network']; + break; + } + } + + $serverdata['registered-users'] = max($serverdata['registered-users'], count($contacts)); + + return $serverdata; + } + + /** + * Checks if the given server does have a '/poco' endpoint. + * This is used for the 'PortableContact' functionality, + * which is used by both Friendica and Hubzilla. + * + * @param string $url URL of the given server + * @param array $serverdata array with server data + * + * @return array server data + */ + private static function checkPoCo(string $url, array $serverdata) + { + $serverdata['poco'] = ''; + + $curlResult = Network::curl($url. '/poco'); + if (!$curlResult->isSuccess()) { + return $serverdata; + } + + $data = json_decode($curlResult->getBody(), true); + if (empty($data)) { + return $serverdata; + } + + if (!empty($data['totalResults'])) { + $registeredUsers = $serverdata['registered-users'] ?? 0; + $serverdata['registered-users'] = max($data['totalResults'], $registeredUsers); + $serverdata['directory-type'] = self::DT_POCO; + $serverdata['poco'] = $url . '/poco'; + } + + return $serverdata; + } + + /** + * Checks if the given server does have a Mastodon style directory endpoint. + * + * @param string $url URL of the given server + * @param array $serverdata array with server data + * + * @return array server data + */ + public static function checkMastodonDirectory(string $url, array $serverdata) + { + $curlResult = Network::curl($url . '/api/v1/directory?limit=1'); + if (!$curlResult->isSuccess()) { + return $serverdata; + } + + $data = json_decode($curlResult->getBody(), true); + if (empty($data)) { + return $serverdata; + } + + if (count($data) == 1) { + $serverdata['directory-type'] = self::DT_MASTODON; + } + + return $serverdata; + } + + /** + * Detects the version number of a given server when it was a NextCloud installation + * + * @param string $url URL of the given server + * @param array $serverdata array with server data + * + * @return array server data + */ + private static function detectNextcloud(string $url, array $serverdata) { $curlResult = Network::curl($url . '/status.php'); @@ -410,7 +915,15 @@ die('Möööp'); return $serverdata; } - private static function detectMastodonAlikes($url, $serverdata) + /** + * Detects data from a given server url if it was a mastodon alike system + * + * @param string $url URL of the given server + * @param array $serverdata array with server data + * + * @return array server data + */ + private static function detectMastodonAlikes(string $url, array $serverdata) { $curlResult = Network::curl($url . '/api/v1/instance'); @@ -423,66 +936,194 @@ die('Möööp'); return $serverdata; } - if (isset($data['version'])) { + if (!empty($data['version'])) { + $serverdata['platform'] = 'mastodon'; + $serverdata['version'] = $data['version'] ?? ''; + $serverdata['network'] = Protocol::ACTIVITYPUB; + } + + if (!empty($data['title'])) { + $serverdata['site_name'] = $data['title']; + } + + if (!empty($data['title']) && empty($serverdata['platform']) && empty($serverdata['network'])) { $serverdata['platform'] = 'mastodon'; - $serverdata['version'] = defaults($data, 'version', ''); - if (!empty($data['title'])) { - $serverdata['site_name'] = $data['title']; - } - if (!empty($data['description'])) { - $serverdata['info'] = trim($data['description']); - } $serverdata['network'] = Protocol::ACTIVITYPUB; } + if (!empty($data['description'])) { + $serverdata['info'] = trim($data['description']); + } + if (!empty($data['stats']['user_count'])) { $serverdata['registered-users'] = $data['stats']['user_count']; } - if (strstr($serverdata['version'], 'Pleroma')) { + if (!empty($serverdata['version']) && preg_match('/.*?\(compatible;\s(.*)\s(.*)\)/ism', $serverdata['version'], $matches)) { + $serverdata['platform'] = strtolower($matches[1]); + $serverdata['version'] = $matches[2]; + } + + if (!empty($serverdata['version']) && strstr(strtolower($serverdata['version']), 'pleroma')) { $serverdata['platform'] = 'pleroma'; - $serverdata['version'] = trim(str_replace('Pleroma', '', $serverdata['version'])); // 2.7.2 (compatible; Pleroma 1.0.0-1225-gf31ad554-develop) + $serverdata['version'] = trim(str_ireplace('pleroma', '', $serverdata['version'])); } - if (strstr($serverdata['version'], 'Pixelfed')) { - print_r($serverdata); - die(); -// $serverdata['platform'] = 'pixelfed'; -// $serverdata['version'] = trim(str_replace('Pixelfed', '', $serverdata['version'])); // 2.7.2 (compatible; Pixelfed 0.10.5) + if (!empty($serverdata['platform']) && strstr($serverdata['platform'], 'pleroma')) { + $serverdata['version'] = trim(str_ireplace('pleroma', '', $serverdata['platform'])); + $serverdata['platform'] = 'pleroma'; } return $serverdata; } - private static function detectGNUSocial($url, $serverdata) + /** + * Detects data from typical Hubzilla endpoints + * + * @param string $url URL of the given server + * @param array $serverdata array with server data + * + * @return array server data + */ + private static function detectHubzilla(string $url, array $serverdata) { - $curlResult = Network::curl($url . '/api/statusnet/version.json'); + $curlResult = Network::curl($url . '/api/statusnet/config.json'); + if (!$curlResult->isSuccess() || ($curlResult->getBody() == '')) { + return $serverdata; + } + + $data = json_decode($curlResult->getBody(), true); + if (empty($data)) { + return $serverdata; + } + + if (!empty($data['site']['name'])) { + $serverdata['site_name'] = $data['site']['name']; + } + + if (!empty($data['site']['platform'])) { + $serverdata['platform'] = strtolower($data['site']['platform']['PLATFORM_NAME']); + $serverdata['version'] = $data['site']['platform']['STD_VERSION']; + $serverdata['network'] = Protocol::ZOT; + } + + if (!empty($data['site']['hubzilla'])) { + $serverdata['platform'] = strtolower($data['site']['hubzilla']['PLATFORM_NAME']); + $serverdata['version'] = $data['site']['hubzilla']['RED_VERSION']; + $serverdata['network'] = Protocol::ZOT; + } + + if (!empty($data['site']['redmatrix'])) { + if (!empty($data['site']['redmatrix']['PLATFORM_NAME'])) { + $serverdata['platform'] = strtolower($data['site']['redmatrix']['PLATFORM_NAME']); + } elseif (!empty($data['site']['redmatrix']['RED_PLATFORM'])) { + $serverdata['platform'] = strtolower($data['site']['redmatrix']['RED_PLATFORM']); + } + $serverdata['version'] = $data['site']['redmatrix']['RED_VERSION']; + $serverdata['network'] = Protocol::ZOT; + } + + $private = false; + $inviteonly = false; + $closed = false; + + if (!empty($data['site']['closed'])) { + $closed = self::toBoolean($data['site']['closed']); + } + + if (!empty($data['site']['private'])) { + $private = self::toBoolean($data['site']['private']); + } + + if (!empty($data['site']['inviteonly'])) { + $inviteonly = self::toBoolean($data['site']['inviteonly']); + } + + if (!$closed && !$private and $inviteonly) { + $register_policy = Register::APPROVE; + } elseif (!$closed && !$private) { + $register_policy = Register::OPEN; + } else { + $register_policy = Register::CLOSED; + } + + return $serverdata; + } + + /** + * Converts input value to a boolean value + * + * @param string|integer $val + * + * @return boolean + */ + private static function toBoolean($val) + { + if (($val == 'true') || ($val == 1)) { + return true; + } elseif (($val == 'false') || ($val == 0)) { + return false; + } + + return $val; + } + + /** + * Detect if the URL belongs to a GNU Social server + * + * @param string $url URL of the given server + * @param array $serverdata array with server data + * + * @return array server data + */ + private static function detectGNUSocial(string $url, array $serverdata) + { + // Test for GNU Social + $curlResult = Network::curl($url . '/api/gnusocial/version.json'); if ($curlResult->isSuccess() && ($curlResult->getBody() != '{"error":"not implemented"}') && ($curlResult->getBody() != '') && (strlen($curlResult->getBody()) < 30)) { - $serverdata['platform'] = 'StatusNet'; + $serverdata['platform'] = 'gnusocial'; // Remove junk that some GNU Social servers return - $serverdata['version'] = str_replace(chr(239).chr(187).chr(191), '', $curlResult->getBody()); + $serverdata['version'] = str_replace(chr(239) . chr(187) . chr(191), '', $curlResult->getBody()); + $serverdata['version'] = str_replace(["\r", "\n", "\t"], '', $serverdata['version']); $serverdata['version'] = trim($serverdata['version'], '"'); $serverdata['network'] = Protocol::OSTATUS; + return $serverdata; } - // Test for GNU Social - $curlResult = Network::curl($url . '/api/gnusocial/version.json'); - + // Test for Statusnet + $curlResult = Network::curl($url . '/api/statusnet/version.json'); if ($curlResult->isSuccess() && ($curlResult->getBody() != '{"error":"not implemented"}') && ($curlResult->getBody() != '') && (strlen($curlResult->getBody()) < 30)) { - $serverdata['platform'] = 'GNU Social'; + // Remove junk that some GNU Social servers return - $serverdata['version'] = str_replace(chr(239) . chr(187) . chr(191), '', $curlResult->getBody()); + $serverdata['version'] = str_replace(chr(239).chr(187).chr(191), '', $curlResult->getBody()); + $serverdata['version'] = str_replace(["\r", "\n", "\t"], '', $serverdata['version']); $serverdata['version'] = trim($serverdata['version'], '"'); - $serverdata['network'] = Protocol::OSTATUS; + + if (!empty($serverdata['version']) && strtolower(substr($serverdata['version'], 0, 7)) == 'pleroma') { + $serverdata['platform'] = 'pleroma'; + $serverdata['version'] = trim(str_ireplace('pleroma', '', $serverdata['version'])); + $serverdata['network'] = Protocol::ACTIVITYPUB; + } else { + $serverdata['platform'] = 'statusnet'; + $serverdata['network'] = Protocol::OSTATUS; + } } return $serverdata; } - private static function detectFriendica($url, $serverdata) + /** + * Detect if the URL belongs to a Friendica server + * + * @param string $url URL of the given server + * @param array $serverdata array with server data + * + * @return array server data + */ + private static function detectFriendica(string $url, array $serverdata) { $curlResult = Network::curl($url . '/friendica/json'); if (!$curlResult->isSuccess()) { @@ -513,7 +1154,7 @@ die('Möööp'); $serverdata['info'] = trim($data['info']); } - $register_policy = defaults($data, 'register_policy', 'REGISTER_CLOSED'); + $register_policy = ($data['register_policy'] ?? '') ?: 'REGISTER_CLOSED'; switch ($register_policy) { case 'REGISTER_OPEN': $serverdata['register_policy'] = Register::OPEN; @@ -533,12 +1174,21 @@ die('Möööp'); break; } - $serverdata['platform'] = defaults($data, 'platform', ''); + $serverdata['platform'] = strtolower($data['platform'] ?? ''); return $serverdata; } - private static function analyseRootBody($curlResult, $serverdata) + /** + * Analyses the landing page of a given server for hints about type and system of that server + * + * @param object $curlResult result of curl execution + * @param array $serverdata array with server data + * @param string $url Server URL + * + * @return array server data + */ + private static function analyseRootBody($curlResult, array $serverdata, string $url) { $doc = new DOMDocument(); @$doc->loadHTML($curlResult->getBody()); @@ -555,53 +1205,47 @@ die('Möööp'); $attr = []; if ($node->attributes->length) { foreach ($node->attributes as $attribute) { - $attribute->value = trim($attribute->value); - if (empty($attribute->value)) { + $value = trim($attribute->value); + if (empty($value)) { continue; } - $attr[$attribute->name] = $attribute->value; + $attr[$attribute->name] = $value; } if (empty($attr['name']) || empty($attr['content'])) { continue; } } -//print_r($attr); + if ($attr['name'] == 'description') { $serverdata['info'] = $attr['content']; } if ($attr['name'] == 'application-name') { - $serverdata['platform'] = $attr['content']; + $serverdata['platform'] = strtolower($attr['content']); if (in_array($attr['content'], ['Misskey', 'Write.as'])) { $serverdata['network'] = Protocol::ACTIVITYPUB; } } -// - if (($attr['name'] == 'generator') && in_array($attr['content'], ['Write.as'])) { -die('as'); -// $serverdata['platform'] = $attr['content']; -// $serverdata['network'] = Protocol::ACTIVITYPUB; - } elseif ($attr['name'] == 'generator') { - $serverdata['platform'] = $attr['content']; - + if (($attr['name'] == 'generator') && (empty($serverdata['platform']) || (substr(strtolower($attr['content']), 0, 9) == 'wordpress'))) { + $serverdata['platform'] = strtolower($attr['content']); $version_part = explode(' ', $attr['content']); - if (count($version_part) == 3) { - if (($version_part[0] == 'Red') && ($version_part[1] == 'Matrix')) { -// $serverdata['platform'] = $version_part[0] . ' ' . $version_part[1]; -// $serverdata['version'] = $version_part[2]; -// $serverdata['network'] = Protocol::DIASPORA; - } - } elseif (count($version_part) == 2) { + if (count($version_part) == 2) { if (in_array($version_part[0], ['WordPress'])) { - $serverdata['platform'] = $version_part[0]; + $serverdata['platform'] = strtolower($version_part[0]); $serverdata['version'] = $version_part[1]; - $serverdata['network'] = Protocol::ACTIVITYPUB; + + // We still do need a reliable test if some AP plugin is activated + if (DBA::exists('apcontact', ['baseurl' => $url])) { + $serverdata['network'] = Protocol::ACTIVITYPUB; + } else { + $serverdata['network'] = Protocol::FEED; + } } if (in_array($version_part[0], ['Friendika', 'Friendica'])) { - $serverdata['platform'] = $version_part[0]; + $serverdata['platform'] = strtolower($version_part[0]); $serverdata['version'] = $version_part[1]; $serverdata['network'] = Protocol::DFRN; } @@ -615,19 +1259,18 @@ die('as'); $attr = []; if ($node->attributes->length) { foreach ($node->attributes as $attribute) { - $attribute->value = trim($attribute->value); - if (empty($attribute->value)) { + $value = trim($attribute->value); + if (empty($value)) { continue; } - $attr[$attribute->name] = $attribute->value; + $attr[$attribute->name] = $value; } if (empty($attr['property']) || empty($attr['content'])) { continue; } } -//print_r($attr); if ($attr['property'] == 'og:site_name') { $serverdata['site_name'] = $attr['content']; @@ -637,14 +1280,8 @@ die('as'); $serverdata['info'] = $attr['content']; } -// if (($attr['property'] == 'og:title') && in_array($attr['content'], ['pixelfed', 'Socialhome'])) { -// if (($attr['property'] == 'og:title') && in_array($attr['content'], ['Nextcloud'])) { -// $serverdata['platform'] = $attr['content']; -// $serverdata['network'] = Protocol::ACTIVITYPUB; -// } - if ($attr['property'] == 'og:platform') { - $serverdata['platform'] = $attr['content']; + $serverdata['platform'] = strtolower($attr['content']); if (in_array($attr['content'], ['PeerTube'])) { $serverdata['network'] = Protocol::ACTIVITYPUB; @@ -652,7 +1289,7 @@ die('as'); } if ($attr['property'] == 'generator') { - $serverdata['platform'] = $attr['content']; + $serverdata['platform'] = strtolower($attr['content']); if (in_array($attr['content'], ['hubzilla'])) { // We later check which compatible protocol modules are loaded. @@ -664,7 +1301,15 @@ die('as'); return $serverdata; } - private static function analyseRootHeader($curlResult, $serverdata) + /** + * Analyses the header data of a given server for hints about type and system of that server + * + * @param object $curlResult result of curl execution + * @param array $serverdata array with server data + * + * @return array server data + */ + private static function analyseRootHeader($curlResult, array $serverdata) { if ($curlResult->getHeader('server') == 'Mastodon') { $serverdata['platform'] = 'mastodon'; @@ -673,15 +1318,132 @@ die('as'); $serverdata['platform'] = 'diaspora'; $serverdata['network'] = $network = Protocol::DIASPORA; $serverdata['version'] = $curlResult->getHeader('x-diaspora-version'); - } elseif ($curlResult->inHeader('x-friendica-version')) { $serverdata['platform'] = 'friendica'; $serverdata['network'] = $network = Protocol::DFRN; $serverdata['version'] = $curlResult->getHeader('x-friendica-version'); - - } else { -//print_r($curlResult->getHeaderArray()); } return $serverdata; } + + /** + * Test if the body contains valid content + * + * @param string $body + * @return boolean + */ + private static function invalidBody(string $body) + { + // Currently we only test for a HTML element. + // Possibly we enhance this in the future. + return !strpos($body, '>'); + } + + /** + * Update the user directory of a given gserver record + * + * @param array $gserver gserver record + */ + public static function updateDirectory(array $gserver) + { + /// @todo Add Mastodon API directory + + if (!empty($gserver['poco'])) { + PortableContact::discoverSingleServer($gserver['id']); + } + } + + /** + * Update GServer entries + */ + public static function discover() + { + // Update the server list + self::discoverFederation(); + + $no_of_queries = 5; + + $requery_days = intval(DI::config()->get('system', 'poco_requery_days')); + + if ($requery_days == 0) { + $requery_days = 7; + } + + $last_update = date('c', time() - (60 * 60 * 24 * $requery_days)); + + $gservers = DBA::p("SELECT `id`, `url`, `nurl`, `network`, `poco` + FROM `gserver` + WHERE `last_contact` >= `last_failure` + AND `poco` != '' + AND `last_poco_query` < ? + ORDER BY RAND()", $last_update + ); + + while ($gserver = DBA::fetch($gservers)) { + if (!GServer::check($gserver['url'], $gserver['network'])) { + // The server is not reachable? Okay, then we will try it later + $fields = ['last_poco_query' => DateTimeFormat::utcNow()]; + DBA::update('gserver', $fields, ['nurl' => $gserver['nurl']]); + continue; + } + + Logger::info('Update directory', ['server' => $gserver['url'], 'id' => $gserver['id']]); + Worker::add(PRIORITY_LOW, 'UpdateServerDirectory', $gserver); + + if (--$no_of_queries == 0) { + break; + } + } + + DBA::close($gservers); + } + + /** + * Discover federated servers + */ + private static function discoverFederation() + { + $last = DI::config()->get('poco', 'last_federation_discovery'); + + if ($last) { + $next = $last + (24 * 60 * 60); + + if ($next > time()) { + return; + } + } + + // Discover federated servers + $curlResult = Network::fetchUrl("http://the-federation.info/pods.json"); + + if (!empty($curlResult)) { + $servers = json_decode($curlResult, true); + + if (!empty($servers['pods'])) { + foreach ($servers['pods'] as $server) { + Worker::add(PRIORITY_LOW, 'UpdateGServer', 'https://' . $server['host']); + } + } + } + + // Disvover Mastodon servers + $accesstoken = DI::config()->get('system', 'instances_social_key'); + + if (!empty($accesstoken)) { + $api = 'https://instances.social/api/1.0/instances/list?count=0'; + $header = ['Authorization: Bearer '.$accesstoken]; + $curlResult = Network::curl($api, false, ['headers' => $header]); + + if ($curlResult->isSuccess()) { + $servers = json_decode($curlResult->getBody(), true); + + foreach ($servers['instances'] as $server) { + $url = (is_null($server['https_score']) ? 'http' : 'https') . '://' . $server['name']; + Worker::add(PRIORITY_LOW, 'UpdateGServer', $url); + } + } + } + + DI::config()->set('poco', 'last_federation_discovery', time()); + } }