X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;f=src%2FModel%2FGServer.php;h=0f47146eb73705b67613bfa793f594d9ec3696a4;hb=6749b2c887552feef3a671c4063425b11a92014a;hp=f17265a3a3fcc0a8f5093c78150f24e59e85c5e4;hpb=efadab66c539bca3c1fd894443973e3c13ef0127;p=friendica.git diff --git a/src/Model/GServer.php b/src/Model/GServer.php index f17265a3a3..0f47146eb7 100644 --- a/src/Model/GServer.php +++ b/src/Model/GServer.php @@ -1,26 +1,42 @@ . + * */ + namespace Friendica\Model; use DOMDocument; use DOMXPath; -use Friendica\Core\Config; +use Friendica\Core\Logger; use Friendica\Core\Protocol; +use Friendica\Core\System; +use Friendica\Core\Worker; use Friendica\Database\DBA; +use Friendica\DI; use Friendica\Module\Register; use Friendica\Network\CurlResult; -use Friendica\Util\Network; +use Friendica\Protocol\Diaspora; +use Friendica\Protocol\PortableContact; use Friendica\Util\DateTimeFormat; +use Friendica\Util\Network; use Friendica\Util\Strings; use Friendica\Util\XML; -use Friendica\Core\Logger; -use Friendica\Protocol\PortableContact; -use Friendica\Protocol\Diaspora; -use Friendica\Network\Probe; /** * This class handles GServer related functions @@ -31,6 +47,57 @@ class GServer const DT_NONE = 0; const DT_POCO = 1; const DT_MASTODON = 2; + + // Methods to detect server types + + // Non endpoint specific methods + const DETECT_MANUAL = 0; + const DETECT_HEADER = 1; + const DETECT_BODY = 2; + + // Implementation specific endpoints + const DETECT_FRIENDIKA = 10; + const DETECT_FRIENDICA = 11; + const DETECT_STATUSNET = 12; + const DETECT_GNUSOCIAL = 13; + const DETECT_CONFIG_JSON = 14; // Statusnet, GNU Social, Older Hubzilla/Redmatrix + const DETECT_SITEINFO_JSON = 15; // Newer Hubzilla + const DETECT_MASTODON_API = 16; + const DETECT_STATUS_PHP = 17; // Nextcloud + + // Standardized endpoints + const DETECT_STATISTICS_JSON = 100; + const DETECT_NODEINFO_1 = 101; + const DETECT_NODEINFO_2 = 102; + + /** + * Get the ID for the given server URL + * + * @param string $url + * @param boolean $no_check Don't check if the server hadn't been found + * @return int gserver id + */ + public static function getID(string $url, bool $no_check = false) + { + if (empty($url)) { + return null; + } + + $url = self::cleanURL($url); + + $gserver = DBA::selectFirst('gserver', ['id'], ['nurl' => Strings::normaliseLink($url)]); + if (DBA::isResult($gserver)) { + Logger::info('Got ID for URL', ['id' => $gserver['id'], 'url' => $url, 'callstack' => System::callstack(20)]); + return $gserver['id']; + } + + if ($no_check || !self::check($url)) { + return null; + } + + return self::getID($url, true); + } + /** * Checks if the given server is reachable * @@ -44,7 +111,7 @@ class GServer public static function reachable(string $profile, string $server = '', string $network = '', bool $force = false) { if ($server == '') { - $server = Contact::getBasepath($profile); + $server = GContact::getBasepath($profile); } if ($server == '') { @@ -112,17 +179,16 @@ class GServer /** * Checks the state of the given server. * - * @param string $server_url URL of the given server - * @param string $network Network value that is used, when detection failed - * @param boolean $force Force an update. + * @param string $server_url URL of the given server + * @param string $network Network value that is used, when detection failed + * @param boolean $force Force an update. + * @param boolean $only_nodeinfo Only use nodeinfo for server detection * * @return boolean 'true' if server seems vital */ - public static function check(string $server_url, string $network = '', bool $force = false) + public static function check(string $server_url, string $network = '', bool $force = false, bool $only_nodeinfo = false) { - // Unify the server address - $server_url = trim($server_url, '/'); - $server_url = str_replace('/index.php', '', $server_url); + $server_url = self::cleanURL($server_url); if ($server_url == '') { return false; @@ -158,47 +224,103 @@ class GServer Logger::info('Server is unknown. Start discovery.', ['Server' => $server_url]); } - return self::detect($server_url, $network); + return self::detect($server_url, $network, $only_nodeinfo); + } + + /** + * Set failed server status + * + * @param string $url + */ + private static function setFailure(string $url) + { + if (DBA::exists('gserver', ['nurl' => Strings::normaliseLink($url)])) { + DBA::update('gserver', ['failed' => true, 'last_failure' => DateTimeFormat::utcNow(), 'detection-method' => null], + ['nurl' => Strings::normaliseLink($url)]); + Logger::info('Set failed status for existing server', ['url' => $url]); + return; + } + DBA::insert('gserver', ['url' => $url, 'nurl' => Strings::normaliseLink($url), + 'network' => Protocol::PHANTOM, 'created' => DateTimeFormat::utcNow(), + 'failed' => true, 'last_failure' => DateTimeFormat::utcNow()]); + Logger::info('Set failed status for new server', ['url' => $url]); + } + + /** + * Remove unwanted content from the given URL + * + * @param string $url + * @return string cleaned URL + */ + public static function cleanURL(string $url) + { + $url = trim($url, '/'); + $url = str_replace('/index.php', '', $url); + + $urlparts = parse_url($url); + unset($urlparts['user']); + unset($urlparts['pass']); + unset($urlparts['query']); + unset($urlparts['fragment']); + return Network::unparseURL($urlparts); + } + + /** + * Return the base URL + * + * @param string $url + * @return string base URL + */ + private static function getBaseURL(string $url) + { + $urlparts = parse_url(self::cleanURL($url)); + unset($urlparts['path']); + return Network::unparseURL($urlparts); } /** * Detect server data (type, protocol, version number, ...) * The detected data is then updated or inserted in the gserver table. * - * @param string $url URL of the given server - * @param string $network Network value that is used, when detection failed + * @param string $url URL of the given server + * @param string $network Network value that is used, when detection failed + * @param boolean $only_nodeinfo Only use nodeinfo for server detection * * @return boolean 'true' if server could be detected */ - public static function detect(string $url, string $network = '') + public static function detect(string $url, string $network = '', bool $only_nodeinfo = false) { Logger::info('Detect server type', ['server' => $url]); - $serverdata = []; + $serverdata = ['detection-method' => self::DETECT_MANUAL]; $original_url = $url; // Remove URL content that is not supposed to exist for a server url - $urlparts = parse_url($url); - unset($urlparts['user']); - unset($urlparts['pass']); - unset($urlparts['query']); - unset($urlparts['fragment']); - $url = Network::unparseURL($urlparts); + $url = self::cleanURL($url); + + // Get base URL + $baseurl = self::getBaseURL($url); // If the URL missmatches, then we mark the old entry as failure if ($url != $original_url) { - DBA::update('gserver', ['last_failure' => DateTimeFormat::utcNow()], ['nurl' => Strings::normaliseLink($original_url)]); + DBA::update('gserver', ['failed' => true, 'last_failure' => DateTimeFormat::utcNow()], + ['nurl' => Strings::normaliseLink($original_url)]); } // When a nodeinfo is present, we don't need to dig further - $xrd_timeout = Config::get('system', 'xrd_timeout'); - $curlResult = Network::curl($url . '/.well-known/nodeinfo', false, ['timeout' => $xrd_timeout]); + $xrd_timeout = DI::config()->get('system', 'xrd_timeout'); + $curlResult = DI::httpRequest()->get($url . '/.well-known/nodeinfo', false, ['timeout' => $xrd_timeout]); if ($curlResult->isTimeout()) { - DBA::update('gserver', ['last_failure' => DateTimeFormat::utcNow()], ['nurl' => Strings::normaliseLink($url)]); + self::setFailure($url); return false; } $nodeinfo = self::fetchNodeinfo($url, $curlResult); + if ($only_nodeinfo && empty($nodeinfo)) { + Logger::info('Invalid nodeinfo in nodeinfo-mode, server is marked as failure', ['url' => $url]); + self::setFailure($url); + return false; + } // When nodeinfo isn't present, we use the older 'statistics.json' endpoint if (empty($nodeinfo)) { @@ -208,18 +330,53 @@ class GServer // If that didn't work out well, we use some protocol specific endpoints // For Friendica and Zot based networks we have to dive deeper to reveal more details if (empty($nodeinfo['network']) || in_array($nodeinfo['network'], [Protocol::DFRN, Protocol::ZOT])) { + if (!empty($nodeinfo['detection-method'])) { + $serverdata['detection-method'] = $nodeinfo['detection-method']; + } + // Fetch the landing page, possibly it reveals some data if (empty($nodeinfo['network'])) { - $curlResult = Network::curl($url, false, ['timeout' => $xrd_timeout]); + if ($baseurl == $url) { + $basedata = $serverdata; + } else { + $basedata = ['detection-method' => self::DETECT_MANUAL]; + } + + $curlResult = DI::httpRequest()->get($baseurl, false, ['timeout' => $xrd_timeout]); if ($curlResult->isSuccess()) { - $serverdata = self::analyseRootHeader($curlResult, $serverdata); - $serverdata = self::analyseRootBody($curlResult, $serverdata, $url); + $basedata = self::analyseRootHeader($curlResult, $basedata); + $basedata = self::analyseRootBody($curlResult, $basedata, $baseurl); } - if (!$curlResult->isSuccess() || empty($curlResult->getBody())) { - DBA::update('gserver', ['last_failure' => DateTimeFormat::utcNow()], ['nurl' => Strings::normaliseLink($url)]); + if (!$curlResult->isSuccess() || empty($curlResult->getBody()) || self::invalidBody($curlResult->getBody())) { + self::setFailure($url); return false; } + + if ($baseurl == $url) { + $serverdata = $basedata; + } else { + // When the base path doesn't seem to contain a social network we try the complete path. + // Most detectable system have to be installed in the root directory. + // We checked the base to avoid false positives. + $curlResult = DI::httpRequest()->get($url, false, ['timeout' => $xrd_timeout]); + if ($curlResult->isSuccess()) { + $urldata = self::analyseRootHeader($curlResult, $serverdata); + $urldata = self::analyseRootBody($curlResult, $urldata, $url); + + $comparebase = $basedata; + unset($comparebase['info']); + unset($comparebase['site_name']); + $compareurl = $urldata; + unset($compareurl['info']); + unset($compareurl['site_name']); + + // We assume that no one will install the identical system in the root and a subfolder + if (!empty(array_diff($comparebase, $compareurl))) { + $serverdata = $urldata; + } + } + } } if (empty($serverdata['network']) || ($serverdata['network'] == Protocol::ACTIVITYPUB)) { @@ -230,7 +387,7 @@ class GServer // With this check we don't have to waste time and ressources for dead systems. // Also this hopefully prevents us from receiving abuse messages. if (empty($serverdata['network']) && !self::validHostMeta($url)) { - DBA::update('gserver', ['last_failure' => DateTimeFormat::utcNow()], ['nurl' => Strings::normaliseLink($url)]); + self::setFailure($url); return false; } @@ -255,6 +412,8 @@ class GServer if (empty($serverdata['network'])) { $serverdata = self::detectGNUSocial($url, $serverdata); } + + $serverdata = array_merge($nodeinfo, $serverdata); } else { $serverdata = $nodeinfo; } @@ -285,22 +444,19 @@ class GServer $registeredUsers = 1; } - if ($serverdata['network'] != Protocol::PHANTOM) { - $gcontacts = DBA::count('gcontact', ['server_url' => [$url, $serverdata['nurl']]]); - $apcontacts = DBA::count('apcontact', ['baseurl' => [$url, $serverdata['nurl']]]); - $contacts = DBA::count('contact', ['uid' => 0, 'baseurl' => [$url, $serverdata['nurl']]]); - $serverdata['registered-users'] = max($gcontacts, $apcontacts, $contacts, $registeredUsers); - } else { + if ($serverdata['network'] == Protocol::PHANTOM) { $serverdata['registered-users'] = $registeredUsers; $serverdata = self::detectNetworkViaContacts($url, $serverdata); } $serverdata['last_contact'] = DateTimeFormat::utcNow(); + $serverdata['failed'] = false; $gserver = DBA::selectFirst('gserver', ['network'], ['nurl' => Strings::normaliseLink($url)]); if (!DBA::isResult($gserver)) { $serverdata['created'] = DateTimeFormat::utcNow(); $ret = DBA::insert('gserver', $serverdata); + $id = DBA::lastInsertId(); } else { // Don't override the network with 'unknown' when there had been a valid entry before if (($serverdata['network'] == Protocol::PHANTOM) && !empty($gserver['network'])) { @@ -308,11 +464,26 @@ class GServer } $ret = DBA::update('gserver', $serverdata, ['nurl' => $serverdata['nurl']]); + $gserver = DBA::selectFirst('gserver', ['id'], ['nurl' => $serverdata['nurl']]); + if (DBA::isResult($gserver)) { + $id = $gserver['id']; + } + } + + if (!empty($serverdata['network']) && !empty($id) && ($serverdata['network'] != Protocol::PHANTOM)) { + $gcontacts = DBA::count('gcontact', ['gsid' => $id]); + $apcontacts = DBA::count('apcontact', ['gsid' => $id]); + $contacts = DBA::count('contact', ['uid' => 0, 'gsid' => $id]); + $max_users = max($gcontacts, $apcontacts, $contacts, $registeredUsers); + if ($max_users > $registeredUsers) { + Logger::info('Update registered users', ['id' => $id, 'url' => $serverdata['nurl'], 'registered-users' => $max_users]); + DBA::update('gserver', ['registered-users' => $max_users], ['id' => $id]); + } } if (!empty($serverdata['network']) && in_array($serverdata['network'], [Protocol::DFRN, Protocol::DIASPORA])) { - self::discoverRelay($url); - } + self::discoverRelay($url); + } return $ret; } @@ -327,7 +498,7 @@ class GServer { Logger::info('Discover relay data', ['server' => $server_url]); - $curlResult = Network::curl($server_url . '/.well-known/x-social-relay'); + $curlResult = DI::httpRequest()->get($server_url . '/.well-known/x-social-relay'); if (!$curlResult->isSuccess()) { return; } @@ -337,6 +508,15 @@ class GServer return; } + // Sanitize incoming data, see https://github.com/friendica/friendica/issues/8565 + $data['subscribe'] = (bool)$data['subscribe'] ?? false; + + if (!$data['subscribe'] || empty($data['scope']) || !in_array(strtolower($data['scope']), ['all', 'tags'])) { + $data['scope'] = ''; + $data['subscribe'] = false; + $data['tags'] = []; + } + $gserver = DBA::selectFirst('gserver', ['id', 'relay-subscribe', 'relay-scope'], ['nurl' => Strings::normaliseLink($server_url)]); if (!DBA::isResult($gserver)) { return; @@ -399,7 +579,7 @@ class GServer */ private static function fetchStatistics(string $url) { - $curlResult = Network::curl($url . '/statistics.json'); + $curlResult = DI::httpRequest()->get($url . '/statistics.json'); if (!$curlResult->isSuccess()) { return []; } @@ -409,7 +589,7 @@ class GServer return []; } - $serverdata = []; + $serverdata = ['detection-method' => self::DETECT_STATISTICS_JSON]; if (!empty($data['version'])) { $serverdata['version'] = $data['version']; @@ -423,11 +603,11 @@ class GServer } if (!empty($data['network'])) { - $serverdata['platform'] = $data['network']; + $serverdata['platform'] = strtolower($data['network']); - if ($serverdata['platform'] == 'Diaspora') { + if ($serverdata['platform'] == 'diaspora') { $serverdata['network'] = Protocol::DIASPORA; - } elseif ($serverdata['platform'] == 'Friendica') { + } elseif ($serverdata['platform'] == 'friendica') { $serverdata['network'] = Protocol::DFRN; } elseif ($serverdata['platform'] == 'hubzilla') { $serverdata['network'] = Protocol::ZOT; @@ -456,6 +636,10 @@ class GServer */ private static function fetchNodeinfo(string $url, CurlResult $curlResult) { + if (!$curlResult->isSuccess()) { + return []; + } + $nodeinfo = json_decode($curlResult->getBody(), true); if (!is_array($nodeinfo) || empty($nodeinfo['links'])) { @@ -505,7 +689,7 @@ class GServer */ private static function parseNodeinfo1(string $nodeinfo_url) { - $curlResult = Network::curl($nodeinfo_url); + $curlResult = DI::httpRequest()->get($nodeinfo_url); if (!$curlResult->isSuccess()) { return []; @@ -517,9 +701,8 @@ class GServer return []; } - $server = []; - - $server['register_policy'] = Register::CLOSED; + $server = ['detection-method' => self::DETECT_NODEINFO_1, + 'register_policy' => Register::CLOSED]; if (!empty($nodeinfo['openRegistrations'])) { $server['register_policy'] = Register::OPEN; @@ -527,7 +710,7 @@ class GServer if (is_array($nodeinfo['software'])) { if (!empty($nodeinfo['software']['name'])) { - $server['platform'] = $nodeinfo['software']['name']; + $server['platform'] = strtolower($nodeinfo['software']['name']); } if (!empty($nodeinfo['software']['version'])) { @@ -583,7 +766,7 @@ class GServer */ private static function parseNodeinfo2(string $nodeinfo_url) { - $curlResult = Network::curl($nodeinfo_url); + $curlResult = DI::httpRequest()->get($nodeinfo_url); if (!$curlResult->isSuccess()) { return []; } @@ -594,9 +777,8 @@ class GServer return []; } - $server = []; - - $server['register_policy'] = Register::CLOSED; + $server = ['detection-method' => self::DETECT_NODEINFO_2, + 'register_policy' => Register::CLOSED]; if (!empty($nodeinfo['openRegistrations'])) { $server['register_policy'] = Register::OPEN; @@ -604,7 +786,7 @@ class GServer if (is_array($nodeinfo['software'])) { if (!empty($nodeinfo['software']['name'])) { - $server['platform'] = $nodeinfo['software']['name']; + $server['platform'] = strtolower($nodeinfo['software']['name']); } if (!empty($nodeinfo['software']['version'])) { @@ -661,7 +843,7 @@ class GServer */ private static function fetchSiteinfo(string $url, array $serverdata) { - $curlResult = Network::curl($url . '/siteinfo.json'); + $curlResult = DI::httpRequest()->get($url . '/siteinfo.json'); if (!$curlResult->isSuccess()) { return $serverdata; } @@ -671,8 +853,12 @@ class GServer return $serverdata; } + if (in_array($serverdata['detection-method'], [self::DETECT_HEADER, self::DETECT_BODY, self::DETECT_MANUAL])) { + $serverdata['detection-method'] = self::DETECT_SITEINFO_JSON; + } + if (!empty($data['url'])) { - $serverdata['platform'] = $data['platform']; + $serverdata['platform'] = strtolower($data['platform']); $serverdata['version'] = $data['version']; } @@ -725,13 +911,13 @@ class GServer */ private static function validHostMeta(string $url) { - $xrd_timeout = Config::get('system', 'xrd_timeout'); - $curlResult = Network::curl($url . '/.well-known/host-meta', false, ['timeout' => $xrd_timeout]); + $xrd_timeout = DI::config()->get('system', 'xrd_timeout'); + $curlResult = DI::httpRequest()->get($url . '/.well-known/host-meta', false, ['timeout' => $xrd_timeout]); if (!$curlResult->isSuccess()) { return false; } - $xrd = XML::parseString($curlResult->getBody(), false); + $xrd = XML::parseString($curlResult->getBody()); if (!is_object($xrd)) { return false; } @@ -780,13 +966,13 @@ class GServer DBA::close($gcontacts); $apcontacts = DBA::select('apcontact', ['url'], ['baseurl' => [$url, $serverdata['nurl']]]); - while ($gcontact = DBA::fetch($gcontacts)) { + while ($apcontact = DBA::fetch($apcontacts)) { $contacts[Strings::normaliseLink($apcontact['url'])] = $apcontact['url']; } DBA::close($apcontacts); $pcontacts = DBA::select('contact', ['url', 'nurl'], ['uid' => 0, 'baseurl' => [$url, $serverdata['nurl']]]); - while ($gcontact = DBA::fetch($gcontacts)) { + while ($pcontact = DBA::fetch($pcontacts)) { $contacts[$pcontact['nurl']] = $pcontact['url']; } DBA::close($pcontacts); @@ -796,8 +982,8 @@ class GServer } foreach ($contacts as $contact) { - $probed = Probe::uri($contact); - if (in_array($probed['network'], Protocol::FEDERATED)) { + $probed = Contact::getByURL($contact); + if (!empty($probed) && in_array($probed['network'], Protocol::FEDERATED)) { $serverdata['network'] = $probed['network']; break; } @@ -822,7 +1008,7 @@ class GServer { $serverdata['poco'] = ''; - $curlResult = Network::curl($url. '/poco'); + $curlResult = DI::httpRequest()->get($url . '/poco'); if (!$curlResult->isSuccess()) { return $serverdata; } @@ -852,7 +1038,7 @@ class GServer */ public static function checkMastodonDirectory(string $url, array $serverdata) { - $curlResult = Network::curl($url . '/api/v1/directory?limit=1'); + $curlResult = DI::httpRequest()->get($url . '/api/v1/directory?limit=1'); if (!$curlResult->isSuccess()) { return $serverdata; } @@ -879,7 +1065,7 @@ class GServer */ private static function detectNextcloud(string $url, array $serverdata) { - $curlResult = Network::curl($url . '/status.php'); + $curlResult = DI::httpRequest()->get($url . '/status.php'); if (!$curlResult->isSuccess() || ($curlResult->getBody() == '')) { return $serverdata; @@ -894,6 +1080,10 @@ class GServer $serverdata['platform'] = 'nextcloud'; $serverdata['version'] = $data['version']; $serverdata['network'] = Protocol::ACTIVITYPUB; + + if (in_array($serverdata['detection-method'], [self::DETECT_HEADER, self::DETECT_BODY, self::DETECT_MANUAL])) { + $serverdata['detection-method'] = self::DETECT_STATUS_PHP; + } } return $serverdata; @@ -909,7 +1099,7 @@ class GServer */ private static function detectMastodonAlikes(string $url, array $serverdata) { - $curlResult = Network::curl($url . '/api/v1/instance'); + $curlResult = DI::httpRequest()->get($url . '/api/v1/instance'); if (!$curlResult->isSuccess() || ($curlResult->getBody() == '')) { return $serverdata; @@ -920,6 +1110,10 @@ class GServer return $serverdata; } + if (in_array($serverdata['detection-method'], [self::DETECT_HEADER, self::DETECT_BODY, self::DETECT_MANUAL])) { + $serverdata['detection-method'] = self::DETECT_MASTODON_API; + } + if (!empty($data['version'])) { $serverdata['platform'] = 'mastodon'; $serverdata['version'] = $data['version'] ?? ''; @@ -930,6 +1124,11 @@ class GServer $serverdata['site_name'] = $data['title']; } + if (!empty($data['title']) && empty($serverdata['platform']) && empty($serverdata['network'])) { + $serverdata['platform'] = 'mastodon'; + $serverdata['network'] = Protocol::ACTIVITYPUB; + } + if (!empty($data['description'])) { $serverdata['info'] = trim($data['description']); } @@ -939,13 +1138,18 @@ class GServer } if (!empty($serverdata['version']) && preg_match('/.*?\(compatible;\s(.*)\s(.*)\)/ism', $serverdata['version'], $matches)) { - $serverdata['platform'] = $matches[1]; + $serverdata['platform'] = strtolower($matches[1]); $serverdata['version'] = $matches[2]; } - if (!empty($serverdata['version']) && strstr($serverdata['version'], 'Pleroma')) { + if (!empty($serverdata['version']) && strstr(strtolower($serverdata['version']), 'pleroma')) { + $serverdata['platform'] = 'pleroma'; + $serverdata['version'] = trim(str_ireplace('pleroma', '', $serverdata['version'])); + } + + if (!empty($serverdata['platform']) && strstr($serverdata['platform'], 'pleroma')) { + $serverdata['version'] = trim(str_ireplace('pleroma', '', $serverdata['platform'])); $serverdata['platform'] = 'pleroma'; - $serverdata['version'] = trim(str_replace('Pleroma', '', $serverdata['version'])); } return $serverdata; @@ -961,13 +1165,13 @@ class GServer */ private static function detectHubzilla(string $url, array $serverdata) { - $curlResult = Network::curl($url . '/api/statusnet/config.json'); + $curlResult = DI::httpRequest()->get($url . '/api/statusnet/config.json'); if (!$curlResult->isSuccess() || ($curlResult->getBody() == '')) { return $serverdata; } $data = json_decode($curlResult->getBody(), true); - if (empty($data)) { + if (empty($data) || empty($data['site'])) { return $serverdata; } @@ -976,22 +1180,22 @@ class GServer } if (!empty($data['site']['platform'])) { - $serverdata['platform'] = $data['site']['platform']['PLATFORM_NAME']; + $serverdata['platform'] = strtolower($data['site']['platform']['PLATFORM_NAME']); $serverdata['version'] = $data['site']['platform']['STD_VERSION']; $serverdata['network'] = Protocol::ZOT; } if (!empty($data['site']['hubzilla'])) { - $serverdata['platform'] = $data['site']['hubzilla']['PLATFORM_NAME']; + $serverdata['platform'] = strtolower($data['site']['hubzilla']['PLATFORM_NAME']); $serverdata['version'] = $data['site']['hubzilla']['RED_VERSION']; $serverdata['network'] = Protocol::ZOT; } if (!empty($data['site']['redmatrix'])) { if (!empty($data['site']['redmatrix']['PLATFORM_NAME'])) { - $serverdata['platform'] = $data['site']['redmatrix']['PLATFORM_NAME']; + $serverdata['platform'] = strtolower($data['site']['redmatrix']['PLATFORM_NAME']); } elseif (!empty($data['site']['redmatrix']['RED_PLATFORM'])) { - $serverdata['platform'] = $data['site']['redmatrix']['RED_PLATFORM']; + $serverdata['platform'] = strtolower($data['site']['redmatrix']['RED_PLATFORM']); } $serverdata['version'] = $data['site']['redmatrix']['RED_VERSION']; @@ -1015,11 +1219,16 @@ class GServer } if (!$closed && !$private and $inviteonly) { - $register_policy = Register::APPROVE; + $serverdata['register_policy'] = Register::APPROVE; } elseif (!$closed && !$private) { - $register_policy = Register::OPEN; + $serverdata['register_policy'] = Register::OPEN; } else { - $register_policy = Register::CLOSED; + $serverdata['register_policy'] = Register::CLOSED; + } + + if (!empty($serverdata['network']) && in_array($serverdata['detection-method'], + [self::DETECT_HEADER, self::DETECT_BODY, self::DETECT_MANUAL])) { + $serverdata['detection-method'] = self::DETECT_CONFIG_JSON; } return $serverdata; @@ -1054,26 +1263,45 @@ class GServer private static function detectGNUSocial(string $url, array $serverdata) { // Test for GNU Social - $curlResult = Network::curl($url . '/api/gnusocial/version.json'); + $curlResult = DI::httpRequest()->get($url . '/api/gnusocial/version.json'); if ($curlResult->isSuccess() && ($curlResult->getBody() != '{"error":"not implemented"}') && ($curlResult->getBody() != '') && (strlen($curlResult->getBody()) < 30)) { $serverdata['platform'] = 'gnusocial'; // Remove junk that some GNU Social servers return $serverdata['version'] = str_replace(chr(239) . chr(187) . chr(191), '', $curlResult->getBody()); + $serverdata['version'] = str_replace(["\r", "\n", "\t"], '', $serverdata['version']); $serverdata['version'] = trim($serverdata['version'], '"'); $serverdata['network'] = Protocol::OSTATUS; + + if (in_array($serverdata['detection-method'], [self::DETECT_HEADER, self::DETECT_BODY, self::DETECT_MANUAL])) { + $serverdata['detection-method'] = self::DETECT_GNUSOCIAL; + } + return $serverdata; } // Test for Statusnet - $curlResult = Network::curl($url . '/api/statusnet/version.json'); + $curlResult = DI::httpRequest()->get($url . '/api/statusnet/version.json'); if ($curlResult->isSuccess() && ($curlResult->getBody() != '{"error":"not implemented"}') && ($curlResult->getBody() != '') && (strlen($curlResult->getBody()) < 30)) { - $serverdata['platform'] = 'statusnet'; + // Remove junk that some GNU Social servers return $serverdata['version'] = str_replace(chr(239).chr(187).chr(191), '', $curlResult->getBody()); + $serverdata['version'] = str_replace(["\r", "\n", "\t"], '', $serverdata['version']); $serverdata['version'] = trim($serverdata['version'], '"'); - $serverdata['network'] = Protocol::OSTATUS; + + if (!empty($serverdata['version']) && strtolower(substr($serverdata['version'], 0, 7)) == 'pleroma') { + $serverdata['platform'] = 'pleroma'; + $serverdata['version'] = trim(str_ireplace('pleroma', '', $serverdata['version'])); + $serverdata['network'] = Protocol::ACTIVITYPUB; + } else { + $serverdata['platform'] = 'statusnet'; + $serverdata['network'] = Protocol::OSTATUS; + } + + if (in_array($serverdata['detection-method'], [self::DETECT_HEADER, self::DETECT_BODY, self::DETECT_MANUAL])) { + $serverdata['detection-method'] = self::DETECT_STATUSNET; + } } return $serverdata; @@ -1089,9 +1317,14 @@ class GServer */ private static function detectFriendica(string $url, array $serverdata) { - $curlResult = Network::curl($url . '/friendica/json'); + $curlResult = DI::httpRequest()->get($url . '/friendica/json'); if (!$curlResult->isSuccess()) { - $curlResult = Network::curl($url . '/friendika/json'); + $curlResult = DI::httpRequest()->get($url . '/friendika/json'); + $friendika = true; + $platform = 'Friendika'; + } else { + $friendika = false; + $platform = 'Friendica'; } if (!$curlResult->isSuccess()) { @@ -1103,6 +1336,10 @@ class GServer return $serverdata; } + if (in_array($serverdata['detection-method'], [self::DETECT_HEADER, self::DETECT_BODY, self::DETECT_MANUAL])) { + $serverdata['detection-method'] = $friendika ? self::DETECT_FRIENDIKA : self::DETECT_FRIENDICA; + } + $serverdata['network'] = Protocol::DFRN; $serverdata['version'] = $data['version']; @@ -1138,7 +1375,7 @@ class GServer break; } - $serverdata['platform'] = $data['platform'] ?? ''; + $serverdata['platform'] = strtolower($data['platform'] ?? $platform); return $serverdata; } @@ -1186,21 +1423,20 @@ class GServer $serverdata['info'] = $attr['content']; } - if ($attr['name'] == 'application-name') { - $serverdata['platform'] = $attr['content']; + if (in_array($attr['name'], ['application-name', 'al:android:app_name', 'al:ios:app_name', + 'twitter:app:name:googleplay', 'twitter:app:name:iphone', 'twitter:app:name:ipad'])) { + $serverdata['platform'] = strtolower($attr['content']); if (in_array($attr['content'], ['Misskey', 'Write.as'])) { $serverdata['network'] = Protocol::ACTIVITYPUB; } } - - if ($attr['name'] == 'generator') { - $serverdata['platform'] = $attr['content']; - + if (($attr['name'] == 'generator') && (empty($serverdata['platform']) || (substr(strtolower($attr['content']), 0, 9) == 'wordpress'))) { + $serverdata['platform'] = strtolower($attr['content']); $version_part = explode(' ', $attr['content']); if (count($version_part) == 2) { if (in_array($version_part[0], ['WordPress'])) { - $serverdata['platform'] = $version_part[0]; + $serverdata['platform'] = strtolower($version_part[0]); $serverdata['version'] = $version_part[1]; // We still do need a reliable test if some AP plugin is activated @@ -1209,9 +1445,13 @@ class GServer } else { $serverdata['network'] = Protocol::FEED; } + + if ($serverdata['detection-method'] == self::DETECT_MANUAL) { + $serverdata['detection-method'] = self::DETECT_BODY; + } } if (in_array($version_part[0], ['Friendika', 'Friendica'])) { - $serverdata['platform'] = $version_part[0]; + $serverdata['platform'] = strtolower($version_part[0]); $serverdata['version'] = $version_part[1]; $serverdata['network'] = Protocol::DFRN; } @@ -1247,7 +1487,7 @@ class GServer } if ($attr['property'] == 'og:platform') { - $serverdata['platform'] = $attr['content']; + $serverdata['platform'] = strtolower($attr['content']); if (in_array($attr['content'], ['PeerTube'])) { $serverdata['network'] = Protocol::ACTIVITYPUB; @@ -1255,7 +1495,7 @@ class GServer } if ($attr['property'] == 'generator') { - $serverdata['platform'] = $attr['content']; + $serverdata['platform'] = strtolower($attr['content']); if (in_array($attr['content'], ['hubzilla'])) { // We later check which compatible protocol modules are loaded. @@ -1264,6 +1504,10 @@ class GServer } } + if (!empty($serverdata['network']) && ($serverdata['detection-method'] == self::DETECT_MANUAL)) { + $serverdata['detection-method'] = self::DETECT_BODY; + } + return $serverdata; } @@ -1279,20 +1523,39 @@ class GServer { if ($curlResult->getHeader('server') == 'Mastodon') { $serverdata['platform'] = 'mastodon'; - $serverdata['network'] = $network = Protocol::ACTIVITYPUB; + $serverdata['network'] = Protocol::ACTIVITYPUB; } elseif ($curlResult->inHeader('x-diaspora-version')) { $serverdata['platform'] = 'diaspora'; - $serverdata['network'] = $network = Protocol::DIASPORA; + $serverdata['network'] = Protocol::DIASPORA; $serverdata['version'] = $curlResult->getHeader('x-diaspora-version'); - } elseif ($curlResult->inHeader('x-friendica-version')) { $serverdata['platform'] = 'friendica'; - $serverdata['network'] = $network = Protocol::DFRN; + $serverdata['network'] = Protocol::DFRN; $serverdata['version'] = $curlResult->getHeader('x-friendica-version'); + } else { + return $serverdata; } + + if ($serverdata['detection-method'] == self::DETECT_MANUAL) { + $serverdata['detection-method'] = self::DETECT_HEADER; + } + return $serverdata; } + /** + * Test if the body contains valid content + * + * @param string $body + * @return boolean + */ + private static function invalidBody(string $body) + { + // Currently we only test for a HTML element. + // Possibly we enhance this in the future. + return !strpos($body, '>'); + } + /** * Update the user directory of a given gserver record * @@ -1306,4 +1569,101 @@ class GServer PortableContact::discoverSingleServer($gserver['id']); } } + + /** + * Update GServer entries + */ + public static function discover() + { + // Update the server list + self::discoverFederation(); + + $no_of_queries = 5; + + $requery_days = intval(DI::config()->get('system', 'poco_requery_days')); + + if ($requery_days == 0) { + $requery_days = 7; + } + + $last_update = date('c', time() - (60 * 60 * 24 * $requery_days)); + + $gservers = DBA::p("SELECT `id`, `url`, `nurl`, `network`, `poco` + FROM `gserver` + WHERE NOT `failed` + AND `poco` != '' + AND `last_poco_query` < ? + ORDER BY RAND()", $last_update + ); + + while ($gserver = DBA::fetch($gservers)) { + if (!GServer::check($gserver['url'], $gserver['network'])) { + // The server is not reachable? Okay, then we will try it later + $fields = ['last_poco_query' => DateTimeFormat::utcNow()]; + DBA::update('gserver', $fields, ['nurl' => $gserver['nurl']]); + continue; + } + + Logger::info('Update directory', ['server' => $gserver['url'], 'id' => $gserver['id']]); + Worker::add(PRIORITY_LOW, 'UpdateServerDirectory', $gserver); + + if (--$no_of_queries == 0) { + break; + } + } + + DBA::close($gservers); + } + + /** + * Discover federated servers + */ + private static function discoverFederation() + { + $last = DI::config()->get('poco', 'last_federation_discovery'); + + if ($last) { + $next = $last + (24 * 60 * 60); + + if ($next > time()) { + return; + } + } + + // Discover federated servers + $protocols = ['activitypub', 'diaspora', 'dfrn', 'ostatus']; + foreach ($protocols as $protocol) { + $query = '{nodes(protocol:"' . $protocol . '"){host}}'; + $curlResult = DI::httpRequest()->fetch('https://the-federation.info/graphql?query=' . urlencode($query)); + if (!empty($curlResult)) { + $data = json_decode($curlResult, true); + if (!empty($data['data']['nodes'])) { + foreach ($data['data']['nodes'] as $server) { + // Using "only_nodeinfo" since servers that are listed on that page should always have it. + Worker::add(PRIORITY_LOW, 'UpdateGServer', 'https://' . $server['host'], true); + } + } + } + } + + // Disvover Mastodon servers + $accesstoken = DI::config()->get('system', 'instances_social_key'); + + if (!empty($accesstoken)) { + $api = 'https://instances.social/api/1.0/instances/list?count=0'; + $header = ['Authorization: Bearer '.$accesstoken]; + $curlResult = DI::httpRequest()->get($api, false, ['headers' => $header]); + + if ($curlResult->isSuccess()) { + $servers = json_decode($curlResult->getBody(), true); + + foreach ($servers['instances'] as $server) { + $url = (is_null($server['https_score']) ? 'http' : 'https') . '://' . $server['name']; + Worker::add(PRIORITY_LOW, 'UpdateGServer', $url); + } + } + } + + DI::config()->set('poco', 'last_federation_discovery', time()); + } }