X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;f=src%2FModel%2FGServer.php;h=af03041498d56cdb2ecaa36bf9afc3e5ced9a880;hb=2e05dac7dae0a3d028b442a2d5afbd4176a32e99;hp=0b79b3f8fdc1327d11fdd610a4c457f64d8ddd70;hpb=ce52d31afa9cb9f925cac560a04e5999f745498a;p=friendica.git diff --git a/src/Model/GServer.php b/src/Model/GServer.php index 0b79b3f8fd..af03041498 100644 --- a/src/Model/GServer.php +++ b/src/Model/GServer.php @@ -1,6 +1,6 @@ '\_', '%' => '\%', '?' => '_', '*' => '%']); + + // The SUBSTRING_INDEX returns everything before the eventual third /, which effectively trims an + // eventual server path and keep only the server domain which we're matching against the pattern. + $sql = "SELECT `gserver`.*, COUNT(*) AS `contacts` + FROM `gserver` + LEFT JOIN `contact` ON `gserver`.`id` = `contact`.`gsid` + WHERE SUBSTRING_INDEX(`gserver`.`nurl`, '/', 3) LIKE ? + AND NOT `gserver`.`failed` + GROUP BY `gserver`.`id`"; + + $stmt = DI::dba()->p($sql, $likePattern); + + return DI::dba()->toArray($stmt); + } + /** * Checks if the given server is reachable * @@ -169,7 +199,7 @@ class GServer if (($now - $contact_time) < (60 * 60 * 24)) { return DateTimeFormat::utc('now +1 day'); } - + // If the last contact was less than a week before then try again in a week if (($now - $contact_time) < (60 * 60 * 24 * 7)) { return DateTimeFormat::utc('now +1 week'); @@ -312,12 +342,23 @@ class GServer // When a nodeinfo is present, we don't need to dig further $xrd_timeout = DI::config()->get('system', 'xrd_timeout'); - $curlResult = DI::httpRequest()->get($url . '/.well-known/nodeinfo', ['timeout' => $xrd_timeout]); + $curlResult = DI::httpClient()->get($url . '/.well-known/nodeinfo', [HttpClientOptions::TIMEOUT => $xrd_timeout]); if ($curlResult->isTimeout()) { self::setFailure($url); return false; } + // On a redirect follow the new host but mark the old one as failure + if ($curlResult->isSuccess() && (parse_url($url, PHP_URL_HOST) != parse_url($curlResult->getRedirectUrl(), PHP_URL_HOST))) { + $curlResult = DI::httpClient()->get($url, [HttpClientOptions::TIMEOUT => $xrd_timeout]); + if (parse_url($url, PHP_URL_HOST) != parse_url($curlResult->getRedirectUrl(), PHP_URL_HOST)) { + Logger::info('Found redirect. Mark old entry as failure', ['old' => $url, 'new' => $curlResult->getRedirectUrl()]); + self::setFailure($url); + self::detect($curlResult->getRedirectUrl(), $network, $only_nodeinfo); + return false; + } + } + $nodeinfo = self::fetchNodeinfo($url, $curlResult); if ($only_nodeinfo && empty($nodeinfo)) { Logger::info('Invalid nodeinfo in nodeinfo-mode, server is marked as failure', ['url' => $url]); @@ -345,8 +386,15 @@ class GServer $basedata = ['detection-method' => self::DETECT_MANUAL]; } - $curlResult = DI::httpRequest()->get($baseurl, ['timeout' => $xrd_timeout]); + $curlResult = DI::httpClient()->get($baseurl, [HttpClientOptions::TIMEOUT => $xrd_timeout]); if ($curlResult->isSuccess()) { + if ((parse_url($baseurl, PHP_URL_HOST) != parse_url($curlResult->getRedirectUrl(), PHP_URL_HOST))) { + Logger::info('Found redirect. Mark old entry as failure', ['old' => $url, 'new' => $curlResult->getRedirectUrl()]); + self::setFailure($url); + self::detect($curlResult->getRedirectUrl(), $network, $only_nodeinfo); + return false; + } + $basedata = self::analyseRootHeader($curlResult, $basedata); $basedata = self::analyseRootBody($curlResult, $basedata, $baseurl); } @@ -362,7 +410,7 @@ class GServer // When the base path doesn't seem to contain a social network we try the complete path. // Most detectable system have to be installed in the root directory. // We checked the base to avoid false positives. - $curlResult = DI::httpRequest()->get($url, ['timeout' => $xrd_timeout]); + $curlResult = DI::httpClient()->get($url, [HttpClientOptions::TIMEOUT => $xrd_timeout]); if ($curlResult->isSuccess()) { $urldata = self::analyseRootHeader($curlResult, $serverdata); $urldata = self::analyseRootBody($curlResult, $urldata, $url); @@ -408,6 +456,10 @@ class GServer $serverdata = self::detectHubzilla($url, $serverdata); } + if (empty($serverdata['network']) || in_array($serverdata['detection-method'], [self::DETECT_MANUAL, self::DETECT_BODY])) { + $serverdata = self::detectPeertube($url, $serverdata); + } + if (empty($serverdata['network'])) { $serverdata = self::detectNextcloud($url, $serverdata); } @@ -448,7 +500,7 @@ class GServer } if ($serverdata['network'] == Protocol::PHANTOM) { - $serverdata['registered-users'] = $registeredUsers; + $serverdata['registered-users'] = max($registeredUsers, 1); $serverdata = self::detectNetworkViaContacts($url, $serverdata); } @@ -478,7 +530,7 @@ class GServer if (!empty($serverdata['network']) && !empty($id) && ($serverdata['network'] != Protocol::PHANTOM)) { $apcontacts = DBA::count('apcontact', ['gsid' => $id]); $contacts = DBA::count('contact', ['uid' => 0, 'gsid' => $id]); - $max_users = max($apcontacts, $contacts, $registeredUsers); + $max_users = max($apcontacts, $contacts, $registeredUsers, 1); if ($max_users > $registeredUsers) { Logger::info('Update registered users', ['id' => $id, 'url' => $serverdata['nurl'], 'registered-users' => $max_users]); DBA::update('gserver', ['registered-users' => $max_users], ['id' => $id]); @@ -502,7 +554,7 @@ class GServer { Logger::info('Discover relay data', ['server' => $server_url]); - $curlResult = DI::httpRequest()->get($server_url . '/.well-known/x-social-relay'); + $curlResult = DI::httpClient()->get($server_url . '/.well-known/x-social-relay'); if (!$curlResult->isSuccess()) { return; } @@ -597,7 +649,7 @@ class GServer */ private static function fetchStatistics(string $url) { - $curlResult = DI::httpRequest()->get($url . '/statistics.json'); + $curlResult = DI::httpClient()->get($url . '/statistics.json'); if (!$curlResult->isSuccess()) { return []; } @@ -647,18 +699,19 @@ class GServer /** * Detect server type by using the nodeinfo data * - * @param string $url address of the server - * @param CurlResult $curlResult + * @param string $url address of the server + * @param ICanHandleHttpResponses $httpResult + * * @return array Server data * @throws \Friendica\Network\HTTPException\InternalServerErrorException */ - private static function fetchNodeinfo(string $url, CurlResult $curlResult) + private static function fetchNodeinfo(string $url, ICanHandleHttpResponses $httpResult) { - if (!$curlResult->isSuccess()) { + if (!$httpResult->isSuccess()) { return []; } - $nodeinfo = json_decode($curlResult->getBody(), true); + $nodeinfo = json_decode($httpResult->getBody(), true); if (!is_array($nodeinfo) || empty($nodeinfo['links'])) { return []; @@ -707,7 +760,7 @@ class GServer */ private static function parseNodeinfo1(string $nodeinfo_url) { - $curlResult = DI::httpRequest()->get($nodeinfo_url); + $curlResult = DI::httpClient()->get($nodeinfo_url); if (!$curlResult->isSuccess()) { return []; @@ -744,7 +797,7 @@ class GServer } if (!empty($nodeinfo['usage']['users']['total'])) { - $server['registered-users'] = $nodeinfo['usage']['users']['total']; + $server['registered-users'] = max($nodeinfo['usage']['users']['total'], 1); } if (!empty($nodeinfo['protocols']['inbound']) && is_array($nodeinfo['protocols']['inbound'])) { @@ -778,13 +831,14 @@ class GServer /** * Parses Nodeinfo 2 * + * @see https://git.feneas.org/jaywink/nodeinfo2 * @param string $nodeinfo_url address of the nodeinfo path * @return array Server data * @throws \Friendica\Network\HTTPException\InternalServerErrorException */ private static function parseNodeinfo2(string $nodeinfo_url) { - $curlResult = DI::httpRequest()->get($nodeinfo_url); + $curlResult = DI::httpClient()->get($nodeinfo_url); if (!$curlResult->isSuccess()) { return []; } @@ -820,13 +874,15 @@ class GServer } if (!empty($nodeinfo['usage']['users']['total'])) { - $server['registered-users'] = $nodeinfo['usage']['users']['total']; + $server['registered-users'] = max($nodeinfo['usage']['users']['total'], 1); } if (!empty($nodeinfo['protocols'])) { $protocols = []; foreach ($nodeinfo['protocols'] as $protocol) { - $protocols[$protocol] = true; + if (is_string($protocol)) { + $protocols[$protocol] = true; + } } if (!empty($protocols['dfrn'])) { @@ -861,7 +917,7 @@ class GServer */ private static function fetchSiteinfo(string $url, array $serverdata) { - $curlResult = DI::httpRequest()->get($url . '/siteinfo.json'); + $curlResult = DI::httpClient()->get($url . '/siteinfo.json'); if (!$curlResult->isSuccess()) { return $serverdata; } @@ -897,7 +953,7 @@ class GServer } if (!empty($data['channels_total'])) { - $serverdata['registered-users'] = $data['channels_total']; + $serverdata['registered-users'] = max($data['channels_total'], 1); } if (!empty($data['register_policy'])) { @@ -930,7 +986,7 @@ class GServer private static function validHostMeta(string $url) { $xrd_timeout = DI::config()->get('system', 'xrd_timeout'); - $curlResult = DI::httpRequest()->get($url . '/.well-known/host-meta', ['timeout' => $xrd_timeout]); + $curlResult = DI::httpClient()->get($url . '/.well-known/host-meta', [HttpClientOptions::TIMEOUT => $xrd_timeout]); if (!$curlResult->isSuccess()) { return false; } @@ -1001,7 +1057,7 @@ class GServer } } - $serverdata['registered-users'] = max($serverdata['registered-users'], count($contacts)); + $serverdata['registered-users'] = max($serverdata['registered-users'], count($contacts), 1); return $serverdata; } @@ -1020,7 +1076,7 @@ class GServer { $serverdata['poco'] = ''; - $curlResult = DI::httpRequest()->get($url . '/poco'); + $curlResult = DI::httpClient()->get($url . '/poco'); if (!$curlResult->isSuccess()) { return $serverdata; } @@ -1032,7 +1088,7 @@ class GServer if (!empty($data['totalResults'])) { $registeredUsers = $serverdata['registered-users'] ?? 0; - $serverdata['registered-users'] = max($data['totalResults'], $registeredUsers); + $serverdata['registered-users'] = max($data['totalResults'], $registeredUsers, 1); $serverdata['directory-type'] = self::DT_POCO; $serverdata['poco'] = $url . '/poco'; } @@ -1050,7 +1106,7 @@ class GServer */ public static function checkMastodonDirectory(string $url, array $serverdata) { - $curlResult = DI::httpRequest()->get($url . '/api/v1/directory?limit=1'); + $curlResult = DI::httpClient()->get($url . '/api/v1/directory?limit=1'); if (!$curlResult->isSuccess()) { return $serverdata; } @@ -1067,6 +1123,54 @@ class GServer return $serverdata; } + /** + * Detects Peertube via their known endpoint + * + * @param string $url URL of the given server + * @param array $serverdata array with server data + * + * @return array server data + */ + private static function detectPeertube(string $url, array $serverdata) + { + $curlResult = DI::httpClient()->get($url . '/api/v1/config'); + + if (!$curlResult->isSuccess() || ($curlResult->getBody() == '')) { + return $serverdata; + } + + $data = json_decode($curlResult->getBody(), true); + if (empty($data)) { + return $serverdata; + } + + if (!empty($data['instance']) && !empty($data['serverVersion'])) { + $serverdata['platform'] = 'peertube'; + $serverdata['version'] = $data['serverVersion']; + $serverdata['network'] = Protocol::ACTIVITYPUB; + + if (!empty($data['instance']['name'])) { + $serverdata['site_name'] = $data['instance']['name']; + } + + if (!empty($data['instance']['shortDescription'])) { + $serverdata['info'] = $data['instance']['shortDescription']; + } + + if (!empty($data['signup'])) { + if (!empty($data['signup']['allowed'])) { + $serverdata['register_policy'] = Register::OPEN; + } + } + + if (in_array($serverdata['detection-method'], [self::DETECT_HEADER, self::DETECT_BODY, self::DETECT_MANUAL])) { + $serverdata['detection-method'] = self::DETECT_V1_CONFIG; + } + } + + return $serverdata; + } + /** * Detects the version number of a given server when it was a NextCloud installation * @@ -1077,7 +1181,7 @@ class GServer */ private static function detectNextcloud(string $url, array $serverdata) { - $curlResult = DI::httpRequest()->get($url . '/status.php'); + $curlResult = DI::httpClient()->get($url . '/status.php'); if (!$curlResult->isSuccess() || ($curlResult->getBody() == '')) { return $serverdata; @@ -1111,7 +1215,7 @@ class GServer */ private static function detectMastodonAlikes(string $url, array $serverdata) { - $curlResult = DI::httpRequest()->get($url . '/api/v1/instance'); + $curlResult = DI::httpClient()->get($url . '/api/v1/instance'); if (!$curlResult->isSuccess() || ($curlResult->getBody() == '')) { return $serverdata; @@ -1146,7 +1250,7 @@ class GServer } if (!empty($data['stats']['user_count'])) { - $serverdata['registered-users'] = $data['stats']['user_count']; + $serverdata['registered-users'] = max($data['stats']['user_count'], 1); } if (!empty($serverdata['version']) && preg_match('/.*?\(compatible;\s(.*)\s(.*)\)/ism', $serverdata['version'], $matches)) { @@ -1177,7 +1281,7 @@ class GServer */ private static function detectHubzilla(string $url, array $serverdata) { - $curlResult = DI::httpRequest()->get($url . '/api/statusnet/config.json'); + $curlResult = DI::httpClient()->get($url . '/api/statusnet/config.json'); if (!$curlResult->isSuccess() || ($curlResult->getBody() == '')) { return $serverdata; } @@ -1275,7 +1379,7 @@ class GServer private static function detectGNUSocial(string $url, array $serverdata) { // Test for GNU Social - $curlResult = DI::httpRequest()->get($url . '/api/gnusocial/version.json'); + $curlResult = DI::httpClient()->get($url . '/api/gnusocial/version.json'); if ($curlResult->isSuccess() && ($curlResult->getBody() != '{"error":"not implemented"}') && ($curlResult->getBody() != '') && (strlen($curlResult->getBody()) < 30)) { $serverdata['platform'] = 'gnusocial'; @@ -1288,12 +1392,12 @@ class GServer if (in_array($serverdata['detection-method'], [self::DETECT_HEADER, self::DETECT_BODY, self::DETECT_MANUAL])) { $serverdata['detection-method'] = self::DETECT_GNUSOCIAL; } - + return $serverdata; } // Test for Statusnet - $curlResult = DI::httpRequest()->get($url . '/api/statusnet/version.json'); + $curlResult = DI::httpClient()->get($url . '/api/statusnet/version.json'); if ($curlResult->isSuccess() && ($curlResult->getBody() != '{"error":"not implemented"}') && ($curlResult->getBody() != '') && (strlen($curlResult->getBody()) < 30)) { @@ -1329,9 +1433,9 @@ class GServer */ private static function detectFriendica(string $url, array $serverdata) { - $curlResult = DI::httpRequest()->get($url . '/friendica/json'); + $curlResult = DI::httpClient()->get($url . '/friendica/json'); if (!$curlResult->isSuccess()) { - $curlResult = DI::httpRequest()->get($url . '/friendika/json'); + $curlResult = DI::httpClient()->get($url . '/friendika/json'); $friendika = true; $platform = 'Friendika'; } else { @@ -1348,7 +1452,7 @@ class GServer return $serverdata; } - if (in_array($serverdata['detection-method'], [self::DETECT_HEADER, self::DETECT_BODY, self::DETECT_MANUAL])) { + if (in_array($serverdata['detection-method'], [self::DETECT_HEADER, self::DETECT_BODY, self::DETECT_MANUAL])) { $serverdata['detection-method'] = $friendika ? self::DETECT_FRIENDIKA : self::DETECT_FRIENDICA; } @@ -1403,6 +1507,10 @@ class GServer */ private static function analyseRootBody($curlResult, array $serverdata, string $url) { + if (empty($curlResult->getBody())) { + return $serverdata; + } + $doc = new DOMDocument(); @$doc->loadHTML($curlResult->getBody()); $xpath = new DOMXPath($doc); @@ -1448,7 +1556,7 @@ class GServer if (count($version_part) == 2) { if (in_array($version_part[0], ['WordPress'])) { - $serverdata['platform'] = strtolower($version_part[0]); + $serverdata['platform'] = 'wordpress'; $serverdata['version'] = $version_part[1]; // We still do need a reliable test if some AP plugin is activated @@ -1539,11 +1647,11 @@ class GServer } elseif ($curlResult->inHeader('x-diaspora-version')) { $serverdata['platform'] = 'diaspora'; $serverdata['network'] = Protocol::DIASPORA; - $serverdata['version'] = $curlResult->getHeader('x-diaspora-version'); + $serverdata['version'] = $curlResult->getHeader('x-diaspora-version')[0] ?? ''; } elseif ($curlResult->inHeader('x-friendica-version')) { $serverdata['platform'] = 'friendica'; $serverdata['network'] = Protocol::DFRN; - $serverdata['version'] = $curlResult->getHeader('x-friendica-version'); + $serverdata['version'] = $curlResult->getHeader('x-friendica-version')[0] ?? ''; } else { return $serverdata; } @@ -1586,13 +1694,9 @@ class GServer $last_update = date('c', time() - (60 * 60 * 24 * $requery_days)); - $gservers = DBA::p("SELECT `id`, `url`, `nurl`, `network`, `poco`, `directory-type` - FROM `gserver` - WHERE NOT `failed` - AND `directory-type` != ? - AND `last_poco_query` < ? - ORDER BY RAND()", self::DT_NONE, $last_update - ); + $gservers = DBA::select('gserver', ['id', 'url', 'nurl', 'network', 'poco', 'directory-type'], + ["NOT `failed` AND `directory-type` != ? AND `last_poco_query` < ?", GServer::DT_NONE, $last_update], + ['order' => ['RAND()']]); while ($gserver = DBA::fetch($gservers)) { Logger::info('Update peer list', ['server' => $gserver['url'], 'id' => $gserver['id']]); @@ -1603,7 +1707,7 @@ class GServer $fields = ['last_poco_query' => DateTimeFormat::utcNow()]; DBA::update('gserver', $fields, ['nurl' => $gserver['nurl']]); - + if (--$no_of_queries == 0) { break; } @@ -1631,7 +1735,7 @@ class GServer $protocols = ['activitypub', 'diaspora', 'dfrn', 'ostatus']; foreach ($protocols as $protocol) { $query = '{nodes(protocol:"' . $protocol . '"){host}}'; - $curlResult = DI::httpRequest()->fetch('https://the-federation.info/graphql?query=' . urlencode($query)); + $curlResult = DI::httpClient()->fetch('https://the-federation.info/graphql?query=' . urlencode($query)); if (!empty($curlResult)) { $data = json_decode($curlResult, true); if (!empty($data['data']['nodes'])) { @@ -1648,8 +1752,7 @@ class GServer if (!empty($accesstoken)) { $api = 'https://instances.social/api/1.0/instances/list?count=0'; - $header = ['Authorization: Bearer '.$accesstoken]; - $curlResult = DI::httpRequest()->get($api, ['header' => $header]); + $curlResult = DI::httpClient()->get($api, [HttpClientOptions::HEADERS => ['Authorization' => ['Bearer ' . $accesstoken]]]); if ($curlResult->isSuccess()) { $servers = json_decode($curlResult->getBody(), true); @@ -1663,4 +1766,86 @@ class GServer DI::config()->set('poco', 'last_federation_discovery', time()); } + + /** + * Set the protocol for the given server + * + * @param int $gsid Server id + * @param int $protocol Protocol id + * @return void + * @throws Exception + */ + public static function setProtocol(int $gsid, int $protocol) + { + if (empty($gsid)) { + return; + } + + $gserver = DBA::selectFirst('gserver', ['protocol', 'url'], ['id' => $gsid]); + if (!DBA::isResult($gserver)) { + return; + } + + $old = $gserver['protocol']; + + if (!is_null($old)) { + /* + The priority for the protocols is: + 1. ActivityPub + 2. DFRN via Diaspora + 3. Legacy DFRN + 4. Diaspora + 5. OStatus + */ + + // We don't need to change it when nothing is to be changed + if ($old == $protocol) { + return; + } + + // We don't want to mark a server as OStatus when it had been marked with any other protocol before + if ($protocol == Post\DeliveryData::OSTATUS) { + return; + } + + // If the server is marked as ActivityPub then we won't change it to anything different + if ($old == Post\DeliveryData::ACTIVITYPUB) { + return; + } + + // Don't change it to anything lower than DFRN if the new one wasn't ActivityPub + if (($old == Post\DeliveryData::DFRN) && ($protocol != Post\DeliveryData::ACTIVITYPUB)) { + return; + } + + // Don't change it to Diaspora when it is a legacy DFRN server + if (($old == Post\DeliveryData::LEGACY_DFRN) && ($protocol == Post\DeliveryData::DIASPORA)) { + return; + } + } + + Logger::info('Protocol for server', ['protocol' => $protocol, 'old' => $old, 'id' => $gsid, 'url' => $gserver['url'], 'callstack' => System::callstack(20)]); + DBA::update('gserver', ['protocol' => $protocol], ['id' => $gsid]); + } + + /** + * Fetch the protocol of the given server + * + * @param int $gsid Server id + * @return int + * @throws Exception + */ + public static function getProtocol(int $gsid) + { + if (empty($gsid)) { + return null; + } + + $gserver = DBA::selectFirst('gserver', ['protocol'], ['id' => $gsid]); + if (DBA::isResult($gserver)) { + return $gserver['protocol']; + } + + return null; + } }