X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;f=src%2FModel%2FGServer.php;h=8305a3c9650666cff2f8b493efbbd1f852c6fb26;hb=a2a7d04fa1f3073e1bf79703e9516b20e502c225;hp=fc70b4fe4735f9215db462f586d041bc1f478409;hpb=dad05f1c7655c0c4e72fc5736a9cf770e663fe6b;p=friendica.git diff --git a/src/Model/GServer.php b/src/Model/GServer.php index fc70b4fe47..8305a3c965 100644 --- a/src/Model/GServer.php +++ b/src/Model/GServer.php @@ -30,7 +30,6 @@ use Friendica\Core\System; use Friendica\Core\Worker; use Friendica\Database\Database; use Friendica\Database\DBA; -use Friendica\Database\DBStructure; use Friendica\DI; use Friendica\Module\Register; use Friendica\Network\HTTPClient\Client\HttpClientAccept; @@ -44,8 +43,8 @@ use Friendica\Util\JsonLD; use Friendica\Util\Network; use Friendica\Util\Strings; use Friendica\Util\XML; -use GuzzleHttp\Exception\TransferException; use Friendica\Network\HTTPException; +use GuzzleHttp\Psr7\Uri; /** * This class handles GServer related functions @@ -94,6 +93,7 @@ class GServer * * @param string $url * @param boolean $only_nodeinfo + * * @return void */ public static function add(string $url, bool $only_nodeinfo = false) @@ -110,9 +110,10 @@ class GServer * * @param string $url * @param boolean $no_check Don't check if the server hadn't been found - * @return int gserver id + * + * @return int|null gserver id or NULL on empty URL or failed check */ - public static function getID(string $url, bool $no_check = false) + public static function getID(string $url, bool $no_check = false): ?int { if (empty($url)) { return null; @@ -139,7 +140,9 @@ class GServer * The pattern is a simple fnmatch() pattern with ? for single wildcard and * for multiple wildcard * * @param string $pattern + * * @return array + * * @throws Exception */ public static function listByDomainPattern(string $pattern): array @@ -170,7 +173,7 @@ class GServer * * @return boolean 'true' if server seems vital */ - public static function reachable(string $profile, string $server = '', string $network = '', bool $force = false) + public static function reachable(string $profile, string $server = '', string $network = '', bool $force = false): bool { if ($server == '') { $contact = Contact::getByURL($profile, null, ['baseurl']); @@ -186,7 +189,19 @@ class GServer return self::check($server, $network, $force); } - public static function getNextUpdateDate(bool $success, string $created = '', string $last_contact = '', bool $undetected = false) + /** + * Calculate the next update day + * + * @param bool $success + * @param string $created + * @param string $last_contact + * @param bool $undetected + * + * @return string + * + * @throws Exception + */ + public static function getNextUpdateDate(bool $success, string $created = '', string $last_contact = '', bool $undetected = false): string { // On successful contact process check again next week when it is a detected system. // When we haven't detected the system, it could be a static website or a really old system. @@ -246,7 +261,7 @@ class GServer * * @return boolean 'true' if server seems vital */ - public static function check(string $server_url, string $network = '', bool $force = false, bool $only_nodeinfo = false) + public static function check(string $server_url, string $network = '', bool $force = false, bool $only_nodeinfo = false): bool { $server_url = self::cleanURL($server_url); if ($server_url == '') { @@ -258,7 +273,7 @@ class GServer if ($gserver['created'] <= DBA::NULL_DATETIME) { $fields = ['created' => DateTimeFormat::utcNow()]; $condition = ['nurl' => Strings::normaliseLink($server_url)]; - DBA::update('gserver', $fields, $condition); + self::update($fields, $condition); } if (!$force && (strtotime($gserver['next_contact']) > time())) { @@ -283,7 +298,7 @@ class GServer $gserver = DBA::selectFirst('gserver', [], ['nurl' => Strings::normaliseLink($url)]); if (DBA::isResult($gserver)) { $next_update = self::getNextUpdateDate(false, $gserver['created'], $gserver['last_contact']); - DBA::update('gserver', ['url' => $url, 'failed' => true, 'last_failure' => DateTimeFormat::utcNow(), + self::update(['url' => $url, 'failed' => true, 'last_failure' => DateTimeFormat::utcNow(), 'next_contact' => $next_update, 'network' => Protocol::PHANTOM, 'detection-method' => null], ['nurl' => Strings::normaliseLink($url)]); Logger::info('Set failed status for existing server', ['url' => $url]); @@ -299,9 +314,10 @@ class GServer * Remove unwanted content from the given URL * * @param string $url + * * @return string cleaned URL */ - public static function cleanURL(string $url) + public static function cleanURL(string $url): string { $url = trim($url, '/'); $url = str_replace('/index.php', '', $url); @@ -311,7 +327,7 @@ class GServer unset($urlparts['pass']); unset($urlparts['query']); unset($urlparts['fragment']); - return Network::unparseURL($urlparts); + return (string)Uri::fromParts($urlparts); } /** @@ -324,7 +340,7 @@ class GServer * * @return boolean 'true' if server could be detected */ - public static function detect(string $url, string $network = '', bool $only_nodeinfo = false) + public static function detect(string $url, string $network = '', bool $only_nodeinfo = false): bool { Logger::info('Detect server type', ['server' => $url]); @@ -337,50 +353,56 @@ class GServer return false; } - if (!Network::isUrlValid($url)) { - self::setFailure($url); - return false; - } - // If the URL missmatches, then we mark the old entry as failure - if (Strings::normaliseLink($url) != Strings::normaliseLink($original_url)) { + if (!Strings::compareLink($url, $original_url)) { self::setFailure($original_url); - self::detect($url, $network, $only_nodeinfo); + if (!self::getID($url, true)) { + self::detect($url, $network, $only_nodeinfo); + } return false; } - // On a redirect follow the new host but mark the old one as failure - try { - $finalurl = rtrim(DI::httpClient()->finalUrl($url), '/'); - } catch (TransferException $exception) { - Logger::notice('Error fetching final URL.', ['url' => $url, 'exception' => $exception]); + $valid_url = Network::isUrlValid($url); + if (!$valid_url) { self::setFailure($url); return false; - } + } else { + $valid_url = rtrim($valid_url, '/'); + } + + if (!Strings::compareLink($url, $valid_url)) { + // We only follow redirects when the path stays the same or the target url has no path. + // Some systems have got redirects on their landing page to a single account page. This check handles it. + if (((parse_url($url, PHP_URL_HOST) != parse_url($valid_url, PHP_URL_HOST)) && (parse_url($url, PHP_URL_PATH) == parse_url($valid_url, PHP_URL_PATH))) || + (((parse_url($url, PHP_URL_HOST) != parse_url($valid_url, PHP_URL_HOST)) || (parse_url($url, PHP_URL_PATH) != parse_url($valid_url, PHP_URL_PATH))) && empty(parse_url($valid_url, PHP_URL_PATH)))) { + Logger::debug('Found redirect. Mark old entry as failure', ['old' => $url, 'new' => $valid_url]); + self::setFailure($url); + if (!self::getID($valid_url, true)) { + self::detect($valid_url, $network, $only_nodeinfo); + } + return false; + } - if (empty($finalurl)) { - Logger::notice('Empty redirected URL.', ['url' => $url]); - return false; - } + if ((parse_url($url, PHP_URL_HOST) != parse_url($valid_url, PHP_URL_HOST)) && (parse_url($url, PHP_URL_PATH) != parse_url($valid_url, PHP_URL_PATH)) && + (parse_url($url, PHP_URL_PATH) == '')) { + Logger::debug('Found redirect. Mark old entry as failure and redirect to the basepath.', ['old' => $url, 'new' => $valid_url]); + $parts = parse_url($valid_url); + unset($parts['path']); + $valid_url = (string)Uri::fromParts($parts); - // We only follow redirects when the path stays the same or the target url has no path. - // Some systems have got redirects on their landing page to a single account page. This check handles it. - if (((parse_url($url, PHP_URL_HOST) != parse_url($finalurl, PHP_URL_HOST)) && (parse_url($url, PHP_URL_PATH) == parse_url($finalurl, PHP_URL_PATH))) || - (((parse_url($url, PHP_URL_HOST) != parse_url($finalurl, PHP_URL_HOST)) || (parse_url($url, PHP_URL_PATH) != parse_url($finalurl, PHP_URL_PATH))) && empty(parse_url($finalurl, PHP_URL_PATH)))) { - Logger::info('Found redirect. Mark old entry as failure', ['old' => $url, 'new' => $finalurl]); - self::setFailure($url); - self::detect($finalurl, $network, $only_nodeinfo); - return false; + self::setFailure($url); + if (!self::getID($valid_url, true)) { + self::detect($valid_url, $network, $only_nodeinfo); + } + return false; + } + Logger::debug('Found redirect, but ignore it.', ['old' => $url, 'new' => $valid_url]); } - if ((parse_url($url, PHP_URL_HOST) == parse_url($finalurl, PHP_URL_HOST)) && - (parse_url($url, PHP_URL_PATH) == parse_url($finalurl, PHP_URL_PATH)) && - (parse_url($url, PHP_URL_SCHEME) != parse_url($finalurl, PHP_URL_SCHEME))) { - if (!Network::isUrlValid($finalurl)) { - self::setFailure($finalurl); - } else { - $url = $finalurl; - } + if ((parse_url($url, PHP_URL_HOST) == parse_url($valid_url, PHP_URL_HOST)) && + (parse_url($url, PHP_URL_PATH) == parse_url($valid_url, PHP_URL_PATH)) && + (parse_url($url, PHP_URL_SCHEME) != parse_url($valid_url, PHP_URL_SCHEME))) { + $url = $valid_url; } $in_webroot = empty(parse_url($url, PHP_URL_PATH)); @@ -553,16 +575,13 @@ class GServer $serverdata['last_contact'] = DateTimeFormat::utcNow(); $serverdata['failed'] = false; - // Limit the length on incoming fields - $serverdata = DBStructure::getFieldsForTable('gserver', $serverdata); - $gserver = DBA::selectFirst('gserver', ['network'], ['nurl' => Strings::normaliseLink($url)]); if (!DBA::isResult($gserver)) { $serverdata['created'] = DateTimeFormat::utcNow(); $ret = DBA::insert('gserver', $serverdata); $id = DBA::lastInsertId(); } else { - $ret = DBA::update('gserver', $serverdata, ['nurl' => $serverdata['nurl']]); + $ret = self::update($serverdata, ['nurl' => $serverdata['nurl']]); $gserver = DBA::selectFirst('gserver', ['id'], ['nurl' => $serverdata['nurl']]); if (DBA::isResult($gserver)) { $id = $gserver['id']; @@ -576,14 +595,14 @@ class GServer $max_users = max($apcontacts, $contacts); if ($max_users > $serverdata['registered-users']) { Logger::info('Update registered users', ['id' => $id, 'url' => $serverdata['nurl'], 'registered-users' => $max_users]); - DBA::update('gserver', ['registered-users' => $max_users], ['id' => $id]); + self::update(['registered-users' => $max_users], ['id' => $id]); } if (empty($serverdata['active-month-users'])) { $contacts = DBA::count('contact', ["`uid` = ? AND `gsid` = ? AND NOT `failed` AND `last-item` > ?", 0, $id, DateTimeFormat::utc('now - 30 days')]); if ($contacts > 0) { Logger::info('Update monthly users', ['id' => $id, 'url' => $serverdata['nurl'], 'monthly-users' => $contacts]); - DBA::update('gserver', ['active-month-users' => $contacts], ['id' => $id]); + self::update(['active-month-users' => $contacts], ['id' => $id]); } } @@ -591,7 +610,7 @@ class GServer $contacts = DBA::count('contact', ["`uid` = ? AND `gsid` = ? AND NOT `failed` AND `last-item` > ?", 0, $id, DateTimeFormat::utc('now - 180 days')]); if ($contacts > 0) { Logger::info('Update halfyear users', ['id' => $id, 'url' => $serverdata['nurl'], 'halfyear-users' => $contacts]); - DBA::update('gserver', ['active-halfyear-users' => $contacts], ['id' => $id]); + self::update(['active-halfyear-users' => $contacts], ['id' => $id]); } } } @@ -612,6 +631,9 @@ class GServer * Fetch relay data from a given server url * * @param string $server_url address of the server + * + * @return void + * * @throws \Friendica\Network\HTTPException\InternalServerErrorException */ private static function discoverRelay(string $server_url) @@ -644,7 +666,7 @@ class GServer if (($gserver['relay-subscribe'] != $data['subscribe']) || ($gserver['relay-scope'] != $data['scope'])) { $fields = ['relay-subscribe' => $data['subscribe'], 'relay-scope' => $data['scope']]; - DBA::update('gserver', $fields, ['id' => $gserver['id']]); + self::update($fields, ['id' => $gserver['id']]); } DBA::delete('gserver-tag', ['gserver-id' => $gserver['id']]); @@ -711,7 +733,7 @@ class GServer * * @return array server data */ - private static function fetchStatistics(string $url, array $serverdata) + private static function fetchStatistics(string $url, array $serverdata): array { $curlResult = DI::httpClient()->get($url . '/statistics.json', HttpClientAccept::JSON); if (!$curlResult->isSuccess()) { @@ -801,9 +823,10 @@ class GServer * @param ICanHandleHttpResponses $httpResult * * @return array Server data + * * @throws \Friendica\Network\HTTPException\InternalServerErrorException */ - private static function fetchNodeinfo(string $url, ICanHandleHttpResponses $httpResult) + private static function fetchNodeinfo(string $url, ICanHandleHttpResponses $httpResult): array { if (!$httpResult->isSuccess()) { return []; @@ -851,10 +874,12 @@ class GServer * Parses Nodeinfo 1 * * @param string $nodeinfo_url address of the nodeinfo path + * * @return array Server data + * * @throws \Friendica\Network\HTTPException\InternalServerErrorException */ - private static function parseNodeinfo1(string $nodeinfo_url) + private static function parseNodeinfo1(string $nodeinfo_url): array { $curlResult = DI::httpClient()->get($nodeinfo_url, HttpClientAccept::JSON); if (!$curlResult->isSuccess()) { @@ -947,11 +972,14 @@ class GServer * Parses Nodeinfo 2 * * @see https://git.feneas.org/jaywink/nodeinfo2 + * * @param string $nodeinfo_url address of the nodeinfo path + * * @return array Server data + * * @throws \Friendica\Network\HTTPException\InternalServerErrorException */ - private static function parseNodeinfo2(string $nodeinfo_url) + private static function parseNodeinfo2(string $nodeinfo_url): array { $curlResult = DI::httpClient()->get($nodeinfo_url, HttpClientAccept::JSON); if (!$curlResult->isSuccess()) { @@ -963,8 +991,11 @@ class GServer return []; } - $server = ['detection-method' => self::DETECT_NODEINFO_2, - 'register_policy' => Register::CLOSED]; + $server = [ + 'detection-method' => self::DETECT_NODEINFO_2, + 'register_policy' => Register::CLOSED, + 'platform' => 'unknown', + ]; if (!empty($nodeinfo['openRegistrations'])) { $server['register_policy'] = Register::OPEN; @@ -1050,11 +1081,14 @@ class GServer * Parses NodeInfo2 protocol 1.0 * * @see https://github.com/jaywink/nodeinfo2/blob/master/PROTOCOL.md + * * @param string $nodeinfo_url address of the nodeinfo path + * * @return array Server data + * * @throws \Friendica\Network\HTTPException\InternalServerErrorException */ - private static function parseNodeinfo210(ICanHandleHttpResponses $httpResult) + private static function parseNodeinfo210(ICanHandleHttpResponses $httpResult): array { if (!$httpResult->isSuccess()) { return []; @@ -1152,7 +1186,7 @@ class GServer * * @return array server data */ - private static function fetchSiteinfo(string $url, array $serverdata) + private static function fetchSiteinfo(string $url, array $serverdata): array { $curlResult = DI::httpClient()->get($url . '/siteinfo.json', HttpClientAccept::JSON); if (!$curlResult->isSuccess()) { @@ -1229,7 +1263,17 @@ class GServer return $serverdata; } - private static function fetchDataFromSystemActor(array $data, array $serverdata) + /** + * Fetches server data via an ActivityPub account with url of that server + * + * @param string $url URL of the given server + * @param array $serverdata array with server data + * + * @return array server data + * + * @throws Exception + */ + private static function fetchDataFromSystemActor(array $data, array $serverdata): array { if (empty($data)) { return ['server' => $serverdata, 'actor' => '']; @@ -1273,7 +1317,7 @@ class GServer * * @return boolean 'true' if the server seems to be vital */ - private static function validHostMeta(string $url) + private static function validHostMeta(string $url): bool { $xrd_timeout = DI::config()->get('system', 'xrd_timeout'); $curlResult = DI::httpClient()->get($url . '/.well-known/host-meta', HttpClientAccept::XRD_XML, [HttpClientOptions::TIMEOUT => $xrd_timeout]); @@ -1319,7 +1363,7 @@ class GServer * * @return array server data */ - private static function detectNetworkViaContacts(string $url, array $serverdata) + private static function detectNetworkViaContacts(string $url, array $serverdata): array { $contacts = []; @@ -1371,7 +1415,7 @@ class GServer * * @return array server data */ - private static function checkPoCo(string $url, array $serverdata) + private static function checkPoCo(string $url, array $serverdata): array { $serverdata['poco'] = ''; @@ -1403,7 +1447,7 @@ class GServer * * @return array server data */ - public static function checkMastodonDirectory(string $url, array $serverdata) + public static function checkMastodonDirectory(string $url, array $serverdata): array { $curlResult = DI::httpClient()->get($url . '/api/v1/directory?limit=1', HttpClientAccept::JSON); if (!$curlResult->isSuccess()) { @@ -1430,7 +1474,7 @@ class GServer * * @return array server data */ - private static function detectPeertube(string $url, array $serverdata) + private static function detectPeertube(string $url, array $serverdata): array { $curlResult = DI::httpClient()->get($url . '/api/v1/config', HttpClientAccept::JSON); if (!$curlResult->isSuccess() || ($curlResult->getBody() == '')) { @@ -1478,7 +1522,7 @@ class GServer * * @return array server data */ - private static function detectNextcloud(string $url, array $serverdata, bool $validHostMeta) + private static function detectNextcloud(string $url, array $serverdata, bool $validHostMeta): array { $curlResult = DI::httpClient()->get($url . '/status.php', HttpClientAccept::JSON); if (!$curlResult->isSuccess() || ($curlResult->getBody() == '')) { @@ -1506,7 +1550,16 @@ class GServer return $serverdata; } - private static function fetchWeeklyUsage(string $url, array $serverdata) { + /** + * Fetches weekly usage data + * + * @param string $url URL of the given server + * @param array $serverdata array with server data + * + * @return array server data + */ + private static function fetchWeeklyUsage(string $url, array $serverdata): array + { $curlResult = DI::httpClient()->get($url . '/api/v1/instance/activity', HttpClientAccept::JSON); if (!$curlResult->isSuccess() || ($curlResult->getBody() == '')) { return $serverdata; @@ -1545,7 +1598,7 @@ class GServer * * @return array server data */ - private static function detectMastodonAlikes(string $url, array $serverdata) + private static function detectMastodonAlikes(string $url, array $serverdata): array { $curlResult = DI::httpClient()->get($url . '/api/v1/instance', HttpClientAccept::JSON); if (!$curlResult->isSuccess() || ($curlResult->getBody() == '')) { @@ -1617,7 +1670,7 @@ class GServer * * @return array server data */ - private static function detectHubzilla(string $url, array $serverdata) + private static function detectHubzilla(string $url, array $serverdata): array { $curlResult = DI::httpClient()->get($url . '/api/statusnet/config.json', HttpClientAccept::JSON); if (!$curlResult->isSuccess() || ($curlResult->getBody() == '')) { @@ -1694,7 +1747,7 @@ class GServer * * @return boolean */ - private static function toBoolean($val) + private static function toBoolean($val): bool { if (($val == 'true') || ($val == 1)) { return true; @@ -1713,7 +1766,7 @@ class GServer * * @return array server data */ - private static function detectGNUSocial(string $url, array $serverdata) + private static function detectGNUSocial(string $url, array $serverdata): array { // Test for GNU Social $curlResult = DI::httpClient()->get($url . '/api/gnusocial/version.json', HttpClientAccept::JSON); @@ -1768,7 +1821,7 @@ class GServer * * @return array server data */ - private static function detectFriendica(string $url, array $serverdata) + private static function detectFriendica(string $url, array $serverdata): array { // There is a bug in some versions of Friendica that will return an ActivityStream actor when the content type "application/json" is requested. // Because of this me must not use ACCEPT_JSON here. @@ -1843,16 +1896,16 @@ class GServer * * @return array server data */ - private static function analyseRootBody($curlResult, array $serverdata) + private static function analyseRootBody($curlResult, array $serverdata): array { if (empty($curlResult->getBody())) { return $serverdata; } - if (file_exists(__DIR__ . '/../../static/generator.config.php')) { - require __DIR__ . '/../../static/generator.config.php'; + if (file_exists(__DIR__ . '/../../static/platforms.config.php')) { + require __DIR__ . '/../../static/platforms.config.php'; } else { - throw new HTTPException\InternalServerErrorException('Invalid generator file'); + throw new HTTPException\InternalServerErrorException('Invalid platform file'); } $platforms = array_merge($ap_platforms, $dfrn_platforms, $zap_platforms, $platforms); @@ -2002,7 +2055,7 @@ class GServer * * @return array server data */ - private static function analyseRootHeader($curlResult, array $serverdata) + private static function analyseRootHeader($curlResult, array $serverdata): array { if ($curlResult->getHeader('server') == 'Mastodon') { $serverdata['platform'] = 'mastodon'; @@ -2056,7 +2109,7 @@ class GServer Worker::add(PRIORITY_LOW, 'UpdateServerDirectory', $gserver); $fields = ['last_poco_query' => DateTimeFormat::utcNow()]; - DBA::update('gserver', $fields, ['nurl' => $gserver['nurl']]); + self::update($fields, ['nurl' => $gserver['nurl']]); if (--$no_of_queries == 0) { break; @@ -2121,7 +2174,7 @@ class GServer * * @param int $gsid Server id * @param int $protocol Protocol id - * @return void + * * @throws Exception */ public static function setProtocol(int $gsid, int $protocol) @@ -2174,17 +2227,19 @@ class GServer } Logger::info('Protocol for server', ['protocol' => $protocol, 'old' => $old, 'id' => $gsid, 'url' => $gserver['url'], 'callstack' => System::callstack(20)]); - DBA::update('gserver', ['protocol' => $protocol], ['id' => $gsid]); + self::update(['protocol' => $protocol], ['id' => $gsid]); } /** * Fetch the protocol of the given server * * @param int $gsid Server id - * @return int + * + * @return ?int One of Post\DeliveryData protocol constants or null if unknown or gserver is missing + * * @throws Exception */ - public static function getProtocol(int $gsid) + public static function getProtocol(int $gsid): ?int { if (empty($gsid)) { return null; @@ -2197,4 +2252,21 @@ class GServer return null; } + + /** + * Enforces gserver table field maximum sizes to avoid "Data too long" database errors + * + * @param array $fields + * @param array $condition + * + * @return bool + * + * @throws Exception + */ + public static function update(array $fields, array $condition): bool + { + $fields = DI::dbaDefinition()->truncateFieldsForTable('gserver', $fields); + + return DBA::update('gserver', $fields, $condition); + } }