src/Model/GServer.php

   1 <?php
   2 /**
   3  * @copyright Copyright (C) 2010-2023, the Friendica project
   4  *
   5  * @license GNU AGPL version 3 or any later version
   6  *
   7  * This program is free software: you can redistribute it and/or modify
   8  * it under the terms of the GNU Affero General Public License as
   9  * published by the Free Software Foundation, either version 3 of the
  10  * License, or (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU Affero General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU Affero General Public License
  18  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  19  *
  20  */
  21
  22 namespace Friendica\Model;
  23
  24 use DOMDocument;
  25 use DOMXPath;
  26 use Exception;
  27 use Friendica\Core\Logger;
  28 use Friendica\Core\Protocol;
  29 use Friendica\Core\System;
  30 use Friendica\Core\Worker;
  31 use Friendica\Database\Database;
  32 use Friendica\Database\DBA;
  33 use Friendica\DI;
  34 use Friendica\Module\Register;
  35 use Friendica\Network\HTTPClient\Client\HttpClientAccept;
  36 use Friendica\Network\HTTPClient\Client\HttpClientOptions;
  37 use Friendica\Network\HTTPClient\Capability\ICanHandleHttpResponses;
  38 use Friendica\Network\Probe;
  39 use Friendica\Protocol\ActivityPub;
  40 use Friendica\Protocol\Relay;
  41 use Friendica\Util\DateTimeFormat;
  42 use Friendica\Util\JsonLD;
  43 use Friendica\Util\Network;
  44 use Friendica\Util\Strings;
  45 use Friendica\Util\XML;
  46 use Friendica\Network\HTTPException;
  47 use Friendica\Worker\UpdateGServer;
  48 use GuzzleHttp\Psr7\Uri;
  49 use Psr\Http\Message\UriInterface;
  50
  51 /**
  52  * This class handles GServer related functions
  53  */
  54 class GServer
  55 {
  56         // Directory types
  57         const DT_NONE = 0;
  58         const DT_POCO = 1;
  59         const DT_MASTODON = 2;
  60
  61         // Methods to detect server types
  62
  63         // Non endpoint specific methods
  64         const DETECT_MANUAL = 0;
  65         const DETECT_HEADER = 1;
  66         const DETECT_BODY = 2;
  67         const DETECT_HOST_META = 3;
  68         const DETECT_CONTACTS = 4;
  69         const DETECT_AP_ACTOR = 5;
  70         const DETECT_AP_COLLECTION = 6;
  71
  72         const DETECT_UNSPECIFIC = [self::DETECT_MANUAL, self::DETECT_HEADER, self::DETECT_BODY, self::DETECT_HOST_META, self::DETECT_CONTACTS, self::DETECT_AP_ACTOR];
  73
  74         // Implementation specific endpoints
  75         // @todo Possibly add Lemmy detection via the endpoint /api/v3/site
  76         const DETECT_FRIENDIKA = 10;
  77         const DETECT_FRIENDICA = 11;
  78         const DETECT_STATUSNET = 12;
  79         const DETECT_GNUSOCIAL = 13;
  80         const DETECT_CONFIG_JSON = 14; // Statusnet, GNU Social, Older Hubzilla/Redmatrix
  81         const DETECT_SITEINFO_JSON = 15; // Newer Hubzilla
  82         const DETECT_MASTODON_API = 16;
  83         const DETECT_STATUS_PHP = 17; // Nextcloud
  84         const DETECT_V1_CONFIG = 18;
  85         const DETECT_PUMPIO = 19; // Deprecated
  86         const DETECT_SYSTEM_ACTOR = 20; // Mistpark, Osada, Roadhouse, Zap
  87
  88         // Standardized endpoints
  89         const DETECT_STATISTICS_JSON = 100;
  90         const DETECT_NODEINFO_1 = 101;
  91         const DETECT_NODEINFO_2 = 102;
  92         const DETECT_NODEINFO_210 = 103;
  93
  94         /**
  95          * Check for the existance of a server and adds it in the background if not existant
  96          *
  97          * @param string $url
  98          * @param boolean $only_nodeinfo
  99          *
 100          * @return void
 101          */
 102         public static function add(string $url, bool $only_nodeinfo = false)
 103         {
 104                 if (self::getID($url)) {
 105                         return;
 106                 }
 107
 108                 UpdateGServer::add(Worker::PRIORITY_LOW, $url, $only_nodeinfo);
 109         }
 110
 111         /**
 112          * Get the ID for the given server URL
 113          *
 114          * @param string $url
 115          * @param boolean $no_check Don't check if the server hadn't been found
 116          *
 117          * @return int|null gserver id or NULL on empty URL or failed check
 118          */
 119         public static function getID(string $url, bool $no_check = false): ?int
 120         {
 121                 $url = self::cleanURL($url);
 122
 123                 if (empty($url)) {
 124                         return null;
 125                 }
 126
 127                 $gserver = DBA::selectFirst('gserver', ['id'], ['nurl' => Strings::normaliseLink($url)]);
 128                 if (DBA::isResult($gserver)) {
 129                         Logger::debug('Got ID for URL', ['id' => $gserver['id'], 'url' => $url, 'callstack' => System::callstack(20)]);
 130
 131                         if (Network::isUrlBlocked($url)) {
 132                                 self::setBlockedById($gserver['id']);
 133                         } else {
 134                                 self::setUnblockedById($gserver['id']);
 135                         }
 136
 137                         return $gserver['id'];
 138                 }
 139
 140                 if ($no_check || !self::check($url)) {
 141                         return null;
 142                 }
 143
 144                 return self::getID($url, true);
 145         }
 146
 147         /**
 148          * Retrieves all the servers which base domain are matching the provided domain pattern
 149          *
 150          * The pattern is a simple fnmatch() pattern with ? for single wildcard and * for multiple wildcard
 151          *
 152          * @param string $pattern
 153          *
 154          * @return array
 155          *
 156          * @throws Exception
 157          */
 158         public static function listByDomainPattern(string $pattern): array
 159         {
 160                 $likePattern = 'http://' . strtr($pattern, ['_' => '\_', '%' => '\%', '?' => '_', '*' => '%']);
 161
 162                 // The SUBSTRING_INDEX returns everything before the eventual third /, which effectively trims an
 163                 // eventual server path and keep only the server domain which we're matching against the pattern.
 164                 $sql = "SELECT `gserver`.*, COUNT(*) AS `contacts`
 165                         FROM `gserver`
 166                         LEFT JOIN `contact` ON `gserver`.`id` = `contact`.`gsid`
 167                         WHERE SUBSTRING_INDEX(`gserver`.`nurl`, '/', 3) LIKE ?
 168                         AND NOT `gserver`.`failed`
 169                         GROUP BY `gserver`.`id`";
 170
 171                 $stmt = DI::dba()->p($sql, $likePattern);
 172
 173                 return DI::dba()->toArray($stmt);
 174         }
 175
 176         /**
 177          * Checks if the given server array is unreachable for a long time now
 178          *
 179          * @param integer $gsid
 180          * @return boolean
 181          */
 182         private static function isDefunct(array $gserver): bool
 183         {
 184                 return ($gserver['failed'] || in_array($gserver['network'], Protocol::FEDERATED)) &&
 185                         ($gserver['last_contact'] >= $gserver['created']) &&
 186                         ($gserver['last_contact'] < $gserver['last_failure']) &&
 187                         ($gserver['last_contact'] < DateTimeFormat::utc('now - 90 days'));
 188         }
 189
 190         /**
 191          * Checks if the given server id is unreachable for a long time now
 192          *
 193          * @param integer $gsid
 194          * @return boolean
 195          */
 196         public static function isDefunctById(int $gsid): bool
 197         {
 198                 $gserver = DBA::selectFirst('gserver', ['url', 'next_contact', 'last_contact', 'last_failure', 'created', 'failed', 'network'], ['id' => $gsid]);
 199                 if (empty($gserver)) {
 200                         return false;
 201                 } else {
 202                         if (strtotime($gserver['next_contact']) < time()) {
 203                                 UpdateGServer::add(Worker::PRIORITY_LOW, $gserver['url']);
 204                         }
 205
 206                         return self::isDefunct($gserver);
 207                 }
 208         }
 209
 210         /**
 211          * Checks if the given server id is reachable
 212          *
 213          * @param integer $gsid
 214          * @return boolean
 215          */
 216         public static function isReachableById(int $gsid): bool
 217         {
 218                 $gserver = DBA::selectFirst('gserver', ['url', 'next_contact', 'failed', 'network'], ['id' => $gsid]);
 219                 if (empty($gserver)) {
 220                         return true;
 221                 } else {
 222                         if (strtotime($gserver['next_contact']) < time()) {
 223                                 UpdateGServer::add(Worker::PRIORITY_LOW, $gserver['url']);
 224                         }
 225
 226                         return !$gserver['failed'] && in_array($gserver['network'], Protocol::FEDERATED);
 227                 }
 228         }
 229
 230         /**
 231          * Checks if the given server is reachable
 232          *
 233          * @param array $contact Contact that should be checked
 234          *
 235          * @return boolean 'true' if server seems vital
 236          */
 237         public static function reachable(array $contact): bool
 238         {
 239                 if (!empty($contact['gsid'])) {
 240                         $gsid = $contact['gsid'];
 241                 } elseif (!empty($contact['baseurl'])) {
 242                         $server = $contact['baseurl'];
 243                 } elseif ($contact['network'] == Protocol::DIASPORA) {
 244                         $parts = parse_url($contact['url']);
 245                         unset($parts['path']);
 246                         $server = (string)Uri::fromParts($parts);
 247                 } else {
 248                         return true;
 249                 }
 250
 251                 if (!empty($gsid)) {
 252                         $condition = ['id' => $gsid];
 253                 } else {
 254                         $condition = ['nurl' => Strings::normaliseLink($server)];
 255                 }
 256
 257                 $gserver = DBA::selectFirst('gserver', ['url', 'next_contact', 'failed', 'network'], $condition);
 258                 if (empty($gserver)) {
 259                         $reachable = true;
 260                 } else {
 261                         $reachable = !$gserver['failed'] && in_array($gserver['network'], Protocol::FEDERATED);
 262                         $server    = $gserver['url'];
 263                 }
 264
 265                 if (!empty($server) && (empty($gserver) || strtotime($gserver['next_contact']) < time())) {
 266                         UpdateGServer::add(Worker::PRIORITY_LOW, $server);
 267                 }
 268
 269                 return $reachable;
 270         }
 271
 272         /**
 273          * Calculate the next update day
 274          *
 275          * @param bool $success
 276          * @param string $created
 277          * @param string $last_contact
 278          * @param bool $undetected
 279          *
 280          * @return string
 281          *
 282          * @throws Exception
 283          */
 284         public static function getNextUpdateDate(bool $success, string $created = '', string $last_contact = '', bool $undetected = false): string
 285         {
 286                 // On successful contact process check again next week when it is a detected system.
 287                 // When we haven't detected the system, it could be a static website or a really old system.
 288                 if ($success) {
 289                         return DateTimeFormat::utc($undetected ? 'now +1 month' : 'now +7 day');
 290                 }
 291
 292                 $now = strtotime(DateTimeFormat::utcNow());
 293
 294                 if ($created > $last_contact) {
 295                         $contact_time = strtotime($created);
 296                 } else {
 297                         $contact_time = strtotime($last_contact);
 298                 }
 299
 300                 // If the last contact was less than 6 hours before then try again in 6 hours
 301                 if (($now - $contact_time) < (60 * 60 * 6)) {
 302                         return DateTimeFormat::utc('now +6 hour');
 303                 }
 304
 305                 // If the last contact was less than 12 hours before then try again in 12 hours
 306                 if (($now - $contact_time) < (60 * 60 * 12)) {
 307                         return DateTimeFormat::utc('now +12 hour');
 308                 }
 309
 310                 // If the last contact was less than 24 hours before then try tomorrow again
 311                 if (($now - $contact_time) < (60 * 60 * 24)) {
 312                         return DateTimeFormat::utc('now +1 day');
 313                 }
 314
 315                 // If the last contact was less than a week before then try again in a week
 316                 if (($now - $contact_time) < (60 * 60 * 24 * 7)) {
 317                         return DateTimeFormat::utc('now +1 week');
 318                 }
 319
 320                 // If the last contact was less than two weeks before then try again in two week
 321                 if (($now - $contact_time) < (60 * 60 * 24 * 14)) {
 322                         return DateTimeFormat::utc('now +2 week');
 323                 }
 324
 325                 // If the last contact was less than a month before then try again in a month
 326                 if (($now - $contact_time) < (60 * 60 * 24 * 30)) {
 327                         return DateTimeFormat::utc('now +1 month');
 328                 }
 329
 330                 // The system hadn't been successul contacted for more than a month, so try again in three months
 331                 return DateTimeFormat::utc('now +3 month');
 332         }
 333
 334         /**
 335          * Checks the state of the given server.
 336          *
 337          * @param string  $server_url    URL of the given server
 338          * @param string  $network       Network value that is used, when detection failed
 339          * @param boolean $force         Force an update.
 340          * @param boolean $only_nodeinfo Only use nodeinfo for server detection
 341          *
 342          * @return boolean 'true' if server seems vital
 343          */
 344         public static function check(string $server_url, string $network = '', bool $force = false, bool $only_nodeinfo = false): bool
 345         {
 346                 $server_url = self::cleanURL($server_url);
 347                 if ($server_url == '') {
 348                         return false;
 349                 }
 350
 351                 if (Network::isUrlBlocked($server_url)) {
 352                         Logger::info('Server is blocked', ['url' => $server_url]);
 353                         self::setBlockedByUrl($server_url);
 354                         return false;
 355                 }
 356
 357                 $gserver = DBA::selectFirst('gserver', [], ['nurl' => Strings::normaliseLink($server_url)]);
 358                 if (DBA::isResult($gserver)) {
 359                         if ($gserver['created'] <= DBA::NULL_DATETIME) {
 360                                 $fields = ['created' => DateTimeFormat::utcNow()];
 361                                 $condition = ['nurl' => Strings::normaliseLink($server_url)];
 362                                 self::update($fields, $condition);
 363                         }
 364
 365                         if (!$force && (strtotime($gserver['next_contact']) > time())) {
 366                                 Logger::info('No update needed', ['server' => $server_url]);
 367                                 return (!$gserver['failed']);
 368                         }
 369                         Logger::info('Server is outdated. Start discovery.', ['Server' => $server_url, 'Force' => $force]);
 370                 } else {
 371                         Logger::info('Server is unknown. Start discovery.', ['Server' => $server_url]);
 372                 }
 373
 374                 return self::detect($server_url, $network, $only_nodeinfo);
 375         }
 376
 377         /**
 378          * Reset failed server status by gserver id
 379          *
 380          * @param int    $gsid
 381          * @param string $network
 382          */
 383         public static function setReachableById(int $gsid, string $network)
 384         {
 385                 $gserver = DBA::selectFirst('gserver', ['url', 'failed', 'next_contact', 'network'], ['id' => $gsid]);
 386                 if (!DBA::isResult($gserver)) {
 387                         return;
 388                 }
 389
 390                 $blocked = Network::isUrlBlocked($gserver['url']);
 391                 if ($gserver['failed']) {
 392                         $fields = ['failed' => false, 'blocked' => $blocked, 'last_contact' => DateTimeFormat::utcNow()];
 393                         if (!empty($network) && !in_array($gserver['network'], Protocol::FEDERATED)) {
 394                                 $fields['network'] = $network;
 395                         }
 396                         self::update($fields, ['id' => $gsid]);
 397                         Logger::info('Reset failed status for server', ['url' => $gserver['url']]);
 398
 399                         if (strtotime($gserver['next_contact']) < time()) {
 400                                 UpdateGServer::add(Worker::PRIORITY_LOW, $gserver['url']);
 401                         }
 402                 } elseif ($blocked) {
 403                         self::setBlockedById($gsid);
 404                 } else {
 405                         self::setUnblockedById($gsid);
 406                 }
 407         }
 408
 409         /**
 410          * Set failed server status by gserver id
 411          *
 412          * @param int $gsid
 413          */
 414         public static function setFailureById(int $gsid)
 415         {
 416                 $gserver = DBA::selectFirst('gserver', ['url', 'failed', 'next_contact'], ['id' => $gsid]);
 417                 if (DBA::isResult($gserver) && !$gserver['failed']) {
 418                         self::update(['failed' => true, 'blocked' => Network::isUrlBlocked($gserver['url']), 'last_failure' => DateTimeFormat::utcNow()], ['id' => $gsid]);
 419                         Logger::info('Set failed status for server', ['url' => $gserver['url']]);
 420
 421                         if (strtotime($gserver['next_contact']) < time()) {
 422                                 UpdateGServer::add(Worker::PRIORITY_LOW, $gserver['url']);
 423                         }
 424                 }
 425         }
 426
 427         public static function setUnblockedById(int $gsid)
 428         {
 429                 $gserver = DBA::selectFirst('gserver', ['url'], ["(`blocked` OR `blocked` IS NULL) AND `id` = ?", $gsid]);
 430                 if (DBA::isResult($gserver)) {
 431                         self::update(['blocked' => false], ['id' => $gsid]);
 432                         Logger::info('Set unblocked status for server', ['url' => $gserver['url']]);
 433                 }
 434         }
 435
 436         public static function setBlockedById(int $gsid)
 437         {
 438                 $gserver = DBA::selectFirst('gserver', ['url'], ["(NOT `blocked` OR `blocked` IS NULL) AND `id` = ?", $gsid]);
 439                 if (DBA::isResult($gserver)) {
 440                         self::update(['blocked' => true, 'failed' => true], ['id' => $gsid]);
 441                         Logger::info('Set blocked status for server', ['url' => $gserver['url']]);
 442                 }
 443         }
 444
 445         public static function setBlockedByUrl(string $url)
 446         {
 447                 $gserver = DBA::selectFirst('gserver', ['url', 'id'], ["(NOT `blocked` OR `blocked` IS NULL) AND `nurl` = ?", Strings::normaliseLink($url)]);
 448                 if (DBA::isResult($gserver)) {
 449                         self::update(['blocked' => true, 'failed' => true], ['id' => $gserver['id']]);
 450                         Logger::info('Set blocked status for server', ['url' => $gserver['url']]);
 451                 }
 452         }
 453
 454         /**
 455          * Set failed server status
 456          *
 457          * @param string $url
 458          */
 459         public static function setFailureByUrl(string $url)
 460         {
 461                 $gserver = DBA::selectFirst('gserver', [], ['nurl' => Strings::normaliseLink($url)]);
 462                 if (DBA::isResult($gserver)) {
 463                         $next_update = self::getNextUpdateDate(false, $gserver['created'], $gserver['last_contact']);
 464                         self::update(['url' => $url, 'failed' => true, 'blocked' => Network::isUrlBlocked($url), 'last_failure' => DateTimeFormat::utcNow(),
 465                         'next_contact' => $next_update, 'network' => Protocol::PHANTOM, 'detection-method' => null],
 466                         ['nurl' => Strings::normaliseLink($url)]);
 467                         Logger::info('Set failed status for existing server', ['url' => $url]);
 468                         if (self::isDefunct($gserver)) {
 469                                 self::archiveContacts($gserver['id']);
 470                         }
 471                         return;
 472                 }
 473                 self::insert(['url' => $url, 'nurl' => Strings::normaliseLink($url),
 474                         'network' => Protocol::PHANTOM, 'created' => DateTimeFormat::utcNow(),
 475                         'failed' => true, 'last_failure' => DateTimeFormat::utcNow()]);
 476                 Logger::info('Set failed status for new server', ['url' => $url]);
 477         }
 478
 479         /**
 480          * Archive server related contacts and inboxes
 481          *
 482          * @param integer $gsid
 483          * @return void
 484          */
 485         private static function archiveContacts(int $gsid)
 486         {
 487                 Contact::update(['archive' => true], ['gsid' => $gsid]);
 488                 DBA::update('inbox-status', ['archive' => true], ['gsid' => $gsid]);
 489         }
 490
 491         /**
 492          * Remove unwanted content from the given URL
 493          *
 494          * @param string $dirtyUrl
 495          *
 496          * @return string cleaned URL
 497          * @throws Exception
 498          * @deprecated since 2023.03 Use cleanUri instead
 499          */
 500         public static function cleanURL(string $dirtyUrl): string
 501         {
 502                 try {
 503                         return (string)self::cleanUri(new Uri($dirtyUrl));
 504                 } catch (\Throwable $e) {
 505                         Logger::warning('Invalid URL', ['dirtyUrl' => $dirtyUrl]);
 506                         return '';
 507                 }
 508         }
 509
 510         /**
 511          * Remove unwanted content from the given URI
 512          *
 513          * @param UriInterface $dirtyUri
 514          *
 515          * @return UriInterface cleaned URI
 516          * @throws Exception
 517          */
 518         public static function cleanUri(UriInterface $dirtyUri): string
 519         {
 520                 return $dirtyUri
 521                         ->withUserInfo('')
 522                         ->withQuery('')
 523                         ->withFragment('')
 524                         ->withPath(
 525                                 preg_replace(
 526                                         '#(?:^|/)index\.php#',
 527                                         '',
 528                                         rtrim($dirtyUri->getPath(), '/')
 529                                 )
 530                         );
 531         }
 532
 533         /**
 534          * Detect server data (type, protocol, version number, ...)
 535          * The detected data is then updated or inserted in the gserver table.
 536          *
 537          * @param string  $url           URL of the given server
 538          * @param string  $network       Network value that is used, when detection failed
 539          * @param boolean $only_nodeinfo Only use nodeinfo for server detection
 540          *
 541          * @return boolean 'true' if server could be detected
 542          */
 543         private static function detect(string $url, string $network = '', bool $only_nodeinfo = false): bool
 544         {
 545                 Logger::info('Detect server type', ['server' => $url]);
 546
 547                 $original_url = $url;
 548
 549                 // Remove URL content that is not supposed to exist for a server url
 550                 $url = rtrim(self::cleanURL($url), '/');
 551                 if (empty($url)) {
 552                         Logger::notice('Empty URL.');
 553                         return false;
 554                 }
 555
 556                 // If the URL missmatches, then we mark the old entry as failure
 557                 if (!Strings::compareLink($url, $original_url)) {
 558                         self::setFailureByUrl($original_url);
 559                         if (!self::getID($url, true)) {
 560                                 self::detect($url, $network, $only_nodeinfo);
 561                         }
 562                         return false;
 563                 }
 564
 565                 $valid_url = Network::isUrlValid($url);
 566                 if (!$valid_url) {
 567                         self::setFailureByUrl($url);
 568                         return false;
 569                 } else {
 570                         $valid_url = rtrim($valid_url, '/');
 571                 }
 572
 573                 if (!Strings::compareLink($url, $valid_url)) {
 574                         // We only follow redirects when the path stays the same or the target url has no path.
 575                         // Some systems have got redirects on their landing page to a single account page. This check handles it.
 576                         if (((parse_url($url, PHP_URL_HOST) != parse_url($valid_url, PHP_URL_HOST)) && (parse_url($url, PHP_URL_PATH) == parse_url($valid_url, PHP_URL_PATH))) ||
 577                                 (((parse_url($url, PHP_URL_HOST) != parse_url($valid_url, PHP_URL_HOST)) || (parse_url($url, PHP_URL_PATH) != parse_url($valid_url, PHP_URL_PATH))) && empty(parse_url($valid_url, PHP_URL_PATH)))) {
 578                                 Logger::debug('Found redirect. Mark old entry as failure', ['old' => $url, 'new' => $valid_url]);
 579                                 self::setFailureByUrl($url);
 580                                 if (!self::getID($valid_url, true)) {
 581                                         self::detect($valid_url, $network, $only_nodeinfo);
 582                                 }
 583                                 return false;
 584                         }
 585
 586                         if ((parse_url($url, PHP_URL_HOST) != parse_url($valid_url, PHP_URL_HOST)) && (parse_url($url, PHP_URL_PATH) != parse_url($valid_url, PHP_URL_PATH)) &&
 587                                 (parse_url($url, PHP_URL_PATH) == '')) {
 588                                 Logger::debug('Found redirect. Mark old entry as failure and redirect to the basepath.', ['old' => $url, 'new' => $valid_url]);
 589                                 $parts = parse_url($valid_url);
 590                                 unset($parts['path']);
 591                                 $valid_url = (string)Uri::fromParts($parts);
 592
 593                                 self::setFailureByUrl($url);
 594                                 if (!self::getID($valid_url, true)) {
 595                                         self::detect($valid_url, $network, $only_nodeinfo);
 596                                 }
 597                                 return false;
 598                         }
 599                         Logger::debug('Found redirect, but ignore it.', ['old' => $url, 'new' => $valid_url]);
 600                 }
 601
 602                 if ((parse_url($url, PHP_URL_HOST) == parse_url($valid_url, PHP_URL_HOST)) &&
 603                         (parse_url($url, PHP_URL_PATH) == parse_url($valid_url, PHP_URL_PATH)) &&
 604                         (parse_url($url, PHP_URL_SCHEME) != parse_url($valid_url, PHP_URL_SCHEME))) {
 605                         $url = $valid_url;
 606                 }
 607
 608                 $in_webroot = empty(parse_url($url, PHP_URL_PATH));
 609
 610                 // When a nodeinfo is present, we don't need to dig further
 611                 $curlResult = DI::httpClient()->get($url . '/.well-known/x-nodeinfo2', HttpClientAccept::JSON);
 612                 if ($curlResult->isTimeout()) {
 613                         self::setFailureByUrl($url);
 614                         return false;
 615                 }
 616
 617                 $serverdata = self::parseNodeinfo210($curlResult);
 618                 if (empty($serverdata)) {
 619                         $curlResult = DI::httpClient()->get($url . '/.well-known/nodeinfo', HttpClientAccept::JSON);
 620                         $serverdata = self::fetchNodeinfo($url, $curlResult);
 621                 }
 622
 623                 if ($only_nodeinfo && empty($serverdata)) {
 624                         Logger::info('Invalid nodeinfo in nodeinfo-mode, server is marked as failure', ['url' => $url]);
 625                         self::setFailureByUrl($url);
 626                         return false;
 627                 } elseif (empty($serverdata)) {
 628                         $serverdata = ['detection-method' => self::DETECT_MANUAL, 'network' => Protocol::PHANTOM, 'platform' => '', 'version' => '', 'site_name' => '', 'info' => ''];
 629                 }
 630
 631                 // When there is no Nodeinfo, then use some protocol specific endpoints
 632                 if ($serverdata['network'] == Protocol::PHANTOM) {
 633                         if ($in_webroot) {
 634                                 // Fetch the landing page, possibly it reveals some data
 635                                 $accept = 'application/activity+json,application/ld+json,application/json,*/*;q=0.9';
 636                                 $curlResult = DI::httpClient()->get($url, $accept);
 637                                 if (!$curlResult->isSuccess() && $curlResult->getReturnCode() == '406') {
 638                                         $curlResult = DI::httpClient()->get($url, HttpClientAccept::HTML);
 639                                         $html_fetched = true;
 640                                 } else {
 641                                         $html_fetched = false;
 642                                 }
 643
 644                                 if ($curlResult->isSuccess()) {
 645                                         $json = json_decode($curlResult->getBody(), true);
 646                                         if (!empty($json) && is_array($json)) {
 647                                                 $data = self::fetchDataFromSystemActor($json, $serverdata);
 648                                                 $serverdata = $data['server'];
 649                                                 $systemactor = $data['actor'];
 650                                                 if (!$html_fetched && !in_array($serverdata['detection-method'], [self::DETECT_SYSTEM_ACTOR, self::DETECT_AP_COLLECTION])) {
 651                                                         $curlResult = DI::httpClient()->get($url, HttpClientAccept::HTML);
 652                                                 }
 653                                         } elseif (!$html_fetched && (strlen($curlResult->getBody()) < 1000)) {
 654                                                 $curlResult = DI::httpClient()->get($url, HttpClientAccept::HTML);
 655                                         }
 656
 657                                         if ($serverdata['detection-method'] != self::DETECT_SYSTEM_ACTOR) {
 658                                                 $serverdata = self::analyseRootHeader($curlResult, $serverdata);
 659                                                 $serverdata = self::analyseRootBody($curlResult, $serverdata);
 660                                         }
 661                                 }
 662
 663                                 if (!$curlResult->isSuccess() || empty($curlResult->getBody())) {
 664                                         self::setFailureByUrl($url);
 665                                         return false;
 666                                 }
 667
 668                                 if (($serverdata['network'] == Protocol::PHANTOM) || in_array($serverdata['detection-method'], self::DETECT_UNSPECIFIC)) {
 669                                         $serverdata = self::detectMastodonAlikes($url, $serverdata);
 670                                 }
 671                         }
 672
 673                         // All following checks are done for systems that always have got a "host-meta" endpoint.
 674                         // With this check we don't have to waste time and ressources for dead systems.
 675                         // Also this hopefully prevents us from receiving abuse messages.
 676                         if (($serverdata['network'] == Protocol::PHANTOM) || in_array($serverdata['detection-method'], self::DETECT_UNSPECIFIC)) {
 677                                 $validHostMeta = self::validHostMeta($url);
 678                         } else {
 679                                 $validHostMeta = false;
 680                         }
 681
 682                         if ($validHostMeta) {
 683                                 if (in_array($serverdata['detection-method'], [self::DETECT_MANUAL, self::DETECT_HEADER, self::DETECT_BODY])) {
 684                                         $serverdata['detection-method'] = self::DETECT_HOST_META;
 685                                 }
 686
 687                                 if (($serverdata['network'] == Protocol::PHANTOM) || in_array($serverdata['detection-method'], self::DETECT_UNSPECIFIC)) {
 688                                         $serverdata = self::detectFriendica($url, $serverdata);
 689                                 }
 690
 691                                 // The following systems have to be installed in the root directory.
 692                                 if ($in_webroot) {
 693                                         // the 'siteinfo.json' is some specific endpoint of Hubzilla and Red
 694                                         if (($serverdata['network'] == Protocol::PHANTOM) || in_array($serverdata['detection-method'], self::DETECT_UNSPECIFIC)) {
 695                                                 $serverdata = self::fetchSiteinfo($url, $serverdata);
 696                                         }
 697
 698                                         // The 'siteinfo.json' doesn't seem to be present on older Hubzilla installations, so we check other endpoints as well
 699                                         if (($serverdata['network'] == Protocol::PHANTOM) || in_array($serverdata['detection-method'], self::DETECT_UNSPECIFIC)) {
 700                                                 $serverdata = self::detectHubzilla($url, $serverdata);
 701                                         }
 702
 703                                         if (($serverdata['network'] == Protocol::PHANTOM) || in_array($serverdata['detection-method'], self::DETECT_UNSPECIFIC)) {
 704                                                 $serverdata = self::detectPeertube($url, $serverdata);
 705                                         }
 706
 707                                         if (($serverdata['network'] == Protocol::PHANTOM) || in_array($serverdata['detection-method'], self::DETECT_UNSPECIFIC)) {
 708                                                 $serverdata = self::detectGNUSocial($url, $serverdata);
 709                                         }
 710                                 }
 711                         } elseif (in_array($serverdata['platform'], ['friendica', 'friendika']) && in_array($serverdata['detection-method'], array_merge(self::DETECT_UNSPECIFIC, [self::DETECT_SYSTEM_ACTOR]))) {
 712                                 $serverdata = self::detectFriendica($url, $serverdata);
 713                         }
 714
 715                         if (($serverdata['network'] == Protocol::PHANTOM) || in_array($serverdata['detection-method'], self::DETECT_UNSPECIFIC)) {
 716                                 $serverdata = self::detectNextcloud($url, $serverdata, $validHostMeta);
 717                         }
 718
 719                         // When nodeinfo isn't present, we use the older 'statistics.json' endpoint
 720                         // Since this endpoint is only rarely used, we query it at a later time
 721                         if (in_array($serverdata['detection-method'], array_merge(self::DETECT_UNSPECIFIC, [self::DETECT_FRIENDICA, self::DETECT_CONFIG_JSON]))) {
 722                                 $serverdata = self::fetchStatistics($url, $serverdata);
 723                         }
 724                 }
 725
 726                 // When we hadn't been able to detect the network type, we use the hint from the parameter
 727                 if (($serverdata['network'] == Protocol::PHANTOM) && !empty($network)) {
 728                         $serverdata['network'] = $network;
 729                 }
 730
 731                 // Most servers aren't installed in a subdirectory, so we declare this entry as failed
 732                 if (($serverdata['network'] == Protocol::PHANTOM) && !empty(parse_url($url, PHP_URL_PATH)) && in_array($serverdata['detection-method'], [self::DETECT_MANUAL])) {
 733                         self::setFailureByUrl($url);
 734                         return false;
 735                 }
 736
 737                 $serverdata['url'] = $url;
 738                 $serverdata['nurl'] = Strings::normaliseLink($url);
 739
 740                 // We have to prevent an endless loop here.
 741                 // When a server is new, then there is no gserver entry yet.
 742                 // But in "detectNetworkViaContacts" it could happen that a contact is updated,
 743                 // and this can call this function here as well.
 744                 if (self::getID($url, true) && (in_array($serverdata['network'], [Protocol::PHANTOM, Protocol::FEED]) ||
 745                         in_array($serverdata['detection-method'], [self::DETECT_MANUAL, self::DETECT_HEADER, self::DETECT_BODY, self::DETECT_HOST_META]))) {
 746                         $serverdata = self::detectNetworkViaContacts($url, $serverdata);
 747                 }
 748
 749                 if (($serverdata['network'] == Protocol::PHANTOM) && in_array($serverdata['detection-method'], [self::DETECT_MANUAL, self::DETECT_BODY])) {
 750                         self::setFailureByUrl($url);
 751                         return false;
 752                 }
 753
 754                 // Detect the directory type
 755                 $serverdata['directory-type'] = self::DT_NONE;
 756
 757                 if (in_array($serverdata['network'], Protocol::FEDERATED)) {
 758                         $serverdata = self::checkMastodonDirectory($url, $serverdata);
 759
 760                         if ($serverdata['directory-type'] == self::DT_NONE) {
 761                                 $serverdata = self::checkPoCo($url, $serverdata);
 762                         }
 763                 }
 764
 765                 if ($serverdata['network'] == Protocol::ACTIVITYPUB) {
 766                         $serverdata = self::fetchWeeklyUsage($url, $serverdata);
 767                 }
 768
 769                 $serverdata['registered-users'] = $serverdata['registered-users'] ?? 0;
 770
 771                 // Numbers above a reasonable value (10 millions) are ignored
 772                 if ($serverdata['registered-users'] > 10000000) {
 773                         $serverdata['registered-users'] = 0;
 774                 }
 775
 776                 // On an active server there has to be at least a single user
 777                 if (!in_array($serverdata['network'], [Protocol::PHANTOM, Protocol::FEED]) && ($serverdata['registered-users'] <= 0)) {
 778                         $serverdata['registered-users'] = 1;
 779                 } elseif (in_array($serverdata['network'], [Protocol::PHANTOM, Protocol::FEED])) {
 780                         $serverdata['registered-users'] = 0;
 781                 }
 782
 783                 $serverdata['next_contact'] = self::getNextUpdateDate(true, '', '', in_array($serverdata['network'], [Protocol::PHANTOM, Protocol::FEED]));
 784                 $serverdata['last_contact'] = DateTimeFormat::utcNow();
 785                 $serverdata['failed']       = false;
 786                 $serverdata['blocked']      = false;
 787
 788                 $gserver = DBA::selectFirst('gserver', ['network'], ['nurl' => Strings::normaliseLink($url)]);
 789                 if (!DBA::isResult($gserver)) {
 790                         $serverdata['created'] = DateTimeFormat::utcNow();
 791                         $ret = self::insert($serverdata);
 792                         $id = DBA::lastInsertId();
 793                 } else {
 794                         $ret = self::update($serverdata, ['nurl' => $serverdata['nurl']]);
 795                         $gserver = DBA::selectFirst('gserver', ['id'], ['nurl' => $serverdata['nurl']]);
 796                         if (DBA::isResult($gserver)) {
 797                                 $id = $gserver['id'];
 798                         }
 799                 }
 800
 801                 // Count the number of known contacts from this server
 802                 if (!empty($id) && !in_array($serverdata['network'], [Protocol::PHANTOM, Protocol::FEED])) {
 803                         $apcontacts = DBA::count('apcontact', ['gsid' => $id]);
 804                         $contacts = DBA::count('contact', ['uid' => 0, 'gsid' => $id, 'failed' => false]);
 805                         $max_users = max($apcontacts, $contacts);
 806                         if ($max_users > $serverdata['registered-users']) {
 807                                 Logger::info('Update registered users', ['id' => $id, 'url' => $serverdata['nurl'], 'registered-users' => $max_users]);
 808                                 self::update(['registered-users' => $max_users], ['id' => $id]);
 809                         }
 810
 811                         if (empty($serverdata['active-month-users'])) {
 812                                 $contacts = DBA::count('contact', ["`uid` = ? AND `gsid` = ? AND NOT `failed` AND `last-item` > ?", 0, $id, DateTimeFormat::utc('now - 30 days')]);
 813                                 if ($contacts > 0) {
 814                                         Logger::info('Update monthly users', ['id' => $id, 'url' => $serverdata['nurl'], 'monthly-users' => $contacts]);
 815                                         self::update(['active-month-users' => $contacts], ['id' => $id]);
 816                                 }
 817                         }
 818
 819                         if (empty($serverdata['active-halfyear-users'])) {
 820                                 $contacts = DBA::count('contact', ["`uid` = ? AND `gsid` = ? AND NOT `failed` AND `last-item` > ?", 0, $id, DateTimeFormat::utc('now - 180 days')]);
 821                                 if ($contacts > 0) {
 822                                         Logger::info('Update halfyear users', ['id' => $id, 'url' => $serverdata['nurl'], 'halfyear-users' => $contacts]);
 823                                         self::update(['active-halfyear-users' => $contacts], ['id' => $id]);
 824                                 }
 825                         }
 826                 }
 827
 828                 if (in_array($serverdata['network'], [Protocol::DFRN, Protocol::DIASPORA])) {
 829                         self::discoverRelay($url);
 830                 }
 831
 832                 if (!empty($systemactor)) {
 833                         $contact = Contact::getByURL($systemactor, true, ['gsid', 'baseurl', 'id', 'network', 'url', 'name']);
 834                         Logger::debug('Fetched system actor',  ['url' => $url, 'gsid' => $id, 'contact' => $contact]);
 835                 }
 836
 837                 return $ret;
 838         }
 839
 840         /**
 841          * Fetch relay data from a given server url
 842          *
 843          * @param string $server_url address of the server
 844          *
 845          * @return void
 846          *
 847          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 848          */
 849         private static function discoverRelay(string $server_url)
 850         {
 851                 Logger::info('Discover relay data', ['server' => $server_url]);
 852
 853                 $curlResult = DI::httpClient()->get($server_url . '/.well-known/x-social-relay', HttpClientAccept::JSON);
 854                 if (!$curlResult->isSuccess()) {
 855                         return;
 856                 }
 857
 858                 $data = json_decode($curlResult->getBody(), true);
 859                 if (!is_array($data)) {
 860                         return;
 861                 }
 862
 863                 // Sanitize incoming data, see https://github.com/friendica/friendica/issues/8565
 864                 $data['subscribe'] = (bool)$data['subscribe'] ?? false;
 865
 866                 if (!$data['subscribe'] || empty($data['scope']) || !in_array(strtolower($data['scope']), ['all', 'tags'])) {
 867                         $data['scope'] = '';
 868                         $data['subscribe'] = false;
 869                         $data['tags'] = [];
 870                 }
 871
 872                 $gserver = DBA::selectFirst('gserver', ['id', 'url', 'network', 'relay-subscribe', 'relay-scope'], ['nurl' => Strings::normaliseLink($server_url)]);
 873                 if (!DBA::isResult($gserver)) {
 874                         return;
 875                 }
 876
 877                 if (($gserver['relay-subscribe'] != $data['subscribe']) || ($gserver['relay-scope'] != $data['scope'])) {
 878                         $fields = ['relay-subscribe' => $data['subscribe'], 'relay-scope' => $data['scope']];
 879                         self::update($fields, ['id' => $gserver['id']]);
 880                 }
 881
 882                 DBA::delete('gserver-tag', ['gserver-id' => $gserver['id']]);
 883
 884                 if ($data['scope'] == 'tags') {
 885                         // Avoid duplicates
 886                         $tags = [];
 887                         foreach ($data['tags'] as $tag) {
 888                                 $tag = mb_strtolower($tag);
 889                                 if (strlen($tag) < 100) {
 890                                         $tags[$tag] = $tag;
 891                                 }
 892                         }
 893
 894                         foreach ($tags as $tag) {
 895                                 DBA::insert('gserver-tag', ['gserver-id' => $gserver['id'], 'tag' => $tag], Database::INSERT_IGNORE);
 896                         }
 897                 }
 898
 899                 // Create or update the relay contact
 900                 $fields = [];
 901                 if (isset($data['protocols'])) {
 902                         if (isset($data['protocols']['diaspora'])) {
 903                                 $fields['network'] = Protocol::DIASPORA;
 904
 905                                 if (isset($data['protocols']['diaspora']['receive'])) {
 906                                         $fields['batch'] = $data['protocols']['diaspora']['receive'];
 907                                 } elseif (is_string($data['protocols']['diaspora'])) {
 908                                         $fields['batch'] = $data['protocols']['diaspora'];
 909                                 }
 910                         }
 911
 912                         if (isset($data['protocols']['dfrn'])) {
 913                                 $fields['network'] = Protocol::DFRN;
 914
 915                                 if (isset($data['protocols']['dfrn']['receive'])) {
 916                                         $fields['batch'] = $data['protocols']['dfrn']['receive'];
 917                                 } elseif (is_string($data['protocols']['dfrn'])) {
 918                                         $fields['batch'] = $data['protocols']['dfrn'];
 919                                 }
 920                         }
 921
 922                         if (isset($data['protocols']['activitypub'])) {
 923                                 $fields['network'] = Protocol::ACTIVITYPUB;
 924
 925                                 if (!empty($data['protocols']['activitypub']['actor'])) {
 926                                         $fields['url'] = $data['protocols']['activitypub']['actor'];
 927                                 }
 928                                 if (!empty($data['protocols']['activitypub']['receive'])) {
 929                                         $fields['batch'] = $data['protocols']['activitypub']['receive'];
 930                                 }
 931                         }
 932                 }
 933
 934                 Logger::info('Discovery ended', ['server' => $server_url, 'data' => $fields]);
 935
 936                 Relay::updateContact($gserver, $fields);
 937         }
 938
 939         /**
 940          * Fetch server data from '/statistics.json' on the given server
 941          *
 942          * @param string $url URL of the given server
 943          *
 944          * @return array server data
 945          */
 946         private static function fetchStatistics(string $url, array $serverdata): array
 947         {
 948                 $curlResult = DI::httpClient()->get($url . '/statistics.json', HttpClientAccept::JSON);
 949                 if (!$curlResult->isSuccess()) {
 950                         return $serverdata;
 951                 }
 952
 953                 $data = json_decode($curlResult->getBody(), true);
 954                 if (empty($data)) {
 955                         return $serverdata;
 956                 }
 957
 958                 // Some AP enabled systems return activity data that we don't expect here.
 959                 if (strpos($curlResult->getContentType(), 'application/activity+json') !== false) {
 960                         return $serverdata;
 961                 }
 962
 963                 $valid = false;
 964                 $old_serverdata = $serverdata;
 965
 966                 $serverdata['detection-method'] = self::DETECT_STATISTICS_JSON;
 967
 968                 if (!empty($data['version'])) {
 969                         $valid = true;
 970                         $serverdata['version'] = $data['version'];
 971                         // Version numbers on statistics.json are presented with additional info, e.g.:
 972                         // 0.6.3.0-p1702cc1c, 0.6.99.0-p1b9ab160 or 3.4.3-2-1191.
 973                         $serverdata['version'] = preg_replace('=(.+)-(.{4,})=ism', '$1', $serverdata['version']);
 974                 }
 975
 976                 if (!empty($data['name'])) {
 977                         $valid = true;
 978                         $serverdata['site_name'] = $data['name'];
 979                 }
 980
 981                 if (!empty($data['network'])) {
 982                         $valid = true;
 983                         $serverdata['platform'] = strtolower($data['network']);
 984
 985                         if ($serverdata['platform'] == 'diaspora') {
 986                                 $serverdata['network'] = Protocol::DIASPORA;
 987                         } elseif ($serverdata['platform'] == 'friendica') {
 988                                 $serverdata['network'] = Protocol::DFRN;
 989                         } elseif ($serverdata['platform'] == 'hubzilla') {
 990                                 $serverdata['network'] = Protocol::ZOT;
 991                         } elseif ($serverdata['platform'] == 'redmatrix') {
 992                                 $serverdata['network'] = Protocol::ZOT;
 993                         }
 994                 }
 995
 996                 if (!empty($data['total_users'])) {
 997                         $valid = true;
 998                         $serverdata['registered-users'] = max($data['total_users'], 1);
 999                 }
1000
1001                 if (!empty($data['active_users_monthly'])) {
1002                         $valid = true;
1003                         $serverdata['active-month-users'] = max($data['active_users_monthly'], 0);
1004                 }
1005
1006                 if (!empty($data['active_users_halfyear'])) {
1007                         $valid = true;
1008                         $serverdata['active-halfyear-users'] = max($data['active_users_halfyear'], 0);
1009                 }
1010
1011                 if (!empty($data['local_posts'])) {
1012                         $valid = true;
1013                         $serverdata['local-posts'] = max($data['local_posts'], 0);
1014                 }
1015
1016                 if (!empty($data['registrations_open'])) {
1017                         $serverdata['register_policy'] = Register::OPEN;
1018                 } else {
1019                         $serverdata['register_policy'] = Register::CLOSED;
1020                 }
1021
1022                 if (!$valid) {
1023                         return $old_serverdata;
1024                 }
1025
1026                 return $serverdata;
1027         }
1028
1029         /**
1030          * Detect server type by using the nodeinfo data
1031          *
1032          * @param string                  $url        address of the server
1033          * @param ICanHandleHttpResponses $httpResult
1034          *
1035          * @return array Server data
1036          *
1037          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
1038          */
1039         private static function fetchNodeinfo(string $url, ICanHandleHttpResponses $httpResult): array
1040         {
1041                 if (!$httpResult->isSuccess()) {
1042                         return [];
1043                 }
1044
1045                 $nodeinfo = json_decode($httpResult->getBody(), true);
1046
1047                 if (!is_array($nodeinfo) || empty($nodeinfo['links'])) {
1048                         return [];
1049                 }
1050
1051                 $nodeinfo1_url = '';
1052                 $nodeinfo2_url = '';
1053
1054                 foreach ($nodeinfo['links'] as $link) {
1055                         if (!is_array($link) || empty($link['rel']) || empty($link['href'])) {
1056                                 Logger::info('Invalid nodeinfo format', ['url' => $url]);
1057                                 continue;
1058                         }
1059                         if ($link['rel'] == 'http://nodeinfo.diaspora.software/ns/schema/1.0') {
1060                                 $nodeinfo1_url = $link['href'];
1061                         } elseif ($link['rel'] == 'http://nodeinfo.diaspora.software/ns/schema/2.0') {
1062                                 $nodeinfo2_url = $link['href'];
1063                         }
1064                 }
1065
1066                 if ($nodeinfo1_url . $nodeinfo2_url == '') {
1067                         return [];
1068                 }
1069
1070                 $server = [];
1071
1072                 if (!empty($nodeinfo2_url)) {
1073                         $server = self::parseNodeinfo2($nodeinfo2_url);
1074                 }
1075
1076                 if (empty($server) && !empty($nodeinfo1_url)) {
1077                         $server = self::parseNodeinfo1($nodeinfo1_url);
1078                 }
1079
1080                 return $server;
1081         }
1082
1083         /**
1084          * Parses Nodeinfo 1
1085          *
1086          * @param string $nodeinfo_url address of the nodeinfo path
1087          *
1088          * @return array Server data
1089          *
1090          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
1091          */
1092         private static function parseNodeinfo1(string $nodeinfo_url): array
1093         {
1094                 $curlResult = DI::httpClient()->get($nodeinfo_url, HttpClientAccept::JSON);
1095                 if (!$curlResult->isSuccess()) {
1096                         return [];
1097                 }
1098
1099                 $nodeinfo = json_decode($curlResult->getBody(), true);
1100
1101                 if (!is_array($nodeinfo)) {
1102                         return [];
1103                 }
1104
1105                 $server = ['detection-method' => self::DETECT_NODEINFO_1,
1106                         'register_policy' => Register::CLOSED];
1107
1108                 if (!empty($nodeinfo['openRegistrations'])) {
1109                         $server['register_policy'] = Register::OPEN;
1110                 }
1111
1112                 if (is_array($nodeinfo['software'])) {
1113                         if (!empty($nodeinfo['software']['name'])) {
1114                                 $server['platform'] = strtolower($nodeinfo['software']['name']);
1115                         }
1116
1117                         if (!empty($nodeinfo['software']['version'])) {
1118                                 $server['version'] = $nodeinfo['software']['version'];
1119                                 // Version numbers on Nodeinfo are presented with additional info, e.g.:
1120                                 // 0.6.3.0-p1702cc1c, 0.6.99.0-p1b9ab160 or 3.4.3-2-1191.
1121                                 $server['version'] = preg_replace('=(.+)-(.{4,})=ism', '$1', $server['version']);
1122                         }
1123                 }
1124
1125                 if (!empty($nodeinfo['metadata']['nodeName'])) {
1126                         $server['site_name'] = $nodeinfo['metadata']['nodeName'];
1127                 }
1128
1129                 if (!empty($nodeinfo['usage']['users']['total'])) {
1130                         $server['registered-users'] = max($nodeinfo['usage']['users']['total'], 1);
1131                 }
1132
1133                 if (!empty($nodeinfo['usage']['users']['activeMonth'])) {
1134                         $server['active-month-users'] = max($nodeinfo['usage']['users']['activeMonth'], 0);
1135                 }
1136
1137                 if (!empty($nodeinfo['usage']['users']['activeHalfyear'])) {
1138                         $server['active-halfyear-users'] = max($nodeinfo['usage']['users']['activeHalfyear'], 0);
1139                 }
1140
1141                 if (!empty($nodeinfo['usage']['localPosts'])) {
1142                         $server['local-posts'] = max($nodeinfo['usage']['localPosts'], 0);
1143                 }
1144
1145                 if (!empty($nodeinfo['usage']['localComments'])) {
1146                         $server['local-comments'] = max($nodeinfo['usage']['localComments'], 0);
1147                 }
1148
1149                 if (!empty($nodeinfo['protocols']['inbound']) && is_array($nodeinfo['protocols']['inbound'])) {
1150                         $protocols = [];
1151                         foreach ($nodeinfo['protocols']['inbound'] as $protocol) {
1152                                 $protocols[$protocol] = true;
1153                         }
1154
1155                         if (!empty($protocols['friendica'])) {
1156                                 $server['network'] = Protocol::DFRN;
1157                         } elseif (!empty($protocols['activitypub'])) {
1158                                 $server['network'] = Protocol::ACTIVITYPUB;
1159                         } elseif (!empty($protocols['diaspora'])) {
1160                                 $server['network'] = Protocol::DIASPORA;
1161                         } elseif (!empty($protocols['ostatus'])) {
1162                                 $server['network'] = Protocol::OSTATUS;
1163                         } elseif (!empty($protocols['gnusocial'])) {
1164                                 $server['network'] = Protocol::OSTATUS;
1165                         } elseif (!empty($protocols['zot'])) {
1166                                 $server['network'] = Protocol::ZOT;
1167                         }
1168                 }
1169
1170                 if (empty($server)) {
1171                         return [];
1172                 }
1173
1174                 if (empty($server['network'])) {
1175                         $server['network'] = Protocol::PHANTOM;
1176                 }
1177
1178                 return $server;
1179         }
1180
1181         /**
1182          * Parses Nodeinfo 2
1183          *
1184          * @see https://git.feneas.org/jaywink/nodeinfo2
1185          *
1186          * @param string $nodeinfo_url address of the nodeinfo path
1187          *
1188          * @return array Server data
1189          *
1190          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
1191          */
1192         private static function parseNodeinfo2(string $nodeinfo_url): array
1193         {
1194                 $curlResult = DI::httpClient()->get($nodeinfo_url, HttpClientAccept::JSON);
1195                 if (!$curlResult->isSuccess()) {
1196                         return [];
1197                 }
1198
1199                 $nodeinfo = json_decode($curlResult->getBody(), true);
1200                 if (!is_array($nodeinfo)) {
1201                         return [];
1202                 }
1203
1204                 $server = [
1205                         'detection-method' => self::DETECT_NODEINFO_2,
1206                         'register_policy' => Register::CLOSED,
1207                         'platform' => 'unknown',
1208                 ];
1209
1210                 if (!empty($nodeinfo['openRegistrations'])) {
1211                         $server['register_policy'] = Register::OPEN;
1212                 }
1213
1214                 if (!empty($nodeinfo['software'])) {
1215                         if (isset($nodeinfo['software']['name'])) {
1216                                 $server['platform'] = strtolower($nodeinfo['software']['name']);
1217                         }
1218
1219                         if (!empty($nodeinfo['software']['version']) && isset($server['platform'])) {
1220                                 $server['version'] = $nodeinfo['software']['version'];
1221                                 // Version numbers on Nodeinfo are presented with additional info, e.g.:
1222                                 // 0.6.3.0-p1702cc1c, 0.6.99.0-p1b9ab160 or 3.4.3-2-1191.
1223                                 $server['version'] = preg_replace('=(.+)-(.{4,})=ism', '$1', $server['version']);
1224
1225                                 // qoto advertises itself as Mastodon
1226                                 if (($server['platform'] == 'mastodon') && substr($nodeinfo['software']['version'], -5) == '-qoto') {
1227                                         $server['platform'] = 'qoto';
1228                                 }
1229                         }
1230                 }
1231
1232                 if (!empty($nodeinfo['metadata']['nodeName'])) {
1233                         $server['site_name'] = $nodeinfo['metadata']['nodeName'];
1234                 }
1235
1236                 if (!empty($nodeinfo['usage']['users']['total'])) {
1237                         $server['registered-users'] = max($nodeinfo['usage']['users']['total'], 1);
1238                 }
1239
1240                 if (!empty($nodeinfo['usage']['users']['activeMonth'])) {
1241                         $server['active-month-users'] = max($nodeinfo['usage']['users']['activeMonth'], 0);
1242                 }
1243
1244                 if (!empty($nodeinfo['usage']['users']['activeHalfyear'])) {
1245                         $server['active-halfyear-users'] = max($nodeinfo['usage']['users']['activeHalfyear'], 0);
1246                 }
1247
1248                 if (!empty($nodeinfo['usage']['localPosts'])) {
1249                         $server['local-posts'] = max($nodeinfo['usage']['localPosts'], 0);
1250                 }
1251
1252                 if (!empty($nodeinfo['usage']['localComments'])) {
1253                         $server['local-comments'] = max($nodeinfo['usage']['localComments'], 0);
1254                 }
1255
1256                 if (!empty($nodeinfo['protocols'])) {
1257                         $protocols = [];
1258                         foreach ($nodeinfo['protocols'] as $protocol) {
1259                                 if (is_string($protocol)) {
1260                                         $protocols[$protocol] = true;
1261                                 }
1262                         }
1263
1264                         if (!empty($protocols['dfrn'])) {
1265                                 $server['network'] = Protocol::DFRN;
1266                         } elseif (!empty($protocols['activitypub'])) {
1267                                 $server['network'] = Protocol::ACTIVITYPUB;
1268                         } elseif (!empty($protocols['diaspora'])) {
1269                                 $server['network'] = Protocol::DIASPORA;
1270                         } elseif (!empty($protocols['ostatus'])) {
1271                                 $server['network'] = Protocol::OSTATUS;
1272                         } elseif (!empty($protocols['gnusocial'])) {
1273                                 $server['network'] = Protocol::OSTATUS;
1274                         } elseif (!empty($protocols['zot'])) {
1275                                 $server['network'] = Protocol::ZOT;
1276                         }
1277                 }
1278
1279                 if (empty($server)) {
1280                         return [];
1281                 }
1282
1283                 if (empty($server['network'])) {
1284                         $server['network'] = Protocol::PHANTOM;
1285                 }
1286
1287                 return $server;
1288         }
1289
1290         /**
1291          * Parses NodeInfo2 protocol 1.0
1292          *
1293          * @see https://github.com/jaywink/nodeinfo2/blob/master/PROTOCOL.md
1294          *
1295          * @param string $nodeinfo_url address of the nodeinfo path
1296          *
1297          * @return array Server data
1298          *
1299          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
1300          */
1301         private static function parseNodeinfo210(ICanHandleHttpResponses $httpResult): array
1302         {
1303                 if (!$httpResult->isSuccess()) {
1304                         return [];
1305                 }
1306
1307                 $nodeinfo = json_decode($httpResult->getBody(), true);
1308
1309                 if (!is_array($nodeinfo)) {
1310                         return [];
1311                 }
1312
1313                 $server = ['detection-method' => self::DETECT_NODEINFO_210,
1314                         'register_policy' => Register::CLOSED];
1315
1316                 if (!empty($nodeinfo['openRegistrations'])) {
1317                         $server['register_policy'] = Register::OPEN;
1318                 }
1319
1320                 if (!empty($nodeinfo['server'])) {
1321                         if (!empty($nodeinfo['server']['software'])) {
1322                                 $server['platform'] = strtolower($nodeinfo['server']['software']);
1323                         }
1324
1325                         if (!empty($nodeinfo['server']['version'])) {
1326                                 $server['version'] = $nodeinfo['server']['version'];
1327                                 // Version numbers on Nodeinfo are presented with additional info, e.g.:
1328                                 // 0.6.3.0-p1702cc1c, 0.6.99.0-p1b9ab160 or 3.4.3-2-1191.
1329                                 $server['version'] = preg_replace('=(.+)-(.{4,})=ism', '$1', $server['version']);
1330                         }
1331
1332                         if (!empty($nodeinfo['server']['name'])) {
1333                                 $server['site_name'] = $nodeinfo['server']['name'];
1334                         }
1335                 }
1336
1337                 if (!empty($nodeinfo['usage']['users']['total'])) {
1338                         $server['registered-users'] = max($nodeinfo['usage']['users']['total'], 1);
1339                 }
1340
1341                 if (!empty($nodeinfo['usage']['users']['activeMonth'])) {
1342                         $server['active-month-users'] = max($nodeinfo['usage']['users']['activeMonth'], 0);
1343                 }
1344
1345                 if (!empty($nodeinfo['usage']['users']['activeHalfyear'])) {
1346                         $server['active-halfyear-users'] = max($nodeinfo['usage']['users']['activeHalfyear'], 0);
1347                 }
1348
1349                 if (!empty($nodeinfo['usage']['localPosts'])) {
1350                         $server['local-posts'] = max($nodeinfo['usage']['localPosts'], 0);
1351                 }
1352
1353                 if (!empty($nodeinfo['usage']['localComments'])) {
1354                         $server['local-comments'] = max($nodeinfo['usage']['localComments'], 0);
1355                 }
1356
1357                 if (!empty($nodeinfo['protocols'])) {
1358                         $protocols = [];
1359                         foreach ($nodeinfo['protocols'] as $protocol) {
1360                                 if (is_string($protocol)) {
1361                                         $protocols[$protocol] = true;
1362                                 }
1363                         }
1364
1365                         if (!empty($protocols['dfrn'])) {
1366                                 $server['network'] = Protocol::DFRN;
1367                         } elseif (!empty($protocols['activitypub'])) {
1368                                 $server['network'] = Protocol::ACTIVITYPUB;
1369                         } elseif (!empty($protocols['diaspora'])) {
1370                                 $server['network'] = Protocol::DIASPORA;
1371                         } elseif (!empty($protocols['ostatus'])) {
1372                                 $server['network'] = Protocol::OSTATUS;
1373                         } elseif (!empty($protocols['gnusocial'])) {
1374                                 $server['network'] = Protocol::OSTATUS;
1375                         } elseif (!empty($protocols['zot'])) {
1376                                 $server['network'] = Protocol::ZOT;
1377                         }
1378                 }
1379
1380                 if (empty($server) || empty($server['platform'])) {
1381                         return [];
1382                 }
1383
1384                 if (empty($server['network'])) {
1385                         $server['network'] = Protocol::PHANTOM;
1386                 }
1387
1388                 return $server;
1389         }
1390
1391         /**
1392          * Fetch server information from a 'siteinfo.json' file on the given server
1393          *
1394          * @param string $url        URL of the given server
1395          * @param array  $serverdata array with server data
1396          *
1397          * @return array server data
1398          */
1399         private static function fetchSiteinfo(string $url, array $serverdata): array
1400         {
1401                 $curlResult = DI::httpClient()->get($url . '/siteinfo.json', HttpClientAccept::JSON);
1402                 if (!$curlResult->isSuccess()) {
1403                         return $serverdata;
1404                 }
1405
1406                 $data = json_decode($curlResult->getBody(), true);
1407                 if (empty($data)) {
1408                         return $serverdata;
1409                 }
1410
1411                 if (in_array($serverdata['detection-method'], self::DETECT_UNSPECIFIC)) {
1412                         $serverdata['detection-method'] = self::DETECT_SITEINFO_JSON;
1413                 }
1414
1415                 if (!empty($data['url'])) {
1416                         $serverdata['platform'] = strtolower($data['platform']);
1417                         $serverdata['version'] = $data['version'] ?? 'N/A';
1418                 }
1419
1420                 if (!empty($data['plugins'])) {
1421                         if (in_array('pubcrawl', $data['plugins'])) {
1422                                 $serverdata['network'] = Protocol::ACTIVITYPUB;
1423                         } elseif (in_array('diaspora', $data['plugins'])) {
1424                                 $serverdata['network'] = Protocol::DIASPORA;
1425                         } elseif (in_array('gnusoc', $data['plugins'])) {
1426                                 $serverdata['network'] = Protocol::OSTATUS;
1427                         } else {
1428                                 $serverdata['network'] = Protocol::ZOT;
1429                         }
1430                 }
1431
1432                 if (!empty($data['site_name'])) {
1433                         $serverdata['site_name'] = $data['site_name'];
1434                 }
1435
1436                 if (!empty($data['channels_total'])) {
1437                         $serverdata['registered-users'] = max($data['channels_total'], 1);
1438                 }
1439
1440                 if (!empty($data['channels_active_monthly'])) {
1441                         $serverdata['active-month-users'] = max($data['channels_active_monthly'], 0);
1442                 }
1443
1444                 if (!empty($data['channels_active_halfyear'])) {
1445                         $serverdata['active-halfyear-users'] = max($data['channels_active_halfyear'], 0);
1446                 }
1447
1448                 if (!empty($data['local_posts'])) {
1449                         $serverdata['local-posts'] = max($data['local_posts'], 0);
1450                 }
1451
1452                 if (!empty($data['local_comments'])) {
1453                         $serverdata['local-comments'] = max($data['local_comments'], 0);
1454                 }
1455
1456                 if (!empty($data['register_policy'])) {
1457                         switch ($data['register_policy']) {
1458                                 case 'REGISTER_OPEN':
1459                                         $serverdata['register_policy'] = Register::OPEN;
1460                                         break;
1461
1462                                 case 'REGISTER_APPROVE':
1463                                         $serverdata['register_policy'] = Register::APPROVE;
1464                                         break;
1465
1466                                 case 'REGISTER_CLOSED':
1467                                 default:
1468                                         $serverdata['register_policy'] = Register::CLOSED;
1469                                         break;
1470                         }
1471                 }
1472
1473                 return $serverdata;
1474         }
1475
1476         /**
1477          * Fetches server data via an ActivityPub account with url of that server
1478          *
1479          * @param string $url        URL of the given server
1480          * @param array  $serverdata array with server data
1481          *
1482          * @return array server data
1483          *
1484          * @throws Exception
1485          */
1486         private static function fetchDataFromSystemActor(array $data, array $serverdata): array
1487         {
1488                 if (empty($data)) {
1489                         return ['server' => $serverdata, 'actor' => ''];
1490                 }
1491
1492                 $actor = JsonLD::compact($data, false);
1493                 if (in_array(JsonLD::fetchElement($actor, '@type'), ActivityPub\Receiver::ACCOUNT_TYPES)) {
1494                         $serverdata['network'] = Protocol::ACTIVITYPUB;
1495                         $serverdata['site_name'] = JsonLD::fetchElement($actor, 'as:name', '@value');
1496                         $serverdata['info'] = JsonLD::fetchElement($actor, 'as:summary', '@value');
1497                         if (!empty($actor['as:generator'])) {
1498                                 $generator = explode(' ', JsonLD::fetchElement($actor['as:generator'], 'as:name', '@value'));
1499                                 $serverdata['platform'] = strtolower(array_shift($generator));
1500                                 $serverdata['detection-method'] = self::DETECT_SYSTEM_ACTOR;
1501                         } else {
1502                                 $serverdata['detection-method'] = self::DETECT_AP_ACTOR;
1503                         }
1504                         return ['server' => $serverdata, 'actor' => $actor['@id']];
1505                 } elseif ((JsonLD::fetchElement($actor, '@type') == 'as:Collection')) {
1506                         // By now only Ktistec seems to provide collections this way
1507                         $serverdata['platform'] = 'ktistec';
1508                         $serverdata['network'] = Protocol::ACTIVITYPUB;
1509                         $serverdata['detection-method'] = self::DETECT_AP_COLLECTION;
1510
1511                         $actors = JsonLD::fetchElementArray($actor, 'as:items');
1512                         if (!empty($actors) && !empty($actors[0]['@id'])) {
1513                                 $actor_url = $actor['@id'] . $actors[0]['@id'];
1514                         } else {
1515                                 $actor_url = '';
1516                         }
1517
1518                         return ['server' => $serverdata, 'actor' => $actor_url];
1519                 }
1520                 return ['server' => $serverdata, 'actor' => ''];
1521         }
1522
1523         /**
1524          * Checks if the server contains a valid host meta file
1525          *
1526          * @param string $url URL of the given server
1527          *
1528          * @return boolean 'true' if the server seems to be vital
1529          */
1530         private static function validHostMeta(string $url): bool
1531         {
1532                 $xrd_timeout = DI::config()->get('system', 'xrd_timeout');
1533                 $curlResult = DI::httpClient()->get($url . Probe::HOST_META, HttpClientAccept::XRD_XML, [HttpClientOptions::TIMEOUT => $xrd_timeout]);
1534                 if (!$curlResult->isSuccess()) {
1535                         return false;
1536                 }
1537
1538                 $xrd = XML::parseString($curlResult->getBody(), true);
1539                 if (!is_object($xrd)) {
1540                         return false;
1541                 }
1542
1543                 $elements = XML::elementToArray($xrd);
1544                 if (empty($elements) || empty($elements['xrd']) || empty($elements['xrd']['link'])) {
1545                         return false;
1546                 }
1547
1548                 $valid = false;
1549                 foreach ($elements['xrd']['link'] as $link) {
1550                         // When there is more than a single "link" element, the array looks slightly different
1551                         if (!empty($link['@attributes'])) {
1552                                 $link = $link['@attributes'];
1553                         }
1554
1555                         if (empty($link['rel']) || empty($link['template'])) {
1556                                 continue;
1557                         }
1558
1559                         if ($link['rel'] == 'lrdd') {
1560                                 // When the webfinger host is the same like the system host, it should be ok.
1561                                 $valid = (parse_url($url, PHP_URL_HOST) == parse_url($link['template'], PHP_URL_HOST));
1562                         }
1563                 }
1564
1565                 return $valid;
1566         }
1567
1568         /**
1569          * Detect the network of the given server via their known contacts
1570          *
1571          * @param string $url        URL of the given server
1572          * @param array  $serverdata array with server data
1573          *
1574          * @return array server data
1575          */
1576         private static function detectNetworkViaContacts(string $url, array $serverdata): array
1577         {
1578                 $contacts = [];
1579
1580                 $nurl = Strings::normaliseLink($url);
1581
1582                 $apcontacts = DBA::select('apcontact', ['url'], ['baseurl' => [$url, $nurl]]);
1583                 while ($apcontact = DBA::fetch($apcontacts)) {
1584                         $contacts[Strings::normaliseLink($apcontact['url'])] = $apcontact['url'];
1585                 }
1586                 DBA::close($apcontacts);
1587
1588                 $pcontacts = DBA::select('contact', ['url', 'nurl'], ['uid' => 0, 'baseurl' => [$url, $nurl]]);
1589                 while ($pcontact = DBA::fetch($pcontacts)) {
1590                         $contacts[$pcontact['nurl']] = $pcontact['url'];
1591                 }
1592                 DBA::close($pcontacts);
1593
1594                 if (empty($contacts)) {
1595                         return $serverdata;
1596                 }
1597
1598                 $time = time();
1599                 foreach ($contacts as $contact) {
1600                         // Endlosschleife verhindern wegen gsid!
1601                         $data = Probe::uri($contact);
1602                         if (in_array($data['network'], Protocol::FEDERATED)) {
1603                                 $serverdata['network'] = $data['network'];
1604
1605                                 if (in_array($serverdata['detection-method'], self::DETECT_UNSPECIFIC)) {
1606                                         $serverdata['detection-method'] = self::DETECT_CONTACTS;
1607                                 }
1608                                 break;
1609                         } elseif ((time() - $time) > 10) {
1610                                 // To reduce the stress on remote systems we probe a maximum of 10 seconds
1611                                 break;
1612                         }
1613                 }
1614
1615                 return $serverdata;
1616         }
1617
1618         /**
1619          * Checks if the given server does have a '/poco' endpoint.
1620          * This is used for the 'PortableContact' functionality,
1621          * which is used by both Friendica and Hubzilla.
1622          *
1623          * @param string $url        URL of the given server
1624          * @param array  $serverdata array with server data
1625          *
1626          * @return array server data
1627          */
1628         private static function checkPoCo(string $url, array $serverdata): array
1629         {
1630                 $serverdata['poco'] = '';
1631
1632                 $curlResult = DI::httpClient()->get($url . '/poco', HttpClientAccept::JSON);
1633                 if (!$curlResult->isSuccess()) {
1634                         return $serverdata;
1635                 }
1636
1637                 $data = json_decode($curlResult->getBody(), true);
1638                 if (empty($data)) {
1639                         return $serverdata;
1640                 }
1641
1642                 if (!empty($data['totalResults'])) {
1643                         $registeredUsers = $serverdata['registered-users'] ?? 0;
1644                         $serverdata['registered-users'] = max($data['totalResults'], $registeredUsers, 1);
1645                         $serverdata['directory-type'] = self::DT_POCO;
1646                         $serverdata['poco'] = $url . '/poco';
1647                 }
1648
1649                 return $serverdata;
1650         }
1651
1652         /**
1653          * Checks if the given server does have a Mastodon style directory endpoint.
1654          *
1655          * @param string $url        URL of the given server
1656          * @param array  $serverdata array with server data
1657          *
1658          * @return array server data
1659          */
1660         public static function checkMastodonDirectory(string $url, array $serverdata): array
1661         {
1662                 $curlResult = DI::httpClient()->get($url . '/api/v1/directory?limit=1', HttpClientAccept::JSON);
1663                 if (!$curlResult->isSuccess()) {
1664                         return $serverdata;
1665                 }
1666
1667                 $data = json_decode($curlResult->getBody(), true);
1668                 if (empty($data)) {
1669                         return $serverdata;
1670                 }
1671
1672                 if (count($data) == 1) {
1673                         $serverdata['directory-type'] = self::DT_MASTODON;
1674                 }
1675
1676                 return $serverdata;
1677         }
1678
1679         /**
1680          * Detects Peertube via their known endpoint
1681          *
1682          * @param string $url        URL of the given server
1683          * @param array  $serverdata array with server data
1684          *
1685          * @return array server data
1686          */
1687         private static function detectPeertube(string $url, array $serverdata): array
1688         {
1689                 $curlResult = DI::httpClient()->get($url . '/api/v1/config', HttpClientAccept::JSON);
1690                 if (!$curlResult->isSuccess() || ($curlResult->getBody() == '')) {
1691                         return $serverdata;
1692                 }
1693
1694                 $data = json_decode($curlResult->getBody(), true);
1695                 if (empty($data)) {
1696                         return $serverdata;
1697                 }
1698
1699                 if (!empty($data['instance']) && !empty($data['serverVersion'])) {
1700                         $serverdata['platform'] = 'peertube';
1701                         $serverdata['version'] = $data['serverVersion'];
1702                         $serverdata['network'] = Protocol::ACTIVITYPUB;
1703
1704                         if (!empty($data['instance']['name'])) {
1705                                 $serverdata['site_name'] = $data['instance']['name'];
1706                         }
1707
1708                         if (!empty($data['instance']['shortDescription'])) {
1709                                 $serverdata['info'] = $data['instance']['shortDescription'];
1710                         }
1711
1712                         if (!empty($data['signup'])) {
1713                                 if (!empty($data['signup']['allowed'])) {
1714                                         $serverdata['register_policy'] = Register::OPEN;
1715                                 }
1716                         }
1717
1718                         if (in_array($serverdata['detection-method'], self::DETECT_UNSPECIFIC)) {
1719                                 $serverdata['detection-method'] = self::DETECT_V1_CONFIG;
1720                         }
1721                 }
1722
1723                 return $serverdata;
1724         }
1725
1726         /**
1727          * Detects the version number of a given server when it was a NextCloud installation
1728          *
1729          * @param string $url        URL of the given server
1730          * @param array  $serverdata array with server data
1731          * @param bool   $validHostMeta
1732          *
1733          * @return array server data
1734          */
1735         private static function detectNextcloud(string $url, array $serverdata, bool $validHostMeta): array
1736         {
1737                 $curlResult = DI::httpClient()->get($url . '/status.php', HttpClientAccept::JSON);
1738                 if (!$curlResult->isSuccess() || ($curlResult->getBody() == '')) {
1739                         return $serverdata;
1740                 }
1741
1742                 $data = json_decode($curlResult->getBody(), true);
1743                 if (empty($data)) {
1744                         return $serverdata;
1745                 }
1746
1747                 if (!empty($data['version'])) {
1748                         $serverdata['platform'] = 'nextcloud';
1749                         $serverdata['version'] = $data['version'];
1750
1751                         if ($validHostMeta) {
1752                                 $serverdata['network'] = Protocol::ACTIVITYPUB;
1753                         }
1754
1755                         if (in_array($serverdata['detection-method'], self::DETECT_UNSPECIFIC)) {
1756                                 $serverdata['detection-method'] = self::DETECT_STATUS_PHP;
1757                         }
1758                 }
1759
1760                 return $serverdata;
1761         }
1762
1763         /**
1764          * Fetches weekly usage data
1765          *
1766          * @param string $url        URL of the given server
1767          * @param array  $serverdata array with server data
1768          *
1769          * @return array server data
1770          */
1771         private static function fetchWeeklyUsage(string $url, array $serverdata): array
1772         {
1773                 $curlResult = DI::httpClient()->get($url . '/api/v1/instance/activity', HttpClientAccept::JSON);
1774                 if (!$curlResult->isSuccess() || ($curlResult->getBody() == '')) {
1775                         return $serverdata;
1776                 }
1777
1778                 $data = json_decode($curlResult->getBody(), true);
1779                 if (empty($data)) {
1780                         return $serverdata;
1781                 }
1782
1783                 $current_week = [];
1784                 foreach ($data as $week) {
1785                         // Use only data from a full week
1786                         if (empty($week['week']) || (time() - $week['week']) < 7 * 24 * 60 * 60) {
1787                                 continue;
1788                         }
1789
1790                         // Most likely the data is sorted correctly. But we better are safe than sorry
1791                         if (empty($current_week['week']) || ($current_week['week'] < $week['week'])) {
1792                                 $current_week = $week;
1793                         }
1794                 }
1795
1796                 if (!empty($current_week['logins'])) {
1797                         $serverdata['active-week-users'] = max($current_week['logins'], 0);
1798                 }
1799
1800                 return $serverdata;
1801         }
1802
1803         /**
1804          * Detects data from a given server url if it was a mastodon alike system
1805          *
1806          * @param string $url        URL of the given server
1807          * @param array  $serverdata array with server data
1808          *
1809          * @return array server data
1810          */
1811         private static function detectMastodonAlikes(string $url, array $serverdata): array
1812         {
1813                 $curlResult = DI::httpClient()->get($url . '/api/v1/instance', HttpClientAccept::JSON);
1814                 if (!$curlResult->isSuccess() || ($curlResult->getBody() == '')) {
1815                         return $serverdata;
1816                 }
1817
1818                 $data = json_decode($curlResult->getBody(), true);
1819                 if (empty($data)) {
1820                         return $serverdata;
1821                 }
1822
1823                 $valid = false;
1824
1825                 if (!empty($data['version'])) {
1826                         $serverdata['platform'] = 'mastodon';
1827                         $serverdata['version'] = $data['version'] ?? '';
1828                         $serverdata['network'] = Protocol::ACTIVITYPUB;
1829                         $valid = true;
1830                 }
1831
1832                 if (!empty($data['title'])) {
1833                         $serverdata['site_name'] = $data['title'];
1834                 }
1835
1836                 if (!empty($data['title']) && empty($serverdata['platform']) && ($serverdata['network'] == Protocol::PHANTOM)) {
1837                         $serverdata['platform'] = 'mastodon';
1838                         $serverdata['network'] = Protocol::ACTIVITYPUB;
1839                         $valid = true;
1840                 }
1841
1842                 if (!empty($data['description'])) {
1843                         $serverdata['info'] = trim($data['description']);
1844                 }
1845
1846                 if (!empty($data['stats']['user_count'])) {
1847                         $serverdata['registered-users'] = max($data['stats']['user_count'], 1);
1848                 }
1849
1850                 if (!empty($serverdata['version']) && preg_match('/.*?\(compatible;\s(.*)\s(.*)\)/ism', $serverdata['version'], $matches)) {
1851                         $serverdata['platform'] = strtolower($matches[1]);
1852                         $serverdata['version'] = $matches[2];
1853                         $valid = true;
1854                 }
1855
1856                 if (!empty($serverdata['version']) && strstr(strtolower($serverdata['version']), 'pleroma')) {
1857                         $serverdata['platform'] = 'pleroma';
1858                         $serverdata['version'] = trim(str_ireplace('pleroma', '', $serverdata['version']));
1859                         $valid = true;
1860                 }
1861
1862                 if (!empty($serverdata['platform']) && strstr($serverdata['platform'], 'pleroma')) {
1863                         $serverdata['version'] = trim(str_ireplace('pleroma', '', $serverdata['platform']));
1864                         $serverdata['platform'] = 'pleroma';
1865                         $valid = true;
1866                 }
1867
1868                 if ($valid && in_array($serverdata['detection-method'], self::DETECT_UNSPECIFIC)) {
1869                         $serverdata['detection-method'] = self::DETECT_MASTODON_API;
1870                 }
1871
1872                 return $serverdata;
1873         }
1874
1875         /**
1876          * Detects data from typical Hubzilla endpoints
1877          *
1878          * @param string $url        URL of the given server
1879          * @param array  $serverdata array with server data
1880          *
1881          * @return array server data
1882          */
1883         private static function detectHubzilla(string $url, array $serverdata): array
1884         {
1885                 $curlResult = DI::httpClient()->get($url . '/api/statusnet/config.json', HttpClientAccept::JSON);
1886                 if (!$curlResult->isSuccess() || ($curlResult->getBody() == '')) {
1887                         return $serverdata;
1888                 }
1889
1890                 $data = json_decode($curlResult->getBody(), true);
1891                 if (empty($data) || empty($data['site'])) {
1892                         return $serverdata;
1893                 }
1894
1895                 if (!empty($data['site']['name'])) {
1896                         $serverdata['site_name'] = $data['site']['name'];
1897                 }
1898
1899                 if (!empty($data['site']['platform'])) {
1900                         $serverdata['platform'] = strtolower($data['site']['platform']['PLATFORM_NAME']);
1901                         $serverdata['version'] = $data['site']['platform']['STD_VERSION'];
1902                         $serverdata['network'] = Protocol::ZOT;
1903                 }
1904
1905                 if (!empty($data['site']['hubzilla'])) {
1906                         $serverdata['platform'] = strtolower($data['site']['hubzilla']['PLATFORM_NAME']);
1907                         $serverdata['version'] = $data['site']['hubzilla']['RED_VERSION'];
1908                         $serverdata['network'] = Protocol::ZOT;
1909                 }
1910
1911                 if (!empty($data['site']['redmatrix'])) {
1912                         if (!empty($data['site']['redmatrix']['PLATFORM_NAME'])) {
1913                                 $serverdata['platform'] = strtolower($data['site']['redmatrix']['PLATFORM_NAME']);
1914                         } elseif (!empty($data['site']['redmatrix']['RED_PLATFORM'])) {
1915                                 $serverdata['platform'] = strtolower($data['site']['redmatrix']['RED_PLATFORM']);
1916                         }
1917
1918                         $serverdata['version'] = $data['site']['redmatrix']['RED_VERSION'];
1919                         $serverdata['network'] = Protocol::ZOT;
1920                 }
1921
1922                 $private = false;
1923                 $inviteonly = false;
1924                 $closed = false;
1925
1926                 if (!empty($data['site']['closed'])) {
1927                         $closed = self::toBoolean($data['site']['closed']);
1928                 }
1929
1930                 if (!empty($data['site']['private'])) {
1931                         $private = self::toBoolean($data['site']['private']);
1932                 }
1933
1934                 if (!empty($data['site']['inviteonly'])) {
1935                         $inviteonly = self::toBoolean($data['site']['inviteonly']);
1936                 }
1937
1938                 if (!$closed && !$private and $inviteonly) {
1939                         $serverdata['register_policy'] = Register::APPROVE;
1940                 } elseif (!$closed && !$private) {
1941                         $serverdata['register_policy'] = Register::OPEN;
1942                 } else {
1943                         $serverdata['register_policy'] = Register::CLOSED;
1944                 }
1945
1946                 if (($serverdata['network'] != Protocol::PHANTOM) && in_array($serverdata['detection-method'], self::DETECT_UNSPECIFIC)) {
1947                         $serverdata['detection-method'] = self::DETECT_CONFIG_JSON;
1948                 }
1949
1950                 return $serverdata;
1951         }
1952
1953         /**
1954          * Converts input value to a boolean value
1955          *
1956          * @param string|integer $val
1957          *
1958          * @return boolean
1959          */
1960         private static function toBoolean($val): bool
1961         {
1962                 if (($val == 'true') || ($val == 1)) {
1963                         return true;
1964                 } elseif (($val == 'false') || ($val == 0)) {
1965                         return false;
1966                 }
1967
1968                 return $val;
1969         }
1970
1971         /**
1972          * Detect if the URL belongs to a GNU Social server
1973          *
1974          * @param string $url        URL of the given server
1975          * @param array  $serverdata array with server data
1976          *
1977          * @return array server data
1978          */
1979         private static function detectGNUSocial(string $url, array $serverdata): array
1980         {
1981                 // Test for GNU Social
1982                 $curlResult = DI::httpClient()->get($url . '/api/gnusocial/version.json', HttpClientAccept::JSON);
1983                 if ($curlResult->isSuccess() && ($curlResult->getBody() != '{"error":"not implemented"}') &&
1984                         ($curlResult->getBody() != '') && (strlen($curlResult->getBody()) < 30)) {
1985                         $serverdata['platform'] = 'gnusocial';
1986                         // Remove junk that some GNU Social servers return
1987                         $serverdata['version'] = str_replace(chr(239) . chr(187) . chr(191), '', $curlResult->getBody());
1988                         $serverdata['version'] = str_replace(["\r", "\n", "\t"], '', $serverdata['version']);
1989                         $serverdata['version'] = trim($serverdata['version'], '"');
1990                         $serverdata['network'] = Protocol::OSTATUS;
1991
1992                         if (in_array($serverdata['detection-method'], self::DETECT_UNSPECIFIC)) {
1993                                 $serverdata['detection-method'] = self::DETECT_GNUSOCIAL;
1994                         }
1995
1996                         return $serverdata;
1997                 }
1998
1999                 // Test for Statusnet
2000                 $curlResult = DI::httpClient()->get($url . '/api/statusnet/version.json', HttpClientAccept::JSON);
2001                 if ($curlResult->isSuccess() && ($curlResult->getBody() != '{"error":"not implemented"}') &&
2002                         ($curlResult->getBody() != '') && (strlen($curlResult->getBody()) < 30)) {
2003
2004                         // Remove junk that some GNU Social servers return
2005                         $serverdata['version'] = str_replace(chr(239).chr(187).chr(191), '', $curlResult->getBody());
2006                         $serverdata['version'] = str_replace(["\r", "\n", "\t"], '', $serverdata['version']);
2007                         $serverdata['version'] = trim($serverdata['version'], '"');
2008
2009                         if (!empty($serverdata['version']) && strtolower(substr($serverdata['version'], 0, 7)) == 'pleroma') {
2010                                 $serverdata['platform'] = 'pleroma';
2011                                 $serverdata['version'] = trim(str_ireplace('pleroma', '', $serverdata['version']));
2012                                 $serverdata['network'] = Protocol::ACTIVITYPUB;
2013                         } else {
2014                                 $serverdata['platform'] = 'statusnet';
2015                                 $serverdata['network'] = Protocol::OSTATUS;
2016                         }
2017
2018                         if (in_array($serverdata['detection-method'], self::DETECT_UNSPECIFIC)) {
2019                                 $serverdata['detection-method'] = self::DETECT_STATUSNET;
2020                         }
2021                 }
2022
2023                 return $serverdata;
2024         }
2025
2026         /**
2027          * Detect if the URL belongs to a Friendica server
2028          *
2029          * @param string $url        URL of the given server
2030          * @param array  $serverdata array with server data
2031          *
2032          * @return array server data
2033          */
2034         private static function detectFriendica(string $url, array $serverdata): array
2035         {
2036                 // There is a bug in some versions of Friendica that will return an ActivityStream actor when the content type "application/json" is requested.
2037                 // Because of this me must not use ACCEPT_JSON here.
2038                 $curlResult = DI::httpClient()->get($url . '/friendica/json');
2039                 if (!$curlResult->isSuccess()) {
2040                         $curlResult = DI::httpClient()->get($url . '/friendika/json');
2041                         $friendika = true;
2042                         $platform = 'Friendika';
2043                 } else {
2044                         $friendika = false;
2045                         $platform = 'Friendica';
2046                 }
2047
2048                 if (!$curlResult->isSuccess()) {
2049                         return $serverdata;
2050                 }
2051
2052                 $data = json_decode($curlResult->getBody(), true);
2053                 if (empty($data) || empty($data['version'])) {
2054                         return $serverdata;
2055                 }
2056
2057                 if (in_array($serverdata['detection-method'], self::DETECT_UNSPECIFIC)) {
2058                         $serverdata['detection-method'] = $friendika ? self::DETECT_FRIENDIKA : self::DETECT_FRIENDICA;
2059                 }
2060
2061                 $serverdata['network'] = Protocol::DFRN;
2062                 $serverdata['version'] = $data['version'];
2063
2064                 if (!empty($data['no_scrape_url'])) {
2065                         $serverdata['noscrape'] = $data['no_scrape_url'];
2066                 }
2067
2068                 if (!empty($data['site_name'])) {
2069                         $serverdata['site_name'] = $data['site_name'];
2070                 }
2071
2072                 if (!empty($data['info'])) {
2073                         $serverdata['info'] = trim($data['info']);
2074                 }
2075
2076                 $register_policy = ($data['register_policy'] ?? '') ?: 'REGISTER_CLOSED';
2077                 switch ($register_policy) {
2078                         case 'REGISTER_OPEN':
2079                                 $serverdata['register_policy'] = Register::OPEN;
2080                                 break;
2081
2082                         case 'REGISTER_APPROVE':
2083                                 $serverdata['register_policy'] = Register::APPROVE;
2084                                 break;
2085
2086                         case 'REGISTER_CLOSED':
2087                         case 'REGISTER_INVITATION':
2088                                 $serverdata['register_policy'] = Register::CLOSED;
2089                                 break;
2090                         default:
2091                                 Logger::info('Register policy is invalid', ['policy' => $register_policy, 'server' => $url]);
2092                                 $serverdata['register_policy'] = Register::CLOSED;
2093                                 break;
2094                 }
2095
2096                 $serverdata['platform'] = strtolower($data['platform'] ?? $platform);
2097
2098                 return $serverdata;
2099         }
2100
2101         /**
2102          * Analyses the landing page of a given server for hints about type and system of that server
2103          *
2104          * @param object $curlResult result of curl execution
2105          * @param array  $serverdata array with server data
2106          *
2107          * @return array server data
2108          */
2109         private static function analyseRootBody($curlResult, array $serverdata): array
2110         {
2111                 if (empty($curlResult->getBody())) {
2112                         return $serverdata;
2113                 }
2114
2115                 if (file_exists(__DIR__ . '/../../static/platforms.config.php')) {
2116                         require __DIR__ . '/../../static/platforms.config.php';
2117                 } else {
2118                         throw new HTTPException\InternalServerErrorException('Invalid platform file');
2119                 }
2120
2121                 $platforms = array_merge($ap_platforms, $dfrn_platforms, $zap_platforms, $platforms);
2122
2123                 $doc = new DOMDocument();
2124                 @$doc->loadHTML($curlResult->getBody());
2125                 $xpath = new DOMXPath($doc);
2126                 $assigned = false;
2127
2128                 // We can only detect honk via some HTML element on their page
2129                 if ($xpath->query('//div[@id="honksonpage"]')->count() == 1) {
2130                         $serverdata['platform'] = 'honk';
2131                         $serverdata['network'] = Protocol::ACTIVITYPUB;
2132                         $assigned = true;
2133                 }
2134
2135                 $title = trim(XML::getFirstNodeValue($xpath, '//head/title/text()'));
2136                 if (!empty($title)) {
2137                         $serverdata['site_name'] = $title;
2138                 }
2139
2140                 $list = $xpath->query('//meta[@name]');
2141
2142                 foreach ($list as $node) {
2143                         $attr = [];
2144                         if ($node->attributes->length) {
2145                                 foreach ($node->attributes as $attribute) {
2146                                         $value = trim($attribute->value);
2147                                         if (empty($value)) {
2148                                                 continue;
2149                                         }
2150
2151                                         $attr[$attribute->name] = $value;
2152                                 }
2153
2154                                 if (empty($attr['name']) || empty($attr['content'])) {
2155                                         continue;
2156                                 }
2157                         }
2158
2159                         if ($attr['name'] == 'description') {
2160                                 $serverdata['info'] = $attr['content'];
2161                         }
2162
2163                         if (in_array($attr['name'], ['application-name', 'al:android:app_name', 'al:ios:app_name',
2164                                 'twitter:app:name:googleplay', 'twitter:app:name:iphone', 'twitter:app:name:ipad', 'generator'])) {
2165                                 $platform = str_ireplace(array_keys($platforms), array_values($platforms), $attr['content']);
2166                                 $platform = str_replace('/', ' ', $platform);
2167                                 $platform_parts = explode(' ', $platform);
2168                                 if ((count($platform_parts) >= 2) && in_array(strtolower($platform_parts[0]), array_values($platforms))) {
2169                                         $platform = $platform_parts[0];
2170                                         $serverdata['version'] = $platform_parts[1];
2171                                 }
2172                                 if (in_array($platform, array_values($dfrn_platforms))) {
2173                                         $serverdata['network'] = Protocol::DFRN;
2174                                 } elseif (in_array($platform, array_values($ap_platforms))) {
2175                                         $serverdata['network'] = Protocol::ACTIVITYPUB;
2176                                 } elseif (in_array($platform, array_values($zap_platforms))) {
2177                                         $serverdata['network'] = Protocol::ZOT;
2178                                 }
2179                                 if (in_array($platform, array_values($platforms))) {
2180                                         $serverdata['platform'] = $platform;
2181                                         $assigned = true;
2182                                 }
2183                         }
2184                 }
2185
2186                 $list = $xpath->query('//meta[@property]');
2187
2188                 foreach ($list as $node) {
2189                         $attr = [];
2190                         if ($node->attributes->length) {
2191                                 foreach ($node->attributes as $attribute) {
2192                                         $value = trim($attribute->value);
2193                                         if (empty($value)) {
2194                                                 continue;
2195                                         }
2196
2197                                         $attr[$attribute->name] = $value;
2198                                 }
2199
2200                                 if (empty($attr['property']) || empty($attr['content'])) {
2201                                         continue;
2202                                 }
2203                         }
2204
2205                         if ($attr['property'] == 'og:site_name') {
2206                                 $serverdata['site_name'] = $attr['content'];
2207                         }
2208
2209                         if ($attr['property'] == 'og:description') {
2210                                 $serverdata['info'] = $attr['content'];
2211                         }
2212
2213                         if (in_array($attr['property'], ['og:platform', 'generator'])) {
2214                                 if (in_array($attr['content'], array_keys($platforms))) {
2215                                         $serverdata['platform'] = $platforms[$attr['content']];
2216                                         $assigned = true;
2217                                 }
2218
2219                                 if (in_array($attr['content'], array_keys($ap_platforms))) {
2220                                         $serverdata['network'] = Protocol::ACTIVITYPUB;
2221                                 } elseif (in_array($attr['content'], array_values($zap_platforms))) {
2222                                         $serverdata['network'] = Protocol::ZOT;
2223                                 }
2224                         }
2225                 }
2226
2227                 $list = $xpath->query('//link[@rel="me"]');
2228                 foreach ($list as $node) {
2229                         foreach ($node->attributes as $attribute) {
2230                                 if (parse_url(trim($attribute->value), PHP_URL_HOST) == 'micro.blog') {
2231                                         $serverdata['version'] = trim($serverdata['platform'] . ' ' . $serverdata['version']);
2232                                         $serverdata['platform'] = 'microblog';
2233                                         $serverdata['network'] = Protocol::ACTIVITYPUB;
2234                                         $assigned = true;
2235                                 }
2236                         }
2237                 }
2238
2239                 if ($serverdata['platform'] != 'microblog') {
2240                         $list = $xpath->query('//link[@rel="micropub"]');
2241                         foreach ($list as $node) {
2242                                 foreach ($node->attributes as $attribute) {
2243                                         if (trim($attribute->value) == 'https://micro.blog/micropub') {
2244                                                 $serverdata['version'] = trim($serverdata['platform'] . ' ' . $serverdata['version']);
2245                                                 $serverdata['platform'] = 'microblog';
2246                                                 $serverdata['network'] = Protocol::ACTIVITYPUB;
2247                                                 $assigned = true;
2248                                         }
2249                                 }
2250                         }
2251                 }
2252
2253                 if ($assigned && in_array($serverdata['detection-method'], [self::DETECT_MANUAL, self::DETECT_HEADER])) {
2254                         $serverdata['detection-method'] = self::DETECT_BODY;
2255                 }
2256
2257                 return $serverdata;
2258         }
2259
2260         /**
2261          * Analyses the header data of a given server for hints about type and system of that server
2262          *
2263          * @param object $curlResult result of curl execution
2264          * @param array  $serverdata array with server data
2265          *
2266          * @return array server data
2267          */
2268         private static function analyseRootHeader($curlResult, array $serverdata): array
2269         {
2270                 if ($curlResult->getHeader('server') == 'Mastodon') {
2271                         $serverdata['platform'] = 'mastodon';
2272                         $serverdata['network'] = Protocol::ACTIVITYPUB;
2273                 } elseif ($curlResult->inHeader('x-diaspora-version')) {
2274                         $serverdata['platform'] = 'diaspora';
2275                         $serverdata['network'] = Protocol::DIASPORA;
2276                         $serverdata['version'] = $curlResult->getHeader('x-diaspora-version')[0] ?? '';
2277                 } elseif ($curlResult->inHeader('x-friendica-version')) {
2278                         $serverdata['platform'] = 'friendica';
2279                         $serverdata['network'] = Protocol::DFRN;
2280                         $serverdata['version'] = $curlResult->getHeader('x-friendica-version')[0] ?? '';
2281                 } else {
2282                         return $serverdata;
2283                 }
2284
2285                 if ($serverdata['detection-method'] == self::DETECT_MANUAL) {
2286                         $serverdata['detection-method'] = self::DETECT_HEADER;
2287                 }
2288
2289                 return $serverdata;
2290         }
2291
2292         /**
2293          * Update GServer entries
2294          */
2295         public static function discover()
2296         {
2297                 // Update the server list
2298                 self::discoverFederation();
2299
2300                 $no_of_queries = 5;
2301
2302                 $requery_days = intval(DI::config()->get('system', 'poco_requery_days'));
2303
2304                 if ($requery_days == 0) {
2305                         $requery_days = 7;
2306                 }
2307
2308                 $last_update = date('c', time() - (60 * 60 * 24 * $requery_days));
2309
2310                 $gservers = DBA::select('gserver', ['id', 'url', 'nurl', 'network', 'poco', 'directory-type'],
2311                         ["NOT `blocked` AND NOT `failed` AND `directory-type` != ? AND `last_poco_query` < ?", GServer::DT_NONE, $last_update],
2312                         ['order' => ['RAND()']]);
2313
2314                 while ($gserver = DBA::fetch($gservers)) {
2315                         Logger::info('Update peer list', ['server' => $gserver['url'], 'id' => $gserver['id']]);
2316                         Worker::add(Worker::PRIORITY_LOW, 'UpdateServerPeers', $gserver['url']);
2317
2318                         Logger::info('Update directory', ['server' => $gserver['url'], 'id' => $gserver['id']]);
2319                         Worker::add(Worker::PRIORITY_LOW, 'UpdateServerDirectory', $gserver);
2320
2321                         $fields = ['last_poco_query' => DateTimeFormat::utcNow()];
2322                         self::update($fields, ['nurl' => $gserver['nurl']]);
2323
2324                         if (--$no_of_queries == 0) {
2325                                 break;
2326                         }
2327                 }
2328
2329                 DBA::close($gservers);
2330         }
2331
2332         /**
2333          * Discover federated servers
2334          */
2335         private static function discoverFederation()
2336         {
2337                 $last = DI::keyValue()->get('poco_last_federation_discovery');
2338
2339                 if ($last) {
2340                         $next = $last + (24 * 60 * 60);
2341
2342                         if ($next > time()) {
2343                                 return;
2344                         }
2345                 }
2346
2347                 // Discover federated servers
2348                 $protocols = ['activitypub', 'diaspora', 'dfrn', 'ostatus'];
2349                 foreach ($protocols as $protocol) {
2350                         $query = '{nodes(protocol:"' . $protocol . '"){host}}';
2351                         $curlResult = DI::httpClient()->fetch('https://the-federation.info/graphql?query=' . urlencode($query), HttpClientAccept::JSON);
2352                         if (!empty($curlResult)) {
2353                                 $data = json_decode($curlResult, true);
2354                                 if (!empty($data['data']['nodes'])) {
2355                                         foreach ($data['data']['nodes'] as $server) {
2356                                                 // Using "only_nodeinfo" since servers that are listed on that page should always have it.
2357                                                 self::add('https://' . $server['host'], true);
2358                                         }
2359                                 }
2360                         }
2361                 }
2362
2363                 // Disvover Mastodon servers
2364                 $accesstoken = DI::config()->get('system', 'instances_social_key');
2365
2366                 if (!empty($accesstoken)) {
2367                         $api = 'https://instances.social/api/1.0/instances/list?count=0';
2368                         $curlResult = DI::httpClient()->get($api, HttpClientAccept::JSON, [HttpClientOptions::HEADERS => ['Authorization' => ['Bearer ' . $accesstoken]]]);
2369                         if ($curlResult->isSuccess()) {
2370                                 $servers = json_decode($curlResult->getBody(), true);
2371
2372                                 if (!empty($servers['instances'])) {
2373                                         foreach ($servers['instances'] as $server) {
2374                                                 $url = (is_null($server['https_score']) ? 'http' : 'https') . '://' . $server['name'];
2375                                                 self::add($url);
2376                                         }
2377                                 }
2378                         }
2379                 }
2380
2381                 DI::keyValue()->set('poco_last_federation_discovery', time());
2382         }
2383
2384         /**
2385          * Set the protocol for the given server
2386          *
2387          * @param int $gsid     Server id
2388          * @param int $protocol Protocol id
2389          *
2390          * @throws Exception
2391          */
2392         public static function setProtocol(int $gsid, int $protocol)
2393         {
2394                 if (empty($gsid)) {
2395                         return;
2396                 }
2397
2398                 $gserver = DBA::selectFirst('gserver', ['protocol', 'url'], ['id' => $gsid]);
2399                 if (!DBA::isResult($gserver)) {
2400                         return;
2401                 }
2402
2403                 $old = $gserver['protocol'];
2404
2405                 if (!is_null($old)) {
2406                         /*
2407                         The priority for the protocols is:
2408                                 1. ActivityPub
2409                                 2. DFRN via Diaspora
2410                                 3. Legacy DFRN
2411                                 4. Diaspora
2412                                 5. OStatus
2413                         */
2414
2415                         // We don't need to change it when nothing is to be changed
2416                         if ($old == $protocol) {
2417                                 return;
2418                         }
2419
2420                         // We don't want to mark a server as OStatus when it had been marked with any other protocol before
2421                         if ($protocol == Post\DeliveryData::OSTATUS) {
2422                                 return;
2423                         }
2424
2425                         // If the server is marked as ActivityPub then we won't change it to anything different
2426                         if ($old == Post\DeliveryData::ACTIVITYPUB) {
2427                                 return;
2428                         }
2429
2430                         // Don't change it to anything lower than DFRN if the new one wasn't ActivityPub
2431                         if (($old == Post\DeliveryData::DFRN) && ($protocol != Post\DeliveryData::ACTIVITYPUB)) {
2432                                 return;
2433                         }
2434
2435                         // Don't change it to Diaspora when it is a legacy DFRN server
2436                         if (($old == Post\DeliveryData::LEGACY_DFRN) && ($protocol == Post\DeliveryData::DIASPORA)) {
2437                                 return;
2438                         }
2439                 }
2440
2441                 Logger::info('Protocol for server', ['protocol' => $protocol, 'old' => $old, 'id' => $gsid, 'url' => $gserver['url'], 'callstack' => System::callstack(20)]);
2442                 self::update(['protocol' => $protocol], ['id' => $gsid]);
2443         }
2444
2445         /**
2446          * Fetch the protocol of the given server
2447          *
2448          * @param int $gsid Server id
2449          *
2450          * @return ?int One of Post\DeliveryData protocol constants or null if unknown or gserver is missing
2451          *
2452          * @throws Exception
2453          */
2454         public static function getProtocol(int $gsid): ?int
2455         {
2456                 if (empty($gsid)) {
2457                         return null;
2458                 }
2459
2460                 $gserver = DBA::selectFirst('gserver', ['protocol'], ['id' => $gsid]);
2461                 if (DBA::isResult($gserver)) {
2462                         return $gserver['protocol'];
2463                 }
2464
2465                 return null;
2466         }
2467
2468         /**
2469          * Update rows in the gserver table.
2470          * Enforces gserver table field maximum sizes to avoid "Data too long" database errors
2471          *
2472          * @param array $fields
2473          * @param array $condition
2474          *
2475          * @return bool
2476          *
2477          * @throws Exception
2478          */
2479         public static function update(array $fields, array $condition): bool
2480         {
2481                 $fields = DI::dbaDefinition()->truncateFieldsForTable('gserver', $fields);
2482
2483                 return DBA::update('gserver', $fields, $condition);
2484         }
2485
2486         /**
2487          * Insert a row into the gserver table.
2488          * Enforces gserver table field maximum sizes to avoid "Data too long" database errors
2489          *
2490          * @param array $fields
2491          * @param int   $duplicate_mode What to do on a duplicated entry
2492          *
2493          * @return bool
2494          *
2495          * @throws Exception
2496          */
2497         public static function insert(array $fields, int $duplicate_mode = Database::INSERT_DEFAULT): bool
2498         {
2499                 $fields = DI::dbaDefinition()->truncateFieldsForTable('gserver', $fields);
2500
2501                 return DBA::insert('gserver', $fields, $duplicate_mode);
2502         }
2503 }