]> git.mxchange.org Git - friendica.git/blob - src/Model/GServer.php
Use rawContent for Special Options to avoid a protected options() method
[friendica.git] / src / Model / GServer.php
1 <?php
2 /**
3  * @copyright Copyright (C) 2010-2022, the Friendica project
4  *
5  * @license GNU AGPL version 3 or any later version
6  *
7  * This program is free software: you can redistribute it and/or modify
8  * it under the terms of the GNU Affero General Public License as
9  * published by the Free Software Foundation, either version 3 of the
10  * License, or (at your option) any later version.
11  *
12  * This program is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15  * GNU Affero General Public License for more details.
16  *
17  * You should have received a copy of the GNU Affero General Public License
18  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
19  *
20  */
21
22 namespace Friendica\Model;
23
24 use DOMDocument;
25 use DOMXPath;
26 use Exception;
27 use Friendica\Core\Logger;
28 use Friendica\Core\Protocol;
29 use Friendica\Core\System;
30 use Friendica\Core\Worker;
31 use Friendica\Database\Database;
32 use Friendica\Database\DBA;
33 use Friendica\DI;
34 use Friendica\Module\Register;
35 use Friendica\Network\HTTPClient\Client\HttpClientOptions;
36 use Friendica\Network\HTTPClient\Capability\ICanHandleHttpResponses;
37 use Friendica\Protocol\Relay;
38 use Friendica\Util\DateTimeFormat;
39 use Friendica\Util\Network;
40 use Friendica\Util\Strings;
41 use Friendica\Util\XML;
42
43 /**
44  * This class handles GServer related functions
45  */
46 class GServer
47 {
48         // Directory types
49         const DT_NONE = 0;
50         const DT_POCO = 1;
51         const DT_MASTODON = 2;
52
53         // Methods to detect server types
54
55         // Non endpoint specific methods
56         const DETECT_MANUAL = 0;
57         const DETECT_HEADER = 1;
58         const DETECT_BODY = 2;
59
60         // Implementation specific endpoints
61         const DETECT_FRIENDIKA = 10;
62         const DETECT_FRIENDICA = 11;
63         const DETECT_STATUSNET = 12;
64         const DETECT_GNUSOCIAL = 13;
65         const DETECT_CONFIG_JSON = 14; // Statusnet, GNU Social, Older Hubzilla/Redmatrix
66         const DETECT_SITEINFO_JSON = 15; // Newer Hubzilla
67         const DETECT_MASTODON_API = 16;
68         const DETECT_STATUS_PHP = 17; // Nextcloud
69         const DETECT_V1_CONFIG = 18;
70
71         // Standardized endpoints
72         const DETECT_STATISTICS_JSON = 100;
73         const DETECT_NODEINFO_1 = 101;
74         const DETECT_NODEINFO_2 = 102;
75
76         /**
77          * Check for the existance of a server and adds it in the background if not existant
78          *
79          * @param string $url
80          * @param boolean $only_nodeinfo
81          * @return void
82          */
83         public static function add(string $url, bool $only_nodeinfo = false)
84         {
85                 if (self::getID($url, false)) {
86                         return;
87                 }
88
89                 Worker::add(PRIORITY_LOW, 'UpdateGServer', $url, $only_nodeinfo);
90         }
91
92         /**
93          * Get the ID for the given server URL
94          *
95          * @param string $url
96          * @param boolean $no_check Don't check if the server hadn't been found
97          * @return int gserver id
98          */
99         public static function getID(string $url, bool $no_check = false)
100         {
101                 if (empty($url)) {
102                         return null;
103                 }
104
105                 $url = self::cleanURL($url);
106
107                 $gserver = DBA::selectFirst('gserver', ['id'], ['nurl' => Strings::normaliseLink($url)]);
108                 if (DBA::isResult($gserver)) {
109                         Logger::info('Got ID for URL', ['id' => $gserver['id'], 'url' => $url, 'callstack' => System::callstack(20)]);
110                         return $gserver['id'];
111                 }
112
113                 if ($no_check || !self::check($url)) {
114                         return null;
115                 }
116
117                 return self::getID($url, true);
118         }
119
120         /**
121          * Retrieves all the servers which base domain are matching the provided domain pattern
122          *
123          * The pattern is a simple fnmatch() pattern with ? for single wildcard and * for multiple wildcard
124          *
125          * @param string $pattern
126          * @return array
127          * @throws Exception
128          */
129         public static function listByDomainPattern(string $pattern): array
130         {
131                 $likePattern = 'http://' . strtr($pattern, ['_' => '\_', '%' => '\%', '?' => '_', '*' => '%']);
132
133                 // The SUBSTRING_INDEX returns everything before the eventual third /, which effectively trims an
134                 // eventual server path and keep only the server domain which we're matching against the pattern.
135                 $sql = "SELECT `gserver`.*, COUNT(*) AS `contacts`
136                         FROM `gserver`
137                         LEFT JOIN `contact` ON `gserver`.`id` = `contact`.`gsid`
138                         WHERE SUBSTRING_INDEX(`gserver`.`nurl`, '/', 3) LIKE ?
139                         AND NOT `gserver`.`failed`
140                         GROUP BY `gserver`.`id`";
141
142                 $stmt = DI::dba()->p($sql, $likePattern);
143
144                 return DI::dba()->toArray($stmt);
145         }
146
147         /**
148          * Checks if the given server is reachable
149          *
150          * @param string  $profile URL of the given profile
151          * @param string  $server  URL of the given server (If empty, taken from profile)
152          * @param string  $network Network value that is used, when detection failed
153          * @param boolean $force   Force an update.
154          *
155          * @return boolean 'true' if server seems vital
156          */
157         public static function reachable(string $profile, string $server = '', string $network = '', bool $force = false)
158         {
159                 if ($server == '') {
160                         $contact = Contact::getByURL($profile, null, ['baseurl']);
161                         if (!empty($contact['baseurl'])) {
162                                 $server = $contact['baseurl'];
163                         }
164                 }
165
166                 if ($server == '') {
167                         return true;
168                 }
169
170                 return self::check($server, $network, $force);
171         }
172
173         public static function getNextUpdateDate(bool $success, string $created = '', string $last_contact = '')
174         {
175                 // On successful contact process check again next week
176                 if ($success) {
177                         return DateTimeFormat::utc('now +7 day');
178                 }
179
180                 $now = strtotime(DateTimeFormat::utcNow());
181
182                 if ($created > $last_contact) {
183                         $contact_time = strtotime($created);
184                 } else {
185                         $contact_time = strtotime($last_contact);
186                 }
187
188                 // If the last contact was less than 6 hours before then try again in 6 hours
189                 if (($now - $contact_time) < (60 * 60 * 6)) {
190                         return DateTimeFormat::utc('now +6 hour');
191                 }
192
193                 // If the last contact was less than 12 hours before then try again in 12 hours
194                 if (($now - $contact_time) < (60 * 60 * 12)) {
195                         return DateTimeFormat::utc('now +12 hour');
196                 }
197
198                 // If the last contact was less than 24 hours before then try tomorrow again
199                 if (($now - $contact_time) < (60 * 60 * 24)) {
200                         return DateTimeFormat::utc('now +1 day');
201                 }
202
203                 // If the last contact was less than a week before then try again in a week
204                 if (($now - $contact_time) < (60 * 60 * 24 * 7)) {
205                         return DateTimeFormat::utc('now +1 week');
206                 }
207
208                 // If the last contact was less than two weeks before then try again in two week
209                 if (($now - $contact_time) < (60 * 60 * 24 * 14)) {
210                         return DateTimeFormat::utc('now +2 week');
211                 }
212
213                 // If the last contact was less than a month before then try again in a month
214                 if (($now - $contact_time) < (60 * 60 * 24 * 30)) {
215                         return DateTimeFormat::utc('now +1 month');
216                 }
217
218                 // The system hadn't been successul contacted for more than a month, so try again in three months
219                 return DateTimeFormat::utc('now +3 month');
220         }
221
222         /**
223          * Checks the state of the given server.
224          *
225          * @param string  $server_url    URL of the given server
226          * @param string  $network       Network value that is used, when detection failed
227          * @param boolean $force         Force an update.
228          * @param boolean $only_nodeinfo Only use nodeinfo for server detection
229          *
230          * @return boolean 'true' if server seems vital
231          */
232         public static function check(string $server_url, string $network = '', bool $force = false, bool $only_nodeinfo = false)
233         {
234                 $server_url = self::cleanURL($server_url);
235                 if ($server_url == '') {
236                         return false;
237                 }
238
239                 $gserver = DBA::selectFirst('gserver', [], ['nurl' => Strings::normaliseLink($server_url)]);
240                 if (DBA::isResult($gserver)) {
241                         if ($gserver['created'] <= DBA::NULL_DATETIME) {
242                                 $fields = ['created' => DateTimeFormat::utcNow()];
243                                 $condition = ['nurl' => Strings::normaliseLink($server_url)];
244                                 DBA::update('gserver', $fields, $condition);
245                         }
246
247                         if (!$force && (strtotime($gserver['next_contact']) > time())) {
248                                 Logger::info('No update needed', ['server' => $server_url]);
249                                 return (!$gserver['failed']);
250                         }
251                         Logger::info('Server is outdated. Start discovery.', ['Server' => $server_url, 'Force' => $force]);
252                 } else {
253                         Logger::info('Server is unknown. Start discovery.', ['Server' => $server_url]);
254                 }
255
256                 return self::detect($server_url, $network, $only_nodeinfo);
257         }
258
259         /**
260          * Set failed server status
261          *
262          * @param string $url
263          */
264         public static function setFailure(string $url)
265         {
266                 $gserver = DBA::selectFirst('gserver', [], ['nurl' => Strings::normaliseLink($url)]);
267                 if (DBA::isResult($gserver)) {
268                         $next_update = self::getNextUpdateDate(false, $gserver['created'], $gserver['last_contact']);
269                         DBA::update('gserver', ['failed' => true, 'last_failure' => DateTimeFormat::utcNow(),
270                         'next_contact' => $next_update, 'detection-method' => null],
271                         ['nurl' => Strings::normaliseLink($url)]);
272                         Logger::info('Set failed status for existing server', ['url' => $url]);
273                         return;
274                 }
275                 DBA::insert('gserver', ['url' => $url, 'nurl' => Strings::normaliseLink($url),
276                         'network' => Protocol::PHANTOM, 'created' => DateTimeFormat::utcNow(),
277                         'failed' => true, 'last_failure' => DateTimeFormat::utcNow()]);
278                 Logger::info('Set failed status for new server', ['url' => $url]);
279         }
280
281         /**
282          * Remove unwanted content from the given URL
283          *
284          * @param string $url
285          * @return string cleaned URL
286          */
287         public static function cleanURL(string $url)
288         {
289                 $url = trim($url, '/');
290                 $url = str_replace('/index.php', '', $url);
291
292                 $urlparts = parse_url($url);
293                 unset($urlparts['user']);
294                 unset($urlparts['pass']);
295                 unset($urlparts['query']);
296                 unset($urlparts['fragment']);
297                 return Network::unparseURL($urlparts);
298         }
299
300         /**
301          * Return the base URL
302          *
303          * @param string $url
304          * @return string base URL
305          */
306         private static function getBaseURL(string $url)
307         {
308                 $urlparts = parse_url(self::cleanURL($url));
309                 unset($urlparts['path']);
310                 return Network::unparseURL($urlparts);
311         }
312
313         /**
314          * Detect server data (type, protocol, version number, ...)
315          * The detected data is then updated or inserted in the gserver table.
316          *
317          * @param string  $url           URL of the given server
318          * @param string  $network       Network value that is used, when detection failed
319          * @param boolean $only_nodeinfo Only use nodeinfo for server detection
320          *
321          * @return boolean 'true' if server could be detected
322          */
323         public static function detect(string $url, string $network = '', bool $only_nodeinfo = false)
324         {
325                 Logger::info('Detect server type', ['server' => $url]);
326                 $serverdata = ['detection-method' => self::DETECT_MANUAL];
327
328                 $original_url = $url;
329
330                 // Remove URL content that is not supposed to exist for a server url
331                 $url = self::cleanURL($url);
332
333                 // Get base URL
334                 $baseurl = self::getBaseURL($url);
335
336                 // If the URL missmatches, then we mark the old entry as failure
337                 if ($url != $original_url) {
338                         /// @todo What to do with "next_contact" here?
339                         DBA::update('gserver', ['failed' => true, 'last_failure' => DateTimeFormat::utcNow()],
340                                 ['nurl' => Strings::normaliseLink($original_url)]);
341                 }
342
343                 // When a nodeinfo is present, we don't need to dig further
344                 $xrd_timeout = DI::config()->get('system', 'xrd_timeout');
345                 $curlResult = DI::httpClient()->get($url . '/.well-known/nodeinfo', [HttpClientOptions::TIMEOUT => $xrd_timeout]);
346                 if ($curlResult->isTimeout()) {
347                         self::setFailure($url);
348                         return false;
349                 }
350
351                 // On a redirect follow the new host but mark the old one as failure
352                 if ($curlResult->isSuccess() && !empty($curlResult->getRedirectUrl()) && (parse_url($url, PHP_URL_HOST) != parse_url($curlResult->getRedirectUrl(), PHP_URL_HOST))) {
353                         $curlResult = DI::httpClient()->get($url, [HttpClientOptions::TIMEOUT => $xrd_timeout]);
354                         if (!empty($curlResult->getRedirectUrl()) && parse_url($url, PHP_URL_HOST) != parse_url($curlResult->getRedirectUrl(), PHP_URL_HOST)) {
355                                 Logger::info('Found redirect. Mark old entry as failure', ['old' => $url, 'new' => $curlResult->getRedirectUrl()]);
356                                 self::setFailure($url);
357                                 self::detect($curlResult->getRedirectUrl(), $network, $only_nodeinfo);
358                                 return false;
359                         }
360                 }
361
362                 $nodeinfo = self::fetchNodeinfo($url, $curlResult);
363                 if ($only_nodeinfo && empty($nodeinfo)) {
364                         Logger::info('Invalid nodeinfo in nodeinfo-mode, server is marked as failure', ['url' => $url]);
365                         self::setFailure($url);
366                         return false;
367                 }
368
369                 // When nodeinfo isn't present, we use the older 'statistics.json' endpoint
370                 if (empty($nodeinfo)) {
371                         $nodeinfo = self::fetchStatistics($url);
372                 }
373
374                 // If that didn't work out well, we use some protocol specific endpoints
375                 // For Friendica and Zot based networks we have to dive deeper to reveal more details
376                 if (empty($nodeinfo['network']) || in_array($nodeinfo['network'], [Protocol::DFRN, Protocol::ZOT])) {
377                         if (!empty($nodeinfo['detection-method'])) {
378                                 $serverdata['detection-method'] = $nodeinfo['detection-method'];
379                         }
380
381                         // Fetch the landing page, possibly it reveals some data
382                         if (empty($nodeinfo['network'])) {
383                                 if ($baseurl == $url) {
384                                         $basedata = $serverdata;
385                                 } else {
386                                         $basedata = ['detection-method' => self::DETECT_MANUAL];
387                                 }
388
389                                 $curlResult = DI::httpClient()->get($baseurl, [HttpClientOptions::TIMEOUT => $xrd_timeout]);
390                                 if ($curlResult->isSuccess()) {
391                                         if (!empty($curlResult->getRedirectUrl()) && (parse_url($baseurl, PHP_URL_HOST) != parse_url($curlResult->getRedirectUrl(), PHP_URL_HOST))) {
392                                                 Logger::info('Found redirect. Mark old entry as failure', ['old' => $url, 'new' => $curlResult->getRedirectUrl()]);
393                                                 self::setFailure($url);
394                                                 self::detect($curlResult->getRedirectUrl(), $network, $only_nodeinfo);
395                                                 return false;
396                                         }
397
398                                         $basedata = self::analyseRootHeader($curlResult, $basedata);
399                                         $basedata = self::analyseRootBody($curlResult, $basedata, $baseurl);
400                                 }
401
402                                 if (!$curlResult->isSuccess() || empty($curlResult->getBody()) || self::invalidBody($curlResult->getBody())) {
403                                         self::setFailure($url);
404                                         return false;
405                                 }
406
407                                 if ($baseurl == $url) {
408                                         $serverdata = $basedata;
409                                 } else {
410                                         // When the base path doesn't seem to contain a social network we try the complete path.
411                                         // Most detectable system have to be installed in the root directory.
412                                         // We checked the base to avoid false positives.
413                                         $curlResult = DI::httpClient()->get($url, [HttpClientOptions::TIMEOUT => $xrd_timeout]);
414                                         if ($curlResult->isSuccess()) {
415                                                 $urldata = self::analyseRootHeader($curlResult, $serverdata);
416                                                 $urldata = self::analyseRootBody($curlResult, $urldata, $url);
417
418                                                 $comparebase = $basedata;
419                                                 unset($comparebase['info']);
420                                                 unset($comparebase['site_name']);
421                                                 $compareurl = $urldata;
422                                                 unset($compareurl['info']);
423                                                 unset($compareurl['site_name']);
424
425                                                 // We assume that no one will install the identical system in the root and a subfolder
426                                                 if (!empty(array_diff($comparebase, $compareurl))) {
427                                                         $serverdata = $urldata;
428                                                 }
429                                         }
430                                 }
431                         }
432
433                         if (empty($serverdata['network']) || ($serverdata['network'] == Protocol::ACTIVITYPUB)) {
434                                 $serverdata = self::detectMastodonAlikes($url, $serverdata);
435                         }
436
437                         // All following checks are done for systems that always have got a "host-meta" endpoint.
438                         // With this check we don't have to waste time and ressources for dead systems.
439                         // Also this hopefully prevents us from receiving abuse messages.
440                         if (empty($serverdata['network']) && !self::validHostMeta($url)) {
441                                 self::setFailure($url);
442                                 return false;
443                         }
444
445                         if (empty($serverdata['network']) || in_array($serverdata['network'], [Protocol::DFRN, Protocol::ACTIVITYPUB])) {
446                                 $serverdata = self::detectFriendica($url, $serverdata);
447                         }
448
449                         // the 'siteinfo.json' is some specific endpoint of Hubzilla and Red
450                         if (empty($serverdata['network']) || ($serverdata['network'] == Protocol::ZOT)) {
451                                 $serverdata = self::fetchSiteinfo($url, $serverdata);
452                         }
453
454                         // The 'siteinfo.json' doesn't seem to be present on older Hubzilla installations
455                         if (empty($serverdata['network'])) {
456                                 $serverdata = self::detectHubzilla($url, $serverdata);
457                         }
458
459                         if (empty($serverdata['network']) || in_array($serverdata['detection-method'], [self::DETECT_MANUAL, self::DETECT_BODY])) {
460                                 $serverdata = self::detectPeertube($url, $serverdata);
461                         }
462
463                         if (empty($serverdata['network'])) {
464                                 $serverdata = self::detectNextcloud($url, $serverdata);
465                         }
466
467                         if (empty($serverdata['network'])) {
468                                 $serverdata = self::detectGNUSocial($url, $serverdata);
469                         }
470
471                         $serverdata = array_merge($nodeinfo, $serverdata);
472                 } else {
473                         $serverdata = $nodeinfo;
474                 }
475
476                 // Detect the directory type
477                 $serverdata['directory-type'] = self::DT_NONE;
478                 $serverdata = self::checkPoCo($url, $serverdata);
479                 $serverdata = self::checkMastodonDirectory($url, $serverdata);
480
481                 // We can't detect the network type. Possibly it is some system that we don't know yet
482                 if (empty($serverdata['network'])) {
483                         $serverdata['network'] = Protocol::PHANTOM;
484                 }
485
486                 // When we hadn't been able to detect the network type, we use the hint from the parameter
487                 if (($serverdata['network'] == Protocol::PHANTOM) && !empty($network)) {
488                         $serverdata['network'] = $network;
489                 }
490
491                 $serverdata['url'] = $url;
492                 $serverdata['nurl'] = Strings::normaliseLink($url);
493
494                 // We take the highest number that we do find
495                 $registeredUsers = $serverdata['registered-users'] ?? 0;
496
497                 // On an active server there has to be at least a single user
498                 if (($serverdata['network'] != Protocol::PHANTOM) && ($registeredUsers == 0)) {
499                         $registeredUsers = 1;
500                 }
501
502                 if ($serverdata['network'] == Protocol::PHANTOM) {
503                         $serverdata['registered-users'] = max($registeredUsers, 1);
504                         $serverdata = self::detectNetworkViaContacts($url, $serverdata);
505                 }
506
507                 $serverdata['next_contact'] = self::getNextUpdateDate(true);
508
509                 $serverdata['last_contact'] = DateTimeFormat::utcNow();
510                 $serverdata['failed'] = false;
511
512                 $gserver = DBA::selectFirst('gserver', ['network'], ['nurl' => Strings::normaliseLink($url)]);
513                 if (!DBA::isResult($gserver)) {
514                         $serverdata['created'] = DateTimeFormat::utcNow();
515                         $ret = DBA::insert('gserver', $serverdata);
516                         $id = DBA::lastInsertId();
517                 } else {
518                         // Don't override the network with 'unknown' when there had been a valid entry before
519                         if (($serverdata['network'] == Protocol::PHANTOM) && !empty($gserver['network'])) {
520                                 unset($serverdata['network']);
521                         }
522
523                         $ret = DBA::update('gserver', $serverdata, ['nurl' => $serverdata['nurl']]);
524                         $gserver = DBA::selectFirst('gserver', ['id'], ['nurl' => $serverdata['nurl']]);
525                         if (DBA::isResult($gserver)) {
526                                 $id = $gserver['id'];
527                         }
528                 }
529
530                 if (!empty($serverdata['network']) && !empty($id) && ($serverdata['network'] != Protocol::PHANTOM)) {
531                         $apcontacts = DBA::count('apcontact', ['gsid' => $id]);
532                         $contacts = DBA::count('contact', ['uid' => 0, 'gsid' => $id]);
533                         $max_users = max($apcontacts, $contacts, $registeredUsers, 1);
534                         if ($max_users > $registeredUsers) {
535                                 Logger::info('Update registered users', ['id' => $id, 'url' => $serverdata['nurl'], 'registered-users' => $max_users]);
536                                 DBA::update('gserver', ['registered-users' => $max_users], ['id' => $id]);
537                         }
538                 }
539
540                 if (!empty($serverdata['network']) && in_array($serverdata['network'], [Protocol::DFRN, Protocol::DIASPORA])) {
541                         self::discoverRelay($url);
542                 }
543
544                 return $ret;
545         }
546
547         /**
548          * Fetch relay data from a given server url
549          *
550          * @param string $server_url address of the server
551          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
552          */
553         private static function discoverRelay(string $server_url)
554         {
555                 Logger::info('Discover relay data', ['server' => $server_url]);
556
557                 $curlResult = DI::httpClient()->get($server_url . '/.well-known/x-social-relay');
558                 if (!$curlResult->isSuccess()) {
559                         return;
560                 }
561
562                 $data = json_decode($curlResult->getBody(), true);
563                 if (!is_array($data)) {
564                         return;
565                 }
566
567                 // Sanitize incoming data, see https://github.com/friendica/friendica/issues/8565
568                 $data['subscribe'] = (bool)$data['subscribe'] ?? false;
569
570                 if (!$data['subscribe'] || empty($data['scope']) || !in_array(strtolower($data['scope']), ['all', 'tags'])) {
571                         $data['scope'] = '';
572                         $data['subscribe'] = false;
573                         $data['tags'] = [];
574                 }
575
576                 $gserver = DBA::selectFirst('gserver', ['id', 'url', 'network', 'relay-subscribe', 'relay-scope'], ['nurl' => Strings::normaliseLink($server_url)]);
577                 if (!DBA::isResult($gserver)) {
578                         return;
579                 }
580
581                 if (($gserver['relay-subscribe'] != $data['subscribe']) || ($gserver['relay-scope'] != $data['scope'])) {
582                         $fields = ['relay-subscribe' => $data['subscribe'], 'relay-scope' => $data['scope']];
583                         DBA::update('gserver', $fields, ['id' => $gserver['id']]);
584                 }
585
586                 DBA::delete('gserver-tag', ['gserver-id' => $gserver['id']]);
587
588                 if ($data['scope'] == 'tags') {
589                         // Avoid duplicates
590                         $tags = [];
591                         foreach ($data['tags'] as $tag) {
592                                 $tag = mb_strtolower($tag);
593                                 if (strlen($tag) < 100) {
594                                         $tags[$tag] = $tag;
595                                 }
596                         }
597
598                         foreach ($tags as $tag) {
599                                 DBA::insert('gserver-tag', ['gserver-id' => $gserver['id'], 'tag' => $tag], Database::INSERT_IGNORE);
600                         }
601                 }
602
603                 // Create or update the relay contact
604                 $fields = [];
605                 if (isset($data['protocols'])) {
606                         if (isset($data['protocols']['diaspora'])) {
607                                 $fields['network'] = Protocol::DIASPORA;
608
609                                 if (isset($data['protocols']['diaspora']['receive'])) {
610                                         $fields['batch'] = $data['protocols']['diaspora']['receive'];
611                                 } elseif (is_string($data['protocols']['diaspora'])) {
612                                         $fields['batch'] = $data['protocols']['diaspora'];
613                                 }
614                         }
615
616                         if (isset($data['protocols']['dfrn'])) {
617                                 $fields['network'] = Protocol::DFRN;
618
619                                 if (isset($data['protocols']['dfrn']['receive'])) {
620                                         $fields['batch'] = $data['protocols']['dfrn']['receive'];
621                                 } elseif (is_string($data['protocols']['dfrn'])) {
622                                         $fields['batch'] = $data['protocols']['dfrn'];
623                                 }
624                         }
625
626                         if (isset($data['protocols']['activitypub'])) {
627                                 $fields['network'] = Protocol::ACTIVITYPUB;
628
629                                 if (!empty($data['protocols']['activitypub']['actor'])) {
630                                         $fields['url'] = $data['protocols']['activitypub']['actor'];
631                                 }
632                                 if (!empty($data['protocols']['activitypub']['receive'])) {
633                                         $fields['batch'] = $data['protocols']['activitypub']['receive'];
634                                 }
635                         }
636                 }
637
638                 Logger::info('Discovery ended', ['server' => $server_url, 'data' => $fields]);
639
640                 Relay::updateContact($gserver, $fields);
641         }
642
643         /**
644          * Fetch server data from '/statistics.json' on the given server
645          *
646          * @param string $url URL of the given server
647          *
648          * @return array server data
649          */
650         private static function fetchStatistics(string $url)
651         {
652                 $curlResult = DI::httpClient()->get($url . '/statistics.json');
653                 if (!$curlResult->isSuccess()) {
654                         return [];
655                 }
656
657                 $data = json_decode($curlResult->getBody(), true);
658                 if (empty($data)) {
659                         return [];
660                 }
661
662                 $serverdata = ['detection-method' => self::DETECT_STATISTICS_JSON];
663
664                 if (!empty($data['version'])) {
665                         $serverdata['version'] = $data['version'];
666                         // Version numbers on statistics.json are presented with additional info, e.g.:
667                         // 0.6.3.0-p1702cc1c, 0.6.99.0-p1b9ab160 or 3.4.3-2-1191.
668                         $serverdata['version'] = preg_replace('=(.+)-(.{4,})=ism', '$1', $serverdata['version']);
669                 }
670
671                 if (!empty($data['name'])) {
672                         $serverdata['site_name'] = $data['name'];
673                 }
674
675                 if (!empty($data['network'])) {
676                         $serverdata['platform'] = strtolower($data['network']);
677
678                         if ($serverdata['platform'] == 'diaspora') {
679                                 $serverdata['network'] = Protocol::DIASPORA;
680                         } elseif ($serverdata['platform'] == 'friendica') {
681                                 $serverdata['network'] = Protocol::DFRN;
682                         } elseif ($serverdata['platform'] == 'hubzilla') {
683                                 $serverdata['network'] = Protocol::ZOT;
684                         } elseif ($serverdata['platform'] == 'redmatrix') {
685                                 $serverdata['network'] = Protocol::ZOT;
686                         }
687                 }
688
689
690                 if (!empty($data['registrations_open'])) {
691                         $serverdata['register_policy'] = Register::OPEN;
692                 } else {
693                         $serverdata['register_policy'] = Register::CLOSED;
694                 }
695
696                 return $serverdata;
697         }
698
699         /**
700          * Detect server type by using the nodeinfo data
701          *
702          * @param string                  $url        address of the server
703          * @param ICanHandleHttpResponses $httpResult
704          *
705          * @return array Server data
706          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
707          */
708         private static function fetchNodeinfo(string $url, ICanHandleHttpResponses $httpResult)
709         {
710                 if (!$httpResult->isSuccess()) {
711                         return [];
712                 }
713
714                 $nodeinfo = json_decode($httpResult->getBody(), true);
715
716                 if (!is_array($nodeinfo) || empty($nodeinfo['links'])) {
717                         return [];
718                 }
719
720                 $nodeinfo1_url = '';
721                 $nodeinfo2_url = '';
722
723                 foreach ($nodeinfo['links'] as $link) {
724                         if (!is_array($link) || empty($link['rel']) || empty($link['href'])) {
725                                 Logger::info('Invalid nodeinfo format', ['url' => $url]);
726                                 continue;
727                         }
728                         if ($link['rel'] == 'http://nodeinfo.diaspora.software/ns/schema/1.0') {
729                                 $nodeinfo1_url = $link['href'];
730                         } elseif ($link['rel'] == 'http://nodeinfo.diaspora.software/ns/schema/2.0') {
731                                 $nodeinfo2_url = $link['href'];
732                         }
733                 }
734
735                 if ($nodeinfo1_url . $nodeinfo2_url == '') {
736                         return [];
737                 }
738
739                 $server = [];
740
741                 // When the nodeinfo url isn't on the same host, then there is obviously something wrong
742                 if (!empty($nodeinfo2_url) && (parse_url($url, PHP_URL_HOST) == parse_url($nodeinfo2_url, PHP_URL_HOST))) {
743                         $server = self::parseNodeinfo2($nodeinfo2_url);
744                 }
745
746                 // When the nodeinfo url isn't on the same host, then there is obviously something wrong
747                 if (empty($server) && !empty($nodeinfo1_url) && (parse_url($url, PHP_URL_HOST) == parse_url($nodeinfo1_url, PHP_URL_HOST))) {
748                         $server = self::parseNodeinfo1($nodeinfo1_url);
749                 }
750
751                 return $server;
752         }
753
754         /**
755          * Parses Nodeinfo 1
756          *
757          * @param string $nodeinfo_url address of the nodeinfo path
758          * @return array Server data
759          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
760          */
761         private static function parseNodeinfo1(string $nodeinfo_url)
762         {
763                 $curlResult = DI::httpClient()->get($nodeinfo_url);
764
765                 if (!$curlResult->isSuccess()) {
766                         return [];
767                 }
768
769                 $nodeinfo = json_decode($curlResult->getBody(), true);
770
771                 if (!is_array($nodeinfo)) {
772                         return [];
773                 }
774
775                 $server = ['detection-method' => self::DETECT_NODEINFO_1,
776                         'register_policy' => Register::CLOSED];
777
778                 if (!empty($nodeinfo['openRegistrations'])) {
779                         $server['register_policy'] = Register::OPEN;
780                 }
781
782                 if (is_array($nodeinfo['software'])) {
783                         if (!empty($nodeinfo['software']['name'])) {
784                                 $server['platform'] = strtolower($nodeinfo['software']['name']);
785                         }
786
787                         if (!empty($nodeinfo['software']['version'])) {
788                                 $server['version'] = $nodeinfo['software']['version'];
789                                 // Version numbers on Nodeinfo are presented with additional info, e.g.:
790                                 // 0.6.3.0-p1702cc1c, 0.6.99.0-p1b9ab160 or 3.4.3-2-1191.
791                                 $server['version'] = preg_replace('=(.+)-(.{4,})=ism', '$1', $server['version']);
792                         }
793                 }
794
795                 if (!empty($nodeinfo['metadata']['nodeName'])) {
796                         $server['site_name'] = $nodeinfo['metadata']['nodeName'];
797                 }
798
799                 if (!empty($nodeinfo['usage']['users']['total'])) {
800                         $server['registered-users'] = max($nodeinfo['usage']['users']['total'], 1);
801                 }
802
803                 if (!empty($nodeinfo['protocols']['inbound']) && is_array($nodeinfo['protocols']['inbound'])) {
804                         $protocols = [];
805                         foreach ($nodeinfo['protocols']['inbound'] as $protocol) {
806                                 $protocols[$protocol] = true;
807                         }
808
809                         if (!empty($protocols['friendica'])) {
810                                 $server['network'] = Protocol::DFRN;
811                         } elseif (!empty($protocols['activitypub'])) {
812                                 $server['network'] = Protocol::ACTIVITYPUB;
813                         } elseif (!empty($protocols['diaspora'])) {
814                                 $server['network'] = Protocol::DIASPORA;
815                         } elseif (!empty($protocols['ostatus'])) {
816                                 $server['network'] = Protocol::OSTATUS;
817                         } elseif (!empty($protocols['gnusocial'])) {
818                                 $server['network'] = Protocol::OSTATUS;
819                         } elseif (!empty($protocols['zot'])) {
820                                 $server['network'] = Protocol::ZOT;
821                         }
822                 }
823
824                 if (empty($server)) {
825                         return [];
826                 }
827
828                 return $server;
829         }
830
831         /**
832          * Parses Nodeinfo 2
833          *
834          * @see https://git.feneas.org/jaywink/nodeinfo2
835          * @param string $nodeinfo_url address of the nodeinfo path
836          * @return array Server data
837          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
838          */
839         private static function parseNodeinfo2(string $nodeinfo_url)
840         {
841                 $curlResult = DI::httpClient()->get($nodeinfo_url);
842                 if (!$curlResult->isSuccess()) {
843                         return [];
844                 }
845
846                 $nodeinfo = json_decode($curlResult->getBody(), true);
847
848                 if (!is_array($nodeinfo)) {
849                         return [];
850                 }
851
852                 $server = ['detection-method' => self::DETECT_NODEINFO_2,
853                         'register_policy' => Register::CLOSED];
854
855                 if (!empty($nodeinfo['openRegistrations'])) {
856                         $server['register_policy'] = Register::OPEN;
857                 }
858
859                 if (is_array($nodeinfo['software'])) {
860                         if (!empty($nodeinfo['software']['name'])) {
861                                 $server['platform'] = strtolower($nodeinfo['software']['name']);
862                         }
863
864                         if (!empty($nodeinfo['software']['version'])) {
865                                 $server['version'] = $nodeinfo['software']['version'];
866                                 // Version numbers on Nodeinfo are presented with additional info, e.g.:
867                                 // 0.6.3.0-p1702cc1c, 0.6.99.0-p1b9ab160 or 3.4.3-2-1191.
868                                 $server['version'] = preg_replace('=(.+)-(.{4,})=ism', '$1', $server['version']);
869                         }
870                 }
871
872                 if (!empty($nodeinfo['metadata']['nodeName'])) {
873                         $server['site_name'] = $nodeinfo['metadata']['nodeName'];
874                 }
875
876                 if (!empty($nodeinfo['usage']['users']['total'])) {
877                         $server['registered-users'] = max($nodeinfo['usage']['users']['total'], 1);
878                 }
879
880                 if (!empty($nodeinfo['protocols'])) {
881                         $protocols = [];
882                         foreach ($nodeinfo['protocols'] as $protocol) {
883                                 if (is_string($protocol)) {
884                                         $protocols[$protocol] = true;
885                                 }
886                         }
887
888                         if (!empty($protocols['dfrn'])) {
889                                 $server['network'] = Protocol::DFRN;
890                         } elseif (!empty($protocols['activitypub'])) {
891                                 $server['network'] = Protocol::ACTIVITYPUB;
892                         } elseif (!empty($protocols['diaspora'])) {
893                                 $server['network'] = Protocol::DIASPORA;
894                         } elseif (!empty($protocols['ostatus'])) {
895                                 $server['network'] = Protocol::OSTATUS;
896                         } elseif (!empty($protocols['gnusocial'])) {
897                                 $server['network'] = Protocol::OSTATUS;
898                         } elseif (!empty($protocols['zot'])) {
899                                 $server['network'] = Protocol::ZOT;
900                         }
901                 }
902
903                 if (empty($server)) {
904                         return [];
905                 }
906
907                 return $server;
908         }
909
910         /**
911          * Fetch server information from a 'siteinfo.json' file on the given server
912          *
913          * @param string $url        URL of the given server
914          * @param array  $serverdata array with server data
915          *
916          * @return array server data
917          */
918         private static function fetchSiteinfo(string $url, array $serverdata)
919         {
920                 $curlResult = DI::httpClient()->get($url . '/siteinfo.json');
921                 if (!$curlResult->isSuccess()) {
922                         return $serverdata;
923                 }
924
925                 $data = json_decode($curlResult->getBody(), true);
926                 if (empty($data)) {
927                         return $serverdata;
928                 }
929
930                 if (in_array($serverdata['detection-method'], [self::DETECT_HEADER, self::DETECT_BODY, self::DETECT_MANUAL])) {
931                         $serverdata['detection-method'] = self::DETECT_SITEINFO_JSON;
932                 }
933
934                 if (!empty($data['url'])) {
935                         $serverdata['platform'] = strtolower($data['platform']);
936                         $serverdata['version'] = $data['version'];
937                 }
938
939                 if (!empty($data['plugins'])) {
940                         if (in_array('pubcrawl', $data['plugins'])) {
941                                 $serverdata['network'] = Protocol::ACTIVITYPUB;
942                         } elseif (in_array('diaspora', $data['plugins'])) {
943                                 $serverdata['network'] = Protocol::DIASPORA;
944                         } elseif (in_array('gnusoc', $data['plugins'])) {
945                                 $serverdata['network'] = Protocol::OSTATUS;
946                         } else {
947                                 $serverdata['network'] = Protocol::ZOT;
948                         }
949                 }
950
951                 if (!empty($data['site_name'])) {
952                         $serverdata['site_name'] = $data['site_name'];
953                 }
954
955                 if (!empty($data['channels_total'])) {
956                         $serverdata['registered-users'] = max($data['channels_total'], 1);
957                 }
958
959                 if (!empty($data['register_policy'])) {
960                         switch ($data['register_policy']) {
961                                 case 'REGISTER_OPEN':
962                                         $serverdata['register_policy'] = Register::OPEN;
963                                         break;
964
965                                 case 'REGISTER_APPROVE':
966                                         $serverdata['register_policy'] = Register::APPROVE;
967                                         break;
968
969                                 case 'REGISTER_CLOSED':
970                                 default:
971                                         $serverdata['register_policy'] = Register::CLOSED;
972                                         break;
973                         }
974                 }
975
976                 return $serverdata;
977         }
978
979         /**
980          * Checks if the server contains a valid host meta file
981          *
982          * @param string $url URL of the given server
983          *
984          * @return boolean 'true' if the server seems to be vital
985          */
986         private static function validHostMeta(string $url)
987         {
988                 $xrd_timeout = DI::config()->get('system', 'xrd_timeout');
989                 $curlResult = DI::httpClient()->get($url . '/.well-known/host-meta', [HttpClientOptions::TIMEOUT => $xrd_timeout]);
990                 if (!$curlResult->isSuccess()) {
991                         return false;
992                 }
993
994                 $xrd = XML::parseString($curlResult->getBody());
995                 if (!is_object($xrd)) {
996                         return false;
997                 }
998
999                 $elements = XML::elementToArray($xrd);
1000                 if (empty($elements) || empty($elements['xrd']) || empty($elements['xrd']['link'])) {
1001                         return false;
1002                 }
1003
1004                 $valid = false;
1005                 foreach ($elements['xrd']['link'] as $link) {
1006                         // When there is more than a single "link" element, the array looks slightly different
1007                         if (!empty($link['@attributes'])) {
1008                                 $link = $link['@attributes'];
1009                         }
1010
1011                         if (empty($link['rel']) || empty($link['template'])) {
1012                                 continue;
1013                         }
1014
1015                         if ($link['rel'] == 'lrdd') {
1016                                 // When the webfinger host is the same like the system host, it should be ok.
1017                                 $valid = (parse_url($url, PHP_URL_HOST) == parse_url($link['template'], PHP_URL_HOST));
1018                         }
1019                 }
1020
1021                 return $valid;
1022         }
1023
1024         /**
1025          * Detect the network of the given server via their known contacts
1026          *
1027          * @param string $url        URL of the given server
1028          * @param array  $serverdata array with server data
1029          *
1030          * @return array server data
1031          */
1032         private static function detectNetworkViaContacts(string $url, array $serverdata)
1033         {
1034                 $contacts = [];
1035
1036                 $apcontacts = DBA::select('apcontact', ['url'], ['baseurl' => [$url, $serverdata['nurl']]]);
1037                 while ($apcontact = DBA::fetch($apcontacts)) {
1038                         $contacts[Strings::normaliseLink($apcontact['url'])] = $apcontact['url'];
1039                 }
1040                 DBA::close($apcontacts);
1041
1042                 $pcontacts = DBA::select('contact', ['url', 'nurl'], ['uid' => 0, 'baseurl' => [$url, $serverdata['nurl']]]);
1043                 while ($pcontact = DBA::fetch($pcontacts)) {
1044                         $contacts[$pcontact['nurl']] = $pcontact['url'];
1045                 }
1046                 DBA::close($pcontacts);
1047
1048                 if (empty($contacts)) {
1049                         return $serverdata;
1050                 }
1051
1052                 foreach ($contacts as $contact) {
1053                         $probed = Contact::getByURL($contact);
1054                         if (!empty($probed) && in_array($probed['network'], Protocol::FEDERATED)) {
1055                                 $serverdata['network'] = $probed['network'];
1056                                 break;
1057                         }
1058                 }
1059
1060                 $serverdata['registered-users'] = max($serverdata['registered-users'], count($contacts), 1);
1061
1062                 return $serverdata;
1063         }
1064
1065         /**
1066          * Checks if the given server does have a '/poco' endpoint.
1067          * This is used for the 'PortableContact' functionality,
1068          * which is used by both Friendica and Hubzilla.
1069          *
1070          * @param string $url        URL of the given server
1071          * @param array  $serverdata array with server data
1072          *
1073          * @return array server data
1074          */
1075         private static function checkPoCo(string $url, array $serverdata)
1076         {
1077                 $serverdata['poco'] = '';
1078
1079                 $curlResult = DI::httpClient()->get($url . '/poco');
1080                 if (!$curlResult->isSuccess()) {
1081                         return $serverdata;
1082                 }
1083
1084                 $data = json_decode($curlResult->getBody(), true);
1085                 if (empty($data)) {
1086                         return $serverdata;
1087                 }
1088
1089                 if (!empty($data['totalResults'])) {
1090                         $registeredUsers = $serverdata['registered-users'] ?? 0;
1091                         $serverdata['registered-users'] = max($data['totalResults'], $registeredUsers, 1);
1092                         $serverdata['directory-type'] = self::DT_POCO;
1093                         $serverdata['poco'] = $url . '/poco';
1094                 }
1095
1096                 return $serverdata;
1097         }
1098
1099         /**
1100          * Checks if the given server does have a Mastodon style directory endpoint.
1101          *
1102          * @param string $url        URL of the given server
1103          * @param array  $serverdata array with server data
1104          *
1105          * @return array server data
1106          */
1107         public static function checkMastodonDirectory(string $url, array $serverdata)
1108         {
1109                 $curlResult = DI::httpClient()->get($url . '/api/v1/directory?limit=1');
1110                 if (!$curlResult->isSuccess()) {
1111                         return $serverdata;
1112                 }
1113
1114                 $data = json_decode($curlResult->getBody(), true);
1115                 if (empty($data)) {
1116                         return $serverdata;
1117                 }
1118
1119                 if (count($data) == 1) {
1120                         $serverdata['directory-type'] = self::DT_MASTODON;
1121                 }
1122
1123                 return $serverdata;
1124         }
1125
1126         /**
1127          * Detects Peertube via their known endpoint
1128          *
1129          * @param string $url        URL of the given server
1130          * @param array  $serverdata array with server data
1131          *
1132          * @return array server data
1133          */
1134         private static function detectPeertube(string $url, array $serverdata)
1135         {
1136                 $curlResult = DI::httpClient()->get($url . '/api/v1/config');
1137
1138                 if (!$curlResult->isSuccess() || ($curlResult->getBody() == '')) {
1139                         return $serverdata;
1140                 }
1141
1142                 $data = json_decode($curlResult->getBody(), true);
1143                 if (empty($data)) {
1144                         return $serverdata;
1145                 }
1146
1147                 if (!empty($data['instance']) && !empty($data['serverVersion'])) {
1148                         $serverdata['platform'] = 'peertube';
1149                         $serverdata['version'] = $data['serverVersion'];
1150                         $serverdata['network'] = Protocol::ACTIVITYPUB;
1151
1152                         if (!empty($data['instance']['name'])) {
1153                                 $serverdata['site_name'] = $data['instance']['name'];
1154                         }
1155
1156                         if (!empty($data['instance']['shortDescription'])) {
1157                                 $serverdata['info'] = $data['instance']['shortDescription'];
1158                         }
1159
1160                         if (!empty($data['signup'])) {
1161                                 if (!empty($data['signup']['allowed'])) {
1162                                         $serverdata['register_policy'] = Register::OPEN;
1163                                 }
1164                         }
1165
1166                         if (in_array($serverdata['detection-method'], [self::DETECT_HEADER, self::DETECT_BODY, self::DETECT_MANUAL])) {
1167                                 $serverdata['detection-method'] = self::DETECT_V1_CONFIG;
1168                         }
1169                 }
1170
1171                 return $serverdata;
1172         }
1173
1174         /**
1175          * Detects the version number of a given server when it was a NextCloud installation
1176          *
1177          * @param string $url        URL of the given server
1178          * @param array  $serverdata array with server data
1179          *
1180          * @return array server data
1181          */
1182         private static function detectNextcloud(string $url, array $serverdata)
1183         {
1184                 $curlResult = DI::httpClient()->get($url . '/status.php');
1185
1186                 if (!$curlResult->isSuccess() || ($curlResult->getBody() == '')) {
1187                         return $serverdata;
1188                 }
1189
1190                 $data = json_decode($curlResult->getBody(), true);
1191                 if (empty($data)) {
1192                         return $serverdata;
1193                 }
1194
1195                 if (!empty($data['version'])) {
1196                         $serverdata['platform'] = 'nextcloud';
1197                         $serverdata['version'] = $data['version'];
1198                         $serverdata['network'] = Protocol::ACTIVITYPUB;
1199
1200                         if (in_array($serverdata['detection-method'], [self::DETECT_HEADER, self::DETECT_BODY, self::DETECT_MANUAL])) {
1201                                 $serverdata['detection-method'] = self::DETECT_STATUS_PHP;
1202                         }
1203                 }
1204
1205                 return $serverdata;
1206         }
1207
1208         /**
1209          * Detects data from a given server url if it was a mastodon alike system
1210          *
1211          * @param string $url        URL of the given server
1212          * @param array  $serverdata array with server data
1213          *
1214          * @return array server data
1215          */
1216         private static function detectMastodonAlikes(string $url, array $serverdata)
1217         {
1218                 $curlResult = DI::httpClient()->get($url . '/api/v1/instance');
1219
1220                 if (!$curlResult->isSuccess() || ($curlResult->getBody() == '')) {
1221                         return $serverdata;
1222                 }
1223
1224                 $data = json_decode($curlResult->getBody(), true);
1225                 if (empty($data)) {
1226                         return $serverdata;
1227                 }
1228
1229                 if (in_array($serverdata['detection-method'], [self::DETECT_HEADER, self::DETECT_BODY, self::DETECT_MANUAL])) {
1230                         $serverdata['detection-method'] = self::DETECT_MASTODON_API;
1231                 }
1232
1233                 if (!empty($data['version'])) {
1234                         $serverdata['platform'] = 'mastodon';
1235                         $serverdata['version'] = $data['version'] ?? '';
1236                         $serverdata['network'] = Protocol::ACTIVITYPUB;
1237                 }
1238
1239                 if (!empty($data['title'])) {
1240                         $serverdata['site_name'] = $data['title'];
1241                 }
1242
1243                 if (!empty($data['title']) && empty($serverdata['platform']) && empty($serverdata['network'])) {
1244                         $serverdata['platform'] = 'mastodon';
1245                         $serverdata['network'] = Protocol::ACTIVITYPUB;
1246                 }
1247
1248                 if (!empty($data['description'])) {
1249                         $serverdata['info'] = trim($data['description']);
1250                 }
1251
1252                 if (!empty($data['stats']['user_count'])) {
1253                         $serverdata['registered-users'] = max($data['stats']['user_count'], 1);
1254                 }
1255
1256                 if (!empty($serverdata['version']) && preg_match('/.*?\(compatible;\s(.*)\s(.*)\)/ism', $serverdata['version'], $matches)) {
1257                         $serverdata['platform'] = strtolower($matches[1]);
1258                         $serverdata['version'] = $matches[2];
1259                 }
1260
1261                 if (!empty($serverdata['version']) && strstr(strtolower($serverdata['version']), 'pleroma')) {
1262                         $serverdata['platform'] = 'pleroma';
1263                         $serverdata['version'] = trim(str_ireplace('pleroma', '', $serverdata['version']));
1264                 }
1265
1266                 if (!empty($serverdata['platform']) && strstr($serverdata['platform'], 'pleroma')) {
1267                         $serverdata['version'] = trim(str_ireplace('pleroma', '', $serverdata['platform']));
1268                         $serverdata['platform'] = 'pleroma';
1269                 }
1270
1271                 return $serverdata;
1272         }
1273
1274         /**
1275          * Detects data from typical Hubzilla endpoints
1276          *
1277          * @param string $url        URL of the given server
1278          * @param array  $serverdata array with server data
1279          *
1280          * @return array server data
1281          */
1282         private static function detectHubzilla(string $url, array $serverdata)
1283         {
1284                 $curlResult = DI::httpClient()->get($url . '/api/statusnet/config.json');
1285                 if (!$curlResult->isSuccess() || ($curlResult->getBody() == '')) {
1286                         return $serverdata;
1287                 }
1288
1289                 $data = json_decode($curlResult->getBody(), true);
1290                 if (empty($data) || empty($data['site'])) {
1291                         return $serverdata;
1292                 }
1293
1294                 if (!empty($data['site']['name'])) {
1295                         $serverdata['site_name'] = $data['site']['name'];
1296                 }
1297
1298                 if (!empty($data['site']['platform'])) {
1299                         $serverdata['platform'] = strtolower($data['site']['platform']['PLATFORM_NAME']);
1300                         $serverdata['version'] = $data['site']['platform']['STD_VERSION'];
1301                         $serverdata['network'] = Protocol::ZOT;
1302                 }
1303
1304                 if (!empty($data['site']['hubzilla'])) {
1305                         $serverdata['platform'] = strtolower($data['site']['hubzilla']['PLATFORM_NAME']);
1306                         $serverdata['version'] = $data['site']['hubzilla']['RED_VERSION'];
1307                         $serverdata['network'] = Protocol::ZOT;
1308                 }
1309
1310                 if (!empty($data['site']['redmatrix'])) {
1311                         if (!empty($data['site']['redmatrix']['PLATFORM_NAME'])) {
1312                                 $serverdata['platform'] = strtolower($data['site']['redmatrix']['PLATFORM_NAME']);
1313                         } elseif (!empty($data['site']['redmatrix']['RED_PLATFORM'])) {
1314                                 $serverdata['platform'] = strtolower($data['site']['redmatrix']['RED_PLATFORM']);
1315                         }
1316
1317                         $serverdata['version'] = $data['site']['redmatrix']['RED_VERSION'];
1318                         $serverdata['network'] = Protocol::ZOT;
1319                 }
1320
1321                 $private = false;
1322                 $inviteonly = false;
1323                 $closed = false;
1324
1325                 if (!empty($data['site']['closed'])) {
1326                         $closed = self::toBoolean($data['site']['closed']);
1327                 }
1328
1329                 if (!empty($data['site']['private'])) {
1330                         $private = self::toBoolean($data['site']['private']);
1331                 }
1332
1333                 if (!empty($data['site']['inviteonly'])) {
1334                         $inviteonly = self::toBoolean($data['site']['inviteonly']);
1335                 }
1336
1337                 if (!$closed && !$private and $inviteonly) {
1338                         $serverdata['register_policy'] = Register::APPROVE;
1339                 } elseif (!$closed && !$private) {
1340                         $serverdata['register_policy'] = Register::OPEN;
1341                 } else {
1342                         $serverdata['register_policy'] = Register::CLOSED;
1343                 }
1344
1345                 if (!empty($serverdata['network']) && in_array($serverdata['detection-method'],
1346                         [self::DETECT_HEADER, self::DETECT_BODY, self::DETECT_MANUAL])) {
1347                         $serverdata['detection-method'] = self::DETECT_CONFIG_JSON;
1348                 }
1349
1350                 return $serverdata;
1351         }
1352
1353         /**
1354          * Converts input value to a boolean value
1355          *
1356          * @param string|integer $val
1357          *
1358          * @return boolean
1359          */
1360         private static function toBoolean($val)
1361         {
1362                 if (($val == 'true') || ($val == 1)) {
1363                         return true;
1364                 } elseif (($val == 'false') || ($val == 0)) {
1365                         return false;
1366                 }
1367
1368                 return $val;
1369         }
1370
1371         /**
1372          * Detect if the URL belongs to a GNU Social server
1373          *
1374          * @param string $url        URL of the given server
1375          * @param array  $serverdata array with server data
1376          *
1377          * @return array server data
1378          */
1379         private static function detectGNUSocial(string $url, array $serverdata)
1380         {
1381                 // Test for GNU Social
1382                 $curlResult = DI::httpClient()->get($url . '/api/gnusocial/version.json');
1383                 if ($curlResult->isSuccess() && ($curlResult->getBody() != '{"error":"not implemented"}') &&
1384                         ($curlResult->getBody() != '') && (strlen($curlResult->getBody()) < 30)) {
1385                         $serverdata['platform'] = 'gnusocial';
1386                         // Remove junk that some GNU Social servers return
1387                         $serverdata['version'] = str_replace(chr(239) . chr(187) . chr(191), '', $curlResult->getBody());
1388                         $serverdata['version'] = str_replace(["\r", "\n", "\t"], '', $serverdata['version']);
1389                         $serverdata['version'] = trim($serverdata['version'], '"');
1390                         $serverdata['network'] = Protocol::OSTATUS;
1391
1392                         if (in_array($serverdata['detection-method'], [self::DETECT_HEADER, self::DETECT_BODY, self::DETECT_MANUAL])) {
1393                                 $serverdata['detection-method'] = self::DETECT_GNUSOCIAL;
1394                         }
1395
1396                         return $serverdata;
1397                 }
1398
1399                 // Test for Statusnet
1400                 $curlResult = DI::httpClient()->get($url . '/api/statusnet/version.json');
1401                 if ($curlResult->isSuccess() && ($curlResult->getBody() != '{"error":"not implemented"}') &&
1402                         ($curlResult->getBody() != '') && (strlen($curlResult->getBody()) < 30)) {
1403
1404                         // Remove junk that some GNU Social servers return
1405                         $serverdata['version'] = str_replace(chr(239).chr(187).chr(191), '', $curlResult->getBody());
1406                         $serverdata['version'] = str_replace(["\r", "\n", "\t"], '', $serverdata['version']);
1407                         $serverdata['version'] = trim($serverdata['version'], '"');
1408
1409                         if (!empty($serverdata['version']) && strtolower(substr($serverdata['version'], 0, 7)) == 'pleroma') {
1410                                 $serverdata['platform'] = 'pleroma';
1411                                 $serverdata['version'] = trim(str_ireplace('pleroma', '', $serverdata['version']));
1412                                 $serverdata['network'] = Protocol::ACTIVITYPUB;
1413                         } else {
1414                                 $serverdata['platform'] = 'statusnet';
1415                                 $serverdata['network'] = Protocol::OSTATUS;
1416                         }
1417
1418                         if (in_array($serverdata['detection-method'], [self::DETECT_HEADER, self::DETECT_BODY, self::DETECT_MANUAL])) {
1419                                 $serverdata['detection-method'] = self::DETECT_STATUSNET;
1420                         }
1421                 }
1422
1423                 return $serverdata;
1424         }
1425
1426         /**
1427          * Detect if the URL belongs to a Friendica server
1428          *
1429          * @param string $url        URL of the given server
1430          * @param array  $serverdata array with server data
1431          *
1432          * @return array server data
1433          */
1434         private static function detectFriendica(string $url, array $serverdata)
1435         {
1436                 $curlResult = DI::httpClient()->get($url . '/friendica/json');
1437                 if (!$curlResult->isSuccess()) {
1438                         $curlResult = DI::httpClient()->get($url . '/friendika/json');
1439                         $friendika = true;
1440                         $platform = 'Friendika';
1441                 } else {
1442                         $friendika = false;
1443                         $platform = 'Friendica';
1444                 }
1445
1446                 if (!$curlResult->isSuccess()) {
1447                         return $serverdata;
1448                 }
1449
1450                 $data = json_decode($curlResult->getBody(), true);
1451                 if (empty($data) || empty($data['version'])) {
1452                         return $serverdata;
1453                 }
1454
1455                 if (in_array($serverdata['detection-method'], [self::DETECT_HEADER, self::DETECT_BODY, self::DETECT_MANUAL])) {
1456                         $serverdata['detection-method'] = $friendika ? self::DETECT_FRIENDIKA : self::DETECT_FRIENDICA;
1457                 }
1458
1459                 $serverdata['network'] = Protocol::DFRN;
1460                 $serverdata['version'] = $data['version'];
1461
1462                 if (!empty($data['no_scrape_url'])) {
1463                         $serverdata['noscrape'] = $data['no_scrape_url'];
1464                 }
1465
1466                 if (!empty($data['site_name'])) {
1467                         $serverdata['site_name'] = $data['site_name'];
1468                 }
1469
1470                 if (!empty($data['info'])) {
1471                         $serverdata['info'] = trim($data['info']);
1472                 }
1473
1474                 $register_policy = ($data['register_policy'] ?? '') ?: 'REGISTER_CLOSED';
1475                 switch ($register_policy) {
1476                         case 'REGISTER_OPEN':
1477                                 $serverdata['register_policy'] = Register::OPEN;
1478                                 break;
1479
1480                         case 'REGISTER_APPROVE':
1481                                 $serverdata['register_policy'] = Register::APPROVE;
1482                                 break;
1483
1484                         case 'REGISTER_CLOSED':
1485                         case 'REGISTER_INVITATION':
1486                                 $serverdata['register_policy'] = Register::CLOSED;
1487                                 break;
1488                         default:
1489                                 Logger::info('Register policy is invalid', ['policy' => $register_policy, 'server' => $url]);
1490                                 $serverdata['register_policy'] = Register::CLOSED;
1491                                 break;
1492                 }
1493
1494                 $serverdata['platform'] = strtolower($data['platform'] ?? $platform);
1495
1496                 return $serverdata;
1497         }
1498
1499         /**
1500          * Analyses the landing page of a given server for hints about type and system of that server
1501          *
1502          * @param object $curlResult result of curl execution
1503          * @param array  $serverdata array with server data
1504          * @param string $url        Server URL
1505          *
1506          * @return array server data
1507          */
1508         private static function analyseRootBody($curlResult, array $serverdata, string $url)
1509         {
1510                 if (empty($curlResult->getBody())) {
1511                         return $serverdata;
1512                 }
1513
1514                 $doc = new DOMDocument();
1515                 @$doc->loadHTML($curlResult->getBody());
1516                 $xpath = new DOMXPath($doc);
1517
1518                 $title = trim(XML::getFirstNodeValue($xpath, '//head/title/text()'));
1519                 if (!empty($title)) {
1520                         $serverdata['site_name'] = $title;
1521                 }
1522
1523                 $list = $xpath->query('//meta[@name]');
1524
1525                 foreach ($list as $node) {
1526                         $attr = [];
1527                         if ($node->attributes->length) {
1528                                 foreach ($node->attributes as $attribute) {
1529                                         $value = trim($attribute->value);
1530                                         if (empty($value)) {
1531                                                 continue;
1532                                         }
1533
1534                                         $attr[$attribute->name] = $value;
1535                                 }
1536
1537                                 if (empty($attr['name']) || empty($attr['content'])) {
1538                                         continue;
1539                                 }
1540                         }
1541
1542                         if ($attr['name'] == 'description') {
1543                                 $serverdata['info'] = $attr['content'];
1544                         }
1545
1546                         if (in_array($attr['name'], ['application-name', 'al:android:app_name', 'al:ios:app_name',
1547                                 'twitter:app:name:googleplay', 'twitter:app:name:iphone', 'twitter:app:name:ipad'])) {
1548                                 $serverdata['platform'] = strtolower($attr['content']);
1549                                 if (in_array($attr['content'], ['Misskey', 'Write.as'])) {
1550                                         $serverdata['network'] = Protocol::ACTIVITYPUB;
1551                                 }
1552                         }
1553                         if (($attr['name'] == 'generator') && (empty($serverdata['platform']) || (substr(strtolower($attr['content']), 0, 9) == 'wordpress'))) {
1554                                 $serverdata['platform'] = strtolower($attr['content']);
1555                                 $version_part = explode(' ', $attr['content']);
1556
1557                                 if (count($version_part) == 2) {
1558                                         if (in_array($version_part[0], ['WordPress'])) {
1559                                                 $serverdata['platform'] = 'wordpress';
1560                                                 $serverdata['version'] = $version_part[1];
1561
1562                                                 // We still do need a reliable test if some AP plugin is activated
1563                                                 if (DBA::exists('apcontact', ['baseurl' => $url])) {
1564                                                         $serverdata['network'] = Protocol::ACTIVITYPUB;
1565                                                 } else {
1566                                                         $serverdata['network'] = Protocol::FEED;
1567                                                 }
1568
1569                                                 if ($serverdata['detection-method'] == self::DETECT_MANUAL) {
1570                                                         $serverdata['detection-method'] = self::DETECT_BODY;
1571                                                 }
1572                                         }
1573                                         if (in_array($version_part[0], ['Friendika', 'Friendica'])) {
1574                                                 $serverdata['platform'] = strtolower($version_part[0]);
1575                                                 $serverdata['version'] = $version_part[1];
1576                                                 $serverdata['network'] = Protocol::DFRN;
1577                                         }
1578                                 }
1579                         }
1580                 }
1581
1582                 $list = $xpath->query('//meta[@property]');
1583
1584                 foreach ($list as $node) {
1585                         $attr = [];
1586                         if ($node->attributes->length) {
1587                                 foreach ($node->attributes as $attribute) {
1588                                         $value = trim($attribute->value);
1589                                         if (empty($value)) {
1590                                                 continue;
1591                                         }
1592
1593                                         $attr[$attribute->name] = $value;
1594                                 }
1595
1596                                 if (empty($attr['property']) || empty($attr['content'])) {
1597                                         continue;
1598                                 }
1599                         }
1600
1601                         if ($attr['property'] == 'og:site_name') {
1602                                 $serverdata['site_name'] = $attr['content'];
1603                         }
1604
1605                         if ($attr['property'] == 'og:description') {
1606                                 $serverdata['info'] = $attr['content'];
1607                         }
1608
1609                         if ($attr['property'] == 'og:platform') {
1610                                 $serverdata['platform'] = strtolower($attr['content']);
1611
1612                                 if (in_array($attr['content'], ['PeerTube'])) {
1613                                         $serverdata['network'] = Protocol::ACTIVITYPUB;
1614                                 }
1615                         }
1616
1617                         if ($attr['property'] == 'generator') {
1618                                 $serverdata['platform'] = strtolower($attr['content']);
1619
1620                                 if (in_array($attr['content'], ['hubzilla'])) {
1621                                         // We later check which compatible protocol modules are loaded.
1622                                         $serverdata['network'] = Protocol::ZOT;
1623                                 }
1624                         }
1625                 }
1626
1627                 if (!empty($serverdata['network']) && ($serverdata['detection-method'] == self::DETECT_MANUAL)) {
1628                         $serverdata['detection-method'] = self::DETECT_BODY;
1629                 }
1630
1631                 return $serverdata;
1632         }
1633
1634         /**
1635          * Analyses the header data of a given server for hints about type and system of that server
1636          *
1637          * @param object $curlResult result of curl execution
1638          * @param array  $serverdata array with server data
1639          *
1640          * @return array server data
1641          */
1642         private static function analyseRootHeader($curlResult, array $serverdata)
1643         {
1644                 if ($curlResult->getHeader('server') == 'Mastodon') {
1645                         $serverdata['platform'] = 'mastodon';
1646                         $serverdata['network'] = Protocol::ACTIVITYPUB;
1647                 } elseif ($curlResult->inHeader('x-diaspora-version')) {
1648                         $serverdata['platform'] = 'diaspora';
1649                         $serverdata['network'] = Protocol::DIASPORA;
1650                         $serverdata['version'] = $curlResult->getHeader('x-diaspora-version')[0] ?? '';
1651                 } elseif ($curlResult->inHeader('x-friendica-version')) {
1652                         $serverdata['platform'] = 'friendica';
1653                         $serverdata['network'] = Protocol::DFRN;
1654                         $serverdata['version'] = $curlResult->getHeader('x-friendica-version')[0] ?? '';
1655                 } else {
1656                         return $serverdata;
1657                 }
1658
1659                 if ($serverdata['detection-method'] == self::DETECT_MANUAL) {
1660                         $serverdata['detection-method'] = self::DETECT_HEADER;
1661                 }
1662
1663                 return $serverdata;
1664         }
1665
1666         /**
1667          * Test if the body contains valid content
1668          *
1669          * @param string $body
1670          * @return boolean
1671          */
1672         private static function invalidBody(string $body)
1673         {
1674                 // Currently we only test for a HTML element.
1675                 // Possibly we enhance this in the future.
1676                 return !strpos($body, '>');
1677         }
1678
1679         /**
1680          * Update GServer entries
1681          */
1682         public static function discover()
1683         {
1684                 // Update the server list
1685                 self::discoverFederation();
1686
1687                 $no_of_queries = 5;
1688
1689                 $requery_days = intval(DI::config()->get('system', 'poco_requery_days'));
1690
1691                 if ($requery_days == 0) {
1692                         $requery_days = 7;
1693                 }
1694
1695                 $last_update = date('c', time() - (60 * 60 * 24 * $requery_days));
1696
1697                 $gservers = DBA::select('gserver', ['id', 'url', 'nurl', 'network', 'poco', 'directory-type'],
1698                         ["NOT `failed` AND `directory-type` != ? AND `last_poco_query` < ?", GServer::DT_NONE, $last_update],
1699                         ['order' => ['RAND()']]);
1700
1701                 while ($gserver = DBA::fetch($gservers)) {
1702                         Logger::info('Update peer list', ['server' => $gserver['url'], 'id' => $gserver['id']]);
1703                         Worker::add(PRIORITY_LOW, 'UpdateServerPeers', $gserver['url']);
1704
1705                         Logger::info('Update directory', ['server' => $gserver['url'], 'id' => $gserver['id']]);
1706                         Worker::add(PRIORITY_LOW, 'UpdateServerDirectory', $gserver);
1707
1708                         $fields = ['last_poco_query' => DateTimeFormat::utcNow()];
1709                         DBA::update('gserver', $fields, ['nurl' => $gserver['nurl']]);
1710
1711                         if (--$no_of_queries == 0) {
1712                                 break;
1713                         }
1714                 }
1715
1716                 DBA::close($gservers);
1717         }
1718
1719         /**
1720          * Discover federated servers
1721          */
1722         private static function discoverFederation()
1723         {
1724                 $last = DI::config()->get('poco', 'last_federation_discovery');
1725
1726                 if ($last) {
1727                         $next = $last + (24 * 60 * 60);
1728
1729                         if ($next > time()) {
1730                                 return;
1731                         }
1732                 }
1733
1734                 // Discover federated servers
1735                 $protocols = ['activitypub', 'diaspora', 'dfrn', 'ostatus'];
1736                 foreach ($protocols as $protocol) {
1737                         $query = '{nodes(protocol:"' . $protocol . '"){host}}';
1738                         $curlResult = DI::httpClient()->fetch('https://the-federation.info/graphql?query=' . urlencode($query));
1739                         if (!empty($curlResult)) {
1740                                 $data = json_decode($curlResult, true);
1741                                 if (!empty($data['data']['nodes'])) {
1742                                         foreach ($data['data']['nodes'] as $server) {
1743                                                 // Using "only_nodeinfo" since servers that are listed on that page should always have it.
1744                                                 self::add('https://' . $server['host'], true);
1745                                         }
1746                                 }
1747                         }
1748                 }
1749
1750                 // Disvover Mastodon servers
1751                 $accesstoken = DI::config()->get('system', 'instances_social_key');
1752
1753                 if (!empty($accesstoken)) {
1754                         $api = 'https://instances.social/api/1.0/instances/list?count=0';
1755                         $curlResult = DI::httpClient()->get($api, [HttpClientOptions::HEADERS => ['Authorization' => ['Bearer ' . $accesstoken]]]);
1756
1757                         if ($curlResult->isSuccess()) {
1758                                 $servers = json_decode($curlResult->getBody(), true);
1759
1760                                 foreach ($servers['instances'] as $server) {
1761                                         $url = (is_null($server['https_score']) ? 'http' : 'https') . '://' . $server['name'];
1762                                         self::add($url);
1763                                 }
1764                         }
1765                 }
1766
1767                 DI::config()->set('poco', 'last_federation_discovery', time());
1768         }
1769
1770         /**
1771          * Set the protocol for the given server
1772          *
1773          * @param int $gsid     Server id
1774          * @param int $protocol Protocol id
1775          * @return void
1776          * @throws Exception
1777          */
1778         public static function setProtocol(int $gsid, int $protocol)
1779         {
1780                 if (empty($gsid)) {
1781                         return;
1782                 }
1783
1784                 $gserver = DBA::selectFirst('gserver', ['protocol', 'url'], ['id' => $gsid]);
1785                 if (!DBA::isResult($gserver)) {
1786                         return;
1787                 }
1788
1789                 $old = $gserver['protocol'];
1790
1791                 if (!is_null($old)) {
1792                         /*
1793                         The priority for the protocols is:
1794                                 1. ActivityPub
1795                                 2. DFRN via Diaspora
1796                                 3. Legacy DFRN
1797                                 4. Diaspora
1798                                 5. OStatus
1799                         */
1800
1801                         // We don't need to change it when nothing is to be changed
1802                         if ($old == $protocol) {
1803                                 return;
1804                         }
1805
1806                         // We don't want to mark a server as OStatus when it had been marked with any other protocol before
1807                         if ($protocol == Post\DeliveryData::OSTATUS) {
1808                                 return;
1809                         }
1810
1811                         // If the server is marked as ActivityPub then we won't change it to anything different
1812                         if ($old == Post\DeliveryData::ACTIVITYPUB) {
1813                                 return;
1814                         }
1815
1816                         // Don't change it to anything lower than DFRN if the new one wasn't ActivityPub
1817                         if (($old == Post\DeliveryData::DFRN) && ($protocol != Post\DeliveryData::ACTIVITYPUB)) {
1818                                 return;
1819                         }
1820
1821                         // Don't change it to Diaspora when it is a legacy DFRN server
1822                         if (($old == Post\DeliveryData::LEGACY_DFRN) && ($protocol == Post\DeliveryData::DIASPORA)) {
1823                                 return;
1824                         }
1825                 }
1826
1827                 Logger::info('Protocol for server', ['protocol' => $protocol, 'old' => $old, 'id' => $gsid, 'url' => $gserver['url'], 'callstack' => System::callstack(20)]);
1828                 DBA::update('gserver', ['protocol' => $protocol], ['id' => $gsid]);
1829         }
1830
1831         /**
1832          * Fetch the protocol of the given server
1833          *
1834          * @param int $gsid Server id
1835          * @return int
1836          * @throws Exception
1837          */
1838         public static function getProtocol(int $gsid)
1839         {
1840                 if (empty($gsid)) {
1841                         return null;
1842                 }
1843
1844                 $gserver = DBA::selectFirst('gserver', ['protocol'], ['id' => $gsid]);
1845                 if (DBA::isResult($gserver)) {
1846                         return $gserver['protocol'];
1847                 }
1848
1849                 return null;
1850         }
1851 }