src/Protocol/PortableContact.php

   1 <?php
   2 /**
   3  * @file src/Protocol/PortableContact.php
   4  *
   5  * @todo Move GNU Social URL schemata (http://server.tld/user/number) to http://server.tld/username
   6  * @todo Fetch profile data from profile page for Redmatrix users
   7  * @todo Detect if it is a forum
   8  */
   9
  10 namespace Friendica\Protocol;
  11
  12 use DOMDocument;
  13 use DOMXPath;
  14 use Exception;
  15 use Friendica\Content\Text\HTML;
  16 use Friendica\Core\Config;
  17 use Friendica\Core\Logger;
  18 use Friendica\Core\Protocol;
  19 use Friendica\Core\Worker;
  20 use Friendica\Database\DBA;
  21 use Friendica\Model\Contact;
  22 use Friendica\Model\GContact;
  23 use Friendica\Model\Profile;
  24 use Friendica\Module\Register;
  25 use Friendica\Network\Probe;
  26 use Friendica\Util\DateTimeFormat;
  27 use Friendica\Util\Network;
  28 use Friendica\Util\Strings;
  29 use Friendica\Util\XML;
  30
  31 class PortableContact
  32 {
  33         const DISABLED = 0;
  34         const USERS = 1;
  35         const USERS_GCONTACTS = 2;
  36         const USERS_GCONTACTS_FALLBACK = 3;
  37
  38         /**
  39          * @brief Fetch POCO data
  40          *
  41          * @param integer $cid  Contact ID
  42          * @param integer $uid  User ID
  43          * @param integer $zcid Global Contact ID
  44          * @param integer $url  POCO address that should be polled
  45          *
  46          * Given a contact-id (minimum), load the PortableContacts friend list for that contact,
  47          * and add the entries to the gcontact (Global Contact) table, or update existing entries
  48          * if anything (name or photo) has changed.
  49          * We use normalised urls for comparison which ignore http vs https and www.domain vs domain
  50          *
  51          * Once the global contact is stored add (if necessary) the contact linkage which associates
  52          * the given uid, cid to the global contact entry. There can be many uid/cid combinations
  53          * pointing to the same global contact id.
  54          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
  55          */
  56         public static function loadWorker($cid, $uid = 0, $zcid = 0, $url = null)
  57         {
  58                 // Call the function "load" via the worker
  59                 Worker::add(PRIORITY_LOW, "DiscoverPoCo", "load", (int)$cid, (int)$uid, (int)$zcid, $url);
  60         }
  61
  62         /**
  63          * @brief Fetch POCO data from the worker
  64          *
  65          * @param integer $cid  Contact ID
  66          * @param integer $uid  User ID
  67          * @param integer $zcid Global Contact ID
  68          * @param integer $url  POCO address that should be polled
  69          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
  70          */
  71         public static function load($cid, $uid, $zcid, $url)
  72         {
  73                 if ($cid) {
  74                         if (!$url || !$uid) {
  75                                 $contact = DBA::selectFirst('contact', ['poco', 'uid'], ['id' => $cid]);
  76                                 if (DBA::isResult($contact)) {
  77                                         $url = $contact['poco'];
  78                                         $uid = $contact['uid'];
  79                                 }
  80                         }
  81                         if (!$uid) {
  82                                 return;
  83                         }
  84                 }
  85
  86                 if (!$url) {
  87                         return;
  88                 }
  89
  90                 $url = $url . (($uid) ? '/@me/@all?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,contactType,generation' : '?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,contactType,generation');
  91
  92                 Logger::log('load: ' . $url, Logger::DEBUG);
  93
  94                 $fetchresult = Network::fetchUrlFull($url);
  95                 $s = $fetchresult->getBody();
  96
  97                 Logger::log('load: returns ' . $s, Logger::DATA);
  98
  99                 Logger::log('load: return code: ' . $fetchresult->getReturnCode(), Logger::DEBUG);
 100
 101                 if (($fetchresult->getReturnCode() > 299) || (! $s)) {
 102                         return;
 103                 }
 104
 105                 $j = json_decode($s, true);
 106
 107                 Logger::log('load: json: ' . print_r($j, true), Logger::DATA);
 108
 109                 if (!isset($j['entry'])) {
 110                         return;
 111                 }
 112
 113                 $total = 0;
 114                 foreach ($j['entry'] as $entry) {
 115                         $total ++;
 116                         $profile_url = '';
 117                         $profile_photo = '';
 118                         $connect_url = '';
 119                         $name = '';
 120                         $network = '';
 121                         $updated = DBA::NULL_DATETIME;
 122                         $location = '';
 123                         $about = '';
 124                         $keywords = '';
 125                         $gender = '';
 126                         $contact_type = -1;
 127                         $generation = 0;
 128
 129                         if (!empty($entry['displayName'])) {
 130                                 $name = $entry['displayName'];
 131                         }
 132
 133                         if (isset($entry['urls'])) {
 134                                 foreach ($entry['urls'] as $url) {
 135                                         if ($url['type'] == 'profile') {
 136                                                 $profile_url = $url['value'];
 137                                                 continue;
 138                                         }
 139                                         if ($url['type'] == 'webfinger') {
 140                                                 $connect_url = str_replace('acct:', '', $url['value']);
 141                                                 continue;
 142                                         }
 143                                 }
 144                         }
 145                         if (isset($entry['photos'])) {
 146                                 foreach ($entry['photos'] as $photo) {
 147                                         if ($photo['type'] == 'profile') {
 148                                                 $profile_photo = $photo['value'];
 149                                                 continue;
 150                                         }
 151                                 }
 152                         }
 153
 154                         if (isset($entry['updated'])) {
 155                                 $updated = date(DateTimeFormat::MYSQL, strtotime($entry['updated']));
 156                         }
 157
 158                         if (isset($entry['network'])) {
 159                                 $network = $entry['network'];
 160                         }
 161
 162                         if (isset($entry['currentLocation'])) {
 163                                 $location = $entry['currentLocation'];
 164                         }
 165
 166                         if (isset($entry['aboutMe'])) {
 167                                 $about = HTML::toBBCode($entry['aboutMe']);
 168                         }
 169
 170                         if (isset($entry['gender'])) {
 171                                 $gender = $entry['gender'];
 172                         }
 173
 174                         if (isset($entry['generation']) && ($entry['generation'] > 0)) {
 175                                 $generation = ++$entry['generation'];
 176                         }
 177
 178                         if (isset($entry['tags'])) {
 179                                 foreach ($entry['tags'] as $tag) {
 180                                         $keywords = implode(", ", $tag);
 181                                 }
 182                         }
 183
 184                         if (isset($entry['contactType']) && ($entry['contactType'] >= 0)) {
 185                                 $contact_type = $entry['contactType'];
 186                         }
 187
 188                         $gcontact = ["url" => $profile_url,
 189                                         "name" => $name,
 190                                         "network" => $network,
 191                                         "photo" => $profile_photo,
 192                                         "about" => $about,
 193                                         "location" => $location,
 194                                         "gender" => $gender,
 195                                         "keywords" => $keywords,
 196                                         "connect" => $connect_url,
 197                                         "updated" => $updated,
 198                                         "contact-type" => $contact_type,
 199                                         "generation" => $generation];
 200
 201                         try {
 202                                 $gcontact = GContact::sanitize($gcontact);
 203                                 $gcid = GContact::update($gcontact);
 204
 205                                 GContact::link($gcid, $uid, $cid, $zcid);
 206                         } catch (Exception $e) {
 207                                 Logger::log($e->getMessage(), Logger::DEBUG);
 208                         }
 209                 }
 210                 Logger::log("load: loaded $total entries", Logger::DEBUG);
 211
 212                 $condition = ["`cid` = ? AND `uid` = ? AND `zcid` = ? AND `updated` < UTC_TIMESTAMP - INTERVAL 2 DAY", $cid, $uid, $zcid];
 213                 DBA::delete('glink', $condition);
 214         }
 215
 216         public static function reachable($profile, $server = "", $network = "", $force = false)
 217         {
 218                 if ($server == "") {
 219                         $server = Contact::getBasepath($profile);
 220                 }
 221
 222                 if ($server == "") {
 223                         return true;
 224                 }
 225
 226                 return self::checkServer($server, $network, $force);
 227         }
 228
 229         public static function alternateOStatusUrl($url)
 230         {
 231                 return(preg_match("=https?://.+/user/\d+=ism", $url, $matches));
 232         }
 233
 234         public static function lastUpdated($profile, $force = false)
 235         {
 236                 $gcontacts = q(
 237                         "SELECT * FROM `gcontact` WHERE `nurl` = '%s'",
 238                         DBA::escape(Strings::normaliseLink($profile))
 239                 );
 240
 241                 if (!DBA::isResult($gcontacts)) {
 242                         return false;
 243                 }
 244
 245                 $contact = ["url" => $profile];
 246
 247                 if ($gcontacts[0]["created"] <= DBA::NULL_DATETIME) {
 248                         $contact['created'] = DateTimeFormat::utcNow();
 249                 }
 250
 251                 $server_url = '';
 252                 if ($force) {
 253                         $server_url = Strings::normaliseLink(Contact::getBasepath($profile));
 254                 }
 255
 256                 if (($server_url == '') && ($gcontacts[0]["server_url"] != "")) {
 257                         $server_url = $gcontacts[0]["server_url"];
 258                 }
 259
 260                 if (!$force && (($server_url == '') || ($gcontacts[0]["server_url"] == $gcontacts[0]["nurl"]))) {
 261                         $server_url = Strings::normaliseLink(Contact::getBasepath($profile));
 262                 }
 263
 264                 if (!in_array($gcontacts[0]["network"], [Protocol::DFRN, Protocol::DIASPORA, Protocol::FEED, Protocol::OSTATUS, ""])) {
 265                         Logger::log("Profile ".$profile.": Network type ".$gcontacts[0]["network"]." can't be checked", Logger::DEBUG);
 266                         return false;
 267                 }
 268
 269                 if ($server_url != "") {
 270                         if (!self::checkServer($server_url, $gcontacts[0]["network"], $force)) {
 271                                 if ($force) {
 272                                         $fields = ['last_failure' => DateTimeFormat::utcNow()];
 273                                         DBA::update('gcontact', $fields, ['nurl' => Strings::normaliseLink($profile)]);
 274                                 }
 275
 276                                 Logger::log("Profile ".$profile.": Server ".$server_url." wasn't reachable.", Logger::DEBUG);
 277                                 return false;
 278                         }
 279                         $contact['server_url'] = $server_url;
 280                 }
 281
 282                 if (in_array($gcontacts[0]["network"], ["", Protocol::FEED])) {
 283                         $server = q(
 284                                 "SELECT `network` FROM `gserver` WHERE `nurl` = '%s' AND `network` != ''",
 285                                 DBA::escape(Strings::normaliseLink($server_url))
 286                         );
 287
 288                         if ($server) {
 289                                 $contact['network'] = $server[0]["network"];
 290                         } else {
 291                                 return false;
 292                         }
 293                 }
 294
 295                 // noscrape is really fast so we don't cache the call.
 296                 if (($server_url != "") && ($gcontacts[0]["nick"] != "")) {
 297                         //  Use noscrape if possible
 298                         $server = q("SELECT `noscrape`, `network` FROM `gserver` WHERE `nurl` = '%s' AND `noscrape` != ''", DBA::escape(Strings::normaliseLink($server_url)));
 299
 300                         if ($server) {
 301                                 $curlResult = Network::curl($server[0]["noscrape"]."/".$gcontacts[0]["nick"]);
 302
 303                                 if ($curlResult->isSuccess() && ($curlResult->getBody() != "")) {
 304                                         $noscrape = json_decode($curlResult->getBody(), true);
 305
 306                                         if (is_array($noscrape)) {
 307                                                 $contact["network"] = $server[0]["network"];
 308
 309                                                 if (isset($noscrape["fn"])) {
 310                                                         $contact["name"] = $noscrape["fn"];
 311                                                 }
 312                                                 if (isset($noscrape["comm"])) {
 313                                                         $contact["community"] = $noscrape["comm"];
 314                                                 }
 315                                                 if (isset($noscrape["tags"])) {
 316                                                         $keywords = implode(" ", $noscrape["tags"]);
 317                                                         if ($keywords != "") {
 318                                                                 $contact["keywords"] = $keywords;
 319                                                         }
 320                                                 }
 321
 322                                                 $location = Profile::formatLocation($noscrape);
 323                                                 if ($location) {
 324                                                         $contact["location"] = $location;
 325                                                 }
 326                                                 if (isset($noscrape["dfrn-notify"])) {
 327                                                         $contact["notify"] = $noscrape["dfrn-notify"];
 328                                                 }
 329                                                 // Remove all fields that are not present in the gcontact table
 330                                                 unset($noscrape["fn"]);
 331                                                 unset($noscrape["key"]);
 332                                                 unset($noscrape["homepage"]);
 333                                                 unset($noscrape["comm"]);
 334                                                 unset($noscrape["tags"]);
 335                                                 unset($noscrape["locality"]);
 336                                                 unset($noscrape["region"]);
 337                                                 unset($noscrape["country-name"]);
 338                                                 unset($noscrape["contacts"]);
 339                                                 unset($noscrape["dfrn-request"]);
 340                                                 unset($noscrape["dfrn-confirm"]);
 341                                                 unset($noscrape["dfrn-notify"]);
 342                                                 unset($noscrape["dfrn-poll"]);
 343
 344                                                 // Set the date of the last contact
 345                                                 /// @todo By now the function "update_gcontact" doesn't work with this field
 346                                                 //$contact["last_contact"] = DateTimeFormat::utcNow();
 347
 348                                                 $contact = array_merge($contact, $noscrape);
 349
 350                                                 GContact::update($contact);
 351
 352                                                 if (!empty($noscrape["updated"])) {
 353                                                         $fields = ['last_contact' => DateTimeFormat::utcNow()];
 354                                                         DBA::update('gcontact', $fields, ['nurl' => Strings::normaliseLink($profile)]);
 355
 356                                                         Logger::log("Profile ".$profile." was last updated at ".$noscrape["updated"]." (noscrape)", Logger::DEBUG);
 357
 358                                                         return $noscrape["updated"];
 359                                                 }
 360                                         }
 361                                 }
 362                         }
 363                 }
 364
 365                 // If we only can poll the feed, then we only do this once a while
 366                 if (!$force && !self::updateNeeded($gcontacts[0]["created"], $gcontacts[0]["updated"], $gcontacts[0]["last_failure"], $gcontacts[0]["last_contact"])) {
 367                         Logger::log("Profile ".$profile." was last updated at ".$gcontacts[0]["updated"]." (cached)", Logger::DEBUG);
 368
 369                         GContact::update($contact);
 370                         return $gcontacts[0]["updated"];
 371                 }
 372
 373                 $data = Probe::uri($profile);
 374
 375                 // Is the profile link the alternate OStatus link notation? (http://domain.tld/user/4711)
 376                 // Then check the other link and delete this one
 377                 if (($data["network"] == Protocol::OSTATUS) && self::alternateOStatusUrl($profile)
 378                         && (Strings::normaliseLink($profile) == Strings::normaliseLink($data["alias"]))
 379                         && (Strings::normaliseLink($profile) != Strings::normaliseLink($data["url"]))
 380                 ) {
 381                         // Delete the old entry
 382                         DBA::delete('gcontact', ['nurl' => Strings::normaliseLink($profile)]);
 383
 384                         $gcontact = array_merge($gcontacts[0], $data);
 385
 386                         $gcontact["server_url"] = $data["baseurl"];
 387
 388                         try {
 389                                 $gcontact = GContact::sanitize($gcontact);
 390                                 GContact::update($gcontact);
 391
 392                                 self::lastUpdated($data["url"], $force);
 393                         } catch (Exception $e) {
 394                                 Logger::log($e->getMessage(), Logger::DEBUG);
 395                         }
 396
 397                         Logger::log("Profile ".$profile." was deleted", Logger::DEBUG);
 398                         return false;
 399                 }
 400
 401                 if (($data["poll"] == "") || (in_array($data["network"], [Protocol::FEED, Protocol::PHANTOM]))) {
 402                         $fields = ['last_failure' => DateTimeFormat::utcNow()];
 403                         DBA::update('gcontact', $fields, ['nurl' => Strings::normaliseLink($profile)]);
 404
 405                         Logger::log("Profile ".$profile." wasn't reachable (profile)", Logger::DEBUG);
 406                         return false;
 407                 }
 408
 409                 $contact = array_merge($contact, $data);
 410
 411                 $contact["server_url"] = $data["baseurl"];
 412
 413                 GContact::update($contact);
 414
 415                 $curlResult = Network::curl($data["poll"]);
 416
 417                 if (!$curlResult->isSuccess()) {
 418                         $fields = ['last_failure' => DateTimeFormat::utcNow()];
 419                         DBA::update('gcontact', $fields, ['nurl' => Strings::normaliseLink($profile)]);
 420
 421                         Logger::log("Profile ".$profile." wasn't reachable (no feed)", Logger::DEBUG);
 422                         return false;
 423                 }
 424
 425                 $doc = new DOMDocument();
 426                 /// @TODO Avoid error supression here
 427                 @$doc->loadXML($curlResult->getBody());
 428
 429                 $xpath = new DOMXPath($doc);
 430                 $xpath->registerNamespace('atom', "http://www.w3.org/2005/Atom");
 431
 432                 $entries = $xpath->query('/atom:feed/atom:entry');
 433
 434                 $last_updated = "";
 435
 436                 foreach ($entries as $entry) {
 437                         $published_item = $xpath->query('atom:published/text()', $entry)->item(0);
 438                         $updated_item   = $xpath->query('atom:updated/text()'  , $entry)->item(0);
 439                         $published      = isset($published_item->nodeValue) ? DateTimeFormat::utc($published_item->nodeValue) : null;
 440                         $updated        = isset($updated_item->nodeValue) ? DateTimeFormat::utc($updated_item->nodeValue) : null;
 441
 442                         if (!isset($published) || !isset($updated)) {
 443                                 Logger::notice('Invalid entry for XPath.', ['entry' => $entry, 'profile' => $profile]);
 444                                 continue;
 445                         }
 446
 447                         if ($last_updated < $published) {
 448                                 $last_updated = $published;
 449                         }
 450
 451                         if ($last_updated < $updated) {
 452                                 $last_updated = $updated;
 453                         }
 454                 }
 455
 456                 // Maybe there aren't any entries. Then check if it is a valid feed
 457                 if ($last_updated == "") {
 458                         if ($xpath->query('/atom:feed')->length > 0) {
 459                                 $last_updated = DBA::NULL_DATETIME;
 460                         }
 461                 }
 462
 463                 $fields = ['last_contact' => DateTimeFormat::utcNow()];
 464
 465                 if (!empty($last_updated)) {
 466                         $fields['updated'] = $last_updated;
 467                 }
 468
 469                 DBA::update('gcontact', $fields, ['nurl' => Strings::normaliseLink($profile)]);
 470
 471                 if (($gcontacts[0]["generation"] == 0)) {
 472                         $fields = ['generation' => 9];
 473                         DBA::update('gcontact', $fields, ['nurl' => Strings::normaliseLink($profile)]);
 474                 }
 475
 476                 Logger::log("Profile ".$profile." was last updated at ".$last_updated, Logger::DEBUG);
 477
 478                 return $last_updated;
 479         }
 480
 481         public static function updateNeeded($created, $updated, $last_failure, $last_contact)
 482         {
 483                 $now = strtotime(DateTimeFormat::utcNow());
 484
 485                 if ($updated > $last_contact) {
 486                         $contact_time = strtotime($updated);
 487                 } else {
 488                         $contact_time = strtotime($last_contact);
 489                 }
 490
 491                 $failure_time = strtotime($last_failure);
 492                 $created_time = strtotime($created);
 493
 494                 // If there is no "created" time then use the current time
 495                 if ($created_time <= 0) {
 496                         $created_time = $now;
 497                 }
 498
 499                 // If the last contact was less than 24 hours then don't update
 500                 if (($now - $contact_time) < (60 * 60 * 24)) {
 501                         return false;
 502                 }
 503
 504                 // If the last failure was less than 24 hours then don't update
 505                 if (($now - $failure_time) < (60 * 60 * 24)) {
 506                         return false;
 507                 }
 508
 509                 // If the last contact was less than a week ago and the last failure is older than a week then don't update
 510                 //if ((($now - $contact_time) < (60 * 60 * 24 * 7)) && ($contact_time > $failure_time))
 511                 //      return false;
 512
 513                 // If the last contact time was more than a week ago and the contact was created more than a week ago, then only try once a week
 514                 if ((($now - $contact_time) > (60 * 60 * 24 * 7)) && (($now - $created_time) > (60 * 60 * 24 * 7)) && (($now - $failure_time) < (60 * 60 * 24 * 7))) {
 515                         return false;
 516                 }
 517
 518                 // If the last contact time was more than a month ago and the contact was created more than a month ago, then only try once a month
 519                 if ((($now - $contact_time) > (60 * 60 * 24 * 30)) && (($now - $created_time) > (60 * 60 * 24 * 30)) && (($now - $failure_time) < (60 * 60 * 24 * 30))) {
 520                         return false;
 521                 }
 522
 523                 return true;
 524         }
 525
 526         /// @TODO Maybe move this out to an utilities class?
 527         private static function toBoolean($val)
 528         {
 529                 if (($val == "true") || ($val == 1)) {
 530                         return true;
 531                 } elseif (($val == "false") || ($val == 0)) {
 532                         return false;
 533                 }
 534
 535                 return $val;
 536         }
 537
 538         /**
 539          * @brief Detect server type (Hubzilla or Friendica) via the poco data
 540          *
 541          * @param array $data POCO data
 542          * @return array Server data
 543          */
 544         private static function detectPocoData(array $data)
 545         {
 546                 if (!isset($data['entry'])) {
 547                         return false;
 548                 }
 549
 550                 if (count($data['entry']) == 0) {
 551                         return false;
 552                 }
 553
 554                 if (!isset($data['entry'][0]['urls'])) {
 555                         return false;
 556                 }
 557
 558                 if (count($data['entry'][0]['urls']) == 0) {
 559                         return false;
 560                 }
 561
 562                 foreach ($data['entry'][0]['urls'] as $url) {
 563                         if ($url['type'] == 'zot') {
 564                                 $server = [];
 565                                 $server["platform"] = 'Hubzilla';
 566                                 $server["network"] = Protocol::DIASPORA;
 567                                 return $server;
 568                         }
 569                 }
 570                 return false;
 571         }
 572
 573         /**
 574          * @brief Detect server type by using the nodeinfo data
 575          *
 576          * @param string $server_url address of the server
 577          * @return array Server data
 578          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 579          */
 580         private static function fetchNodeinfo($server_url)
 581         {
 582                 $curlResult = Network::curl($server_url."/.well-known/nodeinfo");
 583                 if (!$curlResult->isSuccess()) {
 584                         return false;
 585                 }
 586
 587                 $nodeinfo = json_decode($curlResult->getBody(), true);
 588
 589                 if (!is_array($nodeinfo) || !isset($nodeinfo['links'])) {
 590                         return false;
 591                 }
 592
 593                 $nodeinfo1_url = '';
 594                 $nodeinfo2_url = '';
 595
 596                 foreach ($nodeinfo['links'] as $link) {
 597                         if (!is_array($link) || empty($link['rel']) || empty($link['href'])) {
 598                                 Logger::log('Invalid nodeinfo format for ' . $server_url, Logger::DEBUG);
 599                                 continue;
 600                         }
 601                         if ($link['rel'] == 'http://nodeinfo.diaspora.software/ns/schema/1.0') {
 602                                 $nodeinfo1_url = $link['href'];
 603                         } elseif ($link['rel'] == 'http://nodeinfo.diaspora.software/ns/schema/2.0') {
 604                                 $nodeinfo2_url = $link['href'];
 605                         }
 606                 }
 607
 608                 if ($nodeinfo1_url . $nodeinfo2_url == '') {
 609                         return false;
 610                 }
 611
 612                 $server = [];
 613
 614                 // When the nodeinfo url isn't on the same host, then there is obviously something wrong
 615                 if (!empty($nodeinfo2_url) && (parse_url($server_url, PHP_URL_HOST) == parse_url($nodeinfo2_url, PHP_URL_HOST))) {
 616                         $server = self::parseNodeinfo2($nodeinfo2_url);
 617                 }
 618
 619                 // When the nodeinfo url isn't on the same host, then there is obviously something wrong
 620                 if (empty($server) && !empty($nodeinfo1_url) && (parse_url($server_url, PHP_URL_HOST) == parse_url($nodeinfo1_url, PHP_URL_HOST))) {
 621                         $server = self::parseNodeinfo1($nodeinfo1_url);
 622                 }
 623
 624                 return $server;
 625         }
 626
 627         /**
 628          * @brief Parses Nodeinfo 1
 629          *
 630          * @param string $nodeinfo_url address of the nodeinfo path
 631          * @return array Server data
 632          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 633          */
 634         private static function parseNodeinfo1($nodeinfo_url)
 635         {
 636                 $curlResult = Network::curl($nodeinfo_url);
 637
 638                 if (!$curlResult->isSuccess()) {
 639                         return false;
 640                 }
 641
 642                 $nodeinfo = json_decode($curlResult->getBody(), true);
 643
 644                 if (!is_array($nodeinfo)) {
 645                         return false;
 646                 }
 647
 648                 $server = [];
 649
 650                 $server['register_policy'] = Register::CLOSED;
 651
 652                 if (isset($nodeinfo['openRegistrations']) && is_bool($nodeinfo['openRegistrations']) && $nodeinfo['openRegistrations']) {
 653                         $server['register_policy'] = Register::OPEN;
 654                 }
 655
 656                 if (is_array($nodeinfo['software'])) {
 657                         if (isset($nodeinfo['software']['name'])) {
 658                                 $server['platform'] = $nodeinfo['software']['name'];
 659                         }
 660
 661                         if (isset($nodeinfo['software']['version'])) {
 662                                 $server['version'] = $nodeinfo['software']['version'];
 663                                 // Version numbers on Nodeinfo are presented with additional info, e.g.:
 664                                 // 0.6.3.0-p1702cc1c, 0.6.99.0-p1b9ab160 or 3.4.3-2-1191.
 665                                 $server['version'] = preg_replace("=(.+)-(.{4,})=ism", "$1", $server['version']);
 666                         }
 667                 }
 668
 669                 if (isset($nodeinfo['metadata']['nodeName'])) {
 670                         $server['site_name'] = $nodeinfo['metadata']['nodeName'];
 671                 }
 672
 673                 if (!empty($nodeinfo['usage']['users']['total'])) {
 674                         $server['registered-users'] = $nodeinfo['usage']['users']['total'];
 675                 }
 676
 677                 $diaspora = false;
 678                 $friendica = false;
 679                 $gnusocial = false;
 680
 681                 if (!empty($nodeinfo['protocols']['inbound']) && is_array($nodeinfo['protocols']['inbound'])) {
 682                         foreach ($nodeinfo['protocols']['inbound'] as $inbound) {
 683                                 if ($inbound == 'diaspora') {
 684                                         $diaspora = true;
 685                                 }
 686                                 if ($inbound == 'friendica') {
 687                                         $friendica = true;
 688                                 }
 689                                 if ($inbound == 'gnusocial') {
 690                                         $gnusocial = true;
 691                                 }
 692                         }
 693                 }
 694
 695                 if ($gnusocial) {
 696                         $server['network'] = Protocol::OSTATUS;
 697                 }
 698                 if ($diaspora) {
 699                         $server['network'] = Protocol::DIASPORA;
 700                 }
 701                 if ($friendica) {
 702                         $server['network'] = Protocol::DFRN;
 703                 }
 704
 705                 if (!$server) {
 706                         return false;
 707                 }
 708
 709                 return $server;
 710         }
 711
 712         /**
 713          * @brief Parses Nodeinfo 2
 714          *
 715          * @param string $nodeinfo_url address of the nodeinfo path
 716          * @return array Server data
 717          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 718          */
 719         private static function parseNodeinfo2($nodeinfo_url)
 720         {
 721                 $curlResult = Network::curl($nodeinfo_url);
 722                 if (!$curlResult->isSuccess()) {
 723                         return false;
 724                 }
 725
 726                 $nodeinfo = json_decode($curlResult->getBody(), true);
 727
 728                 if (!is_array($nodeinfo)) {
 729                         return false;
 730                 }
 731
 732                 $server = [];
 733
 734                 $server['register_policy'] = Register::CLOSED;
 735
 736                 if (is_bool($nodeinfo['openRegistrations']) && $nodeinfo['openRegistrations']) {
 737                         $server['register_policy'] = Register::OPEN;
 738                 }
 739
 740                 if (is_array($nodeinfo['software'])) {
 741                         if (isset($nodeinfo['software']['name'])) {
 742                                 $server['platform'] = $nodeinfo['software']['name'];
 743                         }
 744
 745                         if (isset($nodeinfo['software']['version'])) {
 746                                 $server['version'] = $nodeinfo['software']['version'];
 747                                 // Version numbers on Nodeinfo are presented with additional info, e.g.:
 748                                 // 0.6.3.0-p1702cc1c, 0.6.99.0-p1b9ab160 or 3.4.3-2-1191.
 749                                 $server['version'] = preg_replace("=(.+)-(.{4,})=ism", "$1", $server['version']);
 750                         }
 751                 }
 752
 753                 if (isset($nodeinfo['metadata']['nodeName'])) {
 754                         $server['site_name'] = $nodeinfo['metadata']['nodeName'];
 755                 }
 756
 757                 if (!empty($nodeinfo['usage']['users']['total'])) {
 758                         $server['registered-users'] = $nodeinfo['usage']['users']['total'];
 759                 }
 760
 761                 $diaspora = false;
 762                 $friendica = false;
 763                 $gnusocial = false;
 764
 765                 if (!empty($nodeinfo['protocols'])) {
 766                         foreach ($nodeinfo['protocols'] as $protocol) {
 767                                 if ($protocol == 'diaspora') {
 768                                         $diaspora = true;
 769                                 } elseif ($protocol == 'friendica') {
 770                                         $friendica = true;
 771                                 } elseif ($protocol == 'gnusocial') {
 772                                         $gnusocial = true;
 773                                 }
 774                         }
 775                 }
 776
 777                 if ($gnusocial) {
 778                         $server['network'] = Protocol::OSTATUS;
 779                 } elseif ($diaspora) {
 780                         $server['network'] = Protocol::DIASPORA;
 781                 } elseif ($friendica) {
 782                         $server['network'] = Protocol::DFRN;
 783                 }
 784
 785                 if (empty($server)) {
 786                         return false;
 787                 }
 788
 789                 return $server;
 790         }
 791
 792         /**
 793          * @brief Detect server type (Hubzilla or Friendica) via the front page body
 794          *
 795          * @param string $body Front page of the server
 796          * @return array Server data
 797          */
 798         private static function detectServerType($body)
 799         {
 800                 $server = false;
 801
 802                 $doc = new DOMDocument();
 803                 /// @TODO Acoid supressing error
 804                 @$doc->loadHTML($body);
 805                 $xpath = new DOMXPath($doc);
 806
 807                 $list = $xpath->query("//meta[@name]");
 808
 809                 foreach ($list as $node) {
 810                         $attr = [];
 811                         if ($node->attributes->length) {
 812                                 foreach ($node->attributes as $attribute) {
 813                                         $attr[$attribute->name] = $attribute->value;
 814                                 }
 815                         }
 816                         if ($attr['name'] == 'generator') {
 817                                 $version_part = explode(" ", $attr['content']);
 818                                 if (count($version_part) == 2) {
 819                                         if (in_array($version_part[0], ["Friendika", "Friendica"])) {
 820                                                 $server = [];
 821                                                 $server["platform"] = $version_part[0];
 822                                                 $server["version"] = $version_part[1];
 823                                                 $server["network"] = Protocol::DFRN;
 824                                         }
 825                                 }
 826                         }
 827                 }
 828
 829                 if (!$server) {
 830                         $list = $xpath->query("//meta[@property]");
 831
 832                         foreach ($list as $node) {
 833                                 $attr = [];
 834                                 if ($node->attributes->length) {
 835                                         foreach ($node->attributes as $attribute) {
 836                                                 $attr[$attribute->name] = $attribute->value;
 837                                         }
 838                                 }
 839                                 if ($attr['property'] == 'generator' && in_array($attr['content'], ["hubzilla", "BlaBlaNet"])) {
 840                                         $server = [];
 841                                         $server["platform"] = $attr['content'];
 842                                         $server["version"] = "";
 843                                         $server["network"] = Protocol::DIASPORA;
 844                                 }
 845                         }
 846                 }
 847
 848                 if (!$server) {
 849                         return false;
 850                 }
 851
 852                 $server["site_name"] = XML::getFirstNodeValue($xpath, '//head/title/text()');
 853
 854                 return $server;
 855         }
 856
 857         public static function checkServer($server_url, $network = "", $force = false)
 858         {
 859                 // Unify the server address
 860                 $server_url = trim($server_url, "/");
 861                 $server_url = str_replace("/index.php", "", $server_url);
 862
 863                 if ($server_url == "") {
 864                         return false;
 865                 }
 866
 867                 $gserver = DBA::selectFirst('gserver', [], ['nurl' => Strings::normaliseLink($server_url)]);
 868                 if (DBA::isResult($gserver)) {
 869                         if ($gserver["created"] <= DBA::NULL_DATETIME) {
 870                                 $fields = ['created' => DateTimeFormat::utcNow()];
 871                                 $condition = ['nurl' => Strings::normaliseLink($server_url)];
 872                                 DBA::update('gserver', $fields, $condition);
 873                         }
 874                         $poco = $gserver["poco"];
 875                         $noscrape = $gserver["noscrape"];
 876
 877                         if ($network == "") {
 878                                 $network = $gserver["network"];
 879                         }
 880
 881                         $last_contact = $gserver["last_contact"];
 882                         $last_failure = $gserver["last_failure"];
 883                         $version = $gserver["version"];
 884                         $platform = $gserver["platform"];
 885                         $site_name = $gserver["site_name"];
 886                         $info = $gserver["info"];
 887                         $register_policy = $gserver["register_policy"];
 888                         $registered_users = $gserver["registered-users"];
 889
 890                         // See discussion under https://forum.friendi.ca/display/0b6b25a8135aabc37a5a0f5684081633
 891                         // It can happen that a zero date is in the database, but storing it again is forbidden.
 892                         if ($last_contact < DBA::NULL_DATETIME) {
 893                                 $last_contact = DBA::NULL_DATETIME;
 894                         }
 895
 896                         if ($last_failure < DBA::NULL_DATETIME) {
 897                                 $last_failure = DBA::NULL_DATETIME;
 898                         }
 899
 900                         if (!$force && !self::updateNeeded($gserver["created"], "", $last_failure, $last_contact)) {
 901                                 Logger::log("Use cached data for server ".$server_url, Logger::DEBUG);
 902                                 return ($last_contact >= $last_failure);
 903                         }
 904                 } else {
 905                         $poco = "";
 906                         $noscrape = "";
 907                         $version = "";
 908                         $platform = "";
 909                         $site_name = "";
 910                         $info = "";
 911                         $register_policy = -1;
 912                         $registered_users = 0;
 913
 914                         $last_contact = DBA::NULL_DATETIME;
 915                         $last_failure = DBA::NULL_DATETIME;
 916                 }
 917                 Logger::log("Server ".$server_url." is outdated or unknown. Start discovery. Force: ".$force." Created: ".$gserver["created"]." Failure: ".$last_failure." Contact: ".$last_contact, Logger::DEBUG);
 918
 919                 $failure = false;
 920                 $possible_failure = false;
 921                 $orig_last_failure = $last_failure;
 922                 $orig_last_contact = $last_contact;
 923
 924                 // Mastodon uses the "@" for user profiles.
 925                 // But this can be misunderstood.
 926                 if (parse_url($server_url, PHP_URL_USER) != '') {
 927                         DBA::update('gserver', ['last_failure' => DateTimeFormat::utcNow()], ['nurl' => Strings::normaliseLink($server_url)]);
 928                         return false;
 929                 }
 930
 931                 // Check if the page is accessible via SSL.
 932                 $orig_server_url = $server_url;
 933                 $server_url = str_replace("http://", "https://", $server_url);
 934
 935                 // We set the timeout to 20 seconds since this operation should be done in no time if the server was vital
 936                 $curlResult = Network::curl($server_url."/.well-known/host-meta", false, ['timeout' => 20]);
 937
 938                 // Quit if there is a timeout.
 939                 // But we want to make sure to only quit if we are mostly sure that this server url fits.
 940                 if (DBA::isResult($gserver) && ($orig_server_url == $server_url) &&
 941                         ($curlResult->isTimeout())) {
 942                         Logger::log("Connection to server ".$server_url." timed out.", Logger::DEBUG);
 943                         DBA::update('gserver', ['last_failure' => DateTimeFormat::utcNow()], ['nurl' => Strings::normaliseLink($server_url)]);
 944                         return false;
 945                 }
 946
 947                 // Maybe the page is unencrypted only?
 948                 $xmlobj = @simplexml_load_string($curlResult->getBody(), 'SimpleXMLElement', 0, "http://docs.oasis-open.org/ns/xri/xrd-1.0");
 949                 if (!$curlResult->isSuccess() || ($curlResult->getBody() == "") || empty($xmlobj) || !is_object($xmlobj)) {
 950                         $server_url = str_replace("https://", "http://", $server_url);
 951
 952                         // We set the timeout to 20 seconds since this operation should be done in no time if the server was vital
 953                         $curlResult = Network::curl($server_url."/.well-known/host-meta", false, ['timeout' => 20]);
 954
 955                         // Quit if there is a timeout
 956                         if ($curlResult->isTimeout()) {
 957                                 Logger::log("Connection to server " . $server_url . " timed out.", Logger::DEBUG);
 958                                 DBA::update('gserver', ['last_failure' => DateTimeFormat::utcNow()], ['nurl' => Strings::normaliseLink($server_url)]);
 959                                 return false;
 960                         }
 961
 962                         $xmlobj = @simplexml_load_string($curlResult->getBody(), 'SimpleXMLElement', 0, "http://docs.oasis-open.org/ns/xri/xrd-1.0");
 963                 }
 964
 965                 if (!$curlResult->isSuccess() || ($curlResult->getBody() == "") || empty($xmlobj) || !is_object($xmlobj)) {
 966                         // Workaround for bad configured servers (known nginx problem)
 967                         if (!empty($curlResult->getInfo()) && !in_array($curlResult->getInfo()["http_code"], ["403", "404"])) {
 968                                 $failure = true;
 969                         }
 970
 971                         $possible_failure = true;
 972                 }
 973
 974                 // If the server has no possible failure we reset the cached data
 975                 if (!$possible_failure) {
 976                         $version = "";
 977                         $platform = "";
 978                         $site_name = "";
 979                         $info = "";
 980                         $register_policy = -1;
 981                 }
 982
 983                 if (!$failure) {
 984                         // This will be too low, but better than no value at all.
 985                         $registered_users = DBA::count('gcontact', ['server_url' => Strings::normaliseLink($server_url)]);
 986                 }
 987
 988                 // Look for poco
 989                 if (!$failure) {
 990                         $curlResult = Network::curl($server_url."/poco");
 991
 992                         if ($curlResult->isSuccess()) {
 993                                 $data = json_decode($curlResult->getBody(), true);
 994
 995                                 if (isset($data['totalResults'])) {
 996                                         $registered_users = $data['totalResults'];
 997                                         $poco = $server_url . "/poco";
 998                                         $server = self::detectPocoData($data);
 999
1000                                         if (!empty($server)) {
1001                                                 $platform = $server['platform'];
1002                                                 $network = $server['network'];
1003                                                 $version = '';
1004                                                 $site_name = '';
1005                                         }
1006                                 }
1007
1008                                 /*
1009                                  * There are servers out there who don't return 404 on a failure
1010                                  * We have to be sure that don't misunderstand this
1011                                  */
1012                                 if (is_null($data)) {
1013                                         $poco = "";
1014                                         $noscrape = "";
1015                                         $network = "";
1016                                 }
1017                         }
1018                 }
1019
1020                 if (!$failure) {
1021                         // Test for Diaspora, Hubzilla, Mastodon or older Friendica servers
1022                         $curlResult = Network::curl($server_url);
1023
1024                         if (!$curlResult->isSuccess() || ($curlResult->getBody() == "")) {
1025                                 $failure = true;
1026                         } else {
1027                                 $server = self::detectServerType($curlResult->getBody());
1028
1029                                 if (!empty($server)) {
1030                                         $platform = $server['platform'];
1031                                         $network = $server['network'];
1032                                         $version = $server['version'];
1033                                         $site_name = $server['site_name'];
1034                                 }
1035
1036                                 $lines = explode("\n", $curlResult->getHeader());
1037
1038                                 if (count($lines)) {
1039                                         foreach ($lines as $line) {
1040                                                 $line = trim($line);
1041
1042                                                 if (stristr($line, 'X-Diaspora-Version:')) {
1043                                                         $platform = "Diaspora";
1044                                                         $version = trim(str_replace("X-Diaspora-Version:", "", $line));
1045                                                         $version = trim(str_replace("x-diaspora-version:", "", $version));
1046                                                         $network = Protocol::DIASPORA;
1047                                                         $versionparts = explode("-", $version);
1048                                                         $version = $versionparts[0];
1049                                                 }
1050
1051                                                 if (stristr($line, 'Server: Mastodon')) {
1052                                                         $platform = "Mastodon";
1053                                                         $network = Protocol::OSTATUS;
1054                                                 }
1055                                         }
1056                                 }
1057                         }
1058                 }
1059
1060                 if (!$failure && ($poco == "")) {
1061                         // Test for Statusnet
1062                         // Will also return data for Friendica and GNU Social - but it will be overwritten later
1063                         // The "not implemented" is a special treatment for really, really old Friendica versions
1064                         $curlResult = Network::curl($server_url."/api/statusnet/version.json");
1065
1066                         if ($curlResult->isSuccess() && ($curlResult->getBody() != '{"error":"not implemented"}') &&
1067                                 ($curlResult->getBody() != '') && (strlen($curlResult->getBody()) < 30)) {
1068                                 $platform = "StatusNet";
1069                                 // Remove junk that some GNU Social servers return
1070                                 $version = str_replace(chr(239).chr(187).chr(191), "", $curlResult->getBody());
1071                                 $version = trim($version, '"');
1072                                 $network = Protocol::OSTATUS;
1073                         }
1074
1075                         // Test for GNU Social
1076                         $curlResult = Network::curl($server_url."/api/gnusocial/version.json");
1077
1078                         if ($curlResult->isSuccess() && ($curlResult->getBody() != '{"error":"not implemented"}') &&
1079                                 ($curlResult->getBody() != '') && (strlen($curlResult->getBody()) < 30)) {
1080                                 $platform = "GNU Social";
1081                                 // Remove junk that some GNU Social servers return
1082                                 $version = str_replace(chr(239) . chr(187) . chr(191), "", $curlResult->getBody());
1083                                 $version = trim($version, '"');
1084                                 $network = Protocol::OSTATUS;
1085                         }
1086
1087                         // Test for Mastodon
1088                         $orig_version = $version;
1089                         $curlResult = Network::curl($server_url . "/api/v1/instance");
1090
1091                         if ($curlResult->isSuccess() && ($curlResult->getBody() != '')) {
1092                                 $data = json_decode($curlResult->getBody(), true);
1093
1094                                 if (isset($data['version'])) {
1095                                         $platform = "Mastodon";
1096                                         $version = defaults($data, 'version', '');
1097                                         $site_name = defaults($data, 'title', '');
1098                                         $info = defaults($data, 'description', '');
1099                                         $network = Protocol::OSTATUS;
1100                                 }
1101
1102                                 if (!empty($data['stats']['user_count'])) {
1103                                         $registered_users = $data['stats']['user_count'];
1104                                 }
1105                         }
1106
1107                         if (strstr($orig_version . $version, 'Pleroma')) {
1108                                 $platform = 'Pleroma';
1109                                 $version = trim(str_replace('Pleroma', '', $version));
1110                         }
1111                 }
1112
1113                 if (!$failure) {
1114                         // Test for Hubzilla and Red
1115                         $curlResult = Network::curl($server_url . "/siteinfo.json");
1116
1117                         if ($curlResult->isSuccess()) {
1118                                 $data = json_decode($curlResult->getBody(), true);
1119
1120                                 if (isset($data['url'])) {
1121                                         $platform = $data['platform'];
1122                                         $version = $data['version'];
1123                                         $network = Protocol::DIASPORA;
1124                                 }
1125
1126                                 if (!empty($data['site_name'])) {
1127                                         $site_name = $data['site_name'];
1128                                 }
1129
1130                                 if (!empty($data['channels_total'])) {
1131                                         $registered_users = $data['channels_total'];
1132                                 }
1133
1134                                 if (!empty($data['register_policy'])) {
1135                                         switch ($data['register_policy']) {
1136                                                 case "REGISTER_OPEN":
1137                                                         $register_policy = Register::OPEN;
1138                                                         break;
1139
1140                                                 case "REGISTER_APPROVE":
1141                                                         $register_policy = Register::APPROVE;
1142                                                         break;
1143
1144                                                 case "REGISTER_CLOSED":
1145                                                 default:
1146                                                         $register_policy = Register::CLOSED;
1147                                                         break;
1148                                         }
1149                                 }
1150                         } else {
1151                                 // Test for Hubzilla, Redmatrix or Friendica
1152                                 $curlResult = Network::curl($server_url."/api/statusnet/config.json");
1153
1154                                 if ($curlResult->isSuccess()) {
1155                                         $data = json_decode($curlResult->getBody(), true);
1156
1157                                         if (isset($data['site']['server'])) {
1158                                                 if (isset($data['site']['platform'])) {
1159                                                         $platform = $data['site']['platform']['PLATFORM_NAME'];
1160                                                         $version = $data['site']['platform']['STD_VERSION'];
1161                                                         $network = Protocol::DIASPORA;
1162                                                 }
1163
1164                                                 if (isset($data['site']['BlaBlaNet'])) {
1165                                                         $platform = $data['site']['BlaBlaNet']['PLATFORM_NAME'];
1166                                                         $version = $data['site']['BlaBlaNet']['STD_VERSION'];
1167                                                         $network = Protocol::DIASPORA;
1168                                                 }
1169
1170                                                 if (isset($data['site']['hubzilla'])) {
1171                                                         $platform = $data['site']['hubzilla']['PLATFORM_NAME'];
1172                                                         $version = $data['site']['hubzilla']['RED_VERSION'];
1173                                                         $network = Protocol::DIASPORA;
1174                                                 }
1175
1176                                                 if (isset($data['site']['redmatrix'])) {
1177                                                         if (isset($data['site']['redmatrix']['PLATFORM_NAME'])) {
1178                                                                 $platform = $data['site']['redmatrix']['PLATFORM_NAME'];
1179                                                         } elseif (isset($data['site']['redmatrix']['RED_PLATFORM'])) {
1180                                                                 $platform = $data['site']['redmatrix']['RED_PLATFORM'];
1181                                                         }
1182
1183                                                         $version = $data['site']['redmatrix']['RED_VERSION'];
1184                                                         $network = Protocol::DIASPORA;
1185                                                 }
1186
1187                                                 if (isset($data['site']['friendica'])) {
1188                                                         $platform = $data['site']['friendica']['FRIENDICA_PLATFORM'];
1189                                                         $version = $data['site']['friendica']['FRIENDICA_VERSION'];
1190                                                         $network = Protocol::DFRN;
1191                                                 }
1192
1193                                                 $site_name = $data['site']['name'];
1194
1195                                                 $private = false;
1196                                                 $inviteonly = false;
1197                                                 $closed = false;
1198
1199                                                 if (!empty($data['site']['closed'])) {
1200                                                         $closed = self::toBoolean($data['site']['closed']);
1201                                                 }
1202
1203                                                 if (!empty($data['site']['private'])) {
1204                                                         $private = self::toBoolean($data['site']['private']);
1205                                                 }
1206
1207                                                 if (!empty($data['site']['inviteonly'])) {
1208                                                         $inviteonly = self::toBoolean($data['site']['inviteonly']);
1209                                                 }
1210
1211                                                 if (!$closed && !$private and $inviteonly) {
1212                                                         $register_policy = Register::APPROVE;
1213                                                 } elseif (!$closed && !$private) {
1214                                                         $register_policy = Register::OPEN;
1215                                                 } else {
1216                                                         $register_policy = Register::CLOSED;
1217                                                 }
1218                                         }
1219                                 }
1220                         }
1221                 }
1222
1223                 // Query statistics.json. Optional package for Diaspora, Friendica and Redmatrix
1224                 if (!$failure) {
1225                         $curlResult = Network::curl($server_url . "/statistics.json");
1226
1227                         if ($curlResult->isSuccess()) {
1228                                 $data = json_decode($curlResult->getBody(), true);
1229
1230                                 if (isset($data['version'])) {
1231                                         $version = $data['version'];
1232                                         // Version numbers on statistics.json are presented with additional info, e.g.:
1233                                         // 0.6.3.0-p1702cc1c, 0.6.99.0-p1b9ab160 or 3.4.3-2-1191.
1234                                         $version = preg_replace("=(.+)-(.{4,})=ism", "$1", $version);
1235                                 }
1236
1237                                 if (!empty($data['name'])) {
1238                                         $site_name = $data['name'];
1239                                 }
1240
1241                                 if (!empty($data['network'])) {
1242                                         $platform = $data['network'];
1243                                 }
1244
1245                                 if ($platform == "Diaspora") {
1246                                         $network = Protocol::DIASPORA;
1247                                 }
1248
1249                                 if (!empty($data['registrations_open']) && $data['registrations_open']) {
1250                                         $register_policy = Register::OPEN;
1251                                 } else {
1252                                         $register_policy = Register::CLOSED;
1253                                 }
1254                         }
1255                 }
1256
1257                 // Query nodeinfo. Working for (at least) Diaspora and Friendica.
1258                 if (!$failure) {
1259                         $server = self::fetchNodeinfo($server_url);
1260
1261                         if (!empty($server)) {
1262                                 $register_policy = $server['register_policy'];
1263
1264                                 if (isset($server['platform'])) {
1265                                         $platform = $server['platform'];
1266                                 }
1267
1268                                 if (isset($server['network'])) {
1269                                         $network = $server['network'];
1270                                 }
1271
1272                                 if (isset($server['version'])) {
1273                                         $version = $server['version'];
1274                                 }
1275
1276                                 if (isset($server['site_name'])) {
1277                                         $site_name = $server['site_name'];
1278                                 }
1279
1280                                 if (isset($server['registered-users'])) {
1281                                         $registered_users = $server['registered-users'];
1282                                 }
1283                         }
1284                 }
1285
1286                 // Check for noscrape
1287                 // Friendica servers could be detected as OStatus servers
1288                 if (!$failure && in_array($network, [Protocol::DFRN, Protocol::OSTATUS])) {
1289                         $curlResult = Network::curl($server_url . "/friendica/json");
1290
1291                         if (!$curlResult->isSuccess()) {
1292                                 $curlResult = Network::curl($server_url . "/friendika/json");
1293                         }
1294
1295                         if ($curlResult->isSuccess()) {
1296                                 $data = json_decode($curlResult->getBody(), true);
1297
1298                                 if (isset($data['version'])) {
1299                                         $network = Protocol::DFRN;
1300
1301                                         if (!empty($data['no_scrape_url'])) {
1302                                                 $noscrape = $data['no_scrape_url'];
1303                                         }
1304
1305                                         $version = $data['version'];
1306
1307                                         if (!empty($data['site_name'])) {
1308                                                 $site_name = $data['site_name'];
1309                                         }
1310
1311                                         $info = defaults($data, 'info', '');
1312
1313                                         $register_policy = defaults($data, 'register_policy', 'REGISTER_CLOSED');
1314                                         switch ($register_policy) {
1315                                                 case 'REGISTER_OPEN':
1316                                                         $register_policy = Register::OPEN;
1317                                                         break;
1318
1319                                                 case 'REGISTER_APPROVE':
1320                                                         $register_policy = Register::APPROVE;
1321                                                         break;
1322
1323                                                 default:
1324                                                         Logger::log("Register policy '$register_policy' from $server_url is invalid.");
1325                                                         // Defaulting to closed
1326
1327                                                 case 'REGISTER_CLOSED':
1328                                                 case 'REGISTER_INVITATION':
1329                                                         $register_policy = Register::CLOSED;
1330                                                         break;
1331                                         }
1332
1333                                         $platform = defaults($data, 'platform', '');
1334                                 }
1335                         }
1336                 }
1337
1338                 // Every server has got at least an admin account
1339                 if (!$failure && ($registered_users == 0)) {
1340                         $registered_users = 1;
1341                 }
1342
1343                 if ($possible_failure && !$failure) {
1344                         $failure = true;
1345                 }
1346
1347                 if ($failure) {
1348                         $last_contact = $orig_last_contact;
1349                         $last_failure = DateTimeFormat::utcNow();
1350                 } else {
1351                         $last_contact = DateTimeFormat::utcNow();
1352                         $last_failure = $orig_last_failure;
1353                 }
1354
1355                 if (($last_contact <= $last_failure) && !$failure) {
1356                         Logger::log("Server ".$server_url." seems to be alive, but last contact wasn't set - could be a bug", Logger::DEBUG);
1357                 } elseif (($last_contact >= $last_failure) && $failure) {
1358                         Logger::log("Server ".$server_url." seems to be dead, but last failure wasn't set - could be a bug", Logger::DEBUG);
1359                 }
1360
1361                 // Check again if the server exists
1362                 $found = DBA::exists('gserver', ['nurl' => Strings::normaliseLink($server_url)]);
1363
1364                 $version = strip_tags($version);
1365                 $site_name = strip_tags($site_name);
1366                 $info = strip_tags($info);
1367                 $platform = strip_tags($platform);
1368
1369                 $fields = ['url' => $server_url, 'version' => $version,
1370                                 'site_name' => $site_name, 'info' => $info, 'register_policy' => $register_policy,
1371                                 'poco' => $poco, 'noscrape' => $noscrape, 'network' => $network,
1372                                 'platform' => $platform, 'registered-users' => $registered_users,
1373                                 'last_contact' => $last_contact, 'last_failure' => $last_failure];
1374
1375                 if ($found) {
1376                         DBA::update('gserver', $fields, ['nurl' => Strings::normaliseLink($server_url)]);
1377                 } elseif (!$failure) {
1378                         $fields['nurl'] = Strings::normaliseLink($server_url);
1379                         $fields['created'] = DateTimeFormat::utcNow();
1380                         DBA::insert('gserver', $fields);
1381                 }
1382
1383                 if (!$failure && in_array($fields['network'], [Protocol::DFRN, Protocol::DIASPORA])) {
1384                         self::discoverRelay($server_url);
1385                 }
1386
1387                 Logger::log("End discovery for server " . $server_url, Logger::DEBUG);
1388
1389                 return !$failure;
1390         }
1391
1392         /**
1393          * @brief Fetch relay data from a given server url
1394          *
1395          * @param string $server_url address of the server
1396          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
1397          */
1398         private static function discoverRelay($server_url)
1399         {
1400                 Logger::log("Discover relay data for server " . $server_url, Logger::DEBUG);
1401
1402                 $curlResult = Network::curl($server_url . "/.well-known/x-social-relay");
1403
1404                 if (!$curlResult->isSuccess()) {
1405                         return;
1406                 }
1407
1408                 $data = json_decode($curlResult->getBody(), true);
1409
1410                 if (!is_array($data)) {
1411                         return;
1412                 }
1413
1414                 $gserver = DBA::selectFirst('gserver', ['id', 'relay-subscribe', 'relay-scope'], ['nurl' => Strings::normaliseLink($server_url)]);
1415
1416                 if (!DBA::isResult($gserver)) {
1417                         return;
1418                 }
1419
1420                 if (($gserver['relay-subscribe'] != $data['subscribe']) || ($gserver['relay-scope'] != $data['scope'])) {
1421                         $fields = ['relay-subscribe' => $data['subscribe'], 'relay-scope' => $data['scope']];
1422                         DBA::update('gserver', $fields, ['id' => $gserver['id']]);
1423                 }
1424
1425                 DBA::delete('gserver-tag', ['gserver-id' => $gserver['id']]);
1426
1427                 if ($data['scope'] == 'tags') {
1428                         // Avoid duplicates
1429                         $tags = [];
1430                         foreach ($data['tags'] as $tag) {
1431                                 $tag = mb_strtolower($tag);
1432                                 if (strlen($tag) < 100) {
1433                                         $tags[$tag] = $tag;
1434                                 }
1435                         }
1436
1437                         foreach ($tags as $tag) {
1438                                 DBA::insert('gserver-tag', ['gserver-id' => $gserver['id'], 'tag' => $tag], true);
1439                         }
1440                 }
1441
1442                 // Create or update the relay contact
1443                 $fields = [];
1444                 if (isset($data['protocols'])) {
1445                         if (isset($data['protocols']['diaspora'])) {
1446                                 $fields['network'] = Protocol::DIASPORA;
1447
1448                                 if (isset($data['protocols']['diaspora']['receive'])) {
1449                                         $fields['batch'] = $data['protocols']['diaspora']['receive'];
1450                                 } elseif (is_string($data['protocols']['diaspora'])) {
1451                                         $fields['batch'] = $data['protocols']['diaspora'];
1452                                 }
1453                         }
1454
1455                         if (isset($data['protocols']['dfrn'])) {
1456                                 $fields['network'] = Protocol::DFRN;
1457
1458                                 if (isset($data['protocols']['dfrn']['receive'])) {
1459                                         $fields['batch'] = $data['protocols']['dfrn']['receive'];
1460                                 } elseif (is_string($data['protocols']['dfrn'])) {
1461                                         $fields['batch'] = $data['protocols']['dfrn'];
1462                                 }
1463                         }
1464                 }
1465                 Diaspora::setRelayContact($server_url, $fields);
1466         }
1467
1468         /**
1469          * @brief Returns a list of all known servers
1470          * @return array List of server urls
1471          * @throws Exception
1472          */
1473         public static function serverlist()
1474         {
1475                 $r = q(
1476                         "SELECT `url`, `site_name` AS `displayName`, `network`, `platform`, `version` FROM `gserver`
1477                         WHERE `network` IN ('%s', '%s', '%s') AND `last_contact` > `last_failure`
1478                         ORDER BY `last_contact`
1479                         LIMIT 1000",
1480                         DBA::escape(Protocol::DFRN),
1481                         DBA::escape(Protocol::DIASPORA),
1482                         DBA::escape(Protocol::OSTATUS)
1483                 );
1484
1485                 if (!DBA::isResult($r)) {
1486                         return false;
1487                 }
1488
1489                 return $r;
1490         }
1491
1492         /**
1493          * @brief Fetch server list from remote servers and adds them when they are new.
1494          *
1495          * @param string $poco URL to the POCO endpoint
1496          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
1497          */
1498         private static function fetchServerlist($poco)
1499         {
1500                 $curlResult = Network::curl($poco . "/@server");
1501
1502                 if (!$curlResult->isSuccess()) {
1503                         return;
1504                 }
1505
1506                 $serverlist = json_decode($curlResult->getBody(), true);
1507
1508                 if (!is_array($serverlist)) {
1509                         return;
1510                 }
1511
1512                 foreach ($serverlist as $server) {
1513                         $server_url = str_replace("/index.php", "", $server['url']);
1514
1515                         $r = q("SELECT `nurl` FROM `gserver` WHERE `nurl` = '%s'", DBA::escape(Strings::normaliseLink($server_url)));
1516
1517                         if (!DBA::isResult($r)) {
1518                                 Logger::log("Call server check for server ".$server_url, Logger::DEBUG);
1519                                 Worker::add(PRIORITY_LOW, "DiscoverPoCo", "server", $server_url);
1520                         }
1521                 }
1522         }
1523
1524         private static function discoverFederation()
1525         {
1526                 $last = Config::get('poco', 'last_federation_discovery');
1527
1528                 if ($last) {
1529                         $next = $last + (24 * 60 * 60);
1530
1531                         if ($next > time()) {
1532                                 return;
1533                         }
1534                 }
1535
1536                 // Discover Friendica, Hubzilla and Diaspora servers
1537                 $curlResult = Network::fetchUrl("http://the-federation.info/pods.json");
1538
1539                 if (!empty($curlResult)) {
1540                         $servers = json_decode($curlResult, true);
1541
1542                         if (!empty($servers['pods'])) {
1543                                 foreach ($servers['pods'] as $server) {
1544                                         Worker::add(PRIORITY_LOW, "DiscoverPoCo", "server", "https://" . $server['host']);
1545                                 }
1546                         }
1547                 }
1548
1549                 // Disvover Mastodon servers
1550                 if (!Config::get('system', 'ostatus_disabled')) {
1551                         $accesstoken = Config::get('system', 'instances_social_key');
1552
1553                         if (!empty($accesstoken)) {
1554                                 $api = 'https://instances.social/api/1.0/instances/list?count=0';
1555                                 $header = ['Authorization: Bearer '.$accesstoken];
1556                                 $curlResult = Network::curl($api, false, ['headers' => $header]);
1557
1558                                 if ($curlResult->isSuccess()) {
1559                                         $servers = json_decode($curlResult->getBody(), true);
1560
1561                                         foreach ($servers['instances'] as $server) {
1562                                                 $url = (is_null($server['https_score']) ? 'http' : 'https') . '://' . $server['name'];
1563                                                 Worker::add(PRIORITY_LOW, "DiscoverPoCo", "server", $url);
1564                                         }
1565                                 }
1566                         }
1567                 }
1568
1569                 // Currently disabled, since the service isn't available anymore.
1570                 // It is not removed since I hope that there will be a successor.
1571                 // Discover GNU Social Servers.
1572                 //if (!Config::get('system','ostatus_disabled')) {
1573                 //      $serverdata = "http://gstools.org/api/get_open_instances/";
1574
1575                 //      $curlResult = Network::curl($serverdata);
1576                 //      if ($curlResult->isSuccess()) {
1577                 //              $servers = json_decode($result->getBody(), true);
1578
1579                 //              foreach($servers['data'] as $server)
1580                 //                      self::checkServer($server['instance_address']);
1581                 //      }
1582                 //}
1583
1584                 Config::set('poco', 'last_federation_discovery', time());
1585         }
1586
1587         public static function discoverSingleServer($id)
1588         {
1589                 $server = DBA::selectFirst('gserver', ['poco', 'nurl', 'url', 'network'], ['id' => $id]);
1590
1591                 if (!DBA::isResult($server)) {
1592                         return false;
1593                 }
1594
1595                 // Discover new servers out there (Works from Friendica version 3.5.2)
1596                 self::fetchServerlist($server["poco"]);
1597
1598                 // Fetch all users from the other server
1599                 $url = $server["poco"] . "/?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,contactType,generation";
1600
1601                 Logger::info("Fetch all users from the server " . $server["url"]);
1602
1603                 $curlResult = Network::curl($url);
1604
1605                 if ($curlResult->isSuccess() && !empty($curlResult->getBody())) {
1606                         $data = json_decode($curlResult->getBody(), true);
1607
1608                         if (!empty($data)) {
1609                                 self::discoverServer($data, 2);
1610                         }
1611
1612                         if (Config::get('system', 'poco_discovery') >= self::USERS_GCONTACTS) {
1613                                 $timeframe = Config::get('system', 'poco_discovery_since');
1614
1615                                 if ($timeframe == 0) {
1616                                         $timeframe = 30;
1617                                 }
1618
1619                                 $updatedSince = date(DateTimeFormat::MYSQL, time() - $timeframe * 86400);
1620
1621                                 // Fetch all global contacts from the other server (Not working with Redmatrix and Friendica versions before 3.3)
1622                                 $url = $server["poco"]."/@global?updatedSince=".$updatedSince."&fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,contactType,generation";
1623
1624                                 $success = false;
1625
1626                                 $curlResult = Network::curl($url);
1627
1628                                 if ($curlResult->isSuccess() && !empty($curlResult->getBody())) {
1629                                         Logger::info("Fetch all global contacts from the server " . $server["nurl"]);
1630                                         $data = json_decode($curlResult->getBody(), true);
1631
1632                                         if (!empty($data)) {
1633                                                 $success = self::discoverServer($data);
1634                                         }
1635                                 }
1636
1637                                 if (!$success && !empty($data) && Config::get('system', 'poco_discovery') >= self::USERS_GCONTACTS_FALLBACK) {
1638                                         Logger::info("Fetch contacts from users of the server " . $server["nurl"]);
1639                                         self::discoverServerUsers($data, $server);
1640                                 }
1641                         }
1642
1643                         $fields = ['last_poco_query' => DateTimeFormat::utcNow()];
1644                         DBA::update('gserver', $fields, ['nurl' => $server["nurl"]]);
1645
1646                         return true;
1647                 } else {
1648                         // If the server hadn't replied correctly, then force a sanity check
1649                         self::checkServer($server["url"], $server["network"], true);
1650
1651                         // If we couldn't reach the server, we will try it some time later
1652                         $fields = ['last_poco_query' => DateTimeFormat::utcNow()];
1653                         DBA::update('gserver', $fields, ['nurl' => $server["nurl"]]);
1654
1655                         return false;
1656                 }
1657         }
1658
1659         public static function discover($complete = false)
1660         {
1661                 // Update the server list
1662                 self::discoverFederation();
1663
1664                 $no_of_queries = 5;
1665
1666                 $requery_days = intval(Config::get('system', 'poco_requery_days'));
1667
1668                 if ($requery_days == 0) {
1669                         $requery_days = 7;
1670                 }
1671
1672                 $last_update = date('c', time() - (60 * 60 * 24 * $requery_days));
1673
1674                 $gservers = q("SELECT `id`, `url`, `nurl`, `network`
1675                         FROM `gserver`
1676                         WHERE `last_contact` >= `last_failure`
1677                         AND `poco` != ''
1678                         AND `last_poco_query` < '%s'
1679                         ORDER BY RAND()", DBA::escape($last_update)
1680                 );
1681
1682                 if (DBA::isResult($gservers)) {
1683                         foreach ($gservers as $gserver) {
1684                                 if (!self::checkServer($gserver['url'], $gserver['network'])) {
1685                                         // The server is not reachable? Okay, then we will try it later
1686                                         $fields = ['last_poco_query' => DateTimeFormat::utcNow()];
1687                                         DBA::update('gserver', $fields, ['nurl' => $gserver['nurl']]);
1688                                         continue;
1689                                 }
1690
1691                                 Logger::log('Update directory from server ' . $gserver['url'] . ' with ID ' . $gserver['id'], Logger::DEBUG);
1692                                 Worker::add(PRIORITY_LOW, 'DiscoverPoCo', 'update_server_directory', (int) $gserver['id']);
1693
1694                                 if (!$complete && ( --$no_of_queries == 0)) {
1695                                         break;
1696                                 }
1697                         }
1698                 }
1699         }
1700
1701         private static function discoverServerUsers(array $data, array $server)
1702         {
1703                 if (!isset($data['entry'])) {
1704                         return;
1705                 }
1706
1707                 foreach ($data['entry'] as $entry) {
1708                         $username = '';
1709
1710                         if (isset($entry['urls'])) {
1711                                 foreach ($entry['urls'] as $url) {
1712                                         if ($url['type'] == 'profile') {
1713                                                 $profile_url = $url['value'];
1714                                                 $path_array = explode('/', parse_url($profile_url, PHP_URL_PATH));
1715                                                 $username = end($path_array);
1716                                         }
1717                                 }
1718                         }
1719
1720                         if ($username != '') {
1721                                 Logger::log('Fetch contacts for the user ' . $username . ' from the server ' . $server['nurl'], Logger::DEBUG);
1722
1723                                 // Fetch all contacts from a given user from the other server
1724                                 $url = $server['poco'] . '/' . $username . '/?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,contactType,generation';
1725
1726                                 $curlResult = Network::curl($url);
1727
1728                                 if ($curlResult->isSuccess()) {
1729                                         $data = json_decode($curlResult->getBody(), true);
1730
1731                                         if (!empty($data)) {
1732                                                 self::discoverServer($data, 3);
1733                                         }
1734                                 }
1735                         }
1736                 }
1737         }
1738
1739         private static function discoverServer(array $data, $default_generation = 0)
1740         {
1741                 if (empty($data['entry'])) {
1742                         return false;
1743                 }
1744
1745                 $success = false;
1746
1747                 foreach ($data['entry'] as $entry) {
1748                         $profile_url = '';
1749                         $profile_photo = '';
1750                         $connect_url = '';
1751                         $name = '';
1752                         $network = '';
1753                         $updated = DBA::NULL_DATETIME;
1754                         $location = '';
1755                         $about = '';
1756                         $keywords = '';
1757                         $gender = '';
1758                         $contact_type = -1;
1759                         $generation = $default_generation;
1760
1761                         if (!empty($entry['displayName'])) {
1762                                 $name = $entry['displayName'];
1763                         }
1764
1765                         if (isset($entry['urls'])) {
1766                                 foreach ($entry['urls'] as $url) {
1767                                         if ($url['type'] == 'profile') {
1768                                                 $profile_url = $url['value'];
1769                                                 continue;
1770                                         }
1771                                         if ($url['type'] == 'webfinger') {
1772                                                 $connect_url = str_replace('acct:' , '', $url['value']);
1773                                                 continue;
1774                                         }
1775                                 }
1776                         }
1777
1778                         if (isset($entry['photos'])) {
1779                                 foreach ($entry['photos'] as $photo) {
1780                                         if ($photo['type'] == 'profile') {
1781                                                 $profile_photo = $photo['value'];
1782                                                 continue;
1783                                         }
1784                                 }
1785                         }
1786
1787                         if (isset($entry['updated'])) {
1788                                 $updated = date(DateTimeFormat::MYSQL, strtotime($entry['updated']));
1789                         }
1790
1791                         if (isset($entry['network'])) {
1792                                 $network = $entry['network'];
1793                         }
1794
1795                         if (isset($entry['currentLocation'])) {
1796                                 $location = $entry['currentLocation'];
1797                         }
1798
1799                         if (isset($entry['aboutMe'])) {
1800                                 $about = HTML::toBBCode($entry['aboutMe']);
1801                         }
1802
1803                         if (isset($entry['gender'])) {
1804                                 $gender = $entry['gender'];
1805                         }
1806
1807                         if (isset($entry['generation']) && ($entry['generation'] > 0)) {
1808                                 $generation = ++$entry['generation'];
1809                         }
1810
1811                         if (isset($entry['contactType']) && ($entry['contactType'] >= 0)) {
1812                                 $contact_type = $entry['contactType'];
1813                         }
1814
1815                         if (isset($entry['tags'])) {
1816                                 foreach ($entry['tags'] as $tag) {
1817                                         $keywords = implode(", ", $tag);
1818                                 }
1819                         }
1820
1821                         if ($generation > 0) {
1822                                 $success = true;
1823
1824                                 Logger::log("Store profile ".$profile_url, Logger::DEBUG);
1825
1826                                 $gcontact = ["url" => $profile_url,
1827                                                 "name" => $name,
1828                                                 "network" => $network,
1829                                                 "photo" => $profile_photo,
1830                                                 "about" => $about,
1831                                                 "location" => $location,
1832                                                 "gender" => $gender,
1833                                                 "keywords" => $keywords,
1834                                                 "connect" => $connect_url,
1835                                                 "updated" => $updated,
1836                                                 "contact-type" => $contact_type,
1837                                                 "generation" => $generation];
1838
1839                                 try {
1840                                         $gcontact = GContact::sanitize($gcontact);
1841                                         GContact::update($gcontact);
1842                                 } catch (Exception $e) {
1843                                         Logger::log($e->getMessage(), Logger::DEBUG);
1844                                 }
1845
1846                                 Logger::log("Done for profile ".$profile_url, Logger::DEBUG);
1847                         }
1848                 }
1849                 return $success;
1850         }
1851
1852 }