src/Protocol/PortableContact.php

   1 <?php
   2 /**
   3  * @file src/Protocol/PortableContact.php
   4  *
   5  * @todo Move GNU Social URL schemata (http://server.tld/user/number) to http://server.tld/username
   6  * @todo Fetch profile data from profile page for Redmatrix users
   7  * @todo Detect if it is a forum
   8  */
   9
  10 namespace Friendica\Protocol;
  11
  12 use DOMDocument;
  13 use DOMXPath;
  14 use Exception;
  15 use Friendica\Content\Text\HTML;
  16 use Friendica\Core\Config;
  17 use Friendica\Core\Logger;
  18 use Friendica\Core\Protocol;
  19 use Friendica\Core\Worker;
  20 use Friendica\Database\DBA;
  21 use Friendica\DI;
  22 use Friendica\Model\Contact;
  23 use Friendica\Model\GContact;
  24 use Friendica\Model\GServer;
  25 use Friendica\Model\Profile;
  26 use Friendica\Module\Register;
  27 use Friendica\Network\Probe;
  28 use Friendica\Util\DateTimeFormat;
  29 use Friendica\Util\Network;
  30 use Friendica\Util\Strings;
  31 use Friendica\Util\XML;
  32
  33 class PortableContact
  34 {
  35         const DISABLED = 0;
  36         const USERS = 1;
  37         const USERS_GCONTACTS = 2;
  38         const USERS_GCONTACTS_FALLBACK = 3;
  39
  40         /**
  41          * Fetch POCO data
  42          *
  43          * @param integer $cid  Contact ID
  44          * @param integer $uid  User ID
  45          * @param integer $zcid Global Contact ID
  46          * @param integer $url  POCO address that should be polled
  47          *
  48          * Given a contact-id (minimum), load the PortableContacts friend list for that contact,
  49          * and add the entries to the gcontact (Global Contact) table, or update existing entries
  50          * if anything (name or photo) has changed.
  51          * We use normalised urls for comparison which ignore http vs https and www.domain vs domain
  52          *
  53          * Once the global contact is stored add (if necessary) the contact linkage which associates
  54          * the given uid, cid to the global contact entry. There can be many uid/cid combinations
  55          * pointing to the same global contact id.
  56          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
  57          */
  58         public static function loadWorker($cid, $uid = 0, $zcid = 0, $url = null)
  59         {
  60                 // Call the function "load" via the worker
  61                 Worker::add(PRIORITY_LOW, 'FetchPoCo', (int)$cid, (int)$uid, (int)$zcid, $url);
  62         }
  63
  64         /**
  65          * Fetch POCO data from the worker
  66          *
  67          * @param integer $cid  Contact ID
  68          * @param integer $uid  User ID
  69          * @param integer $zcid Global Contact ID
  70          * @param integer $url  POCO address that should be polled
  71          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
  72          */
  73         public static function load($cid, $uid, $zcid, $url)
  74         {
  75                 if ($cid) {
  76                         if (!$url || !$uid) {
  77                                 $contact = DBA::selectFirst('contact', ['poco', 'uid'], ['id' => $cid]);
  78                                 if (DBA::isResult($contact)) {
  79                                         $url = $contact['poco'];
  80                                         $uid = $contact['uid'];
  81                                 }
  82                         }
  83                         if (!$uid) {
  84                                 return;
  85                         }
  86                 }
  87
  88                 if (!$url) {
  89                         return;
  90                 }
  91
  92                 $url = $url . (($uid) ? '/@me/@all?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,contactType,generation' : '?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,contactType,generation');
  93
  94                 Logger::log('load: ' . $url, Logger::DEBUG);
  95
  96                 $fetchresult = Network::fetchUrlFull($url);
  97                 $s = $fetchresult->getBody();
  98
  99                 Logger::log('load: returns ' . $s, Logger::DATA);
 100
 101                 Logger::log('load: return code: ' . $fetchresult->getReturnCode(), Logger::DEBUG);
 102
 103                 if (($fetchresult->getReturnCode() > 299) || (! $s)) {
 104                         return;
 105                 }
 106
 107                 $j = json_decode($s, true);
 108
 109                 Logger::log('load: json: ' . print_r($j, true), Logger::DATA);
 110
 111                 if (!isset($j['entry'])) {
 112                         return;
 113                 }
 114
 115                 $total = 0;
 116                 foreach ($j['entry'] as $entry) {
 117                         $total ++;
 118                         $profile_url = '';
 119                         $profile_photo = '';
 120                         $connect_url = '';
 121                         $name = '';
 122                         $network = '';
 123                         $updated = DBA::NULL_DATETIME;
 124                         $location = '';
 125                         $about = '';
 126                         $keywords = '';
 127                         $gender = '';
 128                         $contact_type = -1;
 129                         $generation = 0;
 130
 131                         if (!empty($entry['displayName'])) {
 132                                 $name = $entry['displayName'];
 133                         }
 134
 135                         if (isset($entry['urls'])) {
 136                                 foreach ($entry['urls'] as $url) {
 137                                         if ($url['type'] == 'profile') {
 138                                                 $profile_url = $url['value'];
 139                                                 continue;
 140                                         }
 141                                         if ($url['type'] == 'webfinger') {
 142                                                 $connect_url = str_replace('acct:', '', $url['value']);
 143                                                 continue;
 144                                         }
 145                                 }
 146                         }
 147                         if (isset($entry['photos'])) {
 148                                 foreach ($entry['photos'] as $photo) {
 149                                         if ($photo['type'] == 'profile') {
 150                                                 $profile_photo = $photo['value'];
 151                                                 continue;
 152                                         }
 153                                 }
 154                         }
 155
 156                         if (isset($entry['updated'])) {
 157                                 $updated = date(DateTimeFormat::MYSQL, strtotime($entry['updated']));
 158                         }
 159
 160                         if (isset($entry['network'])) {
 161                                 $network = $entry['network'];
 162                         }
 163
 164                         if (isset($entry['currentLocation'])) {
 165                                 $location = $entry['currentLocation'];
 166                         }
 167
 168                         if (isset($entry['aboutMe'])) {
 169                                 $about = HTML::toBBCode($entry['aboutMe']);
 170                         }
 171
 172                         if (isset($entry['gender'])) {
 173                                 $gender = $entry['gender'];
 174                         }
 175
 176                         if (isset($entry['generation']) && ($entry['generation'] > 0)) {
 177                                 $generation = ++$entry['generation'];
 178                         }
 179
 180                         if (isset($entry['tags'])) {
 181                                 foreach ($entry['tags'] as $tag) {
 182                                         $keywords = implode(", ", $tag);
 183                                 }
 184                         }
 185
 186                         if (isset($entry['contactType']) && ($entry['contactType'] >= 0)) {
 187                                 $contact_type = $entry['contactType'];
 188                         }
 189
 190                         $gcontact = ["url" => $profile_url,
 191                                         "name" => $name,
 192                                         "network" => $network,
 193                                         "photo" => $profile_photo,
 194                                         "about" => $about,
 195                                         "location" => $location,
 196                                         "gender" => $gender,
 197                                         "keywords" => $keywords,
 198                                         "connect" => $connect_url,
 199                                         "updated" => $updated,
 200                                         "contact-type" => $contact_type,
 201                                         "generation" => $generation];
 202
 203                         try {
 204                                 $gcontact = GContact::sanitize($gcontact);
 205                                 $gcid = GContact::update($gcontact);
 206
 207                                 GContact::link($gcid, $uid, $cid, $zcid);
 208                         } catch (Exception $e) {
 209                                 Logger::log($e->getMessage(), Logger::DEBUG);
 210                         }
 211                 }
 212                 Logger::log("load: loaded $total entries", Logger::DEBUG);
 213
 214                 $condition = ["`cid` = ? AND `uid` = ? AND `zcid` = ? AND `updated` < UTC_TIMESTAMP - INTERVAL 2 DAY", $cid, $uid, $zcid];
 215                 DBA::delete('glink', $condition);
 216         }
 217
 218         /**
 219          * Returns a list of all known servers
 220          * @return array List of server urls
 221          * @throws Exception
 222          */
 223         public static function serverlist()
 224         {
 225                 $r = q(
 226                         "SELECT `url`, `site_name` AS `displayName`, `network`, `platform`, `version` FROM `gserver`
 227                         WHERE `network` IN ('%s', '%s', '%s') AND `last_contact` > `last_failure`
 228                         ORDER BY `last_contact`
 229                         LIMIT 1000",
 230                         DBA::escape(Protocol::DFRN),
 231                         DBA::escape(Protocol::DIASPORA),
 232                         DBA::escape(Protocol::OSTATUS)
 233                 );
 234
 235                 if (!DBA::isResult($r)) {
 236                         return false;
 237                 }
 238
 239                 return $r;
 240         }
 241
 242         /**
 243          * Fetch server list from remote servers and adds them when they are new.
 244          *
 245          * @param string $poco URL to the POCO endpoint
 246          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 247          */
 248         private static function fetchServerlist($poco)
 249         {
 250                 $curlResult = Network::curl($poco . "/@server");
 251
 252                 if (!$curlResult->isSuccess()) {
 253                         return;
 254                 }
 255
 256                 $serverlist = json_decode($curlResult->getBody(), true);
 257
 258                 if (!is_array($serverlist)) {
 259                         return;
 260                 }
 261
 262                 foreach ($serverlist as $server) {
 263                         $server_url = str_replace("/index.php", "", $server['url']);
 264
 265                         $r = q("SELECT `nurl` FROM `gserver` WHERE `nurl` = '%s'", DBA::escape(Strings::normaliseLink($server_url)));
 266
 267                         if (!DBA::isResult($r)) {
 268                                 Logger::log("Call server check for server ".$server_url, Logger::DEBUG);
 269                                 Worker::add(PRIORITY_LOW, 'UpdateGServer', $server_url);
 270                         }
 271                 }
 272         }
 273
 274         public static function discoverSingleServer($id)
 275         {
 276                 $server = DBA::selectFirst('gserver', ['poco', 'nurl', 'url', 'network'], ['id' => $id]);
 277
 278                 if (!DBA::isResult($server)) {
 279                         return false;
 280                 }
 281
 282                 // Discover new servers out there (Works from Friendica version 3.5.2)
 283                 self::fetchServerlist($server["poco"]);
 284
 285                 // Fetch all users from the other server
 286                 $url = $server["poco"] . "/?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,contactType,generation";
 287
 288                 Logger::info("Fetch all users from the server " . $server["url"]);
 289
 290                 $curlResult = Network::curl($url);
 291
 292                 if ($curlResult->isSuccess() && !empty($curlResult->getBody())) {
 293                         $data = json_decode($curlResult->getBody(), true);
 294
 295                         if (!empty($data)) {
 296                                 self::discoverServer($data, 2);
 297                         }
 298
 299                         if (DI::config()->get('system', 'poco_discovery') >= self::USERS_GCONTACTS) {
 300                                 $timeframe = DI::config()->get('system', 'poco_discovery_since');
 301
 302                                 if ($timeframe == 0) {
 303                                         $timeframe = 30;
 304                                 }
 305
 306                                 $updatedSince = date(DateTimeFormat::MYSQL, time() - $timeframe * 86400);
 307
 308                                 // Fetch all global contacts from the other server (Not working with Redmatrix and Friendica versions before 3.3)
 309                                 $url = $server["poco"]."/@global?updatedSince=".$updatedSince."&fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,contactType,generation";
 310
 311                                 $success = false;
 312
 313                                 $curlResult = Network::curl($url);
 314
 315                                 if ($curlResult->isSuccess() && !empty($curlResult->getBody())) {
 316                                         Logger::info("Fetch all global contacts from the server " . $server["nurl"]);
 317                                         $data = json_decode($curlResult->getBody(), true);
 318
 319                                         if (!empty($data)) {
 320                                                 $success = self::discoverServer($data);
 321                                         }
 322                                 }
 323
 324                                 if (!$success && !empty($data) && DI::config()->get('system', 'poco_discovery') >= self::USERS_GCONTACTS_FALLBACK) {
 325                                         Logger::info("Fetch contacts from users of the server " . $server["nurl"]);
 326                                         self::discoverServerUsers($data, $server);
 327                                 }
 328                         }
 329
 330                         $fields = ['last_poco_query' => DateTimeFormat::utcNow()];
 331                         DBA::update('gserver', $fields, ['nurl' => $server["nurl"]]);
 332
 333                         return true;
 334                 } else {
 335                         // If the server hadn't replied correctly, then force a sanity check
 336                         GServer::check($server["url"], $server["network"], true);
 337
 338                         // If we couldn't reach the server, we will try it some time later
 339                         $fields = ['last_poco_query' => DateTimeFormat::utcNow()];
 340                         DBA::update('gserver', $fields, ['nurl' => $server["nurl"]]);
 341
 342                         return false;
 343                 }
 344         }
 345
 346         private static function discoverServerUsers(array $data, array $server)
 347         {
 348                 if (!isset($data['entry'])) {
 349                         return;
 350                 }
 351
 352                 foreach ($data['entry'] as $entry) {
 353                         $username = '';
 354
 355                         if (isset($entry['urls'])) {
 356                                 foreach ($entry['urls'] as $url) {
 357                                         if ($url['type'] == 'profile') {
 358                                                 $profile_url = $url['value'];
 359                                                 $path_array = explode('/', parse_url($profile_url, PHP_URL_PATH));
 360                                                 $username = end($path_array);
 361                                         }
 362                                 }
 363                         }
 364
 365                         if ($username != '') {
 366                                 Logger::log('Fetch contacts for the user ' . $username . ' from the server ' . $server['nurl'], Logger::DEBUG);
 367
 368                                 // Fetch all contacts from a given user from the other server
 369                                 $url = $server['poco'] . '/' . $username . '/?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,contactType,generation';
 370
 371                                 $curlResult = Network::curl($url);
 372
 373                                 if ($curlResult->isSuccess()) {
 374                                         $data = json_decode($curlResult->getBody(), true);
 375
 376                                         if (!empty($data)) {
 377                                                 self::discoverServer($data, 3);
 378                                         }
 379                                 }
 380                         }
 381                 }
 382         }
 383
 384         private static function discoverServer(array $data, $default_generation = 0)
 385         {
 386                 if (empty($data['entry'])) {
 387                         return false;
 388                 }
 389
 390                 $success = false;
 391
 392                 foreach ($data['entry'] as $entry) {
 393                         $profile_url = '';
 394                         $profile_photo = '';
 395                         $connect_url = '';
 396                         $name = '';
 397                         $network = '';
 398                         $updated = DBA::NULL_DATETIME;
 399                         $location = '';
 400                         $about = '';
 401                         $keywords = '';
 402                         $gender = '';
 403                         $contact_type = -1;
 404                         $generation = $default_generation;
 405
 406                         if (!empty($entry['displayName'])) {
 407                                 $name = $entry['displayName'];
 408                         }
 409
 410                         if (isset($entry['urls'])) {
 411                                 foreach ($entry['urls'] as $url) {
 412                                         if ($url['type'] == 'profile') {
 413                                                 $profile_url = $url['value'];
 414                                                 continue;
 415                                         }
 416                                         if ($url['type'] == 'webfinger') {
 417                                                 $connect_url = str_replace('acct:' , '', $url['value']);
 418                                                 continue;
 419                                         }
 420                                 }
 421                         }
 422
 423                         if (isset($entry['photos'])) {
 424                                 foreach ($entry['photos'] as $photo) {
 425                                         if ($photo['type'] == 'profile') {
 426                                                 $profile_photo = $photo['value'];
 427                                                 continue;
 428                                         }
 429                                 }
 430                         }
 431
 432                         if (isset($entry['updated'])) {
 433                                 $updated = date(DateTimeFormat::MYSQL, strtotime($entry['updated']));
 434                         }
 435
 436                         if (isset($entry['network'])) {
 437                                 $network = $entry['network'];
 438                         }
 439
 440                         if (isset($entry['currentLocation'])) {
 441                                 $location = $entry['currentLocation'];
 442                         }
 443
 444                         if (isset($entry['aboutMe'])) {
 445                                 $about = HTML::toBBCode($entry['aboutMe']);
 446                         }
 447
 448                         if (isset($entry['gender'])) {
 449                                 $gender = $entry['gender'];
 450                         }
 451
 452                         if (isset($entry['generation']) && ($entry['generation'] > 0)) {
 453                                 $generation = ++$entry['generation'];
 454                         }
 455
 456                         if (isset($entry['contactType']) && ($entry['contactType'] >= 0)) {
 457                                 $contact_type = $entry['contactType'];
 458                         }
 459
 460                         if (isset($entry['tags'])) {
 461                                 foreach ($entry['tags'] as $tag) {
 462                                         $keywords = implode(", ", $tag);
 463                                 }
 464                         }
 465
 466                         if ($generation > 0) {
 467                                 $success = true;
 468
 469                                 Logger::log("Store profile ".$profile_url, Logger::DEBUG);
 470
 471                                 $gcontact = ["url" => $profile_url,
 472                                                 "name" => $name,
 473                                                 "network" => $network,
 474                                                 "photo" => $profile_photo,
 475                                                 "about" => $about,
 476                                                 "location" => $location,
 477                                                 "gender" => $gender,
 478                                                 "keywords" => $keywords,
 479                                                 "connect" => $connect_url,
 480                                                 "updated" => $updated,
 481                                                 "contact-type" => $contact_type,
 482                                                 "generation" => $generation];
 483
 484                                 try {
 485                                         $gcontact = GContact::sanitize($gcontact);
 486                                         GContact::update($gcontact);
 487                                 } catch (Exception $e) {
 488                                         Logger::log($e->getMessage(), Logger::DEBUG);
 489                                 }
 490
 491                                 Logger::log("Done for profile ".$profile_url, Logger::DEBUG);
 492                         }
 493                 }
 494                 return $success;
 495         }
 496 }