src/Database/PostUpdate.php

   1 <?php
   2 /**
   3  * @copyright Copyright (C) 2010-2022, the Friendica project
   4  *
   5  * @license GNU AGPL version 3 or any later version
   6  *
   7  * This program is free software: you can redistribute it and/or modify
   8  * it under the terms of the GNU Affero General Public License as
   9  * published by the Free Software Foundation, either version 3 of the
  10  * License, or (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU Affero General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU Affero General Public License
  18  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  19  *
  20  */
  21
  22 namespace Friendica\Database;
  23
  24 use Friendica\Core\Logger;
  25 use Friendica\Core\Protocol;
  26 use Friendica\DI;
  27 use Friendica\Model\Contact;
  28 use Friendica\Model\Conversation;
  29 use Friendica\Model\GServer;
  30 use Friendica\Model\Item;
  31 use Friendica\Model\ItemURI;
  32 use Friendica\Model\Photo;
  33 use Friendica\Model\Post;
  34 use Friendica\Model\Post\Category;
  35 use Friendica\Model\Tag;
  36 use Friendica\Model\Verb;
  37 use Friendica\Protocol\ActivityPub\Processor;
  38 use Friendica\Protocol\ActivityPub\Receiver;
  39 use Friendica\Util\JsonLD;
  40 use Friendica\Util\Strings;
  41
  42 /**
  43  * These database-intensive post update routines are meant to be executed in the background by the cronjob.
  44  *
  45  * If there is a need for a intensive migration after a database structure change, update this file
  46  * by adding a new method at the end with the number of the new DB_UPDATE_VERSION.
  47  */
  48 class PostUpdate
  49 {
  50         // Needed for the helper function to read from the legacy term table
  51         const OBJECT_TYPE_POST  = 1;
  52
  53         const VERSION = 1452;
  54
  55         /**
  56          * Calls the post update functions
  57          */
  58         public static function update()
  59         {
  60                 if (!self::update1297()) {
  61                         return false;
  62                 }
  63                 if (!self::update1322()) {
  64                         return false;
  65                 }
  66                 if (!self::update1329()) {
  67                         return false;
  68                 }
  69                 if (!self::update1341()) {
  70                         return false;
  71                 }
  72                 if (!self::update1342()) {
  73                         return false;
  74                 }
  75                 if (!self::update1345()) {
  76                         return false;
  77                 }
  78                 if (!self::update1346()) {
  79                         return false;
  80                 }
  81                 if (!self::update1347()) {
  82                         return false;
  83                 }
  84                 if (!self::update1348()) {
  85                         return false;
  86                 }
  87                 if (!self::update1349()) {
  88                         return false;
  89                 }
  90                 if (!self::update1383()) {
  91                         return false;
  92                 }
  93                 if (!self::update1384()) {
  94                         return false;
  95                 }
  96                 if (!self::update1400()) {
  97                         return false;
  98                 }
  99                 if (!self::update1424()) {
 100                         return false;
 101                 }
 102                 if (!self::update1425()) {
 103                         return false;
 104                 }
 105                 if (!self::update1426()) {
 106                         return false;
 107                 }
 108                 if (!self::update1427()) {
 109                         return false;
 110                 }
 111                 if (!self::update1452()) {
 112                         return false;
 113                 }
 114                 if (!self::update1483()) {
 115                         return false;
 116                 }
 117                 return true;
 118         }
 119
 120         /**
 121          * Set the delivery queue count to a negative value for all items preceding the feature.
 122          *
 123          * @return bool "true" when the job is done
 124          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 125          */
 126         private static function update1297()
 127         {
 128                 // Was the script completed?
 129                 if (DI::config()->get('system', 'post_update_version') >= 1297) {
 130                         return true;
 131                 }
 132
 133                 if (!DBStructure::existsTable('item-delivery-data')) {
 134                         DI::config()->set('system', 'post_update_version', 1297);
 135                         return true;
 136                 }
 137
 138                 $max_item_delivery_data = DBA::selectFirst('item-delivery-data', ['iid'], ['queue_count > 0 OR queue_done > 0'], ['order' => ['iid']]);
 139                 $max_iid = $max_item_delivery_data['iid'] ?? 0;
 140
 141                 Logger::info('Start update1297 with max iid: ' . $max_iid);
 142
 143                 $condition = ['`queue_count` = 0 AND `iid` < ?', $max_iid];
 144
 145                 DBA::update('item-delivery-data', ['queue_count' => -1], $condition);
 146
 147                 if (DBA::errorNo() != 0) {
 148                         Logger::error('Database error ' . DBA::errorNo() . ':' . DBA::errorMessage());
 149                         return false;
 150                 }
 151
 152                 Logger::info('Processed rows: ' . DBA::affectedRows());
 153
 154                 DI::config()->set('system', 'post_update_version', 1297);
 155
 156                 Logger::info('Done');
 157
 158                 return true;
 159         }
 160         /**
 161          * Remove contact duplicates
 162          *
 163          * @return bool "true" when the job is done
 164          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 165          */
 166         private static function update1322()
 167         {
 168                 // Was the script completed?
 169                 if (DI::config()->get('system', 'post_update_version') >= 1322) {
 170                         return true;
 171                 }
 172
 173                 Logger::info('Start');
 174
 175                 $contacts = DBA::p("SELECT `nurl`, `uid` FROM `contact`
 176                         WHERE EXISTS (SELECT `nurl` FROM `contact` AS `c2`
 177                                 WHERE `c2`.`nurl` = `contact`.`nurl` AND `c2`.`id` != `contact`.`id` AND `c2`.`uid` = `contact`.`uid` AND `c2`.`network` IN (?, ?, ?) AND NOT `deleted`)
 178                         AND (`network` IN (?, ?, ?) OR (`uid` = ?)) AND NOT `deleted` GROUP BY `nurl`, `uid`",
 179                         Protocol::DIASPORA, Protocol::OSTATUS, Protocol::ACTIVITYPUB,
 180                         Protocol::DIASPORA, Protocol::OSTATUS, Protocol::ACTIVITYPUB, 0);
 181
 182                 while ($contact = DBA::fetch($contacts)) {
 183                         Logger::info('Remove duplicates', ['nurl' => $contact['nurl'], 'uid' => $contact['uid']]);
 184                         Contact::removeDuplicates($contact['nurl'], $contact['uid']);
 185                 }
 186
 187                 DBA::close($contact);
 188                 DI::config()->set('system', 'post_update_version', 1322);
 189
 190                 Logger::info('Done');
 191
 192                 return true;
 193         }
 194
 195         /**
 196          * update user notification data
 197          *
 198          * @return bool "true" when the job is done
 199          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 200          */
 201         private static function update1329()
 202         {
 203                 // Was the script completed?
 204                 if (DI::config()->get('system', 'post_update_version') >= 1329) {
 205                         return true;
 206                 }
 207
 208                 if (!DBStructure::existsTable('item')) {
 209                         DI::config()->set('system', 'post_update_version', 1329);
 210                         return true;
 211                 }
 212
 213                 $id = DI::config()->get('system', 'post_update_version_1329_id', 0);
 214
 215                 Logger::info('Start', ['item' => $id]);
 216
 217                 $start_id = $id;
 218                 $rows = 0;
 219                 $condition = ["`id` > ?", $id];
 220                 $params = ['order' => ['id'], 'limit' => 10000];
 221                 $items = DBA::select('item', ['id', 'uri-id', 'uid'], $condition, $params);
 222
 223                 if (DBA::errorNo() != 0) {
 224                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
 225                         return false;
 226                 }
 227
 228                 while ($item = DBA::fetch($items)) {
 229                         $id = $item['id'];
 230
 231                         Post\UserNotification::setNotification($item['uri-id'], $item['uid']);
 232
 233                         ++$rows;
 234                 }
 235                 DBA::close($items);
 236
 237                 DI::config()->set('system', 'post_update_version_1329_id', $id);
 238
 239                 Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
 240
 241                 if ($start_id == $id) {
 242                         DI::config()->set('system', 'post_update_version', 1329);
 243                         Logger::info('Done');
 244                         return true;
 245                 }
 246
 247                 return false;
 248         }
 249
 250         /**
 251          * Fill the "tag" table with tags and mentions from the body
 252          *
 253          * @return bool "true" when the job is done
 254          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 255          */
 256         private static function update1341()
 257         {
 258                 // Was the script completed?
 259                 if (DI::config()->get('system', 'post_update_version') >= 1341) {
 260                         return true;
 261                 }
 262
 263                 if (!DBStructure::existsTable('item-content')) {
 264                         DI::config()->set('system', 'post_update_version', 1342);
 265                         return true;
 266                 }
 267
 268                 $id = DI::config()->get('system', 'post_update_version_1341_id', 0);
 269
 270                 Logger::info('Start', ['item' => $id]);
 271
 272                 $rows = 0;
 273
 274                 $items = DBA::p("SELECT `uri-id`,`body` FROM `item-content` WHERE
 275                         (`body` LIKE ? OR `body` LIKE ? OR `body` LIKE ?) AND `uri-id` >= ?
 276                         ORDER BY `uri-id` LIMIT 100000", '%#%', '%@%', '%!%', $id);
 277
 278                 if (DBA::errorNo() != 0) {
 279                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
 280                         return false;
 281                 }
 282
 283                 while ($item = DBA::fetch($items)) {
 284                         Tag::storeFromBody($item['uri-id'], $item['body'], '#!@', false);
 285                         $id = $item['uri-id'];
 286                         ++$rows;
 287                         if ($rows % 1000 == 0) {
 288                                 DI::config()->set('system', 'post_update_version_1341_id', $id);
 289                         }
 290                 }
 291                 DBA::close($items);
 292
 293                 DI::config()->set('system', 'post_update_version_1341_id', $id);
 294
 295                 Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
 296
 297                 // When there are less than 1,000 items processed this means that we reached the end
 298                 // The other entries will then be processed with the regular functionality
 299                 if ($rows < 1000) {
 300                         DI::config()->set('system', 'post_update_version', 1341);
 301                         Logger::info('Done');
 302                         return true;
 303                 }
 304
 305                 return false;
 306         }
 307
 308         /**
 309          * Fill the "tag" table with tags and mentions from the "term" table
 310          *
 311          * @return bool "true" when the job is done
 312          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 313          */
 314         private static function update1342()
 315         {
 316                 // Was the script completed?
 317                 if (DI::config()->get('system', 'post_update_version') >= 1342) {
 318                         return true;
 319                 }
 320
 321                 if (!DBStructure::existsTable('term') || !DBStructure::existsTable('item-content')) {
 322                         DI::config()->set('system', 'post_update_version', 1342);
 323                         return true;
 324                 }
 325
 326                 $id = DI::config()->get('system', 'post_update_version_1342_id', 0);
 327
 328                 Logger::info('Start', ['item' => $id]);
 329
 330                 $rows = 0;
 331
 332                 $terms = DBA::p("SELECT `term`.`tid`, `item`.`uri-id`, `term`.`type`, `term`.`term`, `term`.`url`, `item-content`.`body`
 333                         FROM `term`
 334                         INNER JOIN `item` ON `item`.`id` = `term`.`oid`
 335                         INNER JOIN `item-content` ON `item-content`.`uri-id` = `item`.`uri-id`
 336                         WHERE term.type IN (?, ?, ?, ?) AND `tid` >= ? ORDER BY `tid` LIMIT 100000",
 337                         Tag::HASHTAG, Tag::MENTION, Tag::EXCLUSIVE_MENTION, Tag::IMPLICIT_MENTION, $id);
 338
 339                 if (DBA::errorNo() != 0) {
 340                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
 341                         return false;
 342                 }
 343
 344                 while ($term = DBA::fetch($terms)) {
 345                         if (($term['type'] == Tag::MENTION) && !empty($term['url']) && !strstr($term['body'], $term['url'])) {
 346                 $condition = ['nurl' => Strings::normaliseLink($term['url']), 'uid' => 0, 'deleted' => false];
 347                 $contact = DBA::selectFirst('contact', ['url', 'alias'], $condition, ['order' => ['id']]);
 348                 if (!DBA::isResult($contact)) {
 349                         $ssl_url = str_replace('http://', 'https://', $term['url']);
 350                         $condition = ['`alias` IN (?, ?, ?) AND `uid` = ? AND NOT `deleted`', $term['url'], Strings::normaliseLink($term['url']), $ssl_url, 0];
 351                         $contact = DBA::selectFirst('contact', ['url', 'alias'], $condition, ['order' => ['id']]);
 352                 }
 353
 354                 if (DBA::isResult($contact) && (!strstr($term['body'], $contact['url']) && (empty($contact['alias']) || !strstr($term['body'], $contact['alias'])))) {
 355                         $term['type'] = Tag::IMPLICIT_MENTION;
 356                 }
 357                         }
 358
 359                         Tag::store($term['uri-id'], $term['type'], $term['term'], $term['url'], false);
 360
 361                         $id = $term['tid'];
 362                         ++$rows;
 363                         if ($rows % 1000 == 0) {
 364                                 DI::config()->set('system', 'post_update_version_1342_id', $id);
 365                         }
 366                 }
 367                 DBA::close($terms);
 368
 369                 DI::config()->set('system', 'post_update_version_1342_id', $id);
 370
 371                 Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
 372
 373                 // When there are less than 1,000 items processed this means that we reached the end
 374                 // The other entries will then be processed with the regular functionality
 375                 if ($rows < 1000) {
 376                         DI::config()->set('system', 'post_update_version', 1342);
 377                         Logger::info('Done');
 378                         return true;
 379                 }
 380
 381                 return false;
 382         }
 383
 384         /**
 385          * Fill the "post-delivery-data" table with data from the "item-delivery-data" table
 386          *
 387          * @return bool "true" when the job is done
 388          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 389          */
 390         private static function update1345()
 391         {
 392                 // Was the script completed?
 393                 if (DI::config()->get('system', 'post_update_version') >= 1345) {
 394                         return true;
 395                 }
 396
 397                 if (!DBStructure::existsTable('item-delivery-data')) {
 398                         DI::config()->set('system', 'post_update_version', 1345);
 399                         return true;
 400                 }
 401
 402                 $id = DI::config()->get('system', 'post_update_version_1345_id', 0);
 403
 404                 Logger::info('Start', ['item' => $id]);
 405
 406                 $rows = 0;
 407
 408                 $deliveries = DBA::p("SELECT `uri-id`, `iid`, `item-delivery-data`.`postopts`, `item-delivery-data`.`inform`,
 409                         `queue_count`, `queue_done`, `activitypub`, `dfrn`, `diaspora`, `ostatus`, `legacy_dfrn`, `queue_failed`
 410                         FROM `item-delivery-data`
 411                         INNER JOIN `item` ON `item`.`id` = `item-delivery-data`.`iid`
 412                         WHERE `iid` >= ? ORDER BY `iid` LIMIT 10000", $id);
 413
 414                 if (DBA::errorNo() != 0) {
 415                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
 416                         return false;
 417                 }
 418
 419                 while ($delivery = DBA::fetch($deliveries)) {
 420                         $id = $delivery['iid'];
 421                         unset($delivery['iid']);
 422                         DBA::insert('post-delivery-data', $delivery, Database::INSERT_UPDATE);
 423                         ++$rows;
 424                 }
 425                 DBA::close($deliveries);
 426
 427                 DI::config()->set('system', 'post_update_version_1345_id', $id);
 428
 429                 Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
 430
 431                 // When there are less than 100 items processed this means that we reached the end
 432                 // The other entries will then be processed with the regular functionality
 433                 if ($rows < 100) {
 434                         DI::config()->set('system', 'post_update_version', 1345);
 435                         Logger::info('Done');
 436                         return true;
 437                 }
 438
 439                 return false;
 440         }
 441
 442         /**
 443          * Generates the legacy item.file field string from an item ID.
 444          * Includes only file and category terms.
 445          *
 446          * @param int $item_id
 447          * @return string
 448          * @throws \Exception
 449          */
 450         private static function fileTextFromItemId($item_id)
 451         {
 452                 $file_text = '';
 453
 454                 $condition = ['otype' => self::OBJECT_TYPE_POST, 'oid' => $item_id, 'type' => [Category::FILE, Category::CATEGORY]];
 455                 $tags = DBA::selectToArray('term', ['type', 'term', 'url'], $condition);
 456                 foreach ($tags as $tag) {
 457                         if ($tag['type'] == Category::CATEGORY) {
 458                                 $file_text .= '<' . $tag['term'] . '>';
 459                         } else {
 460                                 $file_text .= '[' . $tag['term'] . ']';
 461                         }
 462                 }
 463
 464                 return $file_text;
 465         }
 466
 467         /**
 468          * Fill the "tag" table with tags and mentions from the "term" table
 469          *
 470          * @return bool "true" when the job is done
 471          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 472          */
 473         private static function update1346()
 474         {
 475                 // Was the script completed?
 476                 if (DI::config()->get('system', 'post_update_version') >= 1346) {
 477                         return true;
 478                 }
 479
 480                 if (!DBStructure::existsTable('term')) {
 481                         DI::config()->set('system', 'post_update_version', 1346);
 482                         return true;
 483                 }
 484
 485                 $id = DI::config()->get('system', 'post_update_version_1346_id', 0);
 486
 487                 Logger::info('Start', ['item' => $id]);
 488
 489                 $rows = 0;
 490
 491                 $terms = DBA::select('term', ['oid'],
 492                         ["`type` IN (?, ?) AND `oid` >= ?", Category::CATEGORY, Category::FILE, $id],
 493                         ['order' => ['oid'], 'limit' => 1000, 'group_by' => ['oid']]);
 494
 495                 if (DBA::errorNo() != 0) {
 496                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
 497                         return false;
 498                 }
 499
 500                 while ($term = DBA::fetch($terms)) {
 501                         $item = Post::selectFirst(['uri-id', 'uid'], ['id' => $term['oid']]);
 502                         if (!DBA::isResult($item)) {
 503                                 continue;
 504                         }
 505
 506                         $file = self::fileTextFromItemId($term['oid']);
 507                         if (!empty($file)) {
 508                                 Category::storeTextByURIId($item['uri-id'], $item['uid'], $file);
 509                         }
 510
 511                         $id = $term['oid'];
 512                         ++$rows;
 513                         if ($rows % 100 == 0) {
 514                                 DI::config()->set('system', 'post_update_version_1346_id', $id);
 515                         }
 516                 }
 517                 DBA::close($terms);
 518
 519                 DI::config()->set('system', 'post_update_version_1346_id', $id);
 520
 521                 Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
 522
 523                 // When there are less than 10 items processed this means that we reached the end
 524                 // The other entries will then be processed with the regular functionality
 525                 if ($rows < 10) {
 526                         DI::config()->set('system', 'post_update_version', 1346);
 527                         Logger::info('Done');
 528                         return true;
 529                 }
 530
 531                 return false;
 532         }
 533
 534         /**
 535          * update the "vid" (verb) field in the item table
 536          *
 537          * @return bool "true" when the job is done
 538          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 539          * @throws \ImagickException
 540          */
 541         private static function update1347()
 542         {
 543                 // Was the script completed?
 544                 if (DI::config()->get('system', 'post_update_version') >= 1347) {
 545                         return true;
 546                 }
 547
 548                 if (!DBStructure::existsTable('item-activity') || !DBStructure::existsTable('item')) {
 549                         DI::config()->set('system', 'post_update_version', 1347);
 550                         return true;
 551                 }
 552
 553                 $id = DI::config()->get('system', 'post_update_version_1347_id', 0);
 554
 555                 Logger::info('Start', ['item' => $id]);
 556
 557                 $start_id = $id;
 558                 $rows = 0;
 559
 560                 $items = DBA::p("SELECT `item`.`id`, `item`.`verb` AS `item-verb`, `item-content`.`verb`, `item-activity`.`activity`
 561                         FROM `item` LEFT JOIN `item-content` ON `item-content`.`uri-id` = `item`.`uri-id`
 562                         LEFT JOIN `item-activity` ON `item-activity`.`uri-id` = `item`.`uri-id` AND `item`.`gravity` = ?
 563                         WHERE `item`.`id` >= ? AND `item`.`vid` IS NULL ORDER BY `item`.`id` LIMIT 10000", GRAVITY_ACTIVITY, $id);
 564
 565                 if (DBA::errorNo() != 0) {
 566                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
 567                         return false;
 568                 }
 569
 570                 while ($item = DBA::fetch($items)) {
 571                         $id = $item['id'];
 572                         $verb = $item['item-verb'];
 573                         if (empty($verb)) {
 574                                 $verb = $item['verb'];
 575                         }
 576                         if (empty($verb) && is_int($item['activity'])) {
 577                                 $verb = Item::ACTIVITIES[$item['activity']];
 578                         }
 579                         if (empty($verb)) {
 580                                 continue;
 581                         }
 582
 583                         DBA::update('item', ['vid' => Verb::getID($verb)], ['id' => $item['id']]);
 584                         ++$rows;
 585                 }
 586                 DBA::close($items);
 587
 588                 DI::config()->set('system', 'post_update_version_1347_id', $id);
 589
 590                 Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
 591
 592                 if ($start_id == $id) {
 593                         DI::config()->set('system', 'post_update_version', 1347);
 594                         Logger::info('Done');
 595                         return true;
 596                 }
 597
 598                 return false;
 599         }
 600
 601         /**
 602          * update the "gsid" (global server id) field in the contact table
 603          *
 604          * @return bool "true" when the job is done
 605          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 606          * @throws \ImagickException
 607          */
 608         private static function update1348()
 609         {
 610                 // Was the script completed?
 611                 if (DI::config()->get('system', 'post_update_version') >= 1348) {
 612                         return true;
 613                 }
 614
 615                 $id = DI::config()->get('system', 'post_update_version_1348_id', 0);
 616
 617                 Logger::info('Start', ['contact' => $id]);
 618
 619                 $start_id = $id;
 620                 $rows = 0;
 621                 $condition = ["`id` > ? AND `gsid` IS NULL AND `baseurl` != '' AND NOT `baseurl` IS NULL", $id];
 622                 $params = ['order' => ['id'], 'limit' => 10000];
 623                 $contacts = DBA::select('contact', ['id', 'baseurl'], $condition, $params);
 624
 625                 if (DBA::errorNo() != 0) {
 626                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
 627                         return false;
 628                 }
 629
 630                 while ($contact = DBA::fetch($contacts)) {
 631                         $id = $contact['id'];
 632
 633                         DBA::update('contact',
 634                                 ['gsid' => GServer::getID($contact['baseurl'], true), 'baseurl' => GServer::cleanURL($contact['baseurl'])],
 635                                 ['id' => $contact['id']]);
 636
 637                         ++$rows;
 638                 }
 639                 DBA::close($contacts);
 640
 641                 DI::config()->set('system', 'post_update_version_1348_id', $id);
 642
 643                 Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
 644
 645                 if ($start_id == $id) {
 646                         DI::config()->set('system', 'post_update_version', 1348);
 647                         Logger::info('Done');
 648                         return true;
 649                 }
 650
 651                 return false;
 652         }
 653
 654         /**
 655          * update the "gsid" (global server id) field in the apcontact table
 656          *
 657          * @return bool "true" when the job is done
 658          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 659          * @throws \ImagickException
 660          */
 661         private static function update1349()
 662         {
 663                 // Was the script completed?
 664                 if (DI::config()->get('system', 'post_update_version') >= 1349) {
 665                         return true;
 666                 }
 667
 668                 $id = DI::config()->get('system', 'post_update_version_1349_id', '');
 669
 670                 Logger::info('Start', ['apcontact' => $id]);
 671
 672                 $start_id = $id;
 673                 $rows = 0;
 674                 $condition = ["`url` > ? AND `gsid` IS NULL AND `baseurl` != '' AND NOT `baseurl` IS NULL", $id];
 675                 $params = ['order' => ['url'], 'limit' => 10000];
 676                 $apcontacts = DBA::select('apcontact', ['url', 'baseurl'], $condition, $params);
 677
 678                 if (DBA::errorNo() != 0) {
 679                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
 680                         return false;
 681                 }
 682
 683                 while ($apcontact = DBA::fetch($apcontacts)) {
 684                         $id = $apcontact['url'];
 685
 686                         DBA::update('apcontact',
 687                                 ['gsid' => GServer::getID($apcontact['baseurl'], true), 'baseurl' => GServer::cleanURL($apcontact['baseurl'])],
 688                                 ['url' => $apcontact['url']]);
 689
 690                         ++$rows;
 691                 }
 692                 DBA::close($apcontacts);
 693
 694                 DI::config()->set('system', 'post_update_version_1349_id', $id);
 695
 696                 Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
 697
 698                 if ($start_id == $id) {
 699                         DI::config()->set('system', 'post_update_version', 1349);
 700                         Logger::info('Done');
 701                         return true;
 702                 }
 703
 704                 return false;
 705         }
 706
 707         /**
 708          * Remove orphaned photo entries
 709          *
 710          * @return bool "true" when the job is done
 711          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 712          * @throws \ImagickException
 713          */
 714         private static function update1383()
 715         {
 716                 // Was the script completed?
 717                 if (DI::config()->get('system', 'post_update_version') >= 1383) {
 718                         return true;
 719                 }
 720
 721                 Logger::info('Start');
 722
 723                 $deleted = 0;
 724                 $avatar = [4 => 'photo', 5 => 'thumb', 6 => 'micro'];
 725
 726                 $photos = DBA::select('photo', ['id', 'contact-id', 'resource-id', 'scale'], ["`contact-id` != ? AND `album` = ?", 0, Photo::CONTACT_PHOTOS]);
 727                 while ($photo = DBA::fetch($photos)) {
 728                         $delete = !in_array($photo['scale'], [4, 5, 6]);
 729
 730                         if (!$delete) {
 731                                 // Check if there is a contact entry with that photo
 732                                 $delete = !DBA::exists('contact', ["`id` = ? AND `" . $avatar[$photo['scale']] . "` LIKE ?",
 733                                         $photo['contact-id'], '%' . $photo['resource-id'] . '%']);
 734                         }
 735
 736                         if ($delete) {
 737                                 Photo::delete(['id' => $photo['id']]);
 738                                 $deleted++;
 739                         }
 740                 }
 741                 DBA::close($photos);
 742
 743                 DI::config()->set('system', 'post_update_version', 1383);
 744                 Logger::info('Done', ['deleted' => $deleted]);
 745                 return true;
 746         }
 747
 748         /**
 749          * update the "hash" field in the photo table
 750          *
 751          * @return bool "true" when the job is done
 752          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 753          * @throws \ImagickException
 754          */
 755         private static function update1384()
 756         {
 757                 // Was the script completed?
 758                 if (DI::config()->get('system', 'post_update_version') >= 1384) {
 759                         return true;
 760                 }
 761
 762                 $condition = ["`hash` IS NULL"];
 763                 Logger::info('Start', ['rest' => DBA::count('photo', $condition)]);
 764
 765                 $rows = 0;
 766                 $photos = DBA::select('photo', [], $condition, ['limit' => 100]);
 767
 768                 if (DBA::errorNo() != 0) {
 769                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
 770                         return false;
 771                 }
 772
 773                 while ($photo = DBA::fetch($photos)) {
 774                         $img = Photo::getImageForPhoto($photo);
 775                         if (!empty($img)) {
 776                                 $md5 = md5($img->asString());
 777                         } else {
 778                                 $md5 = '';
 779                         }
 780                         DBA::update('photo', ['hash' => $md5], ['id' => $photo['id']]);
 781                         ++$rows;
 782                 }
 783                 DBA::close($photos);
 784
 785                 Logger::info('Processed', ['rows' => $rows]);
 786
 787                 if ($rows <= 100) {
 788                         DI::config()->set('system', 'post_update_version', 1384);
 789                         Logger::info('Done');
 790                         return true;
 791                 }
 792
 793                 return false;
 794         }
 795
 796         /**
 797          * update the "external-id" field in the post table
 798          *
 799          * @return bool "true" when the job is done
 800          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 801          * @throws \ImagickException
 802          */
 803         private static function update1400()
 804         {
 805                 // Was the script completed?
 806                 if (DI::config()->get('system', 'post_update_version') >= 1400) {
 807                         return true;
 808                 }
 809
 810                 if (!DBStructure::existsTable('item')) {
 811                         DI::config()->set('system', 'post_update_version', 1400);
 812                         return true;
 813                 }
 814
 815                 $condition = ["`extid` != ? AND EXISTS(SELECT `id` FROM `post-user` WHERE `uri-id` = `item`.`uri-id` AND `uid` = `item`.`uid` AND `external-id` IS NULL)", ''];
 816                 Logger::info('Start', ['rest' => DBA::count('item', $condition)]);
 817
 818                 $rows = 0;
 819                 $items = DBA::select('item', ['uri-id', 'uid', 'extid'], $condition, ['order' => ['id'], 'limit' => 10000]);
 820
 821                 if (DBA::errorNo() != 0) {
 822                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
 823                         return false;
 824                 }
 825
 826                 while ($item = DBA::fetch($items)) {
 827                         Post::update(['external-id' => ItemURI::getIdByURI($item['extid'])], ['uri-id' => $item['uri-id'], 'uid' => $item['uid']]);
 828                         ++$rows;
 829                 }
 830                 DBA::close($items);
 831
 832                 Logger::info('Processed', ['rows' => $rows]);
 833
 834                 if ($rows <= 100) {
 835                         DI::config()->set('system', 'post_update_version', 1400);
 836                         Logger::info('Done');
 837                         return true;
 838                 }
 839
 840                 return false;
 841         }
 842
 843         /**
 844          * update the "uri-id" field in the contact table
 845          *
 846          * @return bool "true" when the job is done
 847          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 848          * @throws \ImagickException
 849          */
 850         private static function update1424()
 851         {
 852                 // Was the script completed?
 853                 if (DI::config()->get('system', 'post_update_version') >= 1424) {
 854                         return true;
 855                 }
 856
 857                 $condition = ["`uri-id` IS NULL"];
 858                 Logger::info('Start', ['rest' => DBA::count('contact', $condition)]);
 859
 860                 $rows = 0;
 861                 $contacts = DBA::select('contact', ['id', 'url'], $condition, ['limit' => 1000]);
 862
 863                 if (DBA::errorNo() != 0) {
 864                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
 865                         return false;
 866                 }
 867
 868                 while ($contact = DBA::fetch($contacts)) {
 869                         DBA::update('contact', ['uri-id' => ItemURI::getIdByURI($contact['url'])], ['id' => $contact['id']]);
 870                         ++$rows;
 871                 }
 872                 DBA::close($contacts);
 873
 874                 Logger::info('Processed', ['rows' => $rows]);
 875
 876                 if ($rows <= 100) {
 877                         DI::config()->set('system', 'post_update_version', 1424);
 878                         Logger::info('Done');
 879                         return true;
 880                 }
 881
 882                 return false;
 883         }
 884
 885         /**
 886          * update the "uri-id" field in the fcontact table
 887          *
 888          * @return bool "true" when the job is done
 889          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 890          * @throws \ImagickException
 891          */
 892         private static function update1425()
 893         {
 894                 // Was the script completed?
 895                 if (DI::config()->get('system', 'post_update_version') >= 1425) {
 896                         return true;
 897                 }
 898
 899                 $condition = ["`uri-id` IS NULL"];
 900                 Logger::info('Start', ['rest' => DBA::count('fcontact', $condition)]);
 901
 902                 $rows = 0;
 903                 $fcontacts = DBA::select('fcontact', ['id', 'url', 'guid'], $condition, ['limit' => 1000]);
 904
 905                 if (DBA::errorNo() != 0) {
 906                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
 907                         return false;
 908                 }
 909
 910                 while ($fcontact = DBA::fetch($fcontacts)) {
 911                         if (!empty($fcontact['guid'])) {
 912                                 $uriid = ItemURI::insert(['uri' => $fcontact['url'], 'guid' => $fcontact['guid']]);
 913                         } else {
 914                                 $uriid = ItemURI::getIdByURI($fcontact['url']);
 915                         }
 916                         DBA::update('fcontact', ['uri-id' => $uriid], ['id' => $fcontact['id']]);
 917                         ++$rows;
 918                 }
 919                 DBA::close($fcontacts);
 920
 921                 Logger::info('Processed', ['rows' => $rows]);
 922
 923                 if ($rows <= 100) {
 924                         DI::config()->set('system', 'post_update_version', 1425);
 925                         Logger::info('Done');
 926                         return true;
 927                 }
 928
 929                 return false;
 930         }
 931
 932         /**
 933          * update the "uri-id" field in the apcontact table
 934          *
 935          * @return bool "true" when the job is done
 936          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 937          * @throws \ImagickException
 938          */
 939         private static function update1426()
 940         {
 941                 // Was the script completed?
 942                 if (DI::config()->get('system', 'post_update_version') >= 1426) {
 943                         return true;
 944                 }
 945
 946                 $condition = ["`uri-id` IS NULL"];
 947                 Logger::info('Start', ['rest' => DBA::count('apcontact', $condition)]);
 948
 949                 $rows = 0;
 950                 $apcontacts = DBA::select('apcontact', ['url', 'uuid'], $condition, ['limit' => 1000]);
 951
 952                 if (DBA::errorNo() != 0) {
 953                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
 954                         return false;
 955                 }
 956
 957                 while ($apcontact = DBA::fetch($apcontacts)) {
 958                         if (!empty($apcontact['uuid'])) {
 959                                 $uriid = ItemURI::insert(['uri' => $apcontact['url'], 'guid' => $apcontact['uuid']]);
 960                         } else {
 961                                 $uriid = ItemURI::getIdByURI($apcontact['url']);
 962                         }
 963                         DBA::update('apcontact', ['uri-id' => $uriid], ['url' => $apcontact['url']]);
 964                         ++$rows;
 965                 }
 966                 DBA::close($apcontacts);
 967
 968                 Logger::info('Processed', ['rows' => $rows]);
 969
 970                 if ($rows <= 100) {
 971                         DI::config()->set('system', 'post_update_version', 1426);
 972                         Logger::info('Done');
 973                         return true;
 974                 }
 975
 976                 return false;
 977         }
 978
 979         /**
 980          * update the "uri-id" field in the event table
 981          *
 982          * @return bool "true" when the job is done
 983          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 984          * @throws \ImagickException
 985          */
 986         private static function update1427()
 987         {
 988                 // Was the script completed?
 989                 if (DI::config()->get('system', 'post_update_version') >= 1427) {
 990                         return true;
 991                 }
 992
 993                 $condition = ["`uri-id` IS NULL"];
 994                 Logger::info('Start', ['rest' => DBA::count('event', $condition)]);
 995
 996                 $rows = 0;
 997                 $events = DBA::select('event', ['id', 'uri', 'guid'], $condition, ['limit' => 1000]);
 998
 999                 if (DBA::errorNo() != 0) {
1000                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
1001                         return false;
1002                 }
1003
1004                 while ($event = DBA::fetch($events)) {
1005                         if (!empty($event['guid'])) {
1006                                 $uriid = ItemURI::insert(['uri' => $event['uri'], 'guid' => $event['guid']]);
1007                         } else {
1008                                 $uriid = ItemURI::getIdByURI($event['uri']);
1009                         }
1010                         DBA::update('event', ['uri-id' => $uriid], ['id' => $event['id']]);
1011                         ++$rows;
1012                 }
1013                 DBA::close($events);
1014
1015                 Logger::info('Processed', ['rows' => $rows]);
1016
1017                 if ($rows <= 100) {
1018                         DI::config()->set('system', 'post_update_version', 1427);
1019                         Logger::info('Done');
1020                         return true;
1021                 }
1022
1023                 return false;
1024         }
1025
1026         /**
1027          * Fill the receivers of the post via the raw source
1028          *
1029          * @return bool "true" when the job is done
1030          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
1031          * @throws \ImagickException
1032          */
1033         private static function update1452()
1034         {
1035                 // Was the script completed?
1036                 if (DI::config()->get('system', 'post_update_version') >= 1452) {
1037                         return true;
1038                 }
1039
1040                 $id = DI::config()->get('system', 'post_update_version_1452_id', 0);
1041
1042                 Logger::info('Start', ['uri-id' => $id]);
1043
1044                 $rows     = 0;
1045                 $received = '';
1046
1047                 $conversations = DBA::p("SELECT `post-view`.`uri-id`, `conversation`.`source`, `conversation`.`received` FROM `conversation`
1048                         INNER JOIN `post-view` ON `post-view`.`uri` = `conversation`.`item-uri`
1049                         WHERE NOT `source` IS NULL AND `conversation`.`protocol` = ? AND `uri-id` > ? LIMIT ?",
1050                         Conversation::PARCEL_ACTIVITYPUB, $id, 1000);
1051
1052                 if (DBA::errorNo() != 0) {
1053                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
1054                         return false;
1055                 }
1056
1057                 while ($conversation = DBA::fetch($conversations)) {
1058                         $id       = $conversation['uri-id'];
1059                         $received = $conversation['received'];
1060
1061                         $raw = json_decode($conversation['source'], true);
1062                         if (empty($raw)) {
1063                                 continue;
1064                         }
1065                         $activity = JsonLD::compact($raw);
1066
1067                         $urls = Receiver::getReceiverURL($activity);
1068                         Processor::storeReceivers($conversation['uri-id'], $urls);
1069
1070                         if (!empty($activity['as:object'])) {
1071                                 $urls = array_merge($urls, Receiver::getReceiverURL($activity['as:object']));
1072                                 Processor::storeReceivers($conversation['uri-id'], $urls);
1073                         }
1074                         ++$rows;
1075                 }
1076
1077                 DBA::close($conversations);
1078
1079                 DI::config()->set('system', 'post_update_version_1452_id', $id);
1080
1081                 Logger::info('Processed', ['rows' => $rows, 'last' => $id, 'last-received' => $received]);
1082
1083                 if ($rows <= 100) {
1084                         DI::config()->set('system', 'post_update_version', 1452);
1085                         Logger::info('Done');
1086                         return true;
1087                 }
1088
1089                 return false;
1090         }
1091
1092         /**
1093          * Correct the parent.
1094          * This fixes a bug that was introduced in the development of version 2022.09
1095          *
1096          * @return bool "true" when the job is done
1097          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
1098          * @throws \ImagickException
1099          */
1100         private static function update1483()
1101         {
1102                 // Was the script completed?
1103                 if (DI::config()->get('system', 'post_update_version') >= 1483) {
1104                         return true;
1105                 }
1106
1107                 Logger::info('Start');
1108
1109                 $posts = DBA::select('post-view', ['uri-id'], ['conversation' => './']);
1110                 while ($post = DBA::fetch($posts)) {
1111                         $parent = Item::getParent($post['uri-id']);
1112                         if ($parent != 0) {
1113                                 DBA::update('post', ['parent-uri-id' => $parent], ['uri-id' => $post['uri-id']]);
1114                                 DBA::update('post-user', ['parent-uri-id' => $parent], ['uri-id' => $post['uri-id']]);
1115                         }
1116                 }
1117                 DBA::close($posts);
1118
1119                 DI::config()->set('system', 'post_update_version', 1483);
1120                 Logger::info('Done');
1121                 return true;
1122         }
1123 }