src/Database/PostUpdate.php

   1 <?php
   2 /**
   3  * @copyright Copyright (C) 2010-2021, the Friendica project
   4  *
   5  * @license GNU AGPL version 3 or any later version
   6  *
   7  * This program is free software: you can redistribute it and/or modify
   8  * it under the terms of the GNU Affero General Public License as
   9  * published by the Free Software Foundation, either version 3 of the
  10  * License, or (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU Affero General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU Affero General Public License
  18  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  19  *
  20  */
  21
  22 namespace Friendica\Database;
  23
  24 use Friendica\Core\Logger;
  25 use Friendica\Core\Protocol;
  26 use Friendica\DI;
  27 use Friendica\Model\Contact;
  28 use Friendica\Model\GServer;
  29 use Friendica\Model\Item;
  30 use Friendica\Model\ItemURI;
  31 use Friendica\Model\Photo;
  32 use Friendica\Model\Post;
  33 use Friendica\Model\Post\Category;
  34 use Friendica\Model\Tag;
  35 use Friendica\Model\Verb;
  36 use Friendica\Util\Strings;
  37
  38 /**
  39  * These database-intensive post update routines are meant to be executed in the background by the cronjob.
  40  *
  41  * If there is a need for a intensive migration after a database structure change, update this file
  42  * by adding a new method at the end with the number of the new DB_UPDATE_VERSION.
  43  */
  44 class PostUpdate
  45 {
  46         // Needed for the helper function to read from the legacy term table
  47         const OBJECT_TYPE_POST  = 1;
  48
  49         const VERSION = 1427;
  50
  51         /**
  52          * Calls the post update functions
  53          */
  54         public static function update()
  55         {
  56                 if (!self::update1297()) {
  57                         return false;
  58                 }
  59                 if (!self::update1322()) {
  60                         return false;
  61                 }
  62                 if (!self::update1329()) {
  63                         return false;
  64                 }
  65                 if (!self::update1341()) {
  66                         return false;
  67                 }
  68                 if (!self::update1342()) {
  69                         return false;
  70                 }
  71                 if (!self::update1345()) {
  72                         return false;
  73                 }
  74                 if (!self::update1346()) {
  75                         return false;
  76                 }
  77                 if (!self::update1347()) {
  78                         return false;
  79                 }
  80                 if (!self::update1348()) {
  81                         return false;
  82                 }
  83                 if (!self::update1349()) {
  84                         return false;
  85                 }
  86                 if (!self::update1383()) {
  87                         return false;
  88                 }
  89                 if (!self::update1384()) {
  90                         return false;
  91                 }
  92                 if (!self::update1400()) {
  93                         return false;
  94                 }
  95                 if (!self::update1424()) {
  96                         return false;
  97                 }
  98                 if (!self::update1425()) {
  99                         return false;
 100                 }
 101                 if (!self::update1426()) {
 102                         return false;
 103                 }
 104                 if (!self::update1427()) {
 105                         return false;
 106                 }
 107                 return true;
 108         }
 109
 110         /**
 111          * Set the delivery queue count to a negative value for all items preceding the feature.
 112          *
 113          * @return bool "true" when the job is done
 114          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 115          */
 116         private static function update1297()
 117         {
 118                 // Was the script completed?
 119                 if (DI::config()->get('system', 'post_update_version') >= 1297) {
 120                         return true;
 121                 }
 122
 123                 if (!DBStructure::existsTable('item-delivery-data')) {
 124                         DI::config()->set('system', 'post_update_version', 1297);
 125                         return true;
 126                 }
 127
 128                 $max_item_delivery_data = DBA::selectFirst('item-delivery-data', ['iid'], ['queue_count > 0 OR queue_done > 0'], ['order' => ['iid']]);
 129                 $max_iid = $max_item_delivery_data['iid'];
 130
 131                 Logger::info('Start update1297 with max iid: ' . $max_iid);
 132
 133                 $condition = ['`queue_count` = 0 AND `iid` < ?', $max_iid];
 134
 135                 DBA::update('item-delivery-data', ['queue_count' => -1], $condition);
 136
 137                 if (DBA::errorNo() != 0) {
 138                         Logger::error('Database error ' . DBA::errorNo() . ':' . DBA::errorMessage());
 139                         return false;
 140                 }
 141
 142                 Logger::info('Processed rows: ' . DBA::affectedRows());
 143
 144                 DI::config()->set('system', 'post_update_version', 1297);
 145
 146                 Logger::info('Done');
 147
 148                 return true;
 149         }
 150         /**
 151          * Remove contact duplicates
 152          *
 153          * @return bool "true" when the job is done
 154          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 155          */
 156         private static function update1322()
 157         {
 158                 // Was the script completed?
 159                 if (DI::config()->get('system', 'post_update_version') >= 1322) {
 160                         return true;
 161                 }
 162
 163                 Logger::info('Start');
 164
 165                 $contacts = DBA::p("SELECT `nurl`, `uid` FROM `contact`
 166                         WHERE EXISTS (SELECT `nurl` FROM `contact` AS `c2`
 167                                 WHERE `c2`.`nurl` = `contact`.`nurl` AND `c2`.`id` != `contact`.`id` AND `c2`.`uid` = `contact`.`uid` AND `c2`.`network` IN (?, ?, ?) AND NOT `deleted`)
 168                         AND (`network` IN (?, ?, ?) OR (`uid` = ?)) AND NOT `deleted` GROUP BY `nurl`, `uid`",
 169                         Protocol::DIASPORA, Protocol::OSTATUS, Protocol::ACTIVITYPUB,
 170                         Protocol::DIASPORA, Protocol::OSTATUS, Protocol::ACTIVITYPUB, 0);
 171
 172                 while ($contact = DBA::fetch($contacts)) {
 173                         Logger::info('Remove duplicates', ['nurl' => $contact['nurl'], 'uid' => $contact['uid']]);
 174                         Contact::removeDuplicates($contact['nurl'], $contact['uid']);
 175                 }
 176
 177                 DBA::close($contact);
 178                 DI::config()->set('system', 'post_update_version', 1322);
 179
 180                 Logger::info('Done');
 181
 182                 return true;
 183         }
 184
 185         /**
 186          * update user notification data
 187          *
 188          * @return bool "true" when the job is done
 189          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 190          */
 191         private static function update1329()
 192         {
 193                 // Was the script completed?
 194                 if (DI::config()->get('system', 'post_update_version') >= 1329) {
 195                         return true;
 196                 }
 197
 198                 if (!DBStructure::existsTable('item')) {
 199                         DI::config()->set('system', 'post_update_version', 1329);
 200                         return true;
 201                 }
 202
 203                 $id = DI::config()->get('system', 'post_update_version_1329_id', 0);
 204
 205                 Logger::info('Start', ['item' => $id]);
 206
 207                 $start_id = $id;
 208                 $rows = 0;
 209                 $condition = ["`id` > ?", $id];
 210                 $params = ['order' => ['id'], 'limit' => 10000];
 211                 $items = DBA::select('item', ['id', 'uri-id', 'uid'], $condition, $params);
 212
 213                 if (DBA::errorNo() != 0) {
 214                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
 215                         return false;
 216                 }
 217
 218                 while ($item = DBA::fetch($items)) {
 219                         $id = $item['id'];
 220
 221                         Post\UserNotification::setNotification($item['uri-id'], $item['uid']);
 222
 223                         ++$rows;
 224                 }
 225                 DBA::close($items);
 226
 227                 DI::config()->set('system', 'post_update_version_1329_id', $id);
 228
 229                 Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
 230
 231                 if ($start_id == $id) {
 232                         DI::config()->set('system', 'post_update_version', 1329);
 233                         Logger::info('Done');
 234                         return true;
 235                 }
 236
 237                 return false;
 238         }
 239
 240         /**
 241          * Fill the "tag" table with tags and mentions from the body
 242          *
 243          * @return bool "true" when the job is done
 244          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 245          */
 246         private static function update1341()
 247         {
 248                 // Was the script completed?
 249                 if (DI::config()->get('system', 'post_update_version') >= 1341) {
 250                         return true;
 251                 }
 252
 253                 if (!DBStructure::existsTable('item-content')) {
 254                         DI::config()->set('system', 'post_update_version', 1342);
 255                         return true;
 256                 }
 257
 258                 $id = DI::config()->get('system', 'post_update_version_1341_id', 0);
 259
 260                 Logger::info('Start', ['item' => $id]);
 261
 262                 $rows = 0;
 263
 264                 $items = DBA::p("SELECT `uri-id`,`body` FROM `item-content` WHERE
 265                         (`body` LIKE ? OR `body` LIKE ? OR `body` LIKE ?) AND `uri-id` >= ?
 266                         ORDER BY `uri-id` LIMIT 100000", '%#%', '%@%', '%!%', $id);
 267
 268                 if (DBA::errorNo() != 0) {
 269                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
 270                         return false;
 271                 }
 272
 273                 while ($item = DBA::fetch($items)) {
 274                         Tag::storeFromBody($item['uri-id'], $item['body'], '#!@', false);
 275                         $id = $item['uri-id'];
 276                         ++$rows;
 277                         if ($rows % 1000 == 0) {
 278                                 DI::config()->set('system', 'post_update_version_1341_id', $id);
 279                         }
 280                 }
 281                 DBA::close($items);
 282
 283                 DI::config()->set('system', 'post_update_version_1341_id', $id);
 284
 285                 Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
 286
 287                 // When there are less than 1,000 items processed this means that we reached the end
 288                 // The other entries will then be processed with the regular functionality
 289                 if ($rows < 1000) {
 290                         DI::config()->set('system', 'post_update_version', 1341);
 291                         Logger::info('Done');
 292                         return true;
 293                 }
 294
 295                 return false;
 296         }
 297
 298         /**
 299          * Fill the "tag" table with tags and mentions from the "term" table
 300          *
 301          * @return bool "true" when the job is done
 302          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 303          */
 304         private static function update1342()
 305         {
 306                 // Was the script completed?
 307                 if (DI::config()->get('system', 'post_update_version') >= 1342) {
 308                         return true;
 309                 }
 310
 311                 if (!DBStructure::existsTable('term') || !DBStructure::existsTable('item-content')) {
 312                         DI::config()->set('system', 'post_update_version', 1342);
 313                         return true;
 314                 }
 315
 316                 $id = DI::config()->get('system', 'post_update_version_1342_id', 0);
 317
 318                 Logger::info('Start', ['item' => $id]);
 319
 320                 $rows = 0;
 321
 322                 $terms = DBA::p("SELECT `term`.`tid`, `item`.`uri-id`, `term`.`type`, `term`.`term`, `term`.`url`, `item-content`.`body`
 323                         FROM `term`
 324                         INNER JOIN `item` ON `item`.`id` = `term`.`oid`
 325                         INNER JOIN `item-content` ON `item-content`.`uri-id` = `item`.`uri-id`
 326                         WHERE term.type IN (?, ?, ?, ?) AND `tid` >= ? ORDER BY `tid` LIMIT 100000",
 327                         Tag::HASHTAG, Tag::MENTION, Tag::EXCLUSIVE_MENTION, Tag::IMPLICIT_MENTION, $id);
 328
 329                 if (DBA::errorNo() != 0) {
 330                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
 331                         return false;
 332                 }
 333
 334                 while ($term = DBA::fetch($terms)) {
 335                         if (($term['type'] == Tag::MENTION) && !empty($term['url']) && !strstr($term['body'], $term['url'])) {
 336                 $condition = ['nurl' => Strings::normaliseLink($term['url']), 'uid' => 0, 'deleted' => false];
 337                 $contact = DBA::selectFirst('contact', ['url', 'alias'], $condition, ['order' => ['id']]);
 338                 if (!DBA::isResult($contact)) {
 339                         $ssl_url = str_replace('http://', 'https://', $term['url']);
 340                         $condition = ['`alias` IN (?, ?, ?) AND `uid` = ? AND NOT `deleted`', $term['url'], Strings::normaliseLink($term['url']), $ssl_url, 0];
 341                         $contact = DBA::selectFirst('contact', ['url', 'alias'], $condition, ['order' => ['id']]);
 342                 }
 343
 344                 if (DBA::isResult($contact) && (!strstr($term['body'], $contact['url']) && (empty($contact['alias']) || !strstr($term['body'], $contact['alias'])))) {
 345                         $term['type'] = Tag::IMPLICIT_MENTION;
 346                 }
 347                         }
 348
 349                         Tag::store($term['uri-id'], $term['type'], $term['term'], $term['url'], false);
 350
 351                         $id = $term['tid'];
 352                         ++$rows;
 353                         if ($rows % 1000 == 0) {
 354                                 DI::config()->set('system', 'post_update_version_1342_id', $id);
 355                         }
 356                 }
 357                 DBA::close($terms);
 358
 359                 DI::config()->set('system', 'post_update_version_1342_id', $id);
 360
 361                 Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
 362
 363                 // When there are less than 1,000 items processed this means that we reached the end
 364                 // The other entries will then be processed with the regular functionality
 365                 if ($rows < 1000) {
 366                         DI::config()->set('system', 'post_update_version', 1342);
 367                         Logger::info('Done');
 368                         return true;
 369                 }
 370
 371                 return false;
 372         }
 373
 374         /**
 375          * Fill the "post-delivery-data" table with data from the "item-delivery-data" table
 376          *
 377          * @return bool "true" when the job is done
 378          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 379          */
 380         private static function update1345()
 381         {
 382                 // Was the script completed?
 383                 if (DI::config()->get('system', 'post_update_version') >= 1345) {
 384                         return true;
 385                 }
 386
 387                 if (!DBStructure::existsTable('item-delivery-data')) {
 388                         DI::config()->set('system', 'post_update_version', 1345);
 389                         return true;
 390                 }
 391
 392                 $id = DI::config()->get('system', 'post_update_version_1345_id', 0);
 393
 394                 Logger::info('Start', ['item' => $id]);
 395
 396                 $rows = 0;
 397
 398                 $deliveries = DBA::p("SELECT `uri-id`, `iid`, `item-delivery-data`.`postopts`, `item-delivery-data`.`inform`,
 399                         `queue_count`, `queue_done`, `activitypub`, `dfrn`, `diaspora`, `ostatus`, `legacy_dfrn`, `queue_failed`
 400                         FROM `item-delivery-data`
 401                         INNER JOIN `item` ON `item`.`id` = `item-delivery-data`.`iid`
 402                         WHERE `iid` >= ? ORDER BY `iid` LIMIT 10000", $id);
 403
 404                 if (DBA::errorNo() != 0) {
 405                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
 406                         return false;
 407                 }
 408
 409                 while ($delivery = DBA::fetch($deliveries)) {
 410                         $id = $delivery['iid'];
 411                         unset($delivery['iid']);
 412                         DBA::insert('post-delivery-data', $delivery, Database::INSERT_UPDATE);
 413                         ++$rows;
 414                 }
 415                 DBA::close($deliveries);
 416
 417                 DI::config()->set('system', 'post_update_version_1345_id', $id);
 418
 419                 Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
 420
 421                 // When there are less than 100 items processed this means that we reached the end
 422                 // The other entries will then be processed with the regular functionality
 423                 if ($rows < 100) {
 424                         DI::config()->set('system', 'post_update_version', 1345);
 425                         Logger::info('Done');
 426                         return true;
 427                 }
 428
 429                 return false;
 430         }
 431
 432         /**
 433          * Generates the legacy item.file field string from an item ID.
 434          * Includes only file and category terms.
 435          *
 436          * @param int $item_id
 437          * @return string
 438          * @throws \Exception
 439          */
 440         private static function fileTextFromItemId($item_id)
 441         {
 442                 $file_text = '';
 443
 444                 $condition = ['otype' => self::OBJECT_TYPE_POST, 'oid' => $item_id, 'type' => [Category::FILE, Category::CATEGORY]];
 445                 $tags = DBA::selectToArray('term', ['type', 'term', 'url'], $condition);
 446                 foreach ($tags as $tag) {
 447                         if ($tag['type'] == Category::CATEGORY) {
 448                                 $file_text .= '<' . $tag['term'] . '>';
 449                         } else {
 450                                 $file_text .= '[' . $tag['term'] . ']';
 451                         }
 452                 }
 453
 454                 return $file_text;
 455         }
 456
 457         /**
 458          * Fill the "tag" table with tags and mentions from the "term" table
 459          *
 460          * @return bool "true" when the job is done
 461          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 462          */
 463         private static function update1346()
 464         {
 465                 // Was the script completed?
 466                 if (DI::config()->get('system', 'post_update_version') >= 1346) {
 467                         return true;
 468                 }
 469
 470                 if (!DBStructure::existsTable('term')) {
 471                         DI::config()->set('system', 'post_update_version', 1346);
 472                         return true;
 473                 }
 474
 475                 $id = DI::config()->get('system', 'post_update_version_1346_id', 0);
 476
 477                 Logger::info('Start', ['item' => $id]);
 478
 479                 $rows = 0;
 480
 481                 $terms = DBA::select('term', ['oid'],
 482                         ["`type` IN (?, ?) AND `oid` >= ?", Category::CATEGORY, Category::FILE, $id],
 483                         ['order' => ['oid'], 'limit' => 1000, 'group_by' => ['oid']]);
 484
 485                 if (DBA::errorNo() != 0) {
 486                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
 487                         return false;
 488                 }
 489
 490                 while ($term = DBA::fetch($terms)) {
 491                         $item = Post::selectFirst(['uri-id', 'uid'], ['id' => $term['oid']]);
 492                         if (!DBA::isResult($item)) {
 493                                 continue;
 494                         }
 495
 496                         $file = self::fileTextFromItemId($term['oid']);
 497                         if (!empty($file)) {
 498                                 Category::storeTextByURIId($item['uri-id'], $item['uid'], $file);
 499                         }
 500
 501                         $id = $term['oid'];
 502                         ++$rows;
 503                         if ($rows % 100 == 0) {
 504                                 DI::config()->set('system', 'post_update_version_1346_id', $id);
 505                         }
 506                 }
 507                 DBA::close($terms);
 508
 509                 DI::config()->set('system', 'post_update_version_1346_id', $id);
 510
 511                 Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
 512
 513                 // When there are less than 10 items processed this means that we reached the end
 514                 // The other entries will then be processed with the regular functionality
 515                 if ($rows < 10) {
 516                         DI::config()->set('system', 'post_update_version', 1346);
 517                         Logger::info('Done');
 518                         return true;
 519                 }
 520
 521                 return false;
 522         }
 523
 524         /**
 525          * update the "vid" (verb) field in the item table
 526          *
 527          * @return bool "true" when the job is done
 528          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 529          * @throws \ImagickException
 530          */
 531         private static function update1347()
 532         {
 533                 // Was the script completed?
 534                 if (DI::config()->get("system", "post_update_version") >= 1347) {
 535                         return true;
 536                 }
 537
 538                 if (!DBStructure::existsTable('item-activity') || !DBStructure::existsTable('item')) {
 539                         DI::config()->set('system', 'post_update_version', 1347);
 540                         return true;
 541                 }
 542
 543                 $id = DI::config()->get("system", "post_update_version_1347_id", 0);
 544
 545                 Logger::info('Start', ['item' => $id]);
 546
 547                 $start_id = $id;
 548                 $rows = 0;
 549
 550                 $items = DBA::p("SELECT `item`.`id`, `item`.`verb` AS `item-verb`, `item-content`.`verb`, `item-activity`.`activity`
 551                         FROM `item` LEFT JOIN `item-content` ON `item-content`.`uri-id` = `item`.`uri-id`
 552                         LEFT JOIN `item-activity` ON `item-activity`.`uri-id` = `item`.`uri-id` AND `item`.`gravity` = ?
 553                         WHERE `item`.`id` >= ? AND `item`.`vid` IS NULL ORDER BY `item`.`id` LIMIT 10000", GRAVITY_ACTIVITY, $id);
 554
 555                 if (DBA::errorNo() != 0) {
 556                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
 557                         return false;
 558                 }
 559
 560                 while ($item = DBA::fetch($items)) {
 561                         $id = $item['id'];
 562                         $verb = $item['item-verb'];
 563                         if (empty($verb)) {
 564                                 $verb = $item['verb'];
 565                         }
 566                         if (empty($verb) && is_int($item['activity'])) {
 567                                 $verb = Item::ACTIVITIES[$item['activity']];
 568                         }
 569                         if (empty($verb)) {
 570                                 continue;
 571                         }
 572
 573                         DBA::update('item', ['vid' => Verb::getID($verb)], ['id' => $item['id']]);
 574                         ++$rows;
 575                 }
 576                 DBA::close($items);
 577
 578                 DI::config()->set("system", "post_update_version_1347_id", $id);
 579
 580                 Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
 581
 582                 if ($start_id == $id) {
 583                         DI::config()->set("system", "post_update_version", 1347);
 584                         Logger::info('Done');
 585                         return true;
 586                 }
 587
 588                 return false;
 589         }
 590
 591         /**
 592          * update the "gsid" (global server id) field in the contact table
 593          *
 594          * @return bool "true" when the job is done
 595          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 596          * @throws \ImagickException
 597          */
 598         private static function update1348()
 599         {
 600                 // Was the script completed?
 601                 if (DI::config()->get("system", "post_update_version") >= 1348) {
 602                         return true;
 603                 }
 604
 605                 $id = DI::config()->get("system", "post_update_version_1348_id", 0);
 606
 607                 Logger::info('Start', ['contact' => $id]);
 608
 609                 $start_id = $id;
 610                 $rows = 0;
 611                 $condition = ["`id` > ? AND `gsid` IS NULL AND `baseurl` != '' AND NOT `baseurl` IS NULL", $id];
 612                 $params = ['order' => ['id'], 'limit' => 10000];
 613                 $contacts = DBA::select('contact', ['id', 'baseurl'], $condition, $params);
 614
 615                 if (DBA::errorNo() != 0) {
 616                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
 617                         return false;
 618                 }
 619
 620                 while ($contact = DBA::fetch($contacts)) {
 621                         $id = $contact['id'];
 622
 623                         DBA::update('contact',
 624                                 ['gsid' => GServer::getID($contact['baseurl'], true), 'baseurl' => GServer::cleanURL($contact['baseurl'])],
 625                                 ['id' => $contact['id']]);
 626
 627                         ++$rows;
 628                 }
 629                 DBA::close($contacts);
 630
 631                 DI::config()->set("system", "post_update_version_1348_id", $id);
 632
 633                 Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
 634
 635                 if ($start_id == $id) {
 636                         DI::config()->set("system", "post_update_version", 1348);
 637                         Logger::info('Done');
 638                         return true;
 639                 }
 640
 641                 return false;
 642         }
 643
 644         /**
 645          * update the "gsid" (global server id) field in the apcontact table
 646          *
 647          * @return bool "true" when the job is done
 648          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 649          * @throws \ImagickException
 650          */
 651         private static function update1349()
 652         {
 653                 // Was the script completed?
 654                 if (DI::config()->get("system", "post_update_version") >= 1349) {
 655                         return true;
 656                 }
 657
 658                 $id = DI::config()->get("system", "post_update_version_1349_id", '');
 659
 660                 Logger::info('Start', ['apcontact' => $id]);
 661
 662                 $start_id = $id;
 663                 $rows = 0;
 664                 $condition = ["`url` > ? AND `gsid` IS NULL AND `baseurl` != '' AND NOT `baseurl` IS NULL", $id];
 665                 $params = ['order' => ['url'], 'limit' => 10000];
 666                 $apcontacts = DBA::select('apcontact', ['url', 'baseurl'], $condition, $params);
 667
 668                 if (DBA::errorNo() != 0) {
 669                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
 670                         return false;
 671                 }
 672
 673                 while ($apcontact = DBA::fetch($apcontacts)) {
 674                         $id = $apcontact['url'];
 675
 676                         DBA::update('apcontact',
 677                                 ['gsid' => GServer::getID($apcontact['baseurl'], true), 'baseurl' => GServer::cleanURL($apcontact['baseurl'])],
 678                                 ['url' => $apcontact['url']]);
 679
 680                         ++$rows;
 681                 }
 682                 DBA::close($apcontacts);
 683
 684                 DI::config()->set("system", "post_update_version_1349_id", $id);
 685
 686                 Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
 687
 688                 if ($start_id == $id) {
 689                         DI::config()->set("system", "post_update_version", 1349);
 690                         Logger::info('Done');
 691                         return true;
 692                 }
 693
 694                 return false;
 695         }
 696
 697         /**
 698          * Remove orphaned photo entries
 699          *
 700          * @return bool "true" when the job is done
 701          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 702          * @throws \ImagickException
 703          */
 704         private static function update1383()
 705         {
 706                 // Was the script completed?
 707                 if (DI::config()->get("system", "post_update_version") >= 1383) {
 708                         return true;
 709                 }
 710
 711                 Logger::info('Start');
 712
 713                 $deleted = 0;
 714                 $avatar = [4 => 'photo', 5 => 'thumb', 6 => 'micro'];
 715
 716                 $photos = DBA::select('photo', ['id', 'contact-id', 'resource-id', 'scale'], ["`contact-id` != ? AND `album` = ?", 0, Photo::CONTACT_PHOTOS]);
 717                 while ($photo = DBA::fetch($photos)) {
 718                         $delete = !in_array($photo['scale'], [4, 5, 6]);
 719
 720                         if (!$delete) {
 721                                 // Check if there is a contact entry with that photo
 722                                 $delete = !DBA::exists('contact', ["`id` = ? AND `" . $avatar[$photo['scale']] . "` LIKE ?",
 723                                         $photo['contact-id'], '%' . $photo['resource-id'] . '%']);
 724                         }
 725
 726                         if ($delete) {
 727                                 Photo::delete(['id' => $photo['id']]);
 728                                 $deleted++;
 729                         }
 730                 }
 731                 DBA::close($photos);
 732
 733                 DI::config()->set("system", "post_update_version", 1383);
 734                 Logger::info('Done', ['deleted' => $deleted]);
 735                 return true;
 736         }
 737
 738         /**
 739          * update the "hash" field in the photo table
 740          *
 741          * @return bool "true" when the job is done
 742          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 743          * @throws \ImagickException
 744          */
 745         private static function update1384()
 746         {
 747                 // Was the script completed?
 748                 if (DI::config()->get("system", "post_update_version") >= 1384) {
 749                         return true;
 750                 }
 751
 752                 $condition = ["`hash` IS NULL"];
 753                 Logger::info('Start', ['rest' => DBA::count('photo', $condition)]);
 754
 755                 $rows = 0;
 756                 $photos = DBA::select('photo', [], $condition, ['limit' => 100]);
 757
 758                 if (DBA::errorNo() != 0) {
 759                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
 760                         return false;
 761                 }
 762
 763                 while ($photo = DBA::fetch($photos)) {
 764                         $img = Photo::getImageForPhoto($photo);
 765                         if (!empty($img)) {
 766                                 $md5 = md5($img->asString());
 767                         } else {
 768                                 $md5 = '';
 769                         }
 770                         DBA::update('photo', ['hash' => $md5], ['id' => $photo['id']]);
 771                         ++$rows;
 772                 }
 773                 DBA::close($photos);
 774
 775                 Logger::info('Processed', ['rows' => $rows]);
 776
 777                 if ($rows <= 100) {
 778                         DI::config()->set("system", "post_update_version", 1384);
 779                         Logger::info('Done');
 780                         return true;
 781                 }
 782
 783                 return false;
 784         }
 785
 786         /**
 787          * update the "external-id" field in the post table
 788          *
 789          * @return bool "true" when the job is done
 790          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 791          * @throws \ImagickException
 792          */
 793         private static function update1400()
 794         {
 795                 // Was the script completed?
 796                 if (DI::config()->get("system", "post_update_version") >= 1400) {
 797                         return true;
 798                 }
 799
 800                 if (!DBStructure::existsTable('item')) {
 801                         DI::config()->set("system", "post_update_version", 1400);
 802                         return true;
 803                 }
 804
 805                 $condition = ["`extid` != ? AND EXISTS(SELECT `id` FROM `post-user` WHERE `uri-id` = `item`.`uri-id` AND `uid` = `item`.`uid` AND `external-id` IS NULL)", ''];
 806                 Logger::info('Start', ['rest' => DBA::count('item', $condition)]);
 807
 808                 $rows = 0;
 809                 $items = DBA::select('item', ['uri-id', 'uid', 'extid'], $condition, ['order' => ['id'], 'limit' => 10000]);
 810
 811                 if (DBA::errorNo() != 0) {
 812                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
 813                         return false;
 814                 }
 815
 816                 while ($item = DBA::fetch($items)) {
 817                         Post::update(['external-id' => ItemURI::getIdByURI($item['extid'])], ['uri-id' => $item['uri-id'], 'uid' => $item['uid']]);
 818                         ++$rows;
 819                 }
 820                 DBA::close($items);
 821
 822                 Logger::info('Processed', ['rows' => $rows]);
 823
 824                 if ($rows <= 100) {
 825                         DI::config()->set("system", "post_update_version", 1400);
 826                         Logger::info('Done');
 827                         return true;
 828                 }
 829
 830                 return false;
 831         }
 832
 833         /**
 834          * update the "uri-id" field in the contact table
 835          *
 836          * @return bool "true" when the job is done
 837          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 838          * @throws \ImagickException
 839          */
 840         private static function update1424()
 841         {
 842                 // Was the script completed?
 843                 if (DI::config()->get("system", "post_update_version") >= 1424) {
 844                         return true;
 845                 }
 846
 847                 $condition = ["`uri-id` IS NULL"];
 848                 Logger::info('Start', ['rest' => DBA::count('contact', $condition)]);
 849
 850                 $rows = 0;
 851                 $contacts = DBA::select('contact', ['id', 'url'], $condition, ['limit' => 1000]);
 852
 853                 if (DBA::errorNo() != 0) {
 854                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
 855                         return false;
 856                 }
 857
 858                 while ($contact = DBA::fetch($contacts)) {
 859                         DBA::update('contact', ['uri-id' => ItemURI::getIdByURI($contact['url'])], ['id' => $contact['id']]);
 860                         ++$rows;
 861                 }
 862                 DBA::close($contacts);
 863
 864                 Logger::info('Processed', ['rows' => $rows]);
 865
 866                 if ($rows <= 100) {
 867                         DI::config()->set("system", "post_update_version", 1424);
 868                         Logger::info('Done');
 869                         return true;
 870                 }
 871
 872                 return false;
 873         }
 874
 875         /**
 876          * update the "uri-id" field in the fcontact table
 877          *
 878          * @return bool "true" when the job is done
 879          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 880          * @throws \ImagickException
 881          */
 882         private static function update1425()
 883         {
 884                 // Was the script completed?
 885                 if (DI::config()->get("system", "post_update_version") >= 1425) {
 886                         return true;
 887                 }
 888
 889                 $condition = ["`uri-id` IS NULL"];
 890                 Logger::info('Start', ['rest' => DBA::count('fcontact', $condition)]);
 891
 892                 $rows = 0;
 893                 $fcontacts = DBA::select('fcontact', ['id', 'url', 'guid'], $condition, ['limit' => 1000]);
 894
 895                 if (DBA::errorNo() != 0) {
 896                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
 897                         return false;
 898                 }
 899
 900                 while ($fcontact = DBA::fetch($fcontacts)) {
 901                         if (!empty($fcontact['guid'])) {
 902                                 $uriid = ItemURI::insert(['uri' => $fcontact['url'], 'guid' => $fcontact['guid']]);
 903                         } else {
 904                                 $uriid = ItemURI::getIdByURI($fcontact['url']);
 905                         }
 906                         DBA::update('fcontact', ['uri-id' => $uriid], ['id' => $fcontact['id']]);
 907                         ++$rows;
 908                 }
 909                 DBA::close($fcontacts);
 910
 911                 Logger::info('Processed', ['rows' => $rows]);
 912
 913                 if ($rows <= 100) {
 914                         DI::config()->set("system", "post_update_version", 1425);
 915                         Logger::info('Done');
 916                         return true;
 917                 }
 918
 919                 return false;
 920         }
 921
 922         /**
 923          * update the "uri-id" field in the apcontact table
 924          *
 925          * @return bool "true" when the job is done
 926          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 927          * @throws \ImagickException
 928          */
 929         private static function update1426()
 930         {
 931                 // Was the script completed?
 932                 if (DI::config()->get("system", "post_update_version") >= 1426) {
 933                         return true;
 934                 }
 935
 936                 $condition = ["`uri-id` IS NULL"];
 937                 Logger::info('Start', ['rest' => DBA::count('apcontact', $condition)]);
 938
 939                 $rows = 0;
 940                 $apcontacts = DBA::select('apcontact', ['url', 'uuid'], $condition, ['limit' => 1000]);
 941
 942                 if (DBA::errorNo() != 0) {
 943                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
 944                         return false;
 945                 }
 946
 947                 while ($apcontact = DBA::fetch($apcontacts)) {
 948                         if (!empty($apcontact['uuid'])) {
 949                                 $uriid = ItemURI::insert(['uri' => $apcontact['url'], 'guid' => $apcontact['uuid']]);
 950                         } else {
 951                                 $uriid = ItemURI::getIdByURI($apcontact['url']);
 952                         }
 953                         DBA::update('apcontact', ['uri-id' => $uriid], ['url' => $apcontact['url']]);
 954                         ++$rows;
 955                 }
 956                 DBA::close($apcontacts);
 957
 958                 Logger::info('Processed', ['rows' => $rows]);
 959
 960                 if ($rows <= 100) {
 961                         DI::config()->set("system", "post_update_version", 1426);
 962                         Logger::info('Done');
 963                         return true;
 964                 }
 965
 966                 return false;
 967         }
 968
 969         /**
 970          * update the "uri-id" field in the event table
 971          *
 972          * @return bool "true" when the job is done
 973          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 974          * @throws \ImagickException
 975          */
 976         private static function update1427()
 977         {
 978                 // Was the script completed?
 979                 if (DI::config()->get("system", "post_update_version") >= 1427) {
 980                         return true;
 981                 }
 982
 983                 $condition = ["`uri-id` IS NULL"];
 984                 Logger::info('Start', ['rest' => DBA::count('event', $condition)]);
 985
 986                 $rows = 0;
 987                 $events = DBA::select('event', ['id', 'uri', 'guid'], $condition, ['limit' => 1000]);
 988
 989                 if (DBA::errorNo() != 0) {
 990                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
 991                         return false;
 992                 }
 993
 994                 while ($event = DBA::fetch($events)) {
 995                         if (!empty($event['guid'])) {
 996                                 $uriid = ItemURI::insert(['uri' => $event['uri'], 'guid' => $event['guid']]);
 997                         } else {
 998                                 $uriid = ItemURI::getIdByURI($event['uri']);
 999                         }
1000                         DBA::update('event', ['uri-id' => $uriid], ['id' => $event['id']]);
1001                         ++$rows;
1002                 }
1003                 DBA::close($events);
1004
1005                 Logger::info('Processed', ['rows' => $rows]);
1006
1007                 if ($rows <= 100) {
1008                         DI::config()->set("system", "post_update_version", 1427);
1009                         Logger::info('Done');
1010                         return true;
1011                 }
1012
1013                 return false;
1014         }
1015 }