]> git.mxchange.org Git - friendica.git/blob - src/Database/PostUpdate.php
New table "post-counts" to precalculate the counts
[friendica.git] / src / Database / PostUpdate.php
1 <?php
2 /**
3  * @copyright Copyright (C) 2010-2023, the Friendica project
4  *
5  * @license GNU AGPL version 3 or any later version
6  *
7  * This program is free software: you can redistribute it and/or modify
8  * it under the terms of the GNU Affero General Public License as
9  * published by the Free Software Foundation, either version 3 of the
10  * License, or (at your option) any later version.
11  *
12  * This program is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15  * GNU Affero General Public License for more details.
16  *
17  * You should have received a copy of the GNU Affero General Public License
18  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
19  *
20  */
21
22 namespace Friendica\Database;
23
24 use Friendica\Core\Logger;
25 use Friendica\Core\Protocol;
26 use Friendica\DI;
27 use Friendica\Model\Contact;
28 use Friendica\Model\Conversation;
29 use Friendica\Model\GServer;
30 use Friendica\Model\Item;
31 use Friendica\Model\ItemURI;
32 use Friendica\Model\Photo;
33 use Friendica\Model\Post;
34 use Friendica\Model\Post\Category;
35 use Friendica\Model\Post\Counts;
36 use Friendica\Model\Tag;
37 use Friendica\Model\Verb;
38 use Friendica\Protocol\ActivityPub\Processor;
39 use Friendica\Protocol\ActivityPub\Receiver;
40 use Friendica\Util\JsonLD;
41 use Friendica\Util\Strings;
42 use GuzzleHttp\Psr7\Uri;
43
44 /**
45  * These database-intensive post update routines are meant to be executed in the background by the cronjob.
46  *
47  * If there is a need for a intensive migration after a database structure change, update this file
48  * by adding a new method at the end with the number of the new DB_UPDATE_VERSION.
49  */
50 class PostUpdate
51 {
52         // Needed for the helper function to read from the legacy term table
53         const OBJECT_TYPE_POST  = 1;
54
55         const VERSION = 1543;
56
57         /**
58          * Calls the post update functions
59          */
60         public static function update()
61         {
62                 if (!self::update1297()) {
63                         return false;
64                 }
65                 if (!self::update1322()) {
66                         return false;
67                 }
68                 if (!self::update1329()) {
69                         return false;
70                 }
71                 if (!self::update1341()) {
72                         return false;
73                 }
74                 if (!self::update1342()) {
75                         return false;
76                 }
77                 if (!self::update1345()) {
78                         return false;
79                 }
80                 if (!self::update1346()) {
81                         return false;
82                 }
83                 if (!self::update1347()) {
84                         return false;
85                 }
86                 if (!self::update1348()) {
87                         return false;
88                 }
89                 if (!self::update1349()) {
90                         return false;
91                 }
92                 if (!self::update1383()) {
93                         return false;
94                 }
95                 if (!self::update1384()) {
96                         return false;
97                 }
98                 if (!self::update1400()) {
99                         return false;
100                 }
101                 if (!self::update1424()) {
102                         return false;
103                 }
104                 if (!self::update1425()) {
105                         return false;
106                 }
107                 if (!self::update1426()) {
108                         return false;
109                 }
110                 if (!self::update1427()) {
111                         return false;
112                 }
113                 if (!self::update1452()) {
114                         return false;
115                 }
116                 if (!self::update1483()) {
117                         return false;
118                 }
119                 if (!self::update1484()) {
120                         return false;
121                 }
122                 if (!self::update1506()) {
123                         return false;
124                 }
125                 if (!self::update1507()) {
126                         return false;
127                 }
128                 if (!self::update1543()) {
129                         return false;
130                 }
131                 return true;
132         }
133
134         /**
135          * Set the delivery queue count to a negative value for all items preceding the feature.
136          *
137          * @return bool "true" when the job is done
138          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
139          */
140         private static function update1297()
141         {
142                 // Was the script completed?
143                 if (DI::keyValue()->get('post_update_version') >= 1297) {
144                         return true;
145                 }
146
147                 if (!DBStructure::existsTable('item-delivery-data')) {
148                         DI::keyValue()->set('post_update_version', 1297);
149                         return true;
150                 }
151
152                 $max_item_delivery_data = DBA::selectFirst('item-delivery-data', ['iid'], ['queue_count > 0 OR queue_done > 0'], ['order' => ['iid']]);
153                 $max_iid = $max_item_delivery_data['iid'] ?? 0;
154
155                 Logger::info('Start update1297 with max iid: ' . $max_iid);
156
157                 $condition = ['`queue_count` = 0 AND `iid` < ?', $max_iid];
158
159                 DBA::update('item-delivery-data', ['queue_count' => -1], $condition);
160
161                 if (DBA::errorNo() != 0) {
162                         Logger::error('Database error ' . DBA::errorNo() . ':' . DBA::errorMessage());
163                         return false;
164                 }
165
166                 Logger::info('Processed rows: ' . DBA::affectedRows());
167
168                 DI::keyValue()->set('post_update_version', 1297);
169
170                 Logger::info('Done');
171
172                 return true;
173         }
174         /**
175          * Remove contact duplicates
176          *
177          * @return bool "true" when the job is done
178          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
179          */
180         private static function update1322()
181         {
182                 // Was the script completed?
183                 if (DI::keyValue()->get('post_update_version') >= 1322) {
184                         return true;
185                 }
186
187                 Logger::info('Start');
188
189                 $contacts = DBA::p("SELECT `nurl`, `uid` FROM `contact`
190                         WHERE EXISTS (SELECT `nurl` FROM `contact` AS `c2`
191                                 WHERE `c2`.`nurl` = `contact`.`nurl` AND `c2`.`id` != `contact`.`id` AND `c2`.`uid` = `contact`.`uid` AND `c2`.`network` IN (?, ?, ?) AND NOT `deleted`)
192                         AND (`network` IN (?, ?, ?) OR (`uid` = ?)) AND NOT `deleted` GROUP BY `nurl`, `uid`",
193                         Protocol::DIASPORA, Protocol::OSTATUS, Protocol::ACTIVITYPUB,
194                         Protocol::DIASPORA, Protocol::OSTATUS, Protocol::ACTIVITYPUB, 0);
195
196                 while ($contact = DBA::fetch($contacts)) {
197                         Logger::info('Remove duplicates', ['nurl' => $contact['nurl'], 'uid' => $contact['uid']]);
198                         Contact::removeDuplicates($contact['nurl'], $contact['uid']);
199                 }
200
201                 DBA::close($contact);
202                 DI::keyValue()->set('post_update_version', 1322);
203
204                 Logger::info('Done');
205
206                 return true;
207         }
208
209         /**
210          * update user notification data
211          *
212          * @return bool "true" when the job is done
213          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
214          */
215         private static function update1329()
216         {
217                 // Was the script completed?
218                 if (DI::keyValue()->get('post_update_version') >= 1329) {
219                         return true;
220                 }
221
222                 if (!DBStructure::existsTable('item')) {
223                         DI::keyValue()->set('post_update_version', 1329);
224                         return true;
225                 }
226
227                 $id = DI::keyValue()->get('post_update_version_1329_id') ?? 0;
228
229                 Logger::info('Start', ['item' => $id]);
230
231                 $start_id = $id;
232                 $rows = 0;
233                 $condition = ["`id` > ?", $id];
234                 $params = ['order' => ['id'], 'limit' => 10000];
235                 $items = DBA::select('item', ['id', 'uri-id', 'uid'], $condition, $params);
236
237                 if (DBA::errorNo() != 0) {
238                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
239                         return false;
240                 }
241
242                 while ($item = DBA::fetch($items)) {
243                         $id = $item['id'];
244
245                         Post\UserNotification::setNotification($item['uri-id'], $item['uid']);
246
247                         ++$rows;
248                 }
249                 DBA::close($items);
250
251                 DI::keyValue()->set('post_update_version_1329_id', $id);
252
253                 Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
254
255                 if ($start_id == $id) {
256                         DI::keyValue()->set('post_update_version', 1329);
257                         Logger::info('Done');
258                         return true;
259                 }
260
261                 return false;
262         }
263
264         /**
265          * Fill the "tag" table with tags and mentions from the body
266          *
267          * @return bool "true" when the job is done
268          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
269          */
270         private static function update1341()
271         {
272                 // Was the script completed?
273                 if (DI::keyValue()->get('post_update_version') >= 1341) {
274                         return true;
275                 }
276
277                 if (!DBStructure::existsTable('item-content')) {
278                         DI::keyValue()->set('post_update_version', 1342);
279                         return true;
280                 }
281
282                 $id = DI::keyValue()->get('post_update_version_1341_id') ?? 0;
283
284                 Logger::info('Start', ['item' => $id]);
285
286                 $rows = 0;
287
288                 $items = DBA::p("SELECT `uri-id`,`body` FROM `item-content` WHERE
289                         (`body` LIKE ? OR `body` LIKE ? OR `body` LIKE ?) AND `uri-id` >= ?
290                         ORDER BY `uri-id` LIMIT 100000", '%#%', '%@%', '%!%', $id);
291
292                 if (DBA::errorNo() != 0) {
293                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
294                         return false;
295                 }
296
297                 while ($item = DBA::fetch($items)) {
298                         Tag::storeFromBody($item['uri-id'], $item['body'], '#!@', false);
299                         $id = $item['uri-id'];
300                         ++$rows;
301                         if ($rows % 1000 == 0) {
302                                 DI::keyValue()->set('post_update_version_1341_id', $id);
303                         }
304                 }
305                 DBA::close($items);
306
307                 DI::keyValue()->set('post_update_version_1341_id', $id);
308
309                 Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
310
311                 // When there are less than 1,000 items processed this means that we reached the end
312                 // The other entries will then be processed with the regular functionality
313                 if ($rows < 1000) {
314                         DI::keyValue()->set('post_update_version', 1341);
315                         Logger::info('Done');
316                         return true;
317                 }
318
319                 return false;
320         }
321
322         /**
323          * Fill the "tag" table with tags and mentions from the "term" table
324          *
325          * @return bool "true" when the job is done
326          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
327          */
328         private static function update1342()
329         {
330                 // Was the script completed?
331                 if (DI::keyValue()->get('post_update_version') >= 1342) {
332                         return true;
333                 }
334
335                 if (!DBStructure::existsTable('term') || !DBStructure::existsTable('item-content')) {
336                         DI::keyValue()->set('post_update_version', 1342);
337                         return true;
338                 }
339
340                 $id = DI::keyValue()->get('post_update_version_1342_id') ?? 0;
341
342                 Logger::info('Start', ['item' => $id]);
343
344                 $rows = 0;
345
346                 $terms = DBA::p("SELECT `term`.`tid`, `item`.`uri-id`, `term`.`type`, `term`.`term`, `term`.`url`, `item-content`.`body`
347                         FROM `term`
348                         INNER JOIN `item` ON `item`.`id` = `term`.`oid`
349                         INNER JOIN `item-content` ON `item-content`.`uri-id` = `item`.`uri-id`
350                         WHERE term.type IN (?, ?, ?, ?) AND `tid` >= ? ORDER BY `tid` LIMIT 100000",
351                         Tag::HASHTAG, Tag::MENTION, Tag::EXCLUSIVE_MENTION, Tag::IMPLICIT_MENTION, $id);
352
353                 if (DBA::errorNo() != 0) {
354                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
355                         return false;
356                 }
357
358                 while ($term = DBA::fetch($terms)) {
359                         if (($term['type'] == Tag::MENTION) && !empty($term['url']) && !strstr($term['body'], $term['url'])) {
360                 $condition = ['nurl' => Strings::normaliseLink($term['url']), 'uid' => 0, 'deleted' => false];
361                 $contact = DBA::selectFirst('contact', ['url', 'alias'], $condition, ['order' => ['id']]);
362                 if (!DBA::isResult($contact)) {
363                         $ssl_url = str_replace('http://', 'https://', $term['url']);
364                         $condition = ['`alias` IN (?, ?, ?) AND `uid` = ? AND NOT `deleted`', $term['url'], Strings::normaliseLink($term['url']), $ssl_url, 0];
365                         $contact = DBA::selectFirst('contact', ['url', 'alias'], $condition, ['order' => ['id']]);
366                 }
367
368                 if (DBA::isResult($contact) && (!strstr($term['body'], $contact['url']) && (empty($contact['alias']) || !strstr($term['body'], $contact['alias'])))) {
369                         $term['type'] = Tag::IMPLICIT_MENTION;
370                 }
371                         }
372
373                         Tag::store($term['uri-id'], $term['type'], $term['term'], $term['url']);
374
375                         $id = $term['tid'];
376                         ++$rows;
377                         if ($rows % 1000 == 0) {
378                                 DI::keyValue()->set('post_update_version_1342_id', $id);
379                         }
380                 }
381                 DBA::close($terms);
382
383                 DI::keyValue()->set('post_update_version_1342_id', $id);
384
385                 Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
386
387                 // When there are less than 1,000 items processed this means that we reached the end
388                 // The other entries will then be processed with the regular functionality
389                 if ($rows < 1000) {
390                         DI::keyValue()->set('post_update_version', 1342);
391                         Logger::info('Done');
392                         return true;
393                 }
394
395                 return false;
396         }
397
398         /**
399          * Fill the "post-delivery-data" table with data from the "item-delivery-data" table
400          *
401          * @return bool "true" when the job is done
402          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
403          */
404         private static function update1345()
405         {
406                 // Was the script completed?
407                 if (DI::keyValue()->get('post_update_version') >= 1345) {
408                         return true;
409                 }
410
411                 if (!DBStructure::existsTable('item-delivery-data')) {
412                         DI::keyValue()->set('post_update_version', 1345);
413                         return true;
414                 }
415
416                 $id = DI::keyValue()->get('post_update_version_1345_id') ?? 0;
417
418                 Logger::info('Start', ['item' => $id]);
419
420                 $rows = 0;
421
422                 $deliveries = DBA::p("SELECT `uri-id`, `iid`, `item-delivery-data`.`postopts`, `item-delivery-data`.`inform`,
423                         `queue_count`, `queue_done`, `activitypub`, `dfrn`, `diaspora`, `ostatus`, `legacy_dfrn`, `queue_failed`
424                         FROM `item-delivery-data`
425                         INNER JOIN `item` ON `item`.`id` = `item-delivery-data`.`iid`
426                         WHERE `iid` >= ? ORDER BY `iid` LIMIT 10000", $id);
427
428                 if (DBA::errorNo() != 0) {
429                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
430                         return false;
431                 }
432
433                 while ($delivery = DBA::fetch($deliveries)) {
434                         $id = $delivery['iid'];
435                         unset($delivery['iid']);
436                         DBA::insert('post-delivery-data', $delivery, Database::INSERT_UPDATE);
437                         ++$rows;
438                 }
439                 DBA::close($deliveries);
440
441                 DI::keyValue()->set('post_update_version_1345_id', $id);
442
443                 Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
444
445                 // When there are less than 100 items processed this means that we reached the end
446                 // The other entries will then be processed with the regular functionality
447                 if ($rows < 100) {
448                         DI::keyValue()->set('post_update_version', 1345);
449                         Logger::info('Done');
450                         return true;
451                 }
452
453                 return false;
454         }
455
456         /**
457          * Generates the legacy item.file field string from an item ID.
458          * Includes only file and category terms.
459          *
460          * @param int $item_id
461          * @return string
462          * @throws \Exception
463          */
464         private static function fileTextFromItemId($item_id)
465         {
466                 $file_text = '';
467
468                 $condition = ['otype' => self::OBJECT_TYPE_POST, 'oid' => $item_id, 'type' => [Category::FILE, Category::CATEGORY]];
469                 $tags = DBA::selectToArray('term', ['type', 'term', 'url'], $condition);
470                 foreach ($tags as $tag) {
471                         if ($tag['type'] == Category::CATEGORY) {
472                                 $file_text .= '<' . $tag['term'] . '>';
473                         } else {
474                                 $file_text .= '[' . $tag['term'] . ']';
475                         }
476                 }
477
478                 return $file_text;
479         }
480
481         /**
482          * Fill the "tag" table with tags and mentions from the "term" table
483          *
484          * @return bool "true" when the job is done
485          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
486          */
487         private static function update1346()
488         {
489                 // Was the script completed?
490                 if (DI::keyValue()->get('post_update_version') >= 1346) {
491                         return true;
492                 }
493
494                 if (!DBStructure::existsTable('term')) {
495                         DI::keyValue()->set('post_update_version', 1346);
496                         return true;
497                 }
498
499                 $id = DI::keyValue()->get('post_update_version_1346_id') ?? 0;
500
501                 Logger::info('Start', ['item' => $id]);
502
503                 $rows = 0;
504
505                 $terms = DBA::select('term', ['oid'],
506                         ["`type` IN (?, ?) AND `oid` >= ?", Category::CATEGORY, Category::FILE, $id],
507                         ['order' => ['oid'], 'limit' => 1000, 'group_by' => ['oid']]);
508
509                 if (DBA::errorNo() != 0) {
510                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
511                         return false;
512                 }
513
514                 while ($term = DBA::fetch($terms)) {
515                         $item = Post::selectFirst(['uri-id', 'uid'], ['id' => $term['oid']]);
516                         if (!DBA::isResult($item)) {
517                                 continue;
518                         }
519
520                         $file = self::fileTextFromItemId($term['oid']);
521                         if (!empty($file)) {
522                                 Category::storeTextByURIId($item['uri-id'], $item['uid'], $file);
523                         }
524
525                         $id = $term['oid'];
526                         ++$rows;
527                         if ($rows % 100 == 0) {
528                                 DI::keyValue()->set('post_update_version_1346_id', $id);
529                         }
530                 }
531                 DBA::close($terms);
532
533                 DI::keyValue()->set('post_update_version_1346_id', $id);
534
535                 Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
536
537                 // When there are less than 10 items processed this means that we reached the end
538                 // The other entries will then be processed with the regular functionality
539                 if ($rows < 10) {
540                         DI::keyValue()->set('post_update_version', 1346);
541                         Logger::info('Done');
542                         return true;
543                 }
544
545                 return false;
546         }
547
548         /**
549          * update the "vid" (verb) field in the item table
550          *
551          * @return bool "true" when the job is done
552          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
553          * @throws \ImagickException
554          */
555         private static function update1347()
556         {
557                 // Was the script completed?
558                 if (DI::keyValue()->get('post_update_version') >= 1347) {
559                         return true;
560                 }
561
562                 if (!DBStructure::existsTable('item-activity') || !DBStructure::existsTable('item')) {
563                         DI::keyValue()->set('post_update_version', 1347);
564                         return true;
565                 }
566
567                 $id = DI::keyValue()->get('post_update_version_1347_id') ?? 0;
568
569                 Logger::info('Start', ['item' => $id]);
570
571                 $start_id = $id;
572                 $rows = 0;
573
574                 $items = DBA::p("SELECT `item`.`id`, `item`.`verb` AS `item-verb`, `item-content`.`verb`, `item-activity`.`activity`
575                         FROM `item` LEFT JOIN `item-content` ON `item-content`.`uri-id` = `item`.`uri-id`
576                         LEFT JOIN `item-activity` ON `item-activity`.`uri-id` = `item`.`uri-id` AND `item`.`gravity` = ?
577                         WHERE `item`.`id` >= ? AND `item`.`vid` IS NULL ORDER BY `item`.`id` LIMIT 10000", Item::GRAVITY_ACTIVITY, $id);
578
579                 if (DBA::errorNo() != 0) {
580                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
581                         return false;
582                 }
583
584                 while ($item = DBA::fetch($items)) {
585                         $id = $item['id'];
586                         $verb = $item['item-verb'];
587                         if (empty($verb)) {
588                                 $verb = $item['verb'];
589                         }
590                         if (empty($verb) && is_int($item['activity'])) {
591                                 $verb = Item::ACTIVITIES[$item['activity']];
592                         }
593                         if (empty($verb)) {
594                                 continue;
595                         }
596
597                         DBA::update('item', ['vid' => Verb::getID($verb)], ['id' => $item['id']]);
598                         ++$rows;
599                 }
600                 DBA::close($items);
601
602                 DI::keyValue()->set('post_update_version_1347_id', $id);
603
604                 Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
605
606                 if ($start_id == $id) {
607                         DI::keyValue()->set('post_update_version', 1347);
608                         Logger::info('Done');
609                         return true;
610                 }
611
612                 return false;
613         }
614
615         /**
616          * update the "gsid" (global server id) field in the contact table
617          *
618          * @return bool "true" when the job is done
619          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
620          * @throws \ImagickException
621          */
622         private static function update1348()
623         {
624                 // Was the script completed?
625                 if (DI::keyValue()->get('post_update_version') >= 1348) {
626                         return true;
627                 }
628
629                 $id = DI::keyValue()->get('post_update_version_1348_id') ?? 0;
630
631                 Logger::info('Start', ['contact' => $id]);
632
633                 $start_id = $id;
634                 $rows = 0;
635                 $condition = ["`id` > ? AND `gsid` IS NULL AND `baseurl` != '' AND NOT `baseurl` IS NULL", $id];
636                 $params = ['order' => ['id'], 'limit' => 10000];
637                 $contacts = DBA::select('contact', ['id', 'baseurl'], $condition, $params);
638
639                 if (DBA::errorNo() != 0) {
640                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
641                         return false;
642                 }
643
644                 while ($contact = DBA::fetch($contacts)) {
645                         $id = $contact['id'];
646
647                         DBA::update('contact',
648                                 ['gsid' => GServer::getID($contact['baseurl'], true), 'baseurl' => GServer::cleanURL($contact['baseurl'])],
649                                 ['id' => $contact['id']]);
650
651                         ++$rows;
652                 }
653                 DBA::close($contacts);
654
655                 DI::keyValue()->set('post_update_version_1348_id', $id);
656
657                 Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
658
659                 if ($start_id == $id) {
660                         DI::keyValue()->set('post_update_version', 1348);
661                         Logger::info('Done');
662                         return true;
663                 }
664
665                 return false;
666         }
667
668         /**
669          * update the "gsid" (global server id) field in the apcontact table
670          *
671          * @return bool "true" when the job is done
672          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
673          * @throws \ImagickException
674          */
675         private static function update1349()
676         {
677                 // Was the script completed?
678                 if (DI::keyValue()->get('post_update_version') >= 1349) {
679                         return true;
680                 }
681
682                 $id = DI::keyValue()->get('post_update_version_1349_id') ?? '';
683
684                 Logger::info('Start', ['apcontact' => $id]);
685
686                 $start_id = $id;
687                 $rows = 0;
688                 $condition = ["`url` > ? AND `gsid` IS NULL AND `baseurl` != '' AND NOT `baseurl` IS NULL", $id];
689                 $params = ['order' => ['url'], 'limit' => 10000];
690                 $apcontacts = DBA::select('apcontact', ['url', 'baseurl'], $condition, $params);
691
692                 if (DBA::errorNo() != 0) {
693                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
694                         return false;
695                 }
696
697                 while ($apcontact = DBA::fetch($apcontacts)) {
698                         $id = $apcontact['url'];
699
700                         DBA::update('apcontact',
701                                 ['gsid' => GServer::getID($apcontact['baseurl'], true), 'baseurl' => GServer::cleanURL($apcontact['baseurl'])],
702                                 ['url' => $apcontact['url']]);
703
704                         ++$rows;
705                 }
706                 DBA::close($apcontacts);
707
708                 DI::keyValue()->set('post_update_version_1349_id', $id);
709
710                 Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
711
712                 if ($start_id == $id) {
713                         DI::keyValue()->set('post_update_version', 1349);
714                         Logger::info('Done');
715                         return true;
716                 }
717
718                 return false;
719         }
720
721         /**
722          * Remove orphaned photo entries
723          *
724          * @return bool "true" when the job is done
725          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
726          * @throws \ImagickException
727          */
728         private static function update1383()
729         {
730                 // Was the script completed?
731                 if (DI::keyValue()->get('post_update_version') >= 1383) {
732                         return true;
733                 }
734
735                 Logger::info('Start');
736
737                 $deleted = 0;
738                 $avatar = [4 => 'photo', 5 => 'thumb', 6 => 'micro'];
739
740                 $photos = DBA::select('photo', ['id', 'contact-id', 'resource-id', 'scale'], ["`contact-id` != ? AND `album` = ?", 0, Photo::CONTACT_PHOTOS]);
741                 while ($photo = DBA::fetch($photos)) {
742                         $delete = !in_array($photo['scale'], [4, 5, 6]);
743
744                         if (!$delete) {
745                                 // Check if there is a contact entry with that photo
746                                 $delete = !DBA::exists('contact', ["`id` = ? AND `" . $avatar[$photo['scale']] . "` LIKE ?",
747                                         $photo['contact-id'], '%' . $photo['resource-id'] . '%']);
748                         }
749
750                         if ($delete) {
751                                 Photo::delete(['id' => $photo['id']]);
752                                 $deleted++;
753                         }
754                 }
755                 DBA::close($photos);
756
757                 DI::keyValue()->set('post_update_version', 1383);
758                 Logger::info('Done', ['deleted' => $deleted]);
759                 return true;
760         }
761
762         /**
763          * update the "hash" field in the photo table
764          *
765          * @return bool "true" when the job is done
766          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
767          * @throws \ImagickException
768          */
769         private static function update1384()
770         {
771                 // Was the script completed?
772                 if (DI::keyValue()->get('post_update_version') >= 1384) {
773                         return true;
774                 }
775
776                 $condition = ["`hash` IS NULL"];
777                 Logger::info('Start', ['rest' => DBA::count('photo', $condition)]);
778
779                 $rows = 0;
780                 $photos = DBA::select('photo', [], $condition, ['limit' => 100]);
781
782                 if (DBA::errorNo() != 0) {
783                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
784                         return false;
785                 }
786
787                 while ($photo = DBA::fetch($photos)) {
788                         $img = Photo::getImageForPhoto($photo);
789                         if (!empty($img)) {
790                                 $md5 = md5($img->asString());
791                         } else {
792                                 $md5 = '';
793                         }
794                         DBA::update('photo', ['hash' => $md5], ['id' => $photo['id']]);
795                         ++$rows;
796                 }
797                 DBA::close($photos);
798
799                 Logger::info('Processed', ['rows' => $rows]);
800
801                 if ($rows <= 100) {
802                         DI::keyValue()->set('post_update_version', 1384);
803                         Logger::info('Done');
804                         return true;
805                 }
806
807                 return false;
808         }
809
810         /**
811          * update the "external-id" field in the post table
812          *
813          * @return bool "true" when the job is done
814          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
815          * @throws \ImagickException
816          */
817         private static function update1400()
818         {
819                 // Was the script completed?
820                 if (DI::keyValue()->get('post_update_version') >= 1400) {
821                         return true;
822                 }
823
824                 if (!DBStructure::existsTable('item')) {
825                         DI::keyValue()->set('post_update_version', 1400);
826                         return true;
827                 }
828
829                 $condition = ["`extid` != ? AND EXISTS(SELECT `id` FROM `post-user` WHERE `uri-id` = `item`.`uri-id` AND `uid` = `item`.`uid` AND `external-id` IS NULL)", ''];
830                 Logger::info('Start', ['rest' => DBA::count('item', $condition)]);
831
832                 $rows = 0;
833                 $items = DBA::select('item', ['uri-id', 'uid', 'extid'], $condition, ['order' => ['id'], 'limit' => 10000]);
834
835                 if (DBA::errorNo() != 0) {
836                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
837                         return false;
838                 }
839
840                 while ($item = DBA::fetch($items)) {
841                         Post::update(['external-id' => ItemURI::getIdByURI($item['extid'])], ['uri-id' => $item['uri-id'], 'uid' => $item['uid']]);
842                         ++$rows;
843                 }
844                 DBA::close($items);
845
846                 Logger::info('Processed', ['rows' => $rows]);
847
848                 if ($rows <= 100) {
849                         DI::keyValue()->set('post_update_version', 1400);
850                         Logger::info('Done');
851                         return true;
852                 }
853
854                 return false;
855         }
856
857         /**
858          * update the "uri-id" field in the contact table
859          *
860          * @return bool "true" when the job is done
861          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
862          * @throws \ImagickException
863          */
864         private static function update1424()
865         {
866                 // Was the script completed?
867                 if (DI::keyValue()->get('post_update_version') >= 1424) {
868                         return true;
869                 }
870
871                 $condition = ["`uri-id` IS NULL"];
872                 Logger::info('Start', ['rest' => DBA::count('contact', $condition)]);
873
874                 $rows = 0;
875                 $contacts = DBA::select('contact', ['id', 'url'], $condition, ['limit' => 1000]);
876
877                 if (DBA::errorNo() != 0) {
878                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
879                         return false;
880                 }
881
882                 while ($contact = DBA::fetch($contacts)) {
883                         DBA::update('contact', ['uri-id' => ItemURI::getIdByURI($contact['url'])], ['id' => $contact['id']]);
884                         ++$rows;
885                 }
886                 DBA::close($contacts);
887
888                 Logger::info('Processed', ['rows' => $rows]);
889
890                 if ($rows <= 100) {
891                         DI::keyValue()->set('post_update_version', 1424);
892                         Logger::info('Done');
893                         return true;
894                 }
895
896                 return false;
897         }
898
899         /**
900          * update the "uri-id" field in the fcontact table
901          *
902          * @return bool "true" when the job is done
903          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
904          * @throws \ImagickException
905          */
906         private static function update1425()
907         {
908                 // Was the script completed?
909                 if (DI::keyValue()->get('post_update_version') >= 1425) {
910                         return true;
911                 }
912
913                 if (!DBStructure::existsTable('fcontact')) {
914                         DI::keyValue()->set('post_update_version', 1425);
915                         return true;
916                 }
917
918                 $condition = ["`uri-id` IS NULL"];
919                 Logger::info('Start', ['rest' => DBA::count('fcontact', $condition)]);
920
921                 $rows = 0;
922                 $fcontacts = DBA::select('fcontact', ['id', 'url', 'guid'], $condition, ['limit' => 1000]);
923
924                 if (DBA::errorNo() != 0) {
925                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
926                         return false;
927                 }
928
929                 while ($fcontact = DBA::fetch($fcontacts)) {
930                         if (!empty($fcontact['guid'])) {
931                                 $uriid = ItemURI::insert(['uri' => $fcontact['url'], 'guid' => $fcontact['guid']]);
932                         } else {
933                                 $uriid = ItemURI::getIdByURI($fcontact['url']);
934                         }
935                         DBA::update('fcontact', ['uri-id' => $uriid], ['id' => $fcontact['id']]);
936                         ++$rows;
937                 }
938                 DBA::close($fcontacts);
939
940                 Logger::info('Processed', ['rows' => $rows]);
941
942                 if ($rows <= 100) {
943                         DI::keyValue()->set('post_update_version', 1425);
944                         Logger::info('Done');
945                         return true;
946                 }
947
948                 return false;
949         }
950
951         /**
952          * update the "uri-id" field in the apcontact table
953          *
954          * @return bool "true" when the job is done
955          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
956          * @throws \ImagickException
957          */
958         private static function update1426()
959         {
960                 // Was the script completed?
961                 if (DI::keyValue()->get('post_update_version') >= 1426) {
962                         return true;
963                 }
964
965                 $condition = ["`uri-id` IS NULL"];
966                 Logger::info('Start', ['rest' => DBA::count('apcontact', $condition)]);
967
968                 $rows = 0;
969                 $apcontacts = DBA::select('apcontact', ['url', 'uuid'], $condition, ['limit' => 1000]);
970
971                 if (DBA::errorNo() != 0) {
972                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
973                         return false;
974                 }
975
976                 while ($apcontact = DBA::fetch($apcontacts)) {
977                         if (!empty($apcontact['uuid'])) {
978                                 $uriid = ItemURI::insert(['uri' => $apcontact['url'], 'guid' => $apcontact['uuid']]);
979                         } else {
980                                 $uriid = ItemURI::getIdByURI($apcontact['url']);
981                         }
982                         DBA::update('apcontact', ['uri-id' => $uriid], ['url' => $apcontact['url']]);
983                         ++$rows;
984                 }
985                 DBA::close($apcontacts);
986
987                 Logger::info('Processed', ['rows' => $rows]);
988
989                 if ($rows <= 100) {
990                         DI::keyValue()->set('post_update_version', 1426);
991                         Logger::info('Done');
992                         return true;
993                 }
994
995                 return false;
996         }
997
998         /**
999          * update the "uri-id" field in the event table
1000          *
1001          * @return bool "true" when the job is done
1002          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
1003          * @throws \ImagickException
1004          */
1005         private static function update1427()
1006         {
1007                 // Was the script completed?
1008                 if (DI::keyValue()->get('post_update_version') >= 1427) {
1009                         return true;
1010                 }
1011
1012                 $condition = ["`uri-id` IS NULL"];
1013                 Logger::info('Start', ['rest' => DBA::count('event', $condition)]);
1014
1015                 $rows = 0;
1016                 $events = DBA::select('event', ['id', 'uri', 'guid'], $condition, ['limit' => 1000]);
1017
1018                 if (DBA::errorNo() != 0) {
1019                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
1020                         return false;
1021                 }
1022
1023                 while ($event = DBA::fetch($events)) {
1024                         if (!empty($event['guid'])) {
1025                                 $uriid = ItemURI::insert(['uri' => $event['uri'], 'guid' => $event['guid']]);
1026                         } else {
1027                                 $uriid = ItemURI::getIdByURI($event['uri']);
1028                         }
1029                         DBA::update('event', ['uri-id' => $uriid], ['id' => $event['id']]);
1030                         ++$rows;
1031                 }
1032                 DBA::close($events);
1033
1034                 Logger::info('Processed', ['rows' => $rows]);
1035
1036                 if ($rows <= 100) {
1037                         DI::keyValue()->set('post_update_version', 1427);
1038                         Logger::info('Done');
1039                         return true;
1040                 }
1041
1042                 return false;
1043         }
1044
1045         /**
1046          * Fill the receivers of the post via the raw source
1047          *
1048          * @return bool "true" when the job is done
1049          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
1050          * @throws \ImagickException
1051          */
1052         private static function update1452()
1053         {
1054                 // Was the script completed?
1055                 if (DI::keyValue()->get('post_update_version') >= 1452) {
1056                         return true;
1057                 }
1058
1059                 if (!DBStructure::existsTable('conversation')) {
1060                         DI::keyValue()->set('post_update_version', 1452);
1061                         return true;
1062                 }
1063
1064                 $id = DI::keyValue()->get('post_update_version_1452_id') ?? 0;
1065
1066                 Logger::info('Start', ['uri-id' => $id]);
1067
1068                 $rows     = 0;
1069                 $received = '';
1070
1071                 $conversations = DBA::p("SELECT `post-view`.`uri-id`, `conversation`.`source`, `conversation`.`received` FROM `conversation`
1072                         INNER JOIN `post-view` ON `post-view`.`uri` = `conversation`.`item-uri`
1073                         WHERE NOT `source` IS NULL AND `conversation`.`protocol` = ? AND `uri-id` > ? LIMIT ?",
1074                         Conversation::PARCEL_ACTIVITYPUB, $id, 1000);
1075
1076                 if (DBA::errorNo() != 0) {
1077                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
1078                         return false;
1079                 }
1080
1081                 while ($conversation = DBA::fetch($conversations)) {
1082                         $id       = $conversation['uri-id'];
1083                         $received = $conversation['received'];
1084
1085                         $raw = json_decode($conversation['source'], true);
1086                         if (empty($raw)) {
1087                                 continue;
1088                         }
1089                         $activity = JsonLD::compact($raw);
1090
1091                         $urls = Receiver::getReceiverURL($activity);
1092                         Processor::storeReceivers($conversation['uri-id'], $urls);
1093
1094                         if (!empty($activity['as:object'])) {
1095                                 $urls = array_merge($urls, Receiver::getReceiverURL($activity['as:object']));
1096                                 Processor::storeReceivers($conversation['uri-id'], $urls);
1097                         }
1098                         ++$rows;
1099                 }
1100
1101                 DBA::close($conversations);
1102
1103                 DI::keyValue()->set('post_update_version_1452_id', $id);
1104
1105                 Logger::info('Processed', ['rows' => $rows, 'last' => $id, 'last-received' => $received]);
1106
1107                 if ($rows <= 100) {
1108                         DI::keyValue()->set('post_update_version', 1452);
1109                         Logger::info('Done');
1110                         return true;
1111                 }
1112
1113                 return false;
1114         }
1115
1116         /**
1117          * Correct the parent.
1118          * This fixes a bug that was introduced in the development of version 2022.09
1119          *
1120          * @return bool "true" when the job is done
1121          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
1122          * @throws \ImagickException
1123          */
1124         private static function update1483()
1125         {
1126                 // Was the script completed?
1127                 if (DI::keyValue()->get('post_update_version') >= 1483) {
1128                         return true;
1129                 }
1130
1131                 Logger::info('Start');
1132
1133                 $posts = DBA::select('post-view', ['uri-id'], ['conversation' => './']);
1134                 while ($post = DBA::fetch($posts)) {
1135                         $parent = Item::getParent($post['uri-id']);
1136                         if ($parent != 0) {
1137                                 DBA::update('post', ['parent-uri-id' => $parent], ['uri-id' => $post['uri-id']]);
1138                                 DBA::update('post-user', ['parent-uri-id' => $parent], ['uri-id' => $post['uri-id']]);
1139                         }
1140                 }
1141                 DBA::close($posts);
1142
1143                 DI::keyValue()->set('post_update_version', 1483);
1144                 Logger::info('Done');
1145                 return true;
1146         }
1147
1148         /**
1149          * Handle duplicate contact entries
1150          *
1151          * @return bool "true" when the job is done
1152          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
1153          * @throws \ImagickException
1154          */
1155         private static function update1484()
1156         {
1157                 // Was the script completed?
1158                 if (DI::keyValue()->get('post_update_version') >= 1484) {
1159                         return true;
1160                 }
1161
1162                 $id = DI::keyValue()->get('post_update_version_1484_id') ?? 0;
1163
1164                 Logger::info('Start', ['id' => $id]);
1165
1166                 $rows = 0;
1167
1168                 $contacts = DBA::select('contact', ['id', 'uid', 'uri-id', 'url'], ["`id` > ?", $id], ['order' => ['id'], 'limit' => 1000]);
1169
1170                 if (DBA::errorNo() != 0) {
1171                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
1172                         return false;
1173                 }
1174
1175                 while ($contact = DBA::fetch($contacts)) {
1176                         $id = $contact['id'];
1177                         if (is_null($contact['uri-id'])) {
1178                                 $contact['uri-id'] = ItemURI::getIdByURI($contact['url']);
1179                                 DBA::update('contact', ['uri-id' => $contact['uri-id']], ['id' => $contact['id']]);
1180                         }
1181                         Contact::setAccountUser($contact['id'], $contact['uid'], $contact['uri-id'], $contact['url']);
1182                         ++$rows;
1183                 }
1184                 DBA::close($contacts);
1185
1186                 DI::keyValue()->set('post_update_version_1484_id', $id);
1187
1188                 Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
1189
1190                 if ($rows <= 100) {
1191                         DI::keyValue()->set('post_update_version', 1484);
1192                         Logger::info('Done');
1193                         return true;
1194                 }
1195
1196                 return false;
1197         }
1198
1199         /**
1200          * update the "gsid" (global server id) field in the contact table
1201          *
1202          * @return bool "true" when the job is done
1203          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
1204          * @throws \ImagickException
1205          */
1206         private static function update1506()
1207         {
1208                 // Was the script completed?
1209                 if (DI::keyValue()->get('post_update_version') >= 1506) {
1210                         return true;
1211                 }
1212
1213                 $id = DI::keyValue()->get('post_update_version_1506_id') ?? 0;
1214
1215                 Logger::info('Start', ['contact' => $id]);
1216
1217                 $start_id = $id;
1218                 $rows = 0;
1219                 $condition = ["`id` > ? AND `gsid` IS NULL AND `network` = ?", $id, Protocol::DIASPORA];
1220                 $params = ['order' => ['id'], 'limit' => 10000];
1221                 $contacts = DBA::select('contact', ['id', 'url'], $condition, $params);
1222
1223                 if (DBA::errorNo() != 0) {
1224                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
1225                         return false;
1226                 }
1227
1228                 while ($contact = DBA::fetch($contacts)) {
1229                         $id = $contact['id'];
1230
1231                         $parts = parse_url($contact['url']);
1232                         unset($parts['path']);
1233                         $server = (string)Uri::fromParts($parts);
1234                 
1235                         DBA::update('contact',
1236                                 ['gsid' => GServer::getID($server, true), 'baseurl' => GServer::cleanURL($server)],
1237                                 ['id' => $contact['id']]);
1238
1239                         ++$rows;
1240                 }
1241                 DBA::close($contacts);
1242
1243                 DI::keyValue()->set('post_update_version_1506_id', $id);
1244
1245                 Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
1246
1247                 if ($start_id == $id) {
1248                         DI::keyValue()->set('post_update_version', 1506);
1249                         Logger::info('Done');
1250                         return true;
1251                 }
1252
1253                 return false;
1254         }
1255
1256         /**
1257          * update the "gsid" (global server id) field in the inbox-status table
1258          *
1259          * @return bool "true" when the job is done
1260          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
1261          * @throws \ImagickException
1262          */
1263         private static function update1507()
1264         {
1265                 // Was the script completed?
1266                 if (DI::keyValue()->get('post_update_version') >= 1507) {
1267                         return true;
1268                 }
1269
1270                 $id = DI::keyValue()->get('post_update_version_1507_id') ?? '';
1271
1272                 Logger::info('Start', ['apcontact' => $id]);
1273
1274                 $start_id = $id;
1275                 $rows = 0;
1276                 $condition = ["`url` > ? AND NOT `gsid` IS NULL", $id];
1277                 $params = ['order' => ['url'], 'limit' => 10000];
1278                 $apcontacts = DBA::select('apcontact', ['url', 'gsid', 'sharedinbox', 'inbox'], $condition, $params);
1279
1280                 if (DBA::errorNo() != 0) {
1281                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
1282                         return false;
1283                 }
1284
1285                 while ($apcontact = DBA::fetch($apcontacts)) {
1286                         $id = $apcontact['url'];
1287
1288                         $inbox = [$apcontact['inbox']];
1289                         if (!empty($apcontact['sharedinbox'])) {
1290                                 $inbox[] = $apcontact['sharedinbox'];
1291                         }
1292                         $condition = DBA::mergeConditions(['url' => $inbox], ["`gsid` IS NULL"]);
1293                         DBA::update('inbox-status', ['gsid' => $apcontact['gsid'], 'archive' => GServer::isDefunctById($apcontact['gsid'])], $condition);
1294                         ++$rows;
1295                 }
1296                 DBA::close($apcontacts);
1297
1298                 DI::keyValue()->set('post_update_version_1507_id', $id);
1299
1300                 Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
1301
1302                 if ($start_id == $id) {
1303                         DI::keyValue()->set('post_update_version', 1507);
1304                         Logger::info('Done');
1305                         return true;
1306                 }
1307
1308                 return false;
1309         }
1310
1311         /**
1312          * Create "post-counts" entries for old entries.
1313          *
1314          * @return bool "true" when the job is done
1315          * @throws \Friendica\Network\HTTPException\InternalServerErrorException
1316          * @throws \ImagickException
1317          */
1318         private static function update1543()
1319         {
1320                 // Was the script completed?
1321                 if (DI::keyValue()->get('post_update_version') >= 1543) {
1322                         return true;
1323                 }
1324
1325                 $id = DI::keyValue()->get('post_update_version_1543_id') ?? 0;
1326
1327                 Logger::info('Start', ['uri-id' => $id]);
1328
1329                 $rows = 0;
1330
1331                 $posts = Post::selectPosts(['uri-id', 'parent-uri-id'], ["`uri-id` > ? AND `gravity` IN (?, ?)", $id, Item::GRAVITY_COMMENT, Item::GRAVITY_PARENT], ['order' => ['uri-id'], 'limit' => 1000]);
1332
1333                 if (DBA::errorNo() != 0) {
1334                         Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
1335                         return false;
1336                 }
1337
1338                 while ($post = Post::fetch($posts)) {
1339                         $id = $post['uri-id'];
1340                         Counts::updateForPost($post['uri-id'], $post['parent-uri-id']);
1341                         ++$rows;
1342                 }
1343                 DBA::close($posts);
1344
1345                 DI::keyValue()->set('post_update_version_1543_id', $id);
1346
1347                 Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
1348
1349                 if ($rows <= 100) {
1350                         DI::keyValue()->set('post_update_version', 1543);
1351                         Logger::info('Done');
1352                         return true;
1353                 }
1354
1355                 return false;
1356         }
1357 }