src/Worker/ExpirePosts.php

   1 <?php
   2 /**
   3  * @copyright Copyright (C) 2010-2023, the Friendica project
   4  *
   5  * @license GNU AGPL version 3 or any later version
   6  *
   7  * This program is free software: you can redistribute it and/or modify
   8  * it under the terms of the GNU Affero General Public License as
   9  * published by the Free Software Foundation, either version 3 of the
  10  * License, or (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU Affero General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU Affero General Public License
  18  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  19  *
  20  */
  21
  22 namespace Friendica\Worker;
  23
  24 use Friendica\Core\Logger;
  25 use Friendica\Core\Worker;
  26 use Friendica\Database\Database;
  27 use Friendica\Database\DBA;
  28 use Friendica\Database\DBStructure;
  29 use Friendica\DI;
  30 use Friendica\Model\Item;
  31 use Friendica\Model\Post;
  32 use Friendica\Util\DateTimeFormat;
  33
  34 class ExpirePosts
  35 {
  36         /**
  37          * Expire posts and remove unused item-uri entries
  38          *
  39          * @return void
  40          */
  41         public static function execute()
  42         {
  43                 self::deleteExpiredOriginPosts();
  44
  45                 self::deleteOrphanedEntries();
  46
  47                 self::deleteUnusedItemUri();
  48
  49                 self::deleteExpiredExternalPosts();
  50
  51                 if (DI::config()->get('system', 'add_missing_posts')) {
  52                         self::addMissingEntries();
  53                 }
  54
  55                 // Set the expiry for origin posta
  56                 Worker::add(Worker::PRIORITY_LOW, 'Expire');
  57
  58                 // update nodeinfo data after everything is cleaned up
  59                 Worker::add(Worker::PRIORITY_LOW, 'NodeInfo');
  60         }
  61
  62         /**
  63          * Delete expired origin posts and orphaned post related table entries
  64          *
  65          * @return void
  66          */
  67         private static function deleteExpiredOriginPosts()
  68         {
  69                 Logger::notice('Delete expired posts');
  70                 // physically remove anything that has been deleted for more than two months
  71                 $condition = ["`gravity` = ? AND `deleted` AND `changed` < ?", Item::GRAVITY_PARENT, DateTimeFormat::utc('now - 60 days')];
  72                 $rows = Post::select(['guid', 'uri-id', 'uid'],  $condition);
  73                 while ($row = Post::fetch($rows)) {
  74                         Logger::info('Delete expired item', ['uri-id' => $row['uri-id'], 'guid' => $row['guid']]);
  75                         Post\User::delete(['parent-uri-id' => $row['uri-id'], 'uid' => $row['uid']]);
  76                 }
  77                 DBA::close($rows);
  78
  79                 Logger::notice('Delete expired posts - done');
  80         }
  81
  82         /**
  83          * Delete orphaned entries in the post related tables
  84          *
  85          * @return void
  86          */
  87         private static function deleteOrphanedEntries()
  88         {
  89                 Logger::notice('Delete orphaned entries');
  90
  91                 // "post-user" is the leading table. So we delete every entry that isn't found there
  92                 $tables = ['item', 'post', 'post-content', 'post-thread', 'post-thread-user'];
  93                 foreach ($tables as $table) {
  94                         if (($table == 'item') && !DBStructure::existsTable('item')) {
  95                                 continue;
  96                         }
  97
  98                         Logger::notice('Start collecting orphaned entries', ['table' => $table]);
  99                         $uris = DBA::select($table, ['uri-id'], ["NOT `uri-id` IN (SELECT `uri-id` FROM `post-user`)"]);
 100                         $affected_count = 0;
 101                         Logger::notice('Deleting orphaned entries - start', ['table' => $table]);
 102                         while ($rows = DBA::toArray($uris, false, 100)) {
 103                                 $ids = array_column($rows, 'uri-id');
 104                                 DBA::delete($table, ['uri-id' => $ids]);
 105                                 $affected_count += DBA::affectedRows();
 106                         }
 107                         DBA::close($uris);
 108                         Logger::notice('Orphaned entries deleted', ['table' => $table, 'rows' => $affected_count]);
 109                 }
 110                 Logger::notice('Delete orphaned entries - done');
 111         }
 112
 113         /**
 114          * Add missing entries in some post related tables
 115          *
 116          * @return void
 117          */
 118         private static function addMissingEntries()
 119         {
 120                 Logger::notice('Adding missing entries');
 121
 122                 $rows = 0;
 123                 $userposts = DBA::select('post-user', [], ["`uri-id` not in (select `uri-id` from `post`)"]);
 124                 while ($fields = DBA::fetch($userposts)) {
 125                         $post_fields = DI::dbaDefinition()->truncateFieldsForTable('post', $fields);
 126                         DBA::insert('post', $post_fields, Database::INSERT_IGNORE);
 127                         $rows++;
 128                 }
 129                 DBA::close($userposts);
 130                 if ($rows > 0) {
 131                         Logger::notice('Added post entries', ['rows' => $rows]);
 132                 } else {
 133                         Logger::notice('No post entries added');
 134                 }
 135
 136                 $rows = 0;
 137                 $userposts = DBA::select('post-user', [], ["`gravity` = ? AND `uri-id` not in (select `uri-id` from `post-thread`)", Item::GRAVITY_PARENT]);
 138                 while ($fields = DBA::fetch($userposts)) {
 139                         $post_fields = DI::dbaDefinition()->truncateFieldsForTable('post-thread', $fields);
 140                         $post_fields['commented'] = $post_fields['changed'] = $post_fields['created'];
 141                         DBA::insert('post-thread', $post_fields, Database::INSERT_IGNORE);
 142                         $rows++;
 143                 }
 144                 DBA::close($userposts);
 145                 if ($rows > 0) {
 146                         Logger::notice('Added post-thread entries', ['rows' => $rows]);
 147                 } else {
 148                         Logger::notice('No post-thread entries added');
 149                 }
 150
 151                 $rows = 0;
 152                 $userposts = DBA::select('post-user', [], ["`gravity` = ? AND `id` not in (select `post-user-id` from `post-thread-user`)", Item::GRAVITY_PARENT]);
 153                 while ($fields = DBA::fetch($userposts)) {
 154                         $post_fields = DI::dbaDefinition()->truncateFieldsForTable('post-thread-user', $fields);
 155                         $post_fields['commented'] = $post_fields['changed'] = $post_fields['created'];
 156                         DBA::insert('post-thread-user', $post_fields, Database::INSERT_IGNORE);
 157                         $rows++;
 158                 }
 159                 DBA::close($userposts);
 160                 if ($rows > 0) {
 161                         Logger::notice('Added post-thread-user entries', ['rows' => $rows]);
 162                 } else {
 163                         Logger::notice('No post-thread-user entries added');
 164                 }
 165         }
 166
 167         /**
 168          * Delete unused item-uri entries
 169          */
 170         private static function deleteUnusedItemUri()
 171         {
 172                 // We have to avoid deleting newly created "item-uri" entries.
 173                 // So we fetch a post that had been stored yesterday and only delete older ones.
 174                 $item = Post::selectFirstThread(['uri-id'], ["`uid` = ? AND `received` < ?", 0, DateTimeFormat::utc('now - 1 day')],
 175                         ['order' => ['received' => true]]);
 176                 if (empty($item['uri-id'])) {
 177                         Logger::warning('No item with uri-id found - we better quit here');
 178                         return;
 179                 }
 180                 Logger::notice('Start collecting orphaned URI-ID', ['last-id' => $item['uri-id']]);
 181                 $uris = DBA::select('item-uri', ['id'], ["`id` < ?
 182                         AND NOT EXISTS(SELECT `uri-id` FROM `post-user` WHERE `uri-id` = `item-uri`.`id`)
 183                         AND NOT EXISTS(SELECT `parent-uri-id` FROM `post-user` WHERE `parent-uri-id` = `item-uri`.`id`)
 184                         AND NOT EXISTS(SELECT `thr-parent-id` FROM `post-user` WHERE `thr-parent-id` = `item-uri`.`id`)
 185                         AND NOT EXISTS(SELECT `external-id` FROM `post-user` WHERE `external-id` = `item-uri`.`id`)
 186                         AND NOT EXISTS(SELECT `conversation-id` FROM `post-thread` WHERE `conversation-id` = `item-uri`.`id`)
 187                         AND NOT EXISTS(SELECT `uri-id` FROM `mail` WHERE `uri-id` = `item-uri`.`id`)
 188                         AND NOT EXISTS(SELECT `uri-id` FROM `event` WHERE `uri-id` = `item-uri`.`id`)
 189                         AND NOT EXISTS(SELECT `uri-id` FROM `user-contact` WHERE `uri-id` = `item-uri`.`id`)
 190                         AND NOT EXISTS(SELECT `uri-id` FROM `contact` WHERE `uri-id` = `item-uri`.`id`)
 191                         AND NOT EXISTS(SELECT `uri-id` FROM `apcontact` WHERE `uri-id` = `item-uri`.`id`)
 192                         AND NOT EXISTS(SELECT `uri-id` FROM `diaspora-contact` WHERE `uri-id` = `item-uri`.`id`)
 193                         AND NOT EXISTS(SELECT `uri-id` FROM `inbox-status` WHERE `uri-id` = `item-uri`.`id`)
 194                         AND NOT EXISTS(SELECT `uri-id` FROM `post-delivery` WHERE `uri-id` = `item-uri`.`id`)
 195                         AND NOT EXISTS(SELECT `uri-id` FROM `post-delivery` WHERE `inbox-id` = `item-uri`.`id`)
 196                         AND NOT EXISTS(SELECT `parent-uri-id` FROM `mail` WHERE `parent-uri-id` = `item-uri`.`id`)
 197                         AND NOT EXISTS(SELECT `thr-parent-id` FROM `mail` WHERE `thr-parent-id` = `item-uri`.`id`)", $item['uri-id']]);
 198
 199                 Logger::notice('Start deleting orphaned URI-ID', ['last-id' => $item['uri-id']]);
 200                 $affected_count = 0;
 201                 while ($rows = DBA::toArray($uris, false, 100)) {
 202                         $ids = array_column($rows, 'id');
 203                         DBA::delete('item-uri', ['id' => $ids]);
 204                         $affected_count += DBA::affectedRows();
 205                         Logger::info('Deleted', ['rows' => $affected_count]);
 206                 }
 207                 DBA::close($uris);
 208                 Logger::notice('Orphaned URI-ID entries removed', ['rows' => $affected_count]);
 209         }
 210
 211         /**
 212          * Delete old external post entries
 213          */
 214         private static function deleteExpiredExternalPosts()
 215         {
 216                 $expire_days = DI::config()->get('system', 'dbclean-expire-days');
 217                 $expire_days_unclaimed = DI::config()->get('system', 'dbclean-expire-unclaimed');
 218                 if (empty($expire_days_unclaimed)) {
 219                         $expire_days_unclaimed = $expire_days;
 220                 }
 221
 222                 $limit = DI::config()->get('system', 'dbclean-expire-limit');
 223                 if (empty($limit)) {
 224                         return;
 225                 }
 226
 227                 if (!empty($expire_days)) {
 228                         Logger::notice('Start collecting expired threads', ['expiry_days' => $expire_days]);
 229                         $uris = DBA::select('item-uri', ['id'], ["`id` IN
 230                                 (SELECT `uri-id` FROM `post-thread` WHERE `received` < ?
 231                                         AND NOT `uri-id` IN (SELECT `uri-id` FROM `post-thread-user`
 232                                                 WHERE (`mention` OR `starred` OR `wall`) AND `uri-id` = `post-thread`.`uri-id`)
 233                                         AND NOT `uri-id` IN (SELECT `uri-id` FROM `post-category`
 234                                                 WHERE `uri-id` = `post-thread`.`uri-id`)
 235                                         AND NOT `uri-id` IN (SELECT `uri-id` FROM `post-collection`
 236                                                 WHERE `uri-id` = `post-thread`.`uri-id`)
 237                                         AND NOT `uri-id` IN (SELECT `uri-id` FROM `post-media`
 238                                                 WHERE `uri-id` = `post-thread`.`uri-id`)
 239                                         AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `post-user` INNER JOIN `contact` ON `contact`.`id` = `contact-id` AND `notify_new_posts`
 240                                                 WHERE `parent-uri-id` = `post-thread`.`uri-id`)
 241                                         AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `post-user`
 242                                                 WHERE (`origin` OR `event-id` != 0 OR `post-type` = ?) AND `parent-uri-id` = `post-thread`.`uri-id`)
 243                                         AND NOT `uri-id` IN (SELECT `uri-id` FROM `post-content`
 244                                                 WHERE `resource-id` != 0 AND `uri-id` = `post-thread`.`uri-id`))",
 245                             DateTimeFormat::utc('now - ' . (int)$expire_days . ' days'), Item::PT_PERSONAL_NOTE]);
 246
 247                         Logger::notice('Start deleting expired threads');
 248                         $affected_count = 0;
 249                         while ($rows = DBA::toArray($uris, false, 100)) {
 250                                 $ids = array_column($rows, 'id');
 251                                 DBA::delete('item-uri', ['id' => $ids]);
 252                                 $affected_count += DBA::affectedRows();
 253                         }
 254                         DBA::close($uris);
 255
 256                         Logger::notice('Deleted expired threads', ['rows' => $affected_count]);
 257                 }
 258
 259                 if (!empty($expire_days_unclaimed)) {
 260                         Logger::notice('Start collecting unclaimed public items', ['expiry_days' => $expire_days_unclaimed]);
 261                         $uris = DBA::select('item-uri', ['id'], ["`id` IN
 262                                 (SELECT `uri-id` FROM `post-user` WHERE `gravity` = ? AND `uid` = ? AND `received` < ?
 263                                         AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `post-user` AS `i` WHERE `i`.`uid` != ?
 264                                                 AND `i`.`parent-uri-id` = `post-user`.`uri-id`)
 265                                         AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `post-user` AS `i` WHERE `i`.`uid` = ?
 266                                                 AND `i`.`parent-uri-id` = `post-user`.`uri-id` AND `i`.`received` > ?))",
 267                                 Item::GRAVITY_PARENT, 0, DateTimeFormat::utc('now - ' . (int)$expire_days_unclaimed . ' days'), 0, 0, DateTimeFormat::utc('now - ' . (int)$expire_days_unclaimed . ' days')]);
 268
 269                         Logger::notice('Start deleting unclaimed public items');
 270                         $affected_count = 0;
 271                         while ($rows = DBA::toArray($uris, false, 100)) {
 272                                 $ids = array_column($rows, 'id');
 273                                 DBA::delete('item-uri', ['id' => $ids]);
 274                                 $affected_count += DBA::affectedRows();
 275                         }
 276                         DBA::close($uris);
 277                         Logger::notice('Deleted unclaimed public items', ['rows' => $affected_count]);
 278                 }
 279         }
 280 }