src/Worker/ExpirePosts.php

   1 <?php
   2 /**
   3  * @copyright Copyright (C) 2020, Friendica
   4  *
   5  * @license GNU AGPL version 3 or any later version
   6  *
   7  * This program is free software: you can redistribute it and/or modify
   8  * it under the terms of the GNU Affero General Public License as
   9  * published by the Free Software Foundation, either version 3 of the
  10  * License, or (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU Affero General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU Affero General Public License
  18  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  19  *
  20  */
  21
  22 namespace Friendica\Worker;
  23
  24 use Friendica\Core\Logger;
  25 use Friendica\Core\Worker;
  26 use Friendica\Database\Database;
  27 use Friendica\Database\DBA;
  28 use Friendica\Database\DBStructure;
  29 use Friendica\DI;
  30 use Friendica\Model\Item;
  31 use Friendica\Model\Post;
  32
  33 class ExpirePosts
  34 {
  35         /**
  36          * Expire posts and remove unused item-uri entries
  37          *
  38          * @return void
  39          */
  40         public static function execute()
  41         {
  42                 self::deleteExpiredOriginPosts();
  43
  44                 self::deleteOrphanedEntries();
  45
  46                 self::deleteUnusedItemUri();
  47
  48                 self::deleteExpiredExternalPosts();
  49
  50                 self::addMissingEntries();
  51
  52                 // Set the expiry for origin posta
  53                 Worker::add(PRIORITY_LOW, 'Expire');
  54
  55                 // update nodeinfo data after everything is cleaned up
  56                 Worker::add(PRIORITY_LOW, 'NodeInfo');
  57         }
  58
  59         /**
  60          * Delete expired origin posts and orphaned post related table entries
  61          *
  62          * @return void
  63          */
  64         private static function deleteExpiredOriginPosts()
  65         {
  66                 Logger::notice('Delete expired posts');
  67                 // physically remove anything that has been deleted for more than two months
  68                 $condition = ["`gravity` = ? AND `deleted` AND `changed` < UTC_TIMESTAMP() - INTERVAL 60 DAY", GRAVITY_PARENT];
  69                 $rows = Post::select(['guid', 'uri-id', 'uid'],  $condition);
  70                 while ($row = Post::fetch($rows)) {
  71                         Logger::info('Delete expired item', ['uri-id' => $row['uri-id'], 'guid' => $row['guid']]);
  72                         Post\User::delete(['parent-uri-id' => $row['uri-id'], 'uid' => $row['uid']]);
  73                 }
  74                 DBA::close($rows);
  75
  76                 Logger::notice('Delete expired posts - done');
  77         }
  78
  79         /**
  80          * Delete orphaned entries in the post related tables
  81          *
  82          * @return void
  83          */
  84         private static function deleteOrphanedEntries()
  85         {
  86                 Logger::notice('Delete orphaned entries');
  87
  88                 // "post-user" is the leading table. So we delete every entry that isn't found there
  89                 $tables = ['item', 'post', 'post-content', 'post-thread', 'post-thread-user'];
  90                 foreach ($tables as $table) {
  91                         if (($table == 'item') && !DBStructure::existsTable('item')) {
  92                                 continue;
  93                         }
  94
  95                         Logger::notice('Start collecting orphaned entries', ['table' => $table]);
  96                         $uris = DBA::select($table, ['uri-id'], ["NOT `uri-id` IN (SELECT `uri-id` FROM `post-user`)"]);
  97                         $affected_count = 0;
  98                         Logger::notice('Deleting orphaned entries - start', ['table' => $table]);
  99                         while ($rows = DBA::toArray($uris, false, 100)) {
 100                                 $ids = array_column($rows, 'uri-id');
 101                                 DBA::delete($table, ['uri-id' => $ids]);
 102                                 $affected_count += DBA::affectedRows();
 103                         }
 104                         DBA::close($uris);
 105                         Logger::notice('Orphaned entries deleted', ['table' => $table, 'rows' => $affected_count]);
 106                 }
 107                 Logger::notice('Delete orphaned entries - done');
 108         }
 109
 110         /**
 111          * Add missing entries in some post related tables
 112          *
 113          * @return void
 114          */
 115         private static function addMissingEntries()
 116         {
 117                 Logger::notice('Adding missing entries');
 118
 119                 $rows = 0;
 120                 $userposts = DBA::select('post-user', [], ["`uri-id` not in (select `uri-id` from `post`)"], ['group_by' => ['uri-id']]);
 121                 while ($fields = DBA::fetch($userposts)) {
 122                         $post_fields = DBStructure::getFieldsForTable('post', $fields);
 123                         DBA::insert('post', $post_fields, Database::INSERT_IGNORE);
 124                         $rows++;
 125                 }
 126                 DBA::close($userposts);
 127                 if ($rows > 0) {
 128                         Logger::notice('Added post entries', ['rows' => $rows]);
 129                 } else {
 130                         Logger::notice('No post entries added');
 131                 }
 132
 133                 $rows = 0;
 134                 $userposts = DBA::select('post-user', [], ["`gravity` = ? AND `uri-id` not in (select `uri-id` from `post-thread`)", GRAVITY_PARENT], ['group_by' => ['uri-id']]);
 135                 while ($fields = DBA::fetch($userposts)) {
 136                         $post_fields = DBStructure::getFieldsForTable('post-thread', $fields);
 137                         $post_fields['commented'] = $post_fields['changed'] = $post_fields['created'];
 138                         DBA::insert('post-thread', $post_fields, Database::INSERT_IGNORE);
 139                         $rows++;
 140                 }
 141                 DBA::close($userposts);
 142                 if ($rows > 0) {
 143                         Logger::notice('Added post-thread entries', ['rows' => $rows]);
 144                 } else {
 145                         Logger::notice('No post-thread entries added');
 146                 }
 147
 148                 $rows = 0;
 149                 $userposts = DBA::select('post-user', [], ["`gravity` = ? AND `id` not in (select `post-user-id` from `post-thread-user`)", GRAVITY_PARENT]);
 150                 while ($fields = DBA::fetch($userposts)) {
 151                         $post_fields = DBStructure::getFieldsForTable('post-thread-user', $fields);
 152                         $post_fields['commented'] = $post_fields['changed'] = $post_fields['created'];
 153                         DBA::insert('post-thread-user', $post_fields, Database::INSERT_IGNORE);
 154                         $rows++;
 155                 }
 156                 DBA::close($userposts);
 157                 if ($rows > 0) {
 158                         Logger::notice('Added post-thread-user entries', ['rows' => $rows]);
 159                 } else {
 160                         Logger::notice('No post-thread-user entries added');
 161                 }
 162         }
 163
 164         /**
 165          * Delete unused item-uri entries
 166          */
 167         private static function deleteUnusedItemUri()
 168         {
 169                 // We have to avoid deleting newly created "item-uri" entries.
 170                 // So we fetch a post that had been stored yesterday and only delete older ones.
 171                 $item = Post::selectFirst(['uri-id'], ["`uid` = ? AND `received` < UTC_TIMESTAMP() - INTERVAL ? DAY", 0, 1],
 172                         ['order' => ['received' => true]]);
 173                 if (empty($item['uri-id'])) {
 174                         Logger::warning('No item with uri-id found - we better quit here');
 175                         return;
 176                 }
 177                 Logger::notice('Start collecting orphaned URI-ID', ['last-id' => $item['uri-id']]);
 178                 $uris = DBA::select('item-uri', ['id'], ["`id` < ?
 179                         AND NOT EXISTS(SELECT `uri-id` FROM `post` WHERE `uri-id` = `item-uri`.`id`)
 180                         AND NOT EXISTS(SELECT `parent-uri-id` FROM `post` WHERE `parent-uri-id` = `item-uri`.`id`)
 181                         AND NOT EXISTS(SELECT `thr-parent-id` FROM `post` WHERE `thr-parent-id` = `item-uri`.`id`)
 182                         AND NOT EXISTS(SELECT `external-id` FROM `post` WHERE `external-id` = `item-uri`.`id`)", $item['uri-id']]);
 183
 184                 Logger::notice('Start deleting orphaned URI-ID', ['last-id' => $item['uri-id']]);
 185                 $affected_count = 0;
 186                 while ($rows = DBA::toArray($uris, false, 100)) {
 187                         $ids = array_column($rows, 'id');
 188                         DBA::delete('item-uri', ['id' => $ids]);
 189                         $affected_count += DBA::affectedRows();
 190                         Logger::info('Deleted', ['rows' => $affected_count]);
 191                 }
 192                 DBA::close($uris);
 193                 Logger::notice('Orphaned URI-ID entries removed', ['rows' => $affected_count]);
 194         }
 195
 196         /**
 197          * Delete old external post entries
 198          */
 199         private static function deleteExpiredExternalPosts()
 200         {
 201                 $expire_days = DI::config()->get('system', 'dbclean-expire-days');
 202                 $expire_days_unclaimed = DI::config()->get('system', 'dbclean-expire-unclaimed');
 203                 if (empty($expire_days_unclaimed)) {
 204                         $expire_days_unclaimed = $expire_days;
 205                 }
 206
 207                 $limit = DI::config()->get('system', 'dbclean-expire-limit');
 208                 if (empty($limit)) {
 209                         return;
 210                 }
 211
 212                 if (!empty($expire_days)) {
 213                         Logger::notice('Start collecting expired threads', ['expiry_days' => $expire_days]);
 214                         $uris = DBA::select('item-uri', ['id'], ["`id` IN
 215                                 (SELECT `uri-id` FROM `post-thread` WHERE `received` < UTC_TIMESTAMP() - INTERVAL ? DAY
 216                                         AND NOT `uri-id` IN (SELECT `uri-id` FROM `post-thread-user`
 217                                                 WHERE (`mention` OR `starred` OR `wall` OR `pinned`) AND `uri-id` = `post-thread`.`uri-id`)
 218                                         AND NOT `uri-id` IN (SELECT `uri-id` FROM `post-category`
 219                                                 WHERE `uri-id` = `post-thread`.`uri-id`)
 220                                         AND NOT `uri-id` IN (SELECT `uri-id` FROM `post-media`
 221                                                 WHERE `uri-id` = `post-thread`.`uri-id`)
 222                                         AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `post-user` INNER JOIN `contact` ON `contact`.`id` = `contact-id` AND `notify_new_posts`
 223                                                 WHERE `parent-uri-id` = `post-thread`.`uri-id`)
 224                                         AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `post-user`
 225                                                 WHERE (`origin` OR `event-id` != 0 OR `post-type` = ?) AND `parent-uri-id` = `post-thread`.`uri-id`)
 226                                         AND NOT `uri-id` IN (SELECT `uri-id` FROM `post-content`
 227                                                 WHERE `resource-id` != 0 AND `uri-id` = `post-thread`.`uri-id`))",
 228                                 $expire_days, Item::PT_PERSONAL_NOTE]);
 229
 230                         Logger::notice('Start deleting expired threads');
 231                         $affected_count = 0;
 232                         while ($rows = DBA::toArray($uris, false, 100)) {
 233                                 $ids = array_column($rows, 'id');
 234                                 DBA::delete('item-uri', ['id' => $ids]);
 235                                 $affected_count += DBA::affectedRows();
 236                         }
 237                         DBA::close($uris);
 238
 239                         Logger::notice('Deleted expired threads', ['rows' => $affected_count]);
 240                 }
 241
 242                 if (!empty($expire_days_unclaimed)) {
 243                         Logger::notice('Start collecting unclaimed public items', ['expiry_days' => $expire_days_unclaimed]);
 244                         $uris = DBA::select('item-uri', ['id'], ["`id` IN
 245                                 (SELECT `uri-id` FROM `post-user` WHERE `gravity` = ? AND `uid` = ? AND `received` < UTC_TIMESTAMP() - INTERVAL ? DAY
 246                                         AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `post-user` AS `i` WHERE `i`.`uid` != ?
 247                                                 AND `i`.`parent-uri-id` = `post-user`.`uri-id`)
 248                                         AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `post-user` AS `i` WHERE `i`.`uid` = ?
 249                                                 AND `i`.`parent-uri-id` = `post-user`.`uri-id` AND `i`.`received` > UTC_TIMESTAMP() - INTERVAL ? DAY))",
 250                                 GRAVITY_PARENT, 0, $expire_days_unclaimed, 0, 0, $expire_days_unclaimed]);
 251
 252                         Logger::notice('Start deleting unclaimed public items');
 253                         $affected_count = 0;
 254                         while ($rows = DBA::toArray($uris, false, 100)) {
 255                                 $ids = array_column($rows, 'id');
 256                                 DBA::delete('item-uri', ['id' => $ids]);
 257                                 $affected_count += DBA::affectedRows();
 258                         }
 259                         DBA::close($uris);
 260                         Logger::notice('Deleted unclaimed public items', ['rows' => $affected_count]);
 261                 }
 262         }
 263 }