3 * @copyright Copyright (C) 2010-2022, the Friendica project
5 * @license GNU AGPL version 3 or any later version
7 * This program is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU Affero General Public License as
9 * published by the Free Software Foundation, either version 3 of the
10 * License, or (at your option) any later version.
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU Affero General Public License for more details.
17 * You should have received a copy of the GNU Affero General Public License
18 * along with this program. If not, see <https://www.gnu.org/licenses/>.
22 namespace Friendica\Worker;
24 use Friendica\Core\Logger;
25 use Friendica\Core\Worker;
26 use Friendica\Database\Database;
27 use Friendica\Database\DBA;
28 use Friendica\Database\DBStructure;
30 use Friendica\Model\Item;
31 use Friendica\Model\Post;
32 use Friendica\Util\DateTimeFormat;
37 * Expire posts and remove unused item-uri entries
41 public static function execute()
43 self::deleteExpiredOriginPosts();
45 self::deleteOrphanedEntries();
47 self::deleteUnusedItemUri();
49 self::deleteExpiredExternalPosts();
51 if (DI::config()->get('system', 'add_missing_posts')) {
52 self::addMissingEntries();
55 // Set the expiry for origin posta
56 Worker::add(PRIORITY_LOW, 'Expire');
58 // update nodeinfo data after everything is cleaned up
59 Worker::add(PRIORITY_LOW, 'NodeInfo');
63 * Delete expired origin posts and orphaned post related table entries
67 private static function deleteExpiredOriginPosts()
69 Logger::notice('Delete expired posts');
70 // physically remove anything that has been deleted for more than two months
71 $condition = ["`gravity` = ? AND `deleted` AND `changed` < ?", GRAVITY_PARENT, DateTimeFormat::utc('now - 60 days')];
72 $rows = Post::select(['guid', 'uri-id', 'uid'], $condition);
73 while ($row = Post::fetch($rows)) {
74 Logger::info('Delete expired item', ['uri-id' => $row['uri-id'], 'guid' => $row['guid']]);
75 Post\User::delete(['parent-uri-id' => $row['uri-id'], 'uid' => $row['uid']]);
79 Logger::notice('Delete expired posts - done');
83 * Delete orphaned entries in the post related tables
87 private static function deleteOrphanedEntries()
89 Logger::notice('Delete orphaned entries');
91 // "post-user" is the leading table. So we delete every entry that isn't found there
92 $tables = ['item', 'post', 'post-content', 'post-thread', 'post-thread-user'];
93 foreach ($tables as $table) {
94 if (($table == 'item') && !DBStructure::existsTable('item')) {
98 Logger::notice('Start collecting orphaned entries', ['table' => $table]);
99 $uris = DBA::select($table, ['uri-id'], ["NOT `uri-id` IN (SELECT `uri-id` FROM `post-user`)"]);
101 Logger::notice('Deleting orphaned entries - start', ['table' => $table]);
102 while ($rows = DBA::toArray($uris, false, 100)) {
103 $ids = array_column($rows, 'uri-id');
104 DBA::delete($table, ['uri-id' => $ids]);
105 $affected_count += DBA::affectedRows();
108 Logger::notice('Orphaned entries deleted', ['table' => $table, 'rows' => $affected_count]);
110 Logger::notice('Delete orphaned entries - done');
114 * Add missing entries in some post related tables
118 private static function addMissingEntries()
120 Logger::notice('Adding missing entries');
123 $userposts = DBA::select('post-user', [], ["`uri-id` not in (select `uri-id` from `post`)"]);
124 while ($fields = DBA::fetch($userposts)) {
125 $post_fields = DBStructure::getFieldsForTable('post', $fields);
126 DBA::insert('post', $post_fields, Database::INSERT_IGNORE);
129 DBA::close($userposts);
131 Logger::notice('Added post entries', ['rows' => $rows]);
133 Logger::notice('No post entries added');
137 $userposts = DBA::select('post-user', [], ["`gravity` = ? AND `uri-id` not in (select `uri-id` from `post-thread`)", GRAVITY_PARENT]);
138 while ($fields = DBA::fetch($userposts)) {
139 $post_fields = DBStructure::getFieldsForTable('post-thread', $fields);
140 $post_fields['commented'] = $post_fields['changed'] = $post_fields['created'];
141 DBA::insert('post-thread', $post_fields, Database::INSERT_IGNORE);
144 DBA::close($userposts);
146 Logger::notice('Added post-thread entries', ['rows' => $rows]);
148 Logger::notice('No post-thread entries added');
152 $userposts = DBA::select('post-user', [], ["`gravity` = ? AND `id` not in (select `post-user-id` from `post-thread-user`)", GRAVITY_PARENT]);
153 while ($fields = DBA::fetch($userposts)) {
154 $post_fields = DBStructure::getFieldsForTable('post-thread-user', $fields);
155 $post_fields['commented'] = $post_fields['changed'] = $post_fields['created'];
156 DBA::insert('post-thread-user', $post_fields, Database::INSERT_IGNORE);
159 DBA::close($userposts);
161 Logger::notice('Added post-thread-user entries', ['rows' => $rows]);
163 Logger::notice('No post-thread-user entries added');
168 * Delete unused item-uri entries
170 private static function deleteUnusedItemUri()
172 // We have to avoid deleting newly created "item-uri" entries.
173 // So we fetch a post that had been stored yesterday and only delete older ones.
174 $item = Post::selectFirstThread(['uri-id'], ["`uid` = ? AND `received` < ?", 0, DateTimeFormat::utc('now - 1 day')],
175 ['order' => ['received' => true]]);
176 if (empty($item['uri-id'])) {
177 Logger::warning('No item with uri-id found - we better quit here');
180 Logger::notice('Start collecting orphaned URI-ID', ['last-id' => $item['uri-id']]);
181 $uris = DBA::select('item-uri', ['id'], ["`id` < ?
182 AND NOT EXISTS(SELECT `uri-id` FROM `post-user` WHERE `uri-id` = `item-uri`.`id`)
183 AND NOT EXISTS(SELECT `parent-uri-id` FROM `post-user` WHERE `parent-uri-id` = `item-uri`.`id`)
184 AND NOT EXISTS(SELECT `thr-parent-id` FROM `post-user` WHERE `thr-parent-id` = `item-uri`.`id`)
185 AND NOT EXISTS(SELECT `external-id` FROM `post-user` WHERE `external-id` = `item-uri`.`id`)
186 AND NOT EXISTS(SELECT `uri-id` FROM `mail` WHERE `uri-id` = `item-uri`.`id`)
187 AND NOT EXISTS(SELECT `uri-id` FROM `event` WHERE `uri-id` = `item-uri`.`id`)
188 AND NOT EXISTS(SELECT `uri-id` FROM `user-contact` WHERE `uri-id` = `item-uri`.`id`)
189 AND NOT EXISTS(SELECT `uri-id` FROM `contact` WHERE `uri-id` = `item-uri`.`id`)
190 AND NOT EXISTS(SELECT `uri-id` FROM `apcontact` WHERE `uri-id` = `item-uri`.`id`)
191 AND NOT EXISTS(SELECT `uri-id` FROM `fcontact` WHERE `uri-id` = `item-uri`.`id`)
192 AND NOT EXISTS(SELECT `parent-uri-id` FROM `mail` WHERE `parent-uri-id` = `item-uri`.`id`)
193 AND NOT EXISTS(SELECT `thr-parent-id` FROM `mail` WHERE `thr-parent-id` = `item-uri`.`id`)", $item['uri-id']]);
195 Logger::notice('Start deleting orphaned URI-ID', ['last-id' => $item['uri-id']]);
197 while ($rows = DBA::toArray($uris, false, 100)) {
198 $ids = array_column($rows, 'id');
199 DBA::delete('item-uri', ['id' => $ids]);
200 $affected_count += DBA::affectedRows();
201 Logger::info('Deleted', ['rows' => $affected_count]);
204 Logger::notice('Orphaned URI-ID entries removed', ['rows' => $affected_count]);
208 * Delete old external post entries
210 private static function deleteExpiredExternalPosts()
212 $expire_days = DI::config()->get('system', 'dbclean-expire-days');
213 $expire_days_unclaimed = DI::config()->get('system', 'dbclean-expire-unclaimed');
214 if (empty($expire_days_unclaimed)) {
215 $expire_days_unclaimed = $expire_days;
218 $limit = DI::config()->get('system', 'dbclean-expire-limit');
223 if (!empty($expire_days)) {
224 Logger::notice('Start collecting expired threads', ['expiry_days' => $expire_days]);
225 $uris = DBA::select('item-uri', ['id'], ["`id` IN
226 (SELECT `uri-id` FROM `post-thread` WHERE `received` < ?
227 AND NOT `uri-id` IN (SELECT `uri-id` FROM `post-thread-user`
228 WHERE (`mention` OR `starred` OR `wall` OR `pinned`) AND `uri-id` = `post-thread`.`uri-id`)
229 AND NOT `uri-id` IN (SELECT `uri-id` FROM `post-category`
230 WHERE `uri-id` = `post-thread`.`uri-id`)
231 AND NOT `uri-id` IN (SELECT `uri-id` FROM `post-media`
232 WHERE `uri-id` = `post-thread`.`uri-id`)
233 AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `post-user` INNER JOIN `contact` ON `contact`.`id` = `contact-id` AND `notify_new_posts`
234 WHERE `parent-uri-id` = `post-thread`.`uri-id`)
235 AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `post-user`
236 WHERE (`origin` OR `event-id` != 0 OR `post-type` = ?) AND `parent-uri-id` = `post-thread`.`uri-id`)
237 AND NOT `uri-id` IN (SELECT `uri-id` FROM `post-content`
238 WHERE `resource-id` != 0 AND `uri-id` = `post-thread`.`uri-id`))",
239 DateTimeFormat::utc('now - ' . (int)$expire_days . ' days'), Item::PT_PERSONAL_NOTE]);
241 Logger::notice('Start deleting expired threads');
243 while ($rows = DBA::toArray($uris, false, 100)) {
244 $ids = array_column($rows, 'id');
245 DBA::delete('item-uri', ['id' => $ids]);
246 $affected_count += DBA::affectedRows();
250 Logger::notice('Deleted expired threads', ['rows' => $affected_count]);
253 if (!empty($expire_days_unclaimed)) {
254 Logger::notice('Start collecting unclaimed public items', ['expiry_days' => $expire_days_unclaimed]);
255 $uris = DBA::select('item-uri', ['id'], ["`id` IN
256 (SELECT `uri-id` FROM `post-user` WHERE `gravity` = ? AND `uid` = ? AND `received` < ?
257 AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `post-user` AS `i` WHERE `i`.`uid` != ?
258 AND `i`.`parent-uri-id` = `post-user`.`uri-id`)
259 AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `post-user` AS `i` WHERE `i`.`uid` = ?
260 AND `i`.`parent-uri-id` = `post-user`.`uri-id` AND `i`.`received` > ?))",
261 GRAVITY_PARENT, 0, DateTimeFormat::utc('now - ' . (int)$expire_days_unclaimed . ' days'), 0, 0, DateTimeFormat::utc('now - ' . (int)$expire_days_unclaimed . ' days')]);
263 Logger::notice('Start deleting unclaimed public items');
265 while ($rows = DBA::toArray($uris, false, 100)) {
266 $ids = array_column($rows, 'id');
267 DBA::delete('item-uri', ['id' => $ids]);
268 $affected_count += DBA::affectedRows();
271 Logger::notice('Deleted unclaimed public items', ['rows' => $affected_count]);