<?php
/**
- * @copyright Copyright (C) 2020, Friendica
+ * @copyright Copyright (C) 2010-2021, the Friendica project
*
* @license GNU AGPL version 3 or any later version
*
namespace Friendica\Worker;
use Friendica\Core\Logger;
+use Friendica\Core\Worker;
+use Friendica\Database\Database;
use Friendica\Database\DBA;
+use Friendica\Database\DBStructure;
use Friendica\DI;
-use Friendica\Util\DateTimeFormat;
+use Friendica\Model\Item;
+use Friendica\Model\Post;
class ExpirePosts
{
/**
- * Delete old post entries
+ * Expire posts and remove unused item-uri entries
+ *
+ * @return void
*/
public static function execute()
+ {
+ self::deleteExpiredOriginPosts();
+
+ self::deleteOrphanedEntries();
+
+ self::deleteUnusedItemUri();
+
+ self::deleteExpiredExternalPosts();
+
+ if (DI::config()->get('system', 'add_missing_posts')) {
+ self::addMissingEntries();
+ }
+
+ // Set the expiry for origin posta
+ Worker::add(PRIORITY_LOW, 'Expire');
+
+ // update nodeinfo data after everything is cleaned up
+ Worker::add(PRIORITY_LOW, 'NodeInfo');
+ }
+
+ /**
+ * Delete expired origin posts and orphaned post related table entries
+ *
+ * @return void
+ */
+ private static function deleteExpiredOriginPosts()
+ {
+ Logger::notice('Delete expired posts');
+ // physically remove anything that has been deleted for more than two months
+ $condition = ["`gravity` = ? AND `deleted` AND `changed` < UTC_TIMESTAMP() - INTERVAL 60 DAY", GRAVITY_PARENT];
+ $rows = Post::select(['guid', 'uri-id', 'uid'], $condition);
+ while ($row = Post::fetch($rows)) {
+ Logger::info('Delete expired item', ['uri-id' => $row['uri-id'], 'guid' => $row['guid']]);
+ Post\User::delete(['parent-uri-id' => $row['uri-id'], 'uid' => $row['uid']]);
+ }
+ DBA::close($rows);
+
+ Logger::notice('Delete expired posts - done');
+ }
+
+ /**
+ * Delete orphaned entries in the post related tables
+ *
+ * @return void
+ */
+ private static function deleteOrphanedEntries()
+ {
+ Logger::notice('Delete orphaned entries');
+
+ // "post-user" is the leading table. So we delete every entry that isn't found there
+ $tables = ['item', 'post', 'post-content', 'post-thread', 'post-thread-user'];
+ foreach ($tables as $table) {
+ if (($table == 'item') && !DBStructure::existsTable('item')) {
+ continue;
+ }
+
+ Logger::notice('Start collecting orphaned entries', ['table' => $table]);
+ $uris = DBA::select($table, ['uri-id'], ["NOT `uri-id` IN (SELECT `uri-id` FROM `post-user`)"]);
+ $affected_count = 0;
+ Logger::notice('Deleting orphaned entries - start', ['table' => $table]);
+ while ($rows = DBA::toArray($uris, false, 100)) {
+ $ids = array_column($rows, 'uri-id');
+ DBA::delete($table, ['uri-id' => $ids]);
+ $affected_count += DBA::affectedRows();
+ }
+ DBA::close($uris);
+ Logger::notice('Orphaned entries deleted', ['table' => $table, 'rows' => $affected_count]);
+ }
+ Logger::notice('Delete orphaned entries - done');
+ }
+
+ /**
+ * Add missing entries in some post related tables
+ *
+ * @return void
+ */
+ private static function addMissingEntries()
+ {
+ Logger::notice('Adding missing entries');
+
+ $rows = 0;
+ $userposts = DBA::select('post-user', [], ["`uri-id` not in (select `uri-id` from `post`)"]);
+ while ($fields = DBA::fetch($userposts)) {
+ $post_fields = DBStructure::getFieldsForTable('post', $fields);
+ DBA::insert('post', $post_fields, Database::INSERT_IGNORE);
+ $rows++;
+ }
+ DBA::close($userposts);
+ if ($rows > 0) {
+ Logger::notice('Added post entries', ['rows' => $rows]);
+ } else {
+ Logger::notice('No post entries added');
+ }
+
+ $rows = 0;
+ $userposts = DBA::select('post-user', [], ["`gravity` = ? AND `uri-id` not in (select `uri-id` from `post-thread`)", GRAVITY_PARENT]);
+ while ($fields = DBA::fetch($userposts)) {
+ $post_fields = DBStructure::getFieldsForTable('post-thread', $fields);
+ $post_fields['commented'] = $post_fields['changed'] = $post_fields['created'];
+ DBA::insert('post-thread', $post_fields, Database::INSERT_IGNORE);
+ $rows++;
+ }
+ DBA::close($userposts);
+ if ($rows > 0) {
+ Logger::notice('Added post-thread entries', ['rows' => $rows]);
+ } else {
+ Logger::notice('No post-thread entries added');
+ }
+
+ $rows = 0;
+ $userposts = DBA::select('post-user', [], ["`gravity` = ? AND `id` not in (select `post-user-id` from `post-thread-user`)", GRAVITY_PARENT]);
+ while ($fields = DBA::fetch($userposts)) {
+ $post_fields = DBStructure::getFieldsForTable('post-thread-user', $fields);
+ $post_fields['commented'] = $post_fields['changed'] = $post_fields['created'];
+ DBA::insert('post-thread-user', $post_fields, Database::INSERT_IGNORE);
+ $rows++;
+ }
+ DBA::close($userposts);
+ if ($rows > 0) {
+ Logger::notice('Added post-thread-user entries', ['rows' => $rows]);
+ } else {
+ Logger::notice('No post-thread-user entries added');
+ }
+ }
+
+ /**
+ * Delete unused item-uri entries
+ */
+ private static function deleteUnusedItemUri()
+ {
+ // We have to avoid deleting newly created "item-uri" entries.
+ // So we fetch a post that had been stored yesterday and only delete older ones.
+ $item = Post::selectFirst(['uri-id'], ["`uid` = ? AND `received` < UTC_TIMESTAMP() - INTERVAL ? DAY", 0, 1],
+ ['order' => ['received' => true]]);
+ if (empty($item['uri-id'])) {
+ Logger::warning('No item with uri-id found - we better quit here');
+ return;
+ }
+ Logger::notice('Start collecting orphaned URI-ID', ['last-id' => $item['uri-id']]);
+ $uris = DBA::select('item-uri', ['id'], ["`id` < ?
+ AND NOT EXISTS(SELECT `uri-id` FROM `post-user` WHERE `uri-id` = `item-uri`.`id`)
+ AND NOT EXISTS(SELECT `parent-uri-id` FROM `post-user` WHERE `parent-uri-id` = `item-uri`.`id`)
+ AND NOT EXISTS(SELECT `thr-parent-id` FROM `post-user` WHERE `thr-parent-id` = `item-uri`.`id`)
+ AND NOT EXISTS(SELECT `external-id` FROM `post-user` WHERE `external-id` = `item-uri`.`id`)
+ AND NOT EXISTS(SELECT `uri-id` FROM `mail` WHERE `uri-id` = `item-uri`.`id`)
+ AND NOT EXISTS(SELECT `uri-id` FROM `event` WHERE `uri-id` = `item-uri`.`id`)
+ AND NOT EXISTS(SELECT `uri-id` FROM `contact` WHERE `uri-id` = `item-uri`.`id`)
+ AND NOT EXISTS(SELECT `uri-id` FROM `apcontact` WHERE `uri-id` = `item-uri`.`id`)
+ AND NOT EXISTS(SELECT `uri-id` FROM `fcontact` WHERE `uri-id` = `item-uri`.`id`)
+ AND NOT EXISTS(SELECT `parent-uri-id` FROM `mail` WHERE `parent-uri-id` = `item-uri`.`id`)
+ AND NOT EXISTS(SELECT `thr-parent-id` FROM `mail` WHERE `thr-parent-id` = `item-uri`.`id`)", $item['uri-id']]);
+
+ Logger::notice('Start deleting orphaned URI-ID', ['last-id' => $item['uri-id']]);
+ $affected_count = 0;
+ while ($rows = DBA::toArray($uris, false, 100)) {
+ $ids = array_column($rows, 'id');
+ DBA::delete('item-uri', ['id' => $ids]);
+ $affected_count += DBA::affectedRows();
+ Logger::info('Deleted', ['rows' => $affected_count]);
+ }
+ DBA::close($uris);
+ Logger::notice('Orphaned URI-ID entries removed', ['rows' => $affected_count]);
+ }
+
+ /**
+ * Delete old external post entries
+ */
+ private static function deleteExpiredExternalPosts()
{
$expire_days = DI::config()->get('system', 'dbclean-expire-days');
$expire_days_unclaimed = DI::config()->get('system', 'dbclean-expire-unclaimed');
$expire_days_unclaimed = $expire_days;
}
+ $limit = DI::config()->get('system', 'dbclean-expire-limit');
+ if (empty($limit)) {
+ return;
+ }
+
if (!empty($expire_days)) {
- Logger::notice('Start deleting expired threads', ['expiry_days' => $expire_days]);
- $ret = DBA::e("DELETE FROM `item-uri` WHERE `id` IN
- (SELECT `uri-id` FROM `thread`
- INNER JOIN `contact` ON `id` = `contact-id` AND NOT `notify_new_posts`
- WHERE `received` < UTC_TIMESTAMP() - INTERVAL ? DAY
- AND NOT `mention` AND NOT `starred` AND NOT `wall` AND NOT `origin`
- AND `thread`.`uid` != 0 AND NOT `iid` IN (SELECT `parent` FROM `item`
- WHERE (`item`.`starred` OR (`item`.`resource-id` != '')
- OR (`item`.`event-id` != '') OR (`item`.`attach` != '')
- OR `item`.`wall` OR `item`.`origin`
- OR `uri-id` IN (SELECT `uri-id` FROM `post-category`
- WHERE `uri-id` = `item`.`uri-id`))
- AND `item`.`parent` = `thread`.`iid`))", $expire_days);
-
- Logger::notice('Deleted expired threads', ['result' => $ret, 'rows' => DBA::affectedRows()]);
+ Logger::notice('Start collecting expired threads', ['expiry_days' => $expire_days]);
+ $uris = DBA::select('item-uri', ['id'], ["`id` IN
+ (SELECT `uri-id` FROM `post-thread` WHERE `received` < UTC_TIMESTAMP() - INTERVAL ? DAY
+ AND NOT `uri-id` IN (SELECT `uri-id` FROM `post-thread-user`
+ WHERE (`mention` OR `starred` OR `wall` OR `pinned`) AND `uri-id` = `post-thread`.`uri-id`)
+ AND NOT `uri-id` IN (SELECT `uri-id` FROM `post-category`
+ WHERE `uri-id` = `post-thread`.`uri-id`)
+ AND NOT `uri-id` IN (SELECT `uri-id` FROM `post-media`
+ WHERE `uri-id` = `post-thread`.`uri-id`)
+ AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `post-user` INNER JOIN `contact` ON `contact`.`id` = `contact-id` AND `notify_new_posts`
+ WHERE `parent-uri-id` = `post-thread`.`uri-id`)
+ AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `post-user`
+ WHERE (`origin` OR `event-id` != 0 OR `post-type` = ?) AND `parent-uri-id` = `post-thread`.`uri-id`)
+ AND NOT `uri-id` IN (SELECT `uri-id` FROM `post-content`
+ WHERE `resource-id` != 0 AND `uri-id` = `post-thread`.`uri-id`))",
+ $expire_days, Item::PT_PERSONAL_NOTE]);
+
+ Logger::notice('Start deleting expired threads');
+ $affected_count = 0;
+ while ($rows = DBA::toArray($uris, false, 100)) {
+ $ids = array_column($rows, 'id');
+ DBA::delete('item-uri', ['id' => $ids]);
+ $affected_count += DBA::affectedRows();
+ }
+ DBA::close($uris);
+
+ Logger::notice('Deleted expired threads', ['rows' => $affected_count]);
}
if (!empty($expire_days_unclaimed)) {
- $expiry_date = DateTimeFormat::utc('now - ' . $expire_days_unclaimed . ' days', DateTimeFormat::MYSQL);
-
- Logger::notice('Start deleting unclaimed public items', ['expiry_days' => $expire_days_unclaimed, 'expired' => $expiry_date]);
- $ret = DBA::e("DELETE FROM `item-uri` WHERE `id` IN
- (SELECT `uri-id` FROM `item` WHERE `gravity` = ? AND `uid` = ? AND `received` < ?
- AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `item` WHERE `uid` != ?)
- AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `item` WHERE `uid` = ? AND `received` > ?))",
- GRAVITY_PARENT, 0, $expiry_date, 0, 0, $expiry_date);
+ Logger::notice('Start collecting unclaimed public items', ['expiry_days' => $expire_days_unclaimed]);
+ $uris = DBA::select('item-uri', ['id'], ["`id` IN
+ (SELECT `uri-id` FROM `post-user` WHERE `gravity` = ? AND `uid` = ? AND `received` < UTC_TIMESTAMP() - INTERVAL ? DAY
+ AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `post-user` AS `i` WHERE `i`.`uid` != ?
+ AND `i`.`parent-uri-id` = `post-user`.`uri-id`)
+ AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `post-user` AS `i` WHERE `i`.`uid` = ?
+ AND `i`.`parent-uri-id` = `post-user`.`uri-id` AND `i`.`received` > UTC_TIMESTAMP() - INTERVAL ? DAY))",
+ GRAVITY_PARENT, 0, $expire_days_unclaimed, 0, 0, $expire_days_unclaimed]);
- Logger::notice('Deleted unclaimed public items', ['result' => $ret, 'rows' => DBA::affectedRows()]);
+ Logger::notice('Start deleting unclaimed public items');
+ $affected_count = 0;
+ while ($rows = DBA::toArray($uris, false, 100)) {
+ $ids = array_column($rows, 'id');
+ DBA::delete('item-uri', ['id' => $ids]);
+ $affected_count += DBA::affectedRows();
+ }
+ DBA::close($uris);
+ Logger::notice('Deleted unclaimed public items', ['rows' => $affected_count]);
}
}
}
projects / friendica.git / blobdiff