From: Michael Date: Mon, 15 Aug 2022 13:23:01 +0000 (+0000) Subject: Several performance improvements X-Git-Url: https://git.mxchange.org/?a=commitdiff_plain;h=5b3145d7ce454db1a3c9e0616bde2e52cb486d66;p=friendica.git Several performance improvements --- diff --git a/database.sql b/database.sql index 51c377bad7..417eddfcde 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ -- Friendica 2022.09-dev (Giant Rhubarb) --- DB_UPDATE_VERSION 1480 +-- DB_UPDATE_VERSION 1481 -- ------------------------------------------ @@ -222,7 +222,7 @@ CREATE TABLE IF NOT EXISTS `contact` ( INDEX `uid_contact-type` (`uid`,`contact-type`), INDEX `uid_self_contact-type` (`uid`,`self`,`contact-type`), INDEX `self_network_uid` (`self`,`network`,`uid`), - INDEX `gsid` (`gsid`), + INDEX `gsid_uid_failed` (`gsid`,`uid`,`failed`), INDEX `uri-id` (`uri-id`), FOREIGN KEY (`uid`) REFERENCES `user` (`uid`) ON UPDATE RESTRICT ON DELETE CASCADE, FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE, @@ -920,6 +920,7 @@ CREATE TABLE IF NOT EXISTS `notification` ( INDEX `target-uri-id` (`target-uri-id`), INDEX `parent-uri-id` (`parent-uri-id`), INDEX `seen_uid` (`seen`,`uid`), + INDEX `uid_type_parent-uri-id_actor-id` (`uid`,`type`,`parent-uri-id`,`actor-id`), FOREIGN KEY (`uid`) REFERENCES `user` (`uid`) ON UPDATE RESTRICT ON DELETE CASCADE, FOREIGN KEY (`vid`) REFERENCES `verb` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT, FOREIGN KEY (`actor-id`) REFERENCES `contact` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE, @@ -1153,9 +1154,12 @@ CREATE TABLE IF NOT EXISTS `post-category` ( CREATE TABLE IF NOT EXISTS `post-collection` ( `uri-id` int unsigned NOT NULL COMMENT 'Id of the item-uri table entry that contains the item uri', `type` tinyint unsigned NOT NULL DEFAULT 0 COMMENT '0 - Featured', + `author-id` int unsigned COMMENT 'Author of the featured post', PRIMARY KEY(`uri-id`,`type`), INDEX `type` (`type`), - FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE + INDEX `author-id` (`author-id`), + FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE, + FOREIGN KEY (`author-id`) REFERENCES `contact` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Collection of posts'; -- @@ -2423,7 +2427,7 @@ DROP VIEW IF EXISTS `collection-view`; CREATE VIEW `collection-view` AS SELECT `post-collection`.`uri-id` AS `uri-id`, `post-collection`.`type` AS `type`, - `post`.`author-id` AS `cid`, + `post-collection`.`author-id` AS `cid`, `post`.`received` AS `received`, `post`.`created` AS `created`, `post-thread`.`commented` AS `commented`, @@ -2431,7 +2435,7 @@ CREATE VIEW `collection-view` AS SELECT `post`.`visible` AS `visible`, `post`.`deleted` AS `deleted`, `post`.`thr-parent-id` AS `thr-parent-id`, - `post`.`author-id` AS `author-id`, + `post-collection`.`author-id` AS `author-id`, `post`.`gravity` AS `gravity` FROM `post-collection` INNER JOIN `post` ON `post-collection`.`uri-id` = `post`.`uri-id` diff --git a/doc/database/db_contact.md b/doc/database/db_contact.md index 8e9e105651..ad3991f307 100644 --- a/doc/database/db_contact.md +++ b/doc/database/db_contact.md @@ -121,7 +121,7 @@ Indexes | uid_contact-type | uid, contact-type | | uid_self_contact-type | uid, self, contact-type | | self_network_uid | self, network, uid | -| gsid | gsid | +| gsid_uid_failed | gsid, uid, failed | | uri-id | uri-id | Foreign Keys diff --git a/doc/database/db_notification.md b/doc/database/db_notification.md index e79778cb5b..a2f319ead4 100644 --- a/doc/database/db_notification.md +++ b/doc/database/db_notification.md @@ -31,6 +31,7 @@ Indexes | target-uri-id | target-uri-id | | parent-uri-id | parent-uri-id | | seen_uid | seen, uid | +| uid_type_parent-uri-id_actor-id | uid, type, parent-uri-id, actor-id | Foreign Keys ------------ diff --git a/doc/database/db_post-collection.md b/doc/database/db_post-collection.md index 3018067c8d..a0197e66bc 100644 --- a/doc/database/db_post-collection.md +++ b/doc/database/db_post-collection.md @@ -6,18 +6,20 @@ Collection of posts Fields ------ -| Field | Description | Type | Null | Key | Default | Extra | -| ------ | --------------------------------------------------------- | ---------------- | ---- | --- | ------- | ----- | -| uri-id | Id of the item-uri table entry that contains the item uri | int unsigned | NO | PRI | NULL | | -| type | 0 - Featured | tinyint unsigned | NO | PRI | 0 | | +| Field | Description | Type | Null | Key | Default | Extra | +| --------- | --------------------------------------------------------- | ---------------- | ---- | --- | ------- | ----- | +| uri-id | Id of the item-uri table entry that contains the item uri | int unsigned | NO | PRI | NULL | | +| type | 0 - Featured | tinyint unsigned | NO | PRI | 0 | | +| author-id | Author of the featured post | int unsigned | YES | | NULL | | Indexes ------------ -| Name | Fields | -| ------- | ------------ | -| PRIMARY | uri-id, type | -| type | type | +| Name | Fields | +| --------- | ------------ | +| PRIMARY | uri-id, type | +| type | type | +| author-id | author-id | Foreign Keys ------------ @@ -25,5 +27,6 @@ Foreign Keys | Field | Target Table | Target Field | |-------|--------------|--------------| | uri-id | [item-uri](help/database/db_item-uri) | id | +| author-id | [contact](help/database/db_contact) | id | Return to [database documentation](help/database) diff --git a/src/Model/APContact.php b/src/Model/APContact.php index 38ec423e89..0448765d08 100644 --- a/src/Model/APContact.php +++ b/src/Model/APContact.php @@ -24,6 +24,7 @@ namespace Friendica\Model; use Friendica\Content\Text\HTML; use Friendica\Core\Cache\Enum\Duration; use Friendica\Core\Logger; +use Friendica\Core\Protocol; use Friendica\Core\System; use Friendica\Database\DBA; use Friendica\DI; @@ -352,17 +353,18 @@ class APContact if (!empty($apcontact['outbox'])) { if (!empty($local_owner)) { - $outbox = ActivityPub\Transmitter::getOutbox($local_owner); + $statuses_count = self::getStatusesCount($local_owner); } else { $outbox = ActivityPub::fetchContent($apcontact['outbox']); + $statuses_count = $outbox['totalItems'] ?? 0; } - if (!empty($outbox['totalItems'])) { + if (!empty($statuses_count)) { // Mastodon seriously allows for this condition? // Jul 20 2021 - See https://chaos.social/@m11 for a negative posts count - if ($outbox['totalItems'] < 0) { - $outbox['totalItems'] = 0; + if ($statuses_count < 0) { + $statuses_count = 0; } - $apcontact['statuses_count'] = $outbox['totalItems']; + $apcontact['statuses_count'] = $statuses_count; } } @@ -483,6 +485,30 @@ class APContact return DBA::selectFirst('apcontact', [], ['url' => $apcontact['url']]) ?: []; } + /** + * Fetch the number of statuses for the given owner + * + * @param array $owner + * + * @return integer + */ + private static function getStatusesCount(array $owner): int + { + $condition = [ + 'private' => [Item::PUBLIC, Item::UNLISTED], + 'author-id' => Contact::getIdForURL($owner['url'], 0, false), + 'gravity' => [GRAVITY_PARENT, GRAVITY_COMMENT], + 'network' => Protocol::DFRN, + 'parent-network' => Protocol::FEDERATED, + 'deleted' => false, + 'visible' => true + ]; + + $count = Post::countPosts($condition); + + return $count; + } + /** * Mark the given AP Contact as "to archive" * diff --git a/src/Model/FContact.php b/src/Model/FContact.php index 713d74dc19..62e609fbee 100644 --- a/src/Model/FContact.php +++ b/src/Model/FContact.php @@ -100,7 +100,7 @@ class FContact $interacted = DBA::count('contact-relation', ["`cid` = ? AND NOT `follows` AND `last-interaction` > ?", $contact['id'], $last_interaction]); $interacting = DBA::count('contact-relation', ["`relation-cid` = ? AND NOT `follows` AND `last-interaction` > ?", $contact['id'], $last_interaction]); - $posts = Post::countPosts(['author-id' => $contact['id'], 'gravity' => [GRAVITY_PARENT, GRAVITY_COMMENT]]); + $posts = DBA::count('post', ['author-id' => $contact['id'], 'gravity' => [GRAVITY_PARENT, GRAVITY_COMMENT]]); } $fields = [ diff --git a/src/Model/Post.php b/src/Model/Post.php index c70237e8f5..59fbb491e8 100644 --- a/src/Model/Post.php +++ b/src/Model/Post.php @@ -400,12 +400,12 @@ class Post AND (NOT `causer-blocked` OR `causer-id` = ? OR `causer-id` IS NULL) AND NOT `contact-blocked` AND ((NOT `contact-readonly` AND NOT `contact-pending` AND (`contact-rel` IN (?, ?))) OR `self` OR `gravity` != ? OR `contact-uid` = ?) - AND NOT EXISTS (SELECT `uri-id` FROM `post-user` WHERE `uid` = ? AND `uri-id` = `" . $view . "`.`uri-id` AND `hidden`) - AND NOT EXISTS (SELECT `cid` FROM `user-contact` WHERE `uid` = ? AND `cid` = `author-id` AND `blocked`) - AND NOT EXISTS (SELECT `cid` FROM `user-contact` WHERE `uid` = ? AND `cid` = `owner-id` AND `blocked`) - AND NOT EXISTS (SELECT `cid` FROM `user-contact` WHERE `uid` = ? AND `cid` = `author-id` AND `ignored` AND `gravity` = ?) - AND NOT EXISTS (SELECT `cid` FROM `user-contact` WHERE `uid` = ? AND `cid` = `owner-id` AND `ignored` AND `gravity` = ?)", - 0, Contact::SHARING, Contact::FRIEND, GRAVITY_PARENT, 0, $uid, $uid, $uid, $uid, GRAVITY_PARENT, $uid, GRAVITY_PARENT]); + AND NOT `" . $view . "`.`uri-id` IN (SELECT `uri-id` FROM `post-user` WHERE `uid` = ? AND `hidden`) + AND NOT `author-id` IN (SELECT `cid` FROM `user-contact` WHERE `uid` = ? AND `blocked`) + AND NOT `owner-id` IN (SELECT `cid` FROM `user-contact` WHERE `uid` = ? AND `blocked`) + AND NOT (`gravity` = ? AND `author-id` IN (SELECT `cid` FROM `user-contact` WHERE `uid` = ? AND `ignored`)) + AND NOT (`gravity` = ? AND `owner-id` IN (SELECT `cid` FROM `user-contact` WHERE `uid` = ? AND `ignored`))", + 0, Contact::SHARING, Contact::FRIEND, GRAVITY_PARENT, 0, $uid, $uid, $uid, GRAVITY_PARENT, $uid, GRAVITY_PARENT, $uid]); $select_string = implode(', ', array_map([DBA::class, 'quoteIdentifier'], $selected)); diff --git a/src/Model/Post/Collection.php b/src/Model/Post/Collection.php index 363761d477..2758d80277 100644 --- a/src/Model/Post/Collection.php +++ b/src/Model/Post/Collection.php @@ -37,15 +37,16 @@ class Collection * * @param integer $uri_id * @param integer $type + * @param integer $author_id * @param integer $cache_uid If set to a non zero value, the featured cache is cleared */ - public static function add(int $uri_id, int $type, int $cache_uid = 0) + public static function add(int $uri_id, int $type, int $author_id, int $cache_uid = 0) { if (empty($uri_id)) { throw new BadMethodCallException('Empty URI_id'); } - DBA::insert('post-collection', ['uri-id' => $uri_id, 'type' => $type], Database::INSERT_IGNORE); + DBA::insert('post-collection', ['uri-id' => $uri_id, 'type' => $type, 'author-id' => $author_id], Database::INSERT_IGNORE); if (!empty($cache_uid) && ($type == self::FEATURED)) { DI::cache()->delete(ActivityPub\Transmitter::CACHEKEY_FEATURED . $cache_uid); diff --git a/src/Module/Api/Mastodon/Statuses/Pin.php b/src/Module/Api/Mastodon/Statuses/Pin.php index 6bf8a67396..a85f0fe72f 100644 --- a/src/Module/Api/Mastodon/Statuses/Pin.php +++ b/src/Module/Api/Mastodon/Statuses/Pin.php @@ -41,12 +41,12 @@ class Pin extends BaseApi DI::mstdnError()->UnprocessableEntity(); } - $item = Post::selectFirstForUser($uid, ['id', 'gravity'], ['uri-id' => $this->parameters['id'], 'uid' => [$uid, 0]]); + $item = Post::selectFirstForUser($uid, ['id', 'gravity', 'author-id'], ['uri-id' => $this->parameters['id'], 'uid' => [$uid, 0]]); if (!DBA::isResult($item)) { DI::mstdnError()->RecordNotFound(); } - Post\Collection::add($this->parameters['id'], Post\Collection::FEATURED, $uid); + Post\Collection::add($this->parameters['id'], Post\Collection::FEATURED, $item['author-id'], $uid); System::jsonExit(DI::mstdnStatus()->createFromUriId($this->parameters['id'], $uid)->toArray()); } diff --git a/src/Module/Item/Pin.php b/src/Module/Item/Pin.php index 9735e495c3..3b52b44a12 100644 --- a/src/Module/Item/Pin.php +++ b/src/Module/Item/Pin.php @@ -48,7 +48,7 @@ class Pin extends BaseModule $itemId = intval($this->parameters['id']); - $item = Post::selectFirst(['uri-id', 'uid', 'featured'], ['id' => $itemId]); + $item = Post::selectFirst(['uri-id', 'uid', 'featured', 'author-id'], ['id' => $itemId]); if (!DBA::isResult($item)) { throw new HTTPException\NotFoundException(); } @@ -60,7 +60,7 @@ class Pin extends BaseModule $pinned = !$item['featured']; if ($pinned) { - Post\Collection::add($item['uri-id'], Post\Collection::FEATURED, local_user()); + Post\Collection::add($item['uri-id'], Post\Collection::FEATURED, $item['author-id'], local_user()); } else { Post\Collection::remove($item['uri-id'], Post\Collection::FEATURED, local_user()); } diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php index 76bab6b81c..5413e32841 100644 --- a/src/Protocol/ActivityPub/Processor.php +++ b/src/Protocol/ActivityPub/Processor.php @@ -679,7 +679,7 @@ class Processor * Fetch the Uri-Id of a post for the "featured" collection * * @param array $activity - * @return null|int + * @return null|array */ private static function getUriIdForFeaturedCollection(array $activity) { @@ -697,7 +697,7 @@ class Processor } } - $parent = Post::selectFirst(['uri-id'], ['uri' => $activity['object_id']]); + $parent = Post::selectFirst(['uri-id', 'author-id'], ['uri' => $activity['object_id']]); if (empty($parent['uri-id'])) { if (self::fetchMissingActivity($activity['object_id'], $activity, '', Receiver::COMPLETION_AUTO)) { $parent = Post::selectFirst(['uri-id'], ['uri' => $activity['object_id']]); @@ -705,7 +705,7 @@ class Processor } if (!empty($parent['uri-id'])) { - return $parent['uri-id']; + $parent; } return null; @@ -718,14 +718,14 @@ class Processor */ public static function addToFeaturedCollection(array $activity) { - $uriid = self::getUriIdForFeaturedCollection($activity); - if (empty($uriid)) { + $post = self::getUriIdForFeaturedCollection($activity); + if (empty($post)) { return; } - Logger::debug('Add post to featured collection', ['uri-id' => $uriid]); + Logger::debug('Add post to featured collection', ['post' => $post]); - Post\Collection::add($uriid, Post\Collection::FEATURED); + Post\Collection::add($post['uri-id'], Post\Collection::FEATURED, $post['author-id']); Queue::remove($activity); } @@ -736,14 +736,14 @@ class Processor */ public static function removeFromFeaturedCollection(array $activity) { - $uriid = self::getUriIdForFeaturedCollection($activity); - if (empty($uriid)) { + $post = self::getUriIdForFeaturedCollection($activity); + if (empty($post)) { return; } - Logger::debug('Remove post from featured collection', ['uri-id' => $uriid]); + Logger::debug('Remove post from featured collection', ['post' => $post]); - Post\Collection::remove($uriid, Post\Collection::FEATURED); + Post\Collection::remove($post['uri-id'], Post\Collection::FEATURED); Queue::remove($activity); } @@ -1335,10 +1335,10 @@ class Processor } $id = Item::fetchByLink($post['id']); if (!empty($id)) { - $item = Post::selectFirst(['uri-id', 'featured'], ['id' => $id]); + $item = Post::selectFirst(['uri-id', 'featured', 'author-id'], ['id' => $id]); if (!empty($item['uri-id'])) { if (!$item['featured']) { - Post\Collection::add($item['uri-id'], Post\Collection::FEATURED); + Post\Collection::add($item['uri-id'], Post\Collection::FEATURED, $item['author-id']); Logger::debug('Added featured post', ['uri-id' => $item['uri-id'], 'contact' => $url]); $new++; } else { diff --git a/src/Protocol/ActivityPub/Transmitter.php b/src/Protocol/ActivityPub/Transmitter.php index ecd1843185..ba4baa267b 100644 --- a/src/Protocol/ActivityPub/Transmitter.php +++ b/src/Protocol/ActivityPub/Transmitter.php @@ -59,7 +59,6 @@ class Transmitter { const CACHEKEY_FEATURED = 'transmitter:getFeatured:'; const CACHEKEY_CONTACTS = 'transmitter:getContacts:'; - const CACHEKEY_OUTBOX = 'transmitter:getOutbox:'; /** * Add relay servers to the list of inboxes @@ -251,14 +250,6 @@ class Transmitter */ public static function getOutbox(array $owner, int $page = null, string $requester = '', bool $nocache = false): array { - if (empty($page)) { - $cachekey = self::CACHEKEY_OUTBOX . $owner['uid']; - $result = DI::cache()->get($cachekey); - if (!$nocache && !is_null($result)) { - return $result; - } - } - $condition = ['private' => [Item::PUBLIC, Item::UNLISTED]]; if (!empty($requester)) { @@ -283,12 +274,12 @@ class Transmitter 'visible' => true ]); - $count = Post::count($condition); + $apcontact = APContact::getByURL($owner['url']); $data = ['@context' => ActivityPub::CONTEXT]; $data['id'] = DI::baseUrl() . '/outbox/' . $owner['nickname']; $data['type'] = 'OrderedCollection'; - $data['totalItems'] = $count; + $data['totalItems'] = $apcontact['statuses_count'] ?? 0; if (!empty($page)) { $data['id'] .= '?' . http_build_query(['page' => $page]); @@ -316,15 +307,17 @@ class Transmitter $data['next'] = DI::baseUrl() . '/outbox/' . $owner['nickname'] . '?page=' . ($page + 1); } + // Fix the cached total item count when it is lower than the real count + $total = (($page - 1) * 20) + $data['totalItems']; + if ($total > $data['totalItems']) { + $data['totalItems'] = $total; + } + $data['partOf'] = DI::baseUrl() . '/outbox/' . $owner['nickname']; $data['orderedItems'] = $list; } - if (!empty($cachekey)) { - DI::cache()->set($cachekey, $data, Duration::DAY); - } - return $data; } diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index 1f2017b28d..fc5be58dd8 100644 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -55,7 +55,7 @@ use Friendica\Database\DBA; if (!defined('DB_UPDATE_VERSION')) { - define('DB_UPDATE_VERSION', 1480); + define('DB_UPDATE_VERSION', 1481); } return [ @@ -284,7 +284,7 @@ return [ "uid_contact-type" => ["uid", "contact-type"], "uid_self_contact-type" => ["uid", "self", "contact-type"], "self_network_uid" => ["self", "network", "uid"], - "gsid" => ["gsid"], + "gsid_uid_failed" => ["gsid", "uid", "failed"], "uri-id" => ["uri-id"], ] ], @@ -971,6 +971,7 @@ return [ "target-uri-id" => ["target-uri-id"], "parent-uri-id" => ["parent-uri-id"], "seen_uid" => ["seen", "uid"], + "uid_type_parent-uri-id_actor-id" => ["uid", "type", "parent-uri-id", "actor-id"], ] ], "notify" => [ @@ -1187,10 +1188,12 @@ return [ "fields" => [ "uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"], "type" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "primary" => "1", "comment" => "0 - Featured"], + "author-id" => ["type" => "int unsigned", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Author of the featured post"], ], "indexes" => [ "PRIMARY" => ["uri-id", "type"], "type" => ["type"], + "author-id" => ["author-id"], ] ], "post-content" => [ diff --git a/static/dbview.config.php b/static/dbview.config.php index 875273cb02..2a7cb54feb 100644 --- a/static/dbview.config.php +++ b/static/dbview.config.php @@ -702,7 +702,7 @@ "fields" => [ "uri-id" => ["post-collection", "uri-id"], "type" => ["post-collection", "type"], - "cid" => ["post", "author-id"], + "cid" => ["post-collection", "author-id"], "received" => ["post", "received"], "created" => ["post", "created"], "commented" => ["post-thread", "commented"], @@ -710,7 +710,7 @@ "visible" => ["post", "visible"], "deleted" => ["post", "deleted"], "thr-parent-id" => ["post", "thr-parent-id"], - "author-id" => ["post", "author-id"], + "author-id" => ["post-collection", "author-id"], "gravity" => ["post", "gravity"], ], "query" => "FROM `post-collection` diff --git a/update.php b/update.php index a28187e0cf..fab51993c1 100644 --- a/update.php +++ b/update.php @@ -1100,9 +1100,9 @@ function update_1451() function update_1457() { - $pinned = DBA::select('post-thread-user', ['uri-id'], ['pinned' => true]); + $pinned = DBA::select('post-thread-user', ['uri-id', 'author-id'], ['pinned' => true]); while ($post = DBA::fetch($pinned)) { - Post\Collection::add($post['uri-id'], Post\Collection::FEATURED); + Post\Collection::add($post['uri-id'], Post\Collection::FEATURED, $post['author-id']); } DBA::close($pinned); @@ -1115,3 +1115,9 @@ function update_1480() DBA::update('post', ['deleted' => false], ["`uri-id` IN (SELECT `uri-id` FROM `post-user` WHERE NOT `deleted`)"]); return Update::SUCCESS; } + +function update_1481() +{ + DBA::e("UPDATE `post-collection` INNER JOIN `post` ON `post`.`uri-id` = `post-collection`.`uri-id` SET `post-collection`.`author-id` = `post`.`author-id` WHERE `post-collection`.`author-id` IS null"); + return Update::SUCCESS; +}