Bulk transmission for AP posts

diff --git a/src/Model/Post/Delivery.php b/src/Model/Post/Delivery.php
new file mode 100644 (file)
index 0000000..8cd01c3
--- /dev/null
+++ b/src/Model/Post/Delivery.php
@@ -0,0 +1,64 @@
+<?php
+/**
+ * @copyright Copyright (C) 2010-2022, the Friendica project
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace Friendica\Model\Post;
+
+use Friendica\Database\DBA;
+use BadMethodCallException;
+use Friendica\Database\Database;
+use Friendica\Model\ItemURI;
+
+class Delivery
+{
+       /**
+        * Add a post to an inbox
+        *
+        * @param integer $uri_id
+        * @param string  $inbox
+        * @param string  $created
+        */
+       public static function add(int $uri_id, int $uid, string $inbox, string $created, string $command)
+       {
+               if (empty($uri_id)) {
+                       throw new BadMethodCallException('Empty URI_id');
+               }
+
+               $fields = ['uri-id' => $uri_id, 'uid' => $uid, 'inbox-id' => ItemURI::getIdByURI($inbox), 'created' => $created, 'command' => $command];
+
+               DBA::insert('post-delivery', $fields, Database::INSERT_IGNORE);
+       }
+
+       /**
+        * Remove post from an inbox after delivery
+        *
+        * @param integer $uri_id
+        * @param string  $inbox
+        */
+       public static function remove(int $uri_id, string $inbox)
+       {
+               DBA::delete('post-delivery', ['uri-id' => $uri_id, 'inbox-id' => ItemURI::getIdByURI($inbox)]);
+       }
+
+       public static function selectForInbox(string $inbox)
+       {
+               return DBA::selectToArray('post-delivery', [], ['inbox-id' => ItemURI::getIdByURI($inbox)], ['order' => ['created']]);
+       }
+}

diff --git a/src/Util/HTTPSignature.php b/src/Util/HTTPSignature.php
index cc25f149188f7aa120c5ea72a9563397bfd78f9f..db1d7a2799a9b46cc714db4162c572be678d34b4 100644 (file)
--- a/src/Util/HTTPSignature.php
+++ b/src/Util/HTTPSignature.php
@@ -27,6 +27,7 @@ use Friendica\Database\DBA;
 use Friendica\DI;
 use Friendica\Model\APContact;
 use Friendica\Model\Contact;
+use Friendica\Model\ItemURI;
 use Friendica\Model\User;
 use Friendica\Network\HTTPClient\Client\HttpClientAccept;
 use Friendica\Network\HTTPClient\Client\HttpClientOptions;
@@ -329,7 +330,7 @@ class HTTPSignature
 
                $status = DBA::selectFirst('inbox-status', [], ['url' => $url]);
                if (!DBA::isResult($status)) {
-                       DBA::insert('inbox-status', ['url' => $url, 'created' => $now, 'shared' => $shared], Database::INSERT_IGNORE);
+                       DBA::insert('inbox-status', ['url' => $url, 'uri-id' => ItemURI::getIdByURI($url), 'created' => $now, 'shared' => $shared], Database::INSERT_IGNORE);
                        $status = DBA::selectFirst('inbox-status', [], ['url' => $url]);
                }
 
@@ -369,6 +370,10 @@ class HTTPSignature
                        $fields['archive'] = false;
                }
 
+               if (empty($status['uri-id'])) {
+                       $fields['uri-id'] = ItemURI::getIdByURI($url);
+               }
+
                DBA::update('inbox-status', $fields, ['url' => $url]);
        }
 

diff --git a/src/Worker/APDelivery.php b/src/Worker/APDelivery.php
index dc826e33159e8aa15b3c0af7bff243cec77663a5..299eeba359f1d77de7b70b2465882824f1316ad8 100644 (file)
--- a/src/Worker/APDelivery.php
+++ b/src/Worker/APDelivery.php
@@ -25,6 +25,7 @@ use Friendica\Core\Logger;
 use Friendica\Core\Worker;
 use Friendica\Model\Contact;
 use Friendica\Model\GServer;
+use Friendica\Model\Item;
 use Friendica\Model\Post;
 use Friendica\Protocol\ActivityPub;
 use Friendica\Util\HTTPSignature;
@@ -56,6 +57,67 @@ class APDelivery
 
                Logger::info('Invoked', ['cmd' => $cmd, 'inbox' => $inbox, 'id' => $item_id, 'uri-id' => $uri_id, 'uid' => $uid]);
 
+               if (empty($uri_id)) {
+                       $result = self::deliver($inbox);
+                       $success = $result['success'];
+                       $uri_ids = $result['uri_ids'];
+               }
+
+               if (empty($uri_ids)) {
+                       $success = self::deliverToInbox($cmd, $item_id, $inbox, $uid, $receivers, $uri_id);
+               }
+
+               if (!$success && !Worker::defer() && in_array($cmd, [Delivery::POST])) {
+                       if ($uri_id) {
+                               Post\Delivery::remove($uri_id, $inbox);
+                               Post\DeliveryData::incrementQueueFailed($uri_id);
+                       } elseif (!empty($uri_ids)) {
+                               foreach ($uri_ids as $uri_id) {
+                                       Post\Delivery::remove($uri_id, $inbox);
+                                       Post\DeliveryData::incrementQueueFailed($uri_id);
+                               }
+                       }
+               } elseif ($success && in_array($cmd, [Delivery::POST])) {
+                       if ($uri_id) {
+                               Post\DeliveryData::incrementQueueDone($uri_id, Post\DeliveryData::ACTIVITYPUB);
+                       } elseif (!empty($uri_ids)) {
+                               foreach ($uri_ids as $uri_id) {
+                                       Post\DeliveryData::incrementQueueDone($uri_id, Post\DeliveryData::ACTIVITYPUB);
+                               }
+                       }
+               }
+       }
+
+       private static function deliver(string $inbox)
+       {
+               $uri_ids = [];
+               $success = true;
+
+               $posts = Post\Delivery::selectForInbox($inbox);
+               foreach ($posts as $post) {
+                       $uri_ids[] = $post['uri-id'];
+                       if ($success) {
+                               $success = self::deliverToInbox($post['command'], 0, $inbox, $post['uid'], [], $post['uri-id']);
+                       }
+               }
+
+               return ['success' => $success, 'uri_ids' => $uri_ids];
+       }
+
+       private static function deliverToInbox(string $cmd, int $item_id, string $inbox, int $uid, array $receivers, int $uri_id)
+       {
+               if (empty($item_id) && !empty($uri_id) && !empty($uid)) {
+                       $item = Post::selectFirst(['id', 'parent'], ['uri-id' => $uri_id, 'uid' => $uid]);
+                       $item_id = $item['id'] ?? 0;
+                       if (empty($receivers) && !empty($item)) {
+                               $parent = Post::selectFirst(Item::DELIVER_FIELDLIST, ['id' => $item['parent']]);
+
+                               $inboxes = ActivityPub\Transmitter::fetchTargetInboxes($parent, $uid, true, $item['id']);
+
+                               $receivers = $inboxes[$inbox] ?? [];
+                       }
+               }
+
                $success = true;
 
                if ($cmd == Delivery::MAIL) {
@@ -77,9 +139,21 @@ class APDelivery
                        $data = ActivityPub\Transmitter::createCachedActivityFromItem($item_id);
                        if (!empty($data)) {
                                $success = HTTPSignature::transmit($data, $inbox, $uid);
+                               if ($success && $uri_id) {
+                                       Post\Delivery::remove($uri_id, $inbox);
+                               }
                        }
                }
 
+               self::setSuccess($receivers, $success);
+
+               Logger::info('Delivered', ['cmd' => $cmd, 'inbox' => $inbox, 'id' => $item_id, 'uri-id' => $uri_id, 'uid' => $uid, 'success' => $success]);
+
+               return $success;
+       }
+
+       private static function setSuccess(array $receivers, bool $success)
+       {
                $gsid = null;
 
                foreach ($receivers as $receiver) {
@@ -100,11 +174,5 @@ class APDelivery
                if (!empty($gsid)) {
                        GServer::setProtocol($gsid, Post\DeliveryData::ACTIVITYPUB);
                }
-
-               if (!$success && !Worker::defer() && in_array($cmd, [Delivery::POST])) {
-                       Post\DeliveryData::incrementQueueFailed($uri_id);
-               } elseif ($success && in_array($cmd, [Delivery::POST])) {
-                       Post\DeliveryData::incrementQueueDone($uri_id, Post\DeliveryData::ACTIVITYPUB);
-               }
        }
 }

diff --git a/src/Worker/ExpirePosts.php b/src/Worker/ExpirePosts.php
index b566976e9ef09d203e42418ce56027e179726fc2..52d792ea6288cb482fd1da3ef33901f90ffcebe9 100644 (file)
--- a/src/Worker/ExpirePosts.php
+++ b/src/Worker/ExpirePosts.php
@@ -189,6 +189,9 @@ class ExpirePosts
                        AND NOT EXISTS(SELECT `uri-id` FROM `contact` WHERE `uri-id` = `item-uri`.`id`)
                        AND NOT EXISTS(SELECT `uri-id` FROM `apcontact` WHERE `uri-id` = `item-uri`.`id`)
                        AND NOT EXISTS(SELECT `uri-id` FROM `fcontact` WHERE `uri-id` = `item-uri`.`id`)
+                       AND NOT EXISTS(SELECT `uri-id` FROM `inbox-status` WHERE `uri-id` = `item-uri`.`id`)
+                       AND NOT EXISTS(SELECT `uri-id` FROM `post-delivery` WHERE `uri-id` = `item-uri`.`id`)
+                       AND NOT EXISTS(SELECT `uri-id` FROM `post-delivery` WHERE `inbox-id` = `item-uri`.`id`)
                        AND NOT EXISTS(SELECT `parent-uri-id` FROM `mail` WHERE `parent-uri-id` = `item-uri`.`id`)
                        AND NOT EXISTS(SELECT `thr-parent-id` FROM `mail` WHERE `thr-parent-id` = `item-uri`.`id`)", $item['uri-id']]);
 

diff --git a/src/Worker/Notifier.php b/src/Worker/Notifier.php
index 80628d7dba08d08fe5e55a3dabc84886e2a27038..fdec949a71b2df3507268f5b822246375c4797d0 100644 (file)
--- a/src/Worker/Notifier.php
+++ b/src/Worker/Notifier.php
@@ -789,8 +789,9 @@ class Notifier
                        Logger::info('Delivery via ActivityPub', ['cmd' => $cmd, 'id' => $target_item['id'], 'inbox' => $inbox]);
 
                        if (Worker::add(['priority' => $priority, 'created' => $created, 'dont_fork' => true],
-                                       'APDelivery', $cmd, $target_item['id'], $inbox, $uid, $receivers, $target_item['uri-id'])) {
+                                       'APDelivery', $cmd, 0, $inbox, $uid)) {
                                $delivery_queue_count++;
+                               Post\Delivery::add($target_item['uri-id'], $uid, $inbox, $target_item['created'], $cmd);
                        }
                }
 
@@ -798,8 +799,9 @@ class Notifier
                foreach ($relay_inboxes as $inbox) {
                        Logger::info('Delivery to relay servers via ActivityPub', ['cmd' => $cmd, 'id' => $target_item['id'], 'inbox' => $inbox]);
 
-                       if (Worker::add(['priority' => $priority, 'dont_fork' => true], 'APDelivery', $cmd, $target_item['id'], $inbox, $uid, [], $target_item['uri-id'])) {
+                       if (Worker::add(['priority' => $priority, 'dont_fork' => true], 'APDelivery', $cmd, 0, $inbox, $uid)) {
                                $delivery_queue_count++;
+                               Post\Delivery::add($target_item['uri-id'], $uid, $inbox, $target_item['created'], $cmd);
                        }
                }
 

diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php
index 03748f36a50c8903f2bd9b302b7a16b9ef1c7fe6..e3435989cf4586aa286372f22db00a5c6b47ec20 100644 (file)
--- a/static/dbstructure.config.php
+++ b/static/dbstructure.config.php
@@ -772,6 +772,7 @@ return [
                "comment" => "Status of ActivityPub inboxes",
                "fields" => [
                        "url" => ["type" => "varbinary(255)", "not null" => "1", "primary" => "1", "comment" => "URL of the inbox"],
+                       "uri-id" => ["type" => "int unsigned", "foreign" => ["item-uri" => "id"], "comment" => "Item-uri id of inbox url"],
                        "created" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Creation date of this entry"],
                        "success" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Date of the last successful delivery"],
                        "failure" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Date of the last failed delivery"],
@@ -780,7 +781,8 @@ return [
                        "shared" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "Is it a shared inbox?"]
                ],
                "indexes" => [
-                       "PRIMARY" => ["url"]
+                       "PRIMARY" => ["url"],
+                       "uri-id" => ["uri-id"],
                ]
        ],
        "intro" => [
@@ -1155,6 +1157,21 @@ return [
                        "title-content-warning-body" => ["FULLTEXT", "title", "content-warning", "body"],
                ]
        ],
+       "post-delivery" => [
+               "comment" => "Status of ActivityPub inboxes",
+               "fields" => [
+                       "uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"],
+                       "inbox-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "foreign" => ["item-uri" => "id"], "comment" => "Item-uri id of inbox url"],
+                       "uid" => ["type" => "mediumint unsigned", "foreign" => ["user" => "uid"], "comment" => "Delivering user"],
+                       "created" => ["type" => "datetime", "default" => DBA::NULL_DATETIME, "comment" => ""],
+                       "command" => ["type" => "varbinary(32)", "comment" => ""],
+               ],
+               "indexes" => [
+                       "PRIMARY" => ["uri-id", "inbox-id"],
+                       "inbox-id_created" => ["inbox-id", "created"],
+                       "uid" => ["uid"],
+               ]
+       ],
        "post-delivery-data" => [
                "comment" => "Delivery data for items",
                "fields" => [