Merge pull request #4418 from annando/no-requeue

Don't add already queued items from Diaspora to the queue again

diff --git a/boot.php b/boot.php
index b2d9f019276462efc85f689ac24dab4ecd85e654..48c9b294440f13fa0d3bf3fd9b8d9c5e868f68e0 100644 (file)
--- a/boot.php
+++ b/boot.php
@@ -39,7 +39,7 @@ define('FRIENDICA_PLATFORM',     'Friendica');
 define('FRIENDICA_CODENAME',     'Asparagus');
 define('FRIENDICA_VERSION',      '3.6-dev');
 define('DFRN_PROTOCOL_VERSION',  '2.23');
-define('DB_UPDATE_VERSION',      1249);
+define('DB_UPDATE_VERSION',      1250);
 define('NEW_UPDATE_ROUTINE_VERSION', 1170);
 
 /**

diff --git a/database.sql b/database.sql
index 85cd811c1b0ac15cf766b466f0e6f39d2aa775a7..7193b59d17a93f53f55ec3efd741705c3b1282d6 100644 (file)
--- a/database.sql
+++ b/database.sql
@@ -1,6 +1,6 @@
 -- ------------------------------------------
 -- Friendica 3.6-dev (Asparagus)
--- DB_UPDATE_VERSION 1248
+-- DB_UPDATE_VERSION 1250
 -- ------------------------------------------
 
 
@@ -855,6 +855,7 @@ CREATE TABLE IF NOT EXISTS `queue` (
        `id` int NOT NULL auto_increment COMMENT '',
        `cid` int NOT NULL DEFAULT 0 COMMENT '',
        `network` varchar(32) NOT NULL DEFAULT '' COMMENT '',
+       `guid` varchar(255) NOT NULL DEFAULT '' COMMENT '',
        `created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '',
        `last` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '',
        `content` mediumtext COMMENT '',

diff --git a/doc/htconfig.md b/doc/htconfig.md
index e09c37b74d196ae1423f4d41fabb8b1ea55f5c90..3a482742d2c25821c8e148fe207410c797c313de 100644 (file)
--- a/doc/htconfig.md
+++ b/doc/htconfig.md
@@ -69,6 +69,7 @@ Example: To set the automatic database cleanup process add this line to your .ht
 * **ostatus_poll_timeframe** - Defines how old an item can be to try to complete the conversation with it.
 * **paranoia** (Boolean) - Log out users if their IP address changed.
 * **permit_crawling** (Boolean) - Restricts the search for not logged in users to one search per minute.
+* **queue_no_dead_check** (Boolean) - Ignore if the target contact or server seems to be dead during queue delivery.
 * **worker_debug** (Boolean) - If enabled, it prints out the number of running processes split by priority.
 * **worker_fetch_limit** - Number of worker tasks that are fetched in a single query. Default is 1.
 * **profiler** (Boolean) - Enable internal timings to help optimize code. Needed for "rendertime" addon. Default is false.

diff --git a/src/Database/DBStructure.php b/src/Database/DBStructure.php
index 4c05782f178b118ef94d69fc5b65439f2ed49bcd..60859e53c3fdad1ead071d0d264182f1735c5c75 100644 (file)
--- a/src/Database/DBStructure.php
+++ b/src/Database/DBStructure.php
@@ -1549,9 +1549,10 @@ class DBStructure
                                "comment" => "Queue for messages that couldn't be delivered",
                                "fields" => [
                                                "id" => ["type" => "int", "not null" => "1", "extra" => "auto_increment", "primary" => "1", "comment" => ""],
-                                               "cid" => ["type" => "int", "not null" => "1", "default" => "0", "relation" => ["contact" => "id"], "comment" => ""],
-                                               "network" => ["type" => "varchar(32)", "not null" => "1", "default" => "", "comment" => ""],
-                                               "created" => ["type" => "datetime", "not null" => "1", "default" => NULL_DATE, "comment" => ""],
+                                               "cid" => ["type" => "int", "not null" => "1", "default" => "0", "relation" => ["contact" => "id"], "comment" => "Message receiver"],
+                                               "network" => ["type" => "varchar(32)", "not null" => "1", "default" => "", "comment" => "Receiver's network"],
+                                               "guid" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "Unique GUID of the message"],
+                                               "created" => ["type" => "datetime", "not null" => "1", "default" => NULL_DATE, "comment" => "Date, when the message was created"],
                                                "last" => ["type" => "datetime", "not null" => "1", "default" => NULL_DATE, "comment" => ""],
                                                "content" => ["type" => "mediumtext", "comment" => ""],
                                                "batch" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => ""],

diff --git a/src/Model/Queue.php b/src/Model/Queue.php
index 3c3cf379a4901497bb5d74fd6e04a89d1f57875b..2d7873a1e054160554aaf645ee286841afaf7339 100644 (file)
--- a/src/Model/Queue.php
+++ b/src/Model/Queue.php
@@ -66,7 +66,7 @@ class Queue
         * @param string  $msg     message
         * @param boolean $batch   batch, default false
         */
-       public static function add($cid, $network, $msg, $batch = false)
+       public static function add($cid, $network, $msg, $batch = false, $guid = '')
        {
 
                $max_queue = Config::get('system', 'max_contact_queue');
@@ -86,10 +86,10 @@ class Queue
 
                if (DBM::is_result($r)) {
                        if ($batch &&  ($r[0]['total'] > $batch_queue)) {
-                               logger('add_to_queue: too many queued items for batch server ' . $cid . ' - discarding message');
+                               logger('too many queued items for batch server ' . $cid . ' - discarding message');
                                return;
                        } elseif ((! $batch) && ($r[0]['total'] > $max_queue)) {
-                               logger('add_to_queue: too many queued items for contact ' . $cid . ' - discarding message');
+                               logger('too many queued items for contact ' . $cid . ' - discarding message');
                                return;
                        }
                }
@@ -97,10 +97,12 @@ class Queue
                dba::insert('queue', [
                        'cid'     => $cid,
                        'network' => $network,
+                       'guid'     => $guid,
                        'created' => DateTimeFormat::utcNow(),
                        'last'    => DateTimeFormat::utcNow(),
                        'content' => $msg,
                        'batch'   =>($batch) ? 1 : 0
                ]);
+               logger('Added item ' . $guid . ' for ' . $cid);
        }
 }

diff --git a/src/Protocol/Diaspora.php b/src/Protocol/Diaspora.php
index 0f5c9c0c9ddac10987b25963cc6eb682126c81df..534f6ed6ba241d9e34dfbdd72404544c6af1e878 100644 (file)
--- a/src/Protocol/Diaspora.php
+++ b/src/Protocol/Diaspora.php
@@ -3181,7 +3181,7 @@ class Diaspora
         *
         * @return int Result of the transmission
         */
-       public static function transmit($owner, $contact, $envelope, $public_batch, $queue_run = false, $guid = "")
+       public static function transmit($owner, $contact, $envelope, $public_batch, $queue_run = false, $guid = "", $no_queue = false)
        {
                $a = get_app();
 
@@ -3224,24 +3224,14 @@ class Diaspora
                logger("transmit: ".$logid."-".$guid." returns: ".$return_code);
 
                if (!$return_code || (($return_code == 503) && (stristr($a->get_curl_headers(), "retry-after")))) {
-                       logger("queue message");
-
-                       $r = q(
-                               "SELECT `id` FROM `queue` WHERE `cid` = %d AND `network` = '%s' AND `content` = '%s' AND `batch` = %d LIMIT 1",
-                               intval($contact["id"]),
-                               dbesc(NETWORK_DIASPORA),
-                               dbesc($envelope),
-                               intval($public_batch)
-                       );
-                       if ($r) {
-                               logger("add_to_queue ignored - identical item already in queue");
-                       } else {
+                       if (!$no_queue) {
+                               logger("queue message");
                                // queue message for redelivery
-                               Queue::add($contact["id"], NETWORK_DIASPORA, $envelope, $public_batch);
-
-                               // The message could not be delivered. We mark the contact as "dead"
-                               Contact::markForArchival($contact);
+                               Queue::add($contact["id"], NETWORK_DIASPORA, $envelope, $public_batch, $guid);
                        }
+
+                       // The message could not be delivered. We mark the contact as "dead"
+                       Contact::markForArchival($contact);
                } elseif (($return_code >= 200) && ($return_code <= 299)) {
                        // We successfully delivered a message, the contact is alive
                        Contact::unmarkForArchival($contact);
@@ -3294,7 +3284,7 @@ class Diaspora
                $envelope = self::buildMessage($msg, $owner, $contact, $owner['uprvkey'], $contact['pubkey'], $public_batch);
 
                if ($spool) {
-                       Queue::add($contact['id'], NETWORK_DIASPORA, $envelope, $public_batch);
+                       Queue::add($contact['id'], NETWORK_DIASPORA, $envelope, $public_batch, $guid);
                        return true;
                } else {
                        $return_code = self::transmit($owner, $contact, $envelope, $public_batch, false, $guid);

diff --git a/src/Worker/Delivery.php b/src/Worker/Delivery.php
index 0a77f7b5be18b4d26848fe718f633139df2a7bad..2b9a00176e658d442cd45aa6397046315d7e331b 100644 (file)
--- a/src/Worker/Delivery.php
+++ b/src/Worker/Delivery.php
@@ -322,7 +322,7 @@ class Delivery {
 
                                if ($deliver_status < 0) {
                                        logger('notifier: delivery failed: queuing message');
-                                       Queue::add($contact['id'], NETWORK_DFRN, $atom);
+                                       Queue::add($contact['id'], NETWORK_DFRN, $atom, false, $target_item['guid']);
 
                                        // The message could not be delivered. We mark the contact as "dead"
                                        Contact::markForArchival($contact);

diff --git a/src/Worker/Queue.php b/src/Worker/Queue.php
index 10a83e5c4bf0be0ca2414002909554bb93538146..d92be19fe3bf8ffac5c7b9db01bc9f17cf4bef74 100644 (file)
--- a/src/Worker/Queue.php
+++ b/src/Worker/Queue.php
@@ -6,6 +6,7 @@ namespace Friendica\Worker;
 
 use Friendica\Core\Addon;
 use Friendica\Core\Cache;
+use Friendica\Core\Config;
 use Friendica\Core\Worker;
 use Friendica\Database\DBM;
 use Friendica\Model\Queue as QueueModel;
@@ -28,6 +29,8 @@ class Queue
                $cachekey_deadguy = 'queue_run:deadguy:';
                $cachekey_server = 'queue_run:server:';
 
+               $no_dead_check = Config::get('system', 'queue_no_dead_check', false);
+
                if (!$queue_id) {
                        logger('queue: start');
 
@@ -80,28 +83,30 @@ class Queue
 
                $dead = Cache::get($cachekey_deadguy . $contact['notify']);
 
-               if (!is_null($dead) && $dead) {
+               if (!is_null($dead) && $dead && !$no_dead_check) {
                        logger('queue: skipping known dead url: ' . $contact['notify']);
                        QueueModel::updateTime($q_item['id']);
                        return;
                }
 
-               $server = PortableContact::detectServer($contact['url']);
+               if (!$no_dead_check) {
+                       $server = PortableContact::detectServer($contact['url']);
 
-               if ($server != "") {
-                       $vital = Cache::get($cachekey_server . $server);
+                       if ($server != "") {
+                               $vital = Cache::get($cachekey_server . $server);
 
-                       if (is_null($vital)) {
-                               logger("Check server " . $server . " (" . $contact["network"] . ")");
+                               if (is_null($vital)) {
+                                       logger("Check server " . $server . " (" . $contact["network"] . ")");
 
-                               $vital = PortableContact::checkServer($server, $contact["network"], true);
-                               Cache::set($cachekey_server . $server, $vital, CACHE_QUARTER_HOUR);
-                       }
+                                       $vital = PortableContact::checkServer($server, $contact["network"], true);
+                                       Cache::set($cachekey_server . $server, $vital, CACHE_QUARTER_HOUR);
+                               }
 
-                       if (!is_null($vital) && !$vital) {
-                               logger('queue: skipping dead server: ' . $server);
-                               QueueModel::updateTime($q_item['id']);
-                               return;
+                               if (!is_null($vital) && !$vital) {
+                                       logger('queue: skipping dead server: ' . $server);
+                                       QueueModel::updateTime($q_item['id']);
+                                       return;
+                               }
                        }
                }
 
@@ -145,7 +150,7 @@ class Queue
                        case NETWORK_DIASPORA:
                                if ($contact['notify']) {
                                        logger('queue: diaspora_delivery: item ' . $q_item['id'] . ' for ' . $contact['name'] . ' <' . $contact['url'] . '>');
-                                       $deliver_status = Diaspora::transmit($owner, $contact, $data, $public, true);
+                                       $deliver_status = Diaspora::transmit($owner, $contact, $data, $public, true, 'Queue:' . $q_item['id'], true);
 
                                        if ($deliver_status == (-1)) {
                                                QueueModel::updateTime($q_item['id']);