Merge pull request #4423 from annando/requeue

We now incrementally calculate the queue delivery time

diff --git a/boot.php b/boot.php
index 48c9b294440f13fa0d3bf3fd9b8d9c5e868f68e0..feca5039978d67ee0c481c627ac057626454e104 100644 (file)
--- a/boot.php
+++ b/boot.php
@@ -39,7 +39,7 @@ define('FRIENDICA_PLATFORM',     'Friendica');
 define('FRIENDICA_CODENAME',     'Asparagus');
 define('FRIENDICA_VERSION',      '3.6-dev');
 define('DFRN_PROTOCOL_VERSION',  '2.23');
-define('DB_UPDATE_VERSION',      1250);
+define('DB_UPDATE_VERSION',      1251);
 define('NEW_UPDATE_ROUTINE_VERSION', 1170);
 
 /**

diff --git a/database.sql b/database.sql
index 7193b59d17a93f53f55ec3efd741705c3b1282d6..3dadc23961fd620b0152f21ac282698b61bd4bc3 100644 (file)
--- a/database.sql
+++ b/database.sql
@@ -853,19 +853,18 @@ CREATE TABLE IF NOT EXISTS `push_subscriber` (
 --
 CREATE TABLE IF NOT EXISTS `queue` (
        `id` int NOT NULL auto_increment COMMENT '',
-       `cid` int NOT NULL DEFAULT 0 COMMENT '',
-       `network` varchar(32) NOT NULL DEFAULT '' COMMENT '',
-       `guid` varchar(255) NOT NULL DEFAULT '' COMMENT '',
-       `created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '',
-       `last` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '',
+       `cid` int NOT NULL DEFAULT 0 COMMENT 'Message receiver',
+       `network` varchar(32) NOT NULL DEFAULT '' COMMENT 'Receiver\'s network',
+       `guid` varchar(255) NOT NULL DEFAULT '' COMMENT 'Unique GUID of the message',
+       `created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date, when the message was created',
+       `last` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of last trial',
+       `next` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Next retrial date',
+       `retrial` tinyint NOT NULL DEFAULT 0 COMMENT 'Retrial counter',
        `content` mediumtext COMMENT '',
        `batch` boolean NOT NULL DEFAULT '0' COMMENT '',
         PRIMARY KEY(`id`),
-        INDEX `cid` (`cid`),
-        INDEX `created` (`created`),
         INDEX `last` (`last`),
-        INDEX `network` (`network`),
-        INDEX `batch` (`batch`)
+        INDEX `next` (`next`)
 ) DEFAULT COLLATE utf8mb4_general_ci;
 
 --

diff --git a/src/Database/DBStructure.php b/src/Database/DBStructure.php
index 60859e53c3fdad1ead071d0d264182f1735c5c75..9ab866de9dc0eb18701f9ffcd27fa080a2175a79 100644 (file)
--- a/src/Database/DBStructure.php
+++ b/src/Database/DBStructure.php
@@ -1553,17 +1553,16 @@ class DBStructure
                                                "network" => ["type" => "varchar(32)", "not null" => "1", "default" => "", "comment" => "Receiver's network"],
                                                "guid" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "Unique GUID of the message"],
                                                "created" => ["type" => "datetime", "not null" => "1", "default" => NULL_DATE, "comment" => "Date, when the message was created"],
-                                               "last" => ["type" => "datetime", "not null" => "1", "default" => NULL_DATE, "comment" => ""],
+                                               "last" => ["type" => "datetime", "not null" => "1", "default" => NULL_DATE, "comment" => "Date of last trial"],
+                                               "next" => ["type" => "datetime", "not null" => "1", "default" => NULL_DATE, "comment" => "Next retrial date"],
+                                               "retrial" => ["type" => "tinyint", "not null" => "1", "default" => "0", "comment" => "Retrial counter"],
                                                "content" => ["type" => "mediumtext", "comment" => ""],
                                                "batch" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => ""],
                                                ],
                                "indexes" => [
                                                "PRIMARY" => ["id"],
-                                               "cid" => ["cid"],
-                                               "created" => ["created"],
                                                "last" => ["last"],
-                                               "network" => ["network"],
-                                               "batch" => ["batch"],
+                                               "next" => ["next"],
                                                ]
                                ];
                $database["register"] = [

diff --git a/src/Model/Queue.php b/src/Model/Queue.php
index 2d7873a1e054160554aaf645ee286841afaf7339..8b9e801c3ffc616af346e94322b4be112bbcc74c 100644 (file)
--- a/src/Model/Queue.php
+++ b/src/Model/Queue.php
@@ -19,7 +19,22 @@ class Queue
        public static function updateTime($id)
        {
                logger('queue: requeue item ' . $id);
-               dba::update('queue', ['last' => DateTimeFormat::utcNow()], ['id' => $id]);
+               $queue = dba::selectFirst('queue', ['retrial'], ['id' => $id]);
+               if (!DBM::is_result($queue)) {
+                       return;
+               }
+
+               $retrial = $queue['retrial'];
+
+               if ($retrial > 14) {
+                       self::removeItem($id);
+               }
+
+               // Calculate the delay until the next trial
+               $delay = (($retrial + 3) ** 4) + (rand(1, 30) * ($retrial + 1));
+               $next = DateTimeFormat::utc('now + ' . $delay . ' seconds');
+
+               dba::update('queue', ['last' => DateTimeFormat::utcNow(), 'retrial' => $retrial + 1, 'next' => $next], ['id' => $id]);
        }
 
        /**

diff --git a/src/Worker/Queue.php b/src/Worker/Queue.php
index d92be19fe3bf8ffac5c7b9db01bc9f17cf4bef74..29be22199f68c3aa21be55835dcd67cd52bd0501 100644 (file)
--- a/src/Worker/Queue.php
+++ b/src/Worker/Queue.php
@@ -32,30 +32,12 @@ class Queue
                $no_dead_check = Config::get('system', 'queue_no_dead_check', false);
 
                if (!$queue_id) {
-                       logger('queue: start');
+                       logger('filling queue jobs - start');
 
                        // Handling the pubsubhubbub requests
                        Worker::add(['priority' => PRIORITY_HIGH, 'dont_fork' => true], 'PubSubPublish');
 
-                       $r = q(
-                               "SELECT `queue`.*, `contact`.`name`, `contact`.`uid` FROM `queue`
-                               INNER JOIN `contact` ON `queue`.`cid` = `contact`.`id`
-                               WHERE `queue`.`created` < UTC_TIMESTAMP() - INTERVAL 3 DAY"
-                       );
-
-                       if (DBM::is_result($r)) {
-                               foreach ($r as $rr) {
-                                       logger('Removing expired queue item for ' . $rr['name'] . ', uid=' . $rr['uid']);
-                                       logger('Expired queue data: ' . $rr['content'], LOGGER_DATA);
-                               }
-                               q("DELETE FROM `queue` WHERE `created` < UTC_TIMESTAMP() - INTERVAL 3 DAY");
-                       }
-
-                       /*
-                        * For the first 12 hours we'll try to deliver every 15 minutes
-                        * After that, we'll only attempt delivery once per hour.
-                        */
-                       $r = q("SELECT `id` FROM `queue` WHERE ((`created` > UTC_TIMESTAMP() - INTERVAL 12 HOUR AND `last` < UTC_TIMESTAMP() - INTERVAL 15 MINUTE) OR (`last` < UTC_TIMESTAMP() - INTERVAL 1 HOUR)) ORDER BY `cid`, `created`");
+                       $r = dba::inArray(dba::p("SELECT `id` FROM `queue` WHERE `next` < UTC_TIMESTAMP()"));
 
                        Addon::callHooks('queue_predeliver', $r);
 
@@ -65,6 +47,7 @@ class Queue
                                        Worker::add(['priority' => PRIORITY_LOW, 'dont_fork' => true], "Queue", (int) $q_item['id']);
                                }
                        }
+                       logger('filling queue jobs - end');
                        return;
                }