Show deferred worker tasks in the admin interface and workerqueue logs

diff --git a/mod/admin.php b/mod/admin.php
index e3b537fcf0564bf00cef53f21b133b3213c0101d..c50927e97cfd397ef16ae43b0da7575b14a49f8b 100644 (file)
--- a/mod/admin.php
+++ b/mod/admin.php
@@ -181,32 +181,33 @@ function admin_content(App $a)
        // array(url, name, extra css classes)
        // not part of $aside to make the template more adjustable
        $aside_sub = [
-               'information' => [ L10n::t('Information'), [
-                       "overview" => ["admin/", L10n::t("Overview"), "overview" ],
-                       'federation'   => ["admin/federation/"  , L10n::t('Federation Statistics'), "federation"] ]],
-               'configuration' => [ L10n::t('Configuration'), [
-                       'site'         => ["admin/site/"        , L10n::t("Site")                 , "site"],
-                       'users'        => ["admin/users/"       , L10n::t("Users")                , "users"],
-                       'addons'       => ["admin/addons/"      , L10n::t("Addons")               , "addons"],
-                       'themes'       => ["admin/themes/"      , L10n::t("Themes")               , "themes"],
-                       'features'     => ["admin/features/"    , L10n::t("Additional features")  , "features"],
-                       'tos'          => ["admin/tos/"         , L10n::t("Terms of Service")     , "tos"] ]],
-               'database' => [ L10n::t('Database'), [
-                       'dbsync'       => ["admin/dbsync/"      , L10n::t('DB updates')           , "dbsync"],
-                       'queue'        => ["admin/queue/"       , L10n::t('Inspect Queue')        , "queue"],
-                       'workerqueue'  => ["admin/workerqueue/" , L10n::t('Inspect worker Queue') , "workerqueue"] ]],
-               'tools' => [ L10n::t('Tools'), [
-                       'contactblock' => ["admin/contactblock/", L10n::t('Contact Blocklist')    , "contactblock"],
-                       'blocklist'    => ["admin/blocklist/"   , L10n::t('Server Blocklist')     , "blocklist"],
-                       'deleteitem'   => ["admin/deleteitem/"  , L10n::t('Delete Item')          , 'deleteitem'], ]],
-               "logs" => [ L10n::t("Logs"), [
-                       "logsconfig" => ["admin/logs/", L10n::t("Logs"), "logs"],
-                       "logsview" => ["admin/viewlogs/", L10n::t("View Logs"), 'viewlogs']
+               'information' => [L10n::t('Information'), [
+                       'overview' => ['admin/', L10n::t('Overview'), 'overview'],
+                       'federation'   => ['admin/federation/'  , L10n::t('Federation Statistics'), 'federation']]],
+               'configuration' => [L10n::t('Configuration'), [
+                       'site'         => ['admin/site/'        , L10n::t('Site')                    , 'site'],
+                       'users'        => ['admin/users/'       , L10n::t('Users')                   , 'users'],
+                       'addons'       => ['admin/addons/'      , L10n::t('Addons')                  , 'addons'],
+                       'themes'       => ['admin/themes/'      , L10n::t('Themes')                  , 'themes'],
+                       'features'     => ['admin/features/'    , L10n::t('Additional features')     , 'features'],
+                       'tos'          => ['admin/tos/'         , L10n::t('Terms of Service')        , 'tos']]],
+               'database' => [L10n::t('Database'), [
+                       'dbsync'       => ['admin/dbsync/'      , L10n::t('DB updates')              , 'dbsync'],
+                       'queue'        => ['admin/queue/'       , L10n::t('Inspect Queue')           , 'queue'],
+                       'deferred'     => ['admin/deferred/'    , L10n::t('Inspect Deferred Workers'), 'deferred'],
+                       'workerqueue'  => ['admin/workerqueue/' , L10n::t('Inspect worker Queue')    , 'workerqueue']]],
+               'tools' => [L10n::t('Tools'), [
+                       'contactblock' => ['admin/contactblock/', L10n::t('Contact Blocklist')       , 'contactblock'],
+                       'blocklist'    => ['admin/blocklist/'   , L10n::t('Server Blocklist')        , 'blocklist'],
+                       'deleteitem'   => ['admin/deleteitem/'  , L10n::t('Delete Item')             , 'deleteitem'],]],
+               'logs' => [L10n::t('Logs'), [
+                       'logsconfig' => ['admin/logs/', L10n::t('Logs'), 'logs'],
+                       'logsview' => ['admin/viewlogs/', L10n::t('View Logs'), 'viewlogs']
                ]],
-               "diagnostics" => [ L10n::t("Diagnostics"), [
-                       "phpinfo" => ['phpinfo/', L10n::t('PHP Info'), 'phpinfo'],
-                       "probe" => ['probe/', L10n::t('probe address'), 'probe'],
-                       "webfinger" =>['webfinger/', L10n::t('check webfinger'), 'webfinger']
+               'diagnostics' => [L10n::t('Diagnostics'), [
+                       'phpinfo' => ['phpinfo/', L10n::t('PHP Info'), 'phpinfo'],
+                       'probe' => ['probe/', L10n::t('probe address'), 'probe'],
+                       'webfinger' =>['webfinger/', L10n::t('check webfinger'), 'webfinger']
                ]]
        ];
 
@@ -264,8 +265,11 @@ function admin_content(App $a)
                        case 'queue':
                                $o = admin_page_queue($a);
                                break;
+                       case 'deferred':
+                               $o = admin_page_workerqueue($a, true);
+                               break;
                        case 'workerqueue':
-                               $o = admin_page_workerqueue($a);
+                               $o = admin_page_workerqueue($a, false);
                                break;
                        case 'federation':
                                $o = admin_page_federation($a);
@@ -790,10 +794,20 @@ function admin_page_queue(App $a)
  * @param App $a
  * @return string
  */
-function admin_page_workerqueue(App $a)
+function admin_page_workerqueue(App $a, $deferred)
 {
        // get jobs from the workerqueue table
-       $entries = DBA::select('workerqueue', ['id', 'parameter', 'created', 'priority'], ['done' => 0], ['order'=> ['priority']]);
+       if ($deferred) {
+               $condition = ["NOT `done` AND `next_try` > ?", DateTimeFormat::utcNow()];
+               $sub_title = L10n::t('Inspect Deferred Worker Queue');
+               $info = L10n::t("This page lists the deferred worker jobs. This are jobs that couldn't be executed at the first time.");
+       } else {
+               $condition = ["NOT `done` AND `next_try` < ?", DateTimeFormat::utcNow()];
+               $sub_title = L10n::t('Inspect Worker Queue');
+               $info = L10n::t('This page lists the currently queued worker jobs. These jobs are handled by the worker cronjob you\'ve set up during install.');
+       }
+
+       $entries = DBA::select('workerqueue', ['id', 'parameter', 'created', 'priority'], $condition, ['order'=> ['priority']]);
 
        $r = [];
        while ($entry = DBA::fetch($entries)) {
@@ -807,13 +821,13 @@ function admin_page_workerqueue(App $a)
        $t = get_markup_template('admin/workerqueue.tpl');
        return replace_macros($t, [
                '$title' => L10n::t('Administration'),
-               '$page' => L10n::t('Inspect Worker Queue'),
+               '$page' => $sub_title,
                '$count' => count($r),
                '$id_header' => L10n::t('ID'),
                '$param_header' => L10n::t('Job Parameters'),
                '$created_header' => L10n::t('Created'),
                '$prio_header' => L10n::t('Priority'),
-               '$info' => L10n::t('This page lists the currently queued worker jobs. These jobs are handled by the worker cronjob you\'ve set up during install.'),
+               '$info' => $info,
                '$entries' => $r,
        ]);
 }
@@ -900,15 +914,17 @@ function admin_page_summary(App $a)
 
        $pending = Register::getPendingCount();
 
-       $r = q("SELECT COUNT(*) AS `total` FROM `queue` WHERE 1");
-       $queue = (($r) ? $r[0]['total'] : 0);
+       $queue = DBA::count('queue', []);
+
+       $deferred = DBA::count('workerqueue', ["`executed` <= ? AND NOT `done` AND `next_try` > ?",
+               DBA::NULL_DATETIME, DateTimeFormat::utcNow()]);
 
-       $r = q("SELECT COUNT(*) AS `total` FROM `workerqueue` WHERE NOT `done`");
-       $workerqueue = (($r) ? $r[0]['total'] : 0);
+       $workerqueue = DBA::count('workerqueue', ["`executed` <= ? AND NOT `done` AND `next_try` < ?",
+               DBA::NULL_DATETIME, DateTimeFormat::utcNow()]);
 
        // We can do better, but this is a quick queue status
 
-       $queues = ['label' => L10n::t('Message queues'), 'queue' => $queue, 'workerq' => $workerqueue];
+       $queues = ['label' => L10n::t('Message queues'), 'queue' => $queue, 'deferred' => $deferred, 'workerq' => $workerqueue];
 
 
        $r = q("SHOW variables LIKE 'max_allowed_packet'");

diff --git a/src/Core/Worker.php b/src/Core/Worker.php
index 76400d469848c4f218d3f19493f6137e94d473b7..a3c5884604debd24cb73125f031b52c4f0b93311 100644 (file)
--- a/src/Core/Worker.php
+++ b/src/Core/Worker.php
@@ -146,6 +146,17 @@ class Worker
                logger("Couldn't select a workerqueue entry, quitting process " . getmypid() . ".", LOGGER_DEBUG);
        }
 
+       /**
+        * @brief Returns the number of deferred entries in the worker queue
+        *
+        * @return integer Number of deferred entries in the worker queue
+        */
+       private static function deferredEntries()
+       {
+               return DBA::count('workerqueue', ["`executed` <= ? AND NOT `done` AND `next_try` > ?",
+                       DBA::NULL_DATETIME, DateTimeFormat::utcNow()]);
+       }
+
        /**
         * @brief Returns the number of non executed entries in the worker queue
         *
@@ -679,6 +690,7 @@ class Worker
                        }
 
                        $entries = self::totalEntries();
+                       $deferred = self::deferredEntries();
 
                        if (Config::get("system", "worker_fastlane", false) && ($queues > 0) && ($entries > 0) && ($active >= $queues)) {
                                $top_priority = self::highestPriority();
@@ -690,7 +702,7 @@ class Worker
                                }
                        }
 
-                       logger("Load: ".$load."/".$maxsysload." - processes: ".$active."/".$entries.$processlist." - maximum: ".$queues."/".$maxqueues, LOGGER_DEBUG);
+                       logger("Load: " . $load ."/" . $maxsysload . " - processes: " . $deferred . "/" . $active . "/" . $entries . $processlist . " - maximum: " . $queues . "/" . $maxqueues, LOGGER_DEBUG);
 
                        // Are there fewer workers running as possible? Then fork a new one.
                        if (!Config::get("system", "worker_dont_fork", false) && ($queues > ($active + 1)) && ($entries > 1)) {
@@ -797,13 +809,14 @@ class Worker
                $queue_length = Config::get('system', 'worker_fetch_limit', 1);
                $lower_job_limit = $worker_queues * $queue_length * 2;
                $jobs = self::totalEntries();
+               $deferred = self::deferredEntries();
 
                // Now do some magic
                $exponent = 2;
                $slope = $queue_length / pow($lower_job_limit, $exponent);
                $limit = min($queue_length, ceil($slope * pow($jobs, $exponent)));
 
-               logger('Total: '.$jobs.' - Maximum: '.$queue_length.' - jobs per queue: '.$limit, LOGGER_DEBUG);
+               logger('Deferred: ' . $deferred . ' - Total: ' . $jobs . ' - Maximum: ' . $queue_length . ' - jobs per queue: ' . $limit, LOGGER_DEBUG);
                $ids = [];
                if (self::passingSlow($highest_priority)) {
                        // Are there waiting processes with a higher priority than the currently highest?

diff --git a/view/templates/admin/summary.tpl b/view/templates/admin/summary.tpl
index 202fd43b3e52ab065647f65d96a7b5eb6daec480..298097f792b6e64fc5ee3e286f3d44a5050a107e 100644 (file)
--- a/view/templates/admin/summary.tpl
+++ b/view/templates/admin/summary.tpl
@@ -11,7 +11,7 @@
 
        <dl>
                <dt>{{$queues.label}}</dt>
-               <dd><a href="{{$baseurl}}/admin/queue">{{$queues.queue}}</a> - <a href="{{$baseurl}}/admin/workerqueue">{{$queues.workerq}}</a></dd>
+               <dd><a href="{{$baseurl}}/admin/queue">{{$queues.queue}}</a> - <a href="{{$baseurl}}/admin/deferred">{{$queues.deferred}} - <a href="{{$baseurl}}/admin/workerqueue">{{$queues.workerq}}</a></dd>
        </dl>
        <dl>
                <dt>{{$pending.0}}</dt>

diff --git a/view/theme/frio/templates/admin/summary.tpl b/view/theme/frio/templates/admin/summary.tpl
index 3c33f92fdbe5dd03cc398861807e1e13144715e5..66afda721b24fdfcf6fc0f84e49d39a4bee1b8c9 100644 (file)
--- a/view/theme/frio/templates/admin/summary.tpl
+++ b/view/theme/frio/templates/admin/summary.tpl
@@ -14,7 +14,7 @@
                {{* The work queues short statistic. *}}
                <div id="admin-summary-queues" class="col-lg-12 col-md-12 col-sm-12 col-xs-12 admin-summary">
                        <div class="col-lg-4 col-md-4 col-sm-4 col-xs-12 admin-summary-label-name text-muted">{{$queues.label}}</div>
-                       <div class="col-lg-8 col-md-8 col-sm-8 col-xs-12 admin-summary-entry"><a href="{{$baseurl}}/admin/queue">{{$queues.queue}}</a> - <a href="{{$baseurl}}/admin/workerqueue">{{$queues.workerq}}</a></div>
+                       <div class="col-lg-8 col-md-8 col-sm-8 col-xs-12 admin-summary-entry"><a href="{{$baseurl}}/admin/queue">{{$queues.queue}}</a> - <a href="{{$baseurl}}/admin/deferred">{{$queues.deferred}} - <a href="{{$baseurl}}/admin/workerqueue">{{$queues.workerq}}</a></div>
                </div>
 
                {{* Number of pending registrations. *}}