X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;f=lib%2Fstompqueuemanager.php;h=4084470e211f9cc0c656c18512f637bf6cd48e19;hb=b53e1439969bfa2c0b551d8cc2fc8fe15652c62a;hp=ec150bbb6e14b0f99ff4d406d8f743dbe409670b;hpb=da34de74fece355785710bc4ff0bab03389a2877;p=quix0rs-gnu-social.git diff --git a/lib/stompqueuemanager.php b/lib/stompqueuemanager.php index ec150bbb6e..4084470e21 100644 --- a/lib/stompqueuemanager.php +++ b/lib/stompqueuemanager.php @@ -31,7 +31,6 @@ require_once 'Stomp.php'; require_once 'Stomp/Exception.php'; - class StompQueueManager extends QueueManager { protected $servers; @@ -40,8 +39,9 @@ class StompQueueManager extends QueueManager protected $base; protected $control; - protected $useTransactions = true; - + protected $useTransactions; + protected $useAcks; + protected $sites = array(); protected $subscriptions = array(); @@ -60,10 +60,13 @@ class StompQueueManager extends QueueManager } else { $this->servers = array($server); } - $this->username = common_config('queue', 'stomp_username'); - $this->password = common_config('queue', 'stomp_password'); - $this->base = common_config('queue', 'queue_basename'); - $this->control = common_config('queue', 'control_channel'); + $this->username = common_config('queue', 'stomp_username'); + $this->password = common_config('queue', 'stomp_password'); + $this->base = common_config('queue', 'queue_basename'); + $this->control = common_config('queue', 'control_channel'); + $this->breakout = common_config('queue', 'breakout'); + $this->useTransactions = common_config('queue', 'stomp_transactions'); + $this->useAcks = common_config('queue', 'stomp_acks'); } /** @@ -75,20 +78,6 @@ class StompQueueManager extends QueueManager return IoManager::INSTANCE_PER_PROCESS; } - /** - * Record each site we'll be handling input for in this process, - * so we can listen to the necessary queues for it. - * - * @fixme possibly actually do subscription here to save another - * loop over all sites later? - * @fixme possibly don't assume it's the current site - */ - public function addSite($server) - { - $this->sites[] = $server; - $this->initialize(); - } - /** * Optional; ping any running queue handler daemons with a notification * such as announcing a new site to handle or requesting clean shutdown. @@ -108,9 +97,10 @@ class StompQueueManager extends QueueManager $message .= ':' . $param; } $this->_connect(); - $result = $this->_send($this->control, - $message, - array ('created' => common_sql_now())); + $con = $this->cons[$this->defaultIdx]; + $result = $con->send($this->control, + $message, + array ('created' => common_sql_now())); if ($result) { $this->_log(LOG_INFO, "Sent control ping to queue daemons: $message"); return true; @@ -121,68 +111,33 @@ class StompQueueManager extends QueueManager } /** - * Instantiate the appropriate QueueHandler class for the given queue. + * Saves an object into the queue item table. * + * @param mixed $object * @param string $queue - * @return mixed QueueHandler or null - */ - function getHandler($queue) - { - $handlers = $this->handlers[$this->currentSite()]; - if (isset($handlers[$queue])) { - $class = $handlers[$queue]; - if (class_exists($class)) { - return new $class(); - } else { - common_log(LOG_ERR, "Nonexistent handler class '$class' for queue '$queue'"); - } - } else { - common_log(LOG_ERR, "Requested handler for unkown queue '$queue'"); - } - return null; - } - - /** - * Get a list of all registered queue transport names. - * - * @return array of strings - */ - function getQueues() - { - $group = $this->activeGroup(); - $site = $this->currentSite(); - if (empty($this->groups[$site][$group])) { - return array(); - } else { - return array_keys($this->groups[$site][$group]); - } - } - - /** - * Register a queue transport name and handler class for your plugin. - * Only registered transports will be reliably picked up! + * @param string $siteNickname optional override to drop into another site's queue * - * @param string $transport - * @param string $class - * @param string $group - */ - public function connect($transport, $class, $group='queuedaemon') - { - $this->handlers[$this->currentSite()][$transport] = $class; - $this->groups[$this->currentSite()][$group][$transport] = $class; - } - - /** - * Saves a notice object reference into the queue item table. * @return boolean true on success * @throws StompException on connection or send error */ - public function enqueue($object, $queue) + public function enqueue($object, $queue, $siteNickname=null) { $this->_connect(); - return $this->_doEnqueue($object, $queue, $this->defaultIdx); + if (common_config('queue', 'stomp_enqueue_on')) { + // We're trying to force all writes to a single server. + // WARNING: this might do odd things if that server connection dies. + $idx = array_search(common_config('queue', 'stomp_enqueue_on'), + $this->servers); + if ($idx === false) { + common_log(LOG_ERR, 'queue stomp_enqueue_on setting does not match our server list.'); + $idx = $this->defaultIdx; + } + } else { + $idx = $this->defaultIdx; + } + return $this->_doEnqueue($object, $queue, $idx, $siteNickname); } - + /** * Saves a notice object reference into the queue item table * on the given connection. @@ -190,10 +145,13 @@ class StompQueueManager extends QueueManager * @return boolean true on success * @throws StompException on connection or send error */ - protected function _doEnqueue($object, $queue, $idx) + protected function _doEnqueue($object, $queue, $idx, $siteNickname=null) { - $msg = $this->encode($object); $rep = $this->logrep($object); + $envelope = array('site' => $siteNickname ? $siteNickname : common_config('site', 'nickname'), + 'handler' => $queue, + 'payload' => $this->encode($object)); + $msg = serialize($envelope); $props = array('created' => common_sql_now()); if ($this->isPersistent($queue)) { @@ -202,14 +160,15 @@ class StompQueueManager extends QueueManager $con = $this->cons[$idx]; $host = $con->getServer(); - $result = $con->send($this->queueName($queue), $msg, $props); + $target = $this->queueName($queue); + $result = $con->send($target, $msg, $props); if (!$result) { - common_log(LOG_ERR, "Error sending $rep to $queue queue on $host"); + $this->_log(LOG_ERR, "Error sending $rep to $queue queue on $host $target"); return false; } - common_log(LOG_DEBUG, "complete remote queueing $rep for $queue on $host"); + $this->_log(LOG_DEBUG, "complete remote queueing $rep for $queue on $host $target"); $this->stats('enqueued', $queue); return true; } @@ -275,12 +234,14 @@ class StompQueueManager extends QueueManager $idx = $this->connectionFromSocket($socket); $con = $this->cons[$idx]; $host = $con->getServer(); + $this->defaultIdx = $idx; $ok = true; try { $frames = $con->readFrames(); } catch (StompException $e) { - common_log(LOG_ERR, "Lost connection to $host: " . $e->getMessage()); + $this->_log(LOG_ERR, "Lost connection to $host: " . $e->getMessage()); + fclose($socket); // ??? $this->cons[$idx] = null; $this->transaction[$idx] = null; $this->disconnect[$idx] = time(); @@ -289,14 +250,17 @@ class StompQueueManager extends QueueManager foreach ($frames as $frame) { $dest = $frame->headers['destination']; if ($dest == $this->control) { - if (!$this->handleControlSignal($idx, $frame)) { + if (!$this->handleControlSignal($frame)) { // We got a control event that requests a shutdown; // close out and stop handling anything else! break; } } else { - $ok = $ok && $this->handleItem($idx, $frame); + $ok = $this->handleItem($frame) && $ok; } + $this->ack($idx, $frame); + $this->commit($idx); + $this->begin($idx); } return $ok; } @@ -333,32 +297,17 @@ class StompQueueManager extends QueueManager parent::start($master); $this->_connectAll(); - common_log(LOG_INFO, "Subscribing to $this->control"); - foreach ($this->cons as $con) { - if ($con) { - $con->subscribe($this->control); - } - } - if ($this->sites) { - foreach ($this->sites as $server) { - StatusNet::init($server); - $this->doSubscribe(); - } - } else { - $this->doSubscribe(); - } foreach ($this->cons as $i => $con) { if ($con) { + $this->doSubscribe($con); $this->begin($i); } } return true; } - + /** - * Subscribe to all the queues we're going to need to handle... - * - * Side effects: in multi-site mode, may reset site configuration. + * Close out any active connections. * * @return bool return false on failure */ @@ -369,29 +318,13 @@ class StompQueueManager extends QueueManager foreach ($this->cons as $i => $con) { if ($con) { $this->rollback($i); - $con->unsubscribe($this->control); + $con->disconnect(); + $this->cons[$i] = null; } } - if ($this->sites) { - foreach ($this->sites as $server) { - StatusNet::init($server); - $this->doUnsubscribe(); - } - } else { - $this->doUnsubscribe(); - } return true; } - /** - * Get identifier of the currently active site configuration - * @return string - */ - protected function currentSite() - { - return common_config('site', 'server'); // @fixme switch to nickname - } - /** * Lazy open a single connection to Stomp queue server. * If multiple servers are configured, we let the Stomp client library @@ -448,6 +381,10 @@ class StompQueueManager extends QueueManager } } + /** + * Attempt to manually reconnect to the Stomp server for the given + * slot. If successful, set up our subscriptions on it. + */ protected function _reconnect($idx) { try { @@ -459,18 +396,8 @@ class StompQueueManager extends QueueManager if ($con) { $this->cons[$idx] = $con; $this->disconnect[$idx] = null; - - // now we have to listen to everything... - // @fixme refactor this nicer. :P - $host = $con->getServer(); - $this->_log(LOG_INFO, "Resubscribing to $this->control on $host"); - $con->subscribe($this->control); - foreach ($this->subscriptions as $site => $queues) { - foreach ($queues as $queue) { - $this->_log(LOG_INFO, "Resubscribing to $queue on $host"); - $con->subscribe($queue); - } - } + + $this->doSubscribe($con); $this->begin($idx); } else { // Try again later... @@ -494,42 +421,47 @@ class StompQueueManager extends QueueManager } /** - * Subscribe to all enabled notice queues for the current site. + * Set up all our raw queue subscriptions on the given connection + * @param LiberalStomp $con */ - protected function doSubscribe() + protected function doSubscribe(LiberalStomp $con) { - $site = $this->currentSite(); - $this->_connect(); - foreach ($this->getQueues() as $queue) { - $rawqueue = $this->queueName($queue); - $this->subscriptions[$site][$queue] = $rawqueue; - $this->_log(LOG_INFO, "Subscribing to $rawqueue"); - foreach ($this->cons as $con) { - if ($con) { - $con->subscribe($rawqueue); - } - } + $host = $con->getServer(); + foreach ($this->subscriptions() as $sub) { + $this->_log(LOG_INFO, "Subscribing to $sub on $host"); + $con->subscribe($sub); } } - + /** - * Subscribe from all enabled notice queues for the current site. + * Grab a full list of stomp-side queue subscriptions. + * Will include: + * - control broadcast channel + * - shared group queues for active groups + * - per-handler and per-site breakouts from $config['queue']['breakout'] + * that are rooted in the active groups. + * + * @return array of strings */ - protected function doUnsubscribe() + protected function subscriptions() { - $site = $this->currentSite(); - $this->_connect(); - if (!empty($this->subscriptions[$site])) { - foreach ($this->subscriptions[$site] as $queue => $rawqueue) { - $this->_log(LOG_INFO, "Unsubscribing from $rawqueue"); - foreach ($this->cons as $con) { - if ($con) { - $con->unsubscribe($rawqueue); - } - } - unset($this->subscriptions[$site][$queue]); + $subs = array(); + $subs[] = $this->control; + + foreach ($this->activeGroups as $group) { + $subs[] = $this->base . $group; + } + + foreach ($this->breakout as $spec) { + $parts = explode('/', $spec); + if (count($parts) < 2 || count($parts) > 3) { + common_log(LOG_ERR, "Bad queue breakout specifier $spec"); + } + if (in_array($parts[0], $this->activeGroups)) { + $subs[] = $this->base . $spec; } } + return array_unique($subs); } /** @@ -541,73 +473,130 @@ class StompQueueManager extends QueueManager * Side effects: in multi-site mode, may reset site configuration to * match the site that queued the event. * - * @param int $idx connection index * @param StompFrame $frame - * @return bool + * @return bool success */ - protected function handleItem($idx, $frame) + protected function handleItem($frame) { - $this->defaultIdx = $idx; + $host = $this->cons[$this->defaultIdx]->getServer(); + $message = unserialize($frame->body); - list($site, $queue) = $this->parseDestination($frame->headers['destination']); - if ($site != $this->currentSite()) { - $this->stats('switch'); - StatusNet::init($site); + if ($message === false) { + $this->_log(LOG_ERR, "Can't unserialize frame: {$frame->body}"); + $this->_log(LOG_ERR, "Unserializable frame length: " . strlen($frame->body)); + return false; } - $host = $this->cons[$idx]->getServer(); - if (is_numeric($frame->body)) { - $id = intval($frame->body); - $info = "notice $id posted at {$frame->headers['created']} in queue $queue from $host"; + $site = $message['site']; + $queue = $message['handler']; - $notice = Notice::staticGet('id', $id); - if (empty($notice)) { - $this->_log(LOG_WARNING, "Skipping missing $info"); - $this->ack($idx, $frame); - $this->commit($idx); - $this->begin($idx); - $this->stats('badnotice', $queue); - return false; - } + if ($this->isDeadletter($frame, $message)) { + $this->stats('deadletter', $queue); + return false; + } - $item = $notice; - } else { - // @fixme should we serialize, or json, or what here? - $info = "string posted at {$frame->headers['created']} in queue $queue from $host"; - $item = $frame->body; + // @fixme detect failing site switches + $this->switchSite($site); + + try { + $item = $this->decode($message['payload']); + } catch (Exception $e) { + $this->_log(LOG_ERR, "Skipping empty or deleted item in queue $queue from $host"); + $this->stats('baditem', $queue); + return false; } + $info = $this->logrep($item) . " posted at " . + $frame->headers['created'] . " in queue $queue from $host"; + $this->_log(LOG_DEBUG, "Dequeued $info"); $handler = $this->getHandler($queue); if (!$handler) { $this->_log(LOG_ERR, "Missing handler class; skipping $info"); - $this->ack($idx, $frame); - $this->commit($idx); - $this->begin($idx); $this->stats('badhandler', $queue); return false; } - $ok = $handler->handle($item); + try { + $ok = $handler->handle($item); + } catch (Exception $e) { + $this->_log(LOG_ERR, "Exception on queue $queue: " . $e->getMessage()); + $ok = false; + } - if (!$ok) { + if ($ok) { + $this->_log(LOG_INFO, "Successfully handled $info"); + $this->stats('handled', $queue); + } else { $this->_log(LOG_WARNING, "Failed handling $info"); - // FIXME we probably shouldn't have to do - // this kind of queue management ourselves; - // if we don't ack, it should resend... - $this->ack($idx, $frame); + // Requeing moves the item to the end of the line for its next try. + // @fixme add a manual retry count $this->enqueue($item, $queue); - $this->commit($idx); - $this->begin($idx); $this->stats('requeued', $queue); - return false; } - $this->_log(LOG_INFO, "Successfully handled $info"); - $this->ack($idx, $frame); - $this->commit($idx); - $this->begin($idx); - $this->stats('handled', $queue); - return true; + return $ok; + } + + /** + * Check if a redelivered message has been run through enough + * that we're going to give up on it. + * + * @param StompFrame $frame + * @param array $message unserialized message body + * @return boolean true if we should discard + */ + protected function isDeadLetter($frame, $message) + { + if (isset($frame->headers['redelivered']) && $frame->headers['redelivered'] == 'true') { + // Message was redelivered, possibly indicating a previous failure. + $msgId = $frame->headers['message-id']; + $site = $message['site']; + $queue = $message['handler']; + $msgInfo = "message $msgId for $site in queue $queue"; + + $deliveries = $this->incDeliveryCount($msgId); + if ($deliveries > common_config('queue', 'max_retries')) { + $info = "DEAD-LETTER FILE: Gave up after retry $deliveries on $msgInfo"; + + $outdir = common_config('queue', 'dead_letter_dir'); + if ($outdir) { + $filename = $outdir . "/$site-$queue-" . rawurlencode($msgId); + $info .= ": dumping to $filename"; + file_put_contents($filename, $message['payload']); + } + + common_log(LOG_ERR, $info); + return true; + } else { + common_log(LOG_INFO, "retry $deliveries on $msgInfo"); + } + } + return false; + } + + /** + * Update count of times we've re-encountered this message recently, + * triggered when we get a message marked as 'redelivered'. + * + * Requires a CLI-friendly cache configuration. + * + * @param string $msgId message-id header from message + * @return int number of retries recorded + */ + function incDeliveryCount($msgId) + { + $count = 0; + $cache = Cache::instance(); + if ($cache) { + $key = 'statusnet:stomp:message-retries:' . $msgId; + $count = $cache->increment($key); + if (!$count) { + $count = 1; + $cache->set($key, $count, null, 3600); + $got = $cache->get($key); + } + } + return $count; } /** @@ -640,86 +629,90 @@ class StompQueueManager extends QueueManager } else { $this->_log(LOG_ERR, "Ignoring unrecognized control message: $message"); } - - $this->ack($idx, $frame); - $this->commit($idx); - $this->begin($idx); return $shutdown; } - + /** - * Set us up with queue subscriptions for a new site added at runtime, + * Switch site, if necessary, and reset current handler assignments + * @param string $site + */ + function switchSite($site) + { + if ($site != GNUsocial::currentSite()) { + $this->stats('switch'); + GNUsocial::switchSite($site); + $this->initialize(); + } + } + + /** + * (Re)load runtime configuration for a given site by nickname, * triggered by a broadcast to the 'statusnet-control' topic. * + * Configuration changes in database should update, but config + * files might not. + * * @param array $frame Stomp frame * @return bool true to continue; false to stop further processing. */ protected function updateSiteConfig($nickname) { - if (empty($this->sites)) { - if ($nickname == common_config('site', 'nickname')) { - StatusNet::init(common_config('site', 'server')); - $this->doUnsubscribe(); - $this->doSubscribe(); - } else { - $this->_log(LOG_INFO, "Ignoring update ping for other site $nickname"); + $sn = Status_network::getKV('nickname', $nickname); + if ($sn) { + $this->switchSite($nickname); + if (!in_array($nickname, $this->sites)) { + $this->addSite(); } + $this->stats('siteupdate'); } else { - $sn = Status_network::staticGet($nickname); - if ($sn) { - $server = $sn->getServerName(); // @fixme do config-by-nick - StatusNet::init($server); - if (empty($this->sites[$server])) { - $this->addSite($server); - } - $this->_log(LOG_INFO, "(Re)subscribing to queues for site $nickname / $server"); - $this->doUnsubscribe(); - $this->doSubscribe(); - $this->stats('siteupdate'); - } else { - $this->_log(LOG_ERR, "Ignoring ping for unrecognized new site $nickname"); - } + $this->_log(LOG_ERR, "Ignoring ping for unrecognized new site $nickname"); } } /** * Combines the queue_basename from configuration with the - * site server name and queue name to give eg: + * group name for this queue to give eg: * - * /queue/statusnet/identi.ca/sms + * /queue/statusnet/main + * /queue/statusnet/main/distrib + * /queue/statusnet/xmpp/xmppout/site01 * * @param string $queue * @return string */ protected function queueName($queue) { - return common_config('queue', 'queue_basename') . - $this->currentSite() . '/' . $queue; + $group = $this->queueGroup($queue); + $site = GNUsocial::currentSite(); + + $specs = array("$group/$queue/$site", + "$group/$queue"); + foreach ($specs as $spec) { + if (in_array($spec, $this->breakout)) { + return $this->base . $spec; + } + } + return $this->base . $group; } /** - * Returns the site and queue name from the server-side queue. + * Get the breakout mode for the given queue on the current site. * - * @param string queue destination (eg '/queue/statusnet/identi.ca/sms') - * @return array of site and queue: ('identi.ca','sms') or false if unrecognized + * @param string $queue + * @return string one of 'shared', 'handler', 'site' */ - protected function parseDestination($dest) + protected function breakoutMode($queue) { - $prefix = common_config('queue', 'queue_basename'); - if (substr($dest, 0, strlen($prefix)) == $prefix) { - $rest = substr($dest, strlen($prefix)); - return explode("/", $rest, 2); + $breakout = common_config('queue', 'breakout'); + if (isset($breakout[$queue])) { + return $breakout[$queue]; + } else if (isset($breakout['*'])) { + return $breakout['*']; } else { - common_log(LOG_ERR, "Got a message from unrecognized stomp queue: $dest"); - return array(false, false); + return 'shared'; } } - function _log($level, $msg) - { - common_log($level, 'StompQueueManager: '.$msg); - } - protected function begin($idx) { if ($this->useTransactions) { @@ -734,13 +727,15 @@ class StompQueueManager extends QueueManager protected function ack($idx, $frame) { - if ($this->useTransactions) { - if (empty($this->transaction[$idx])) { - throw new Exception("Tried to ack but not in a transaction"); + if ($this->useAcks) { + if ($this->useTransactions) { + if (empty($this->transaction[$idx])) { + throw new Exception("Tried to ack but not in a transaction"); + } + $this->cons[$idx]->ack($frame, $this->transaction[$idx]); + } else { + $this->cons[$idx]->ack($frame); } - $this->cons[$idx]->ack($frame, $this->transaction[$idx]); - } else { - $this->cons[$idx]->ack($frame); } }