Better handling and logging for intermittent HTTP client submission errors.

diff --git a/TODO b/TODO
index 7d6ec4d16995df9bd832d23395f8e2448daa8b87..c4920231012c317d78831cf09c89482ef6db2c2a 100644 (file)
--- a/TODO
+++ b/TODO
@@ -1,7 +1,3 @@
-Some last few things to do before release.
-
-- Handle/investigate the HTTP client pipeline errors
-
 Consider what happens when multiple requests for a file are received.
 
 When another request comes in for a file already being downloaded,

diff --git a/apt_p2p/HTTPDownloader.py b/apt_p2p/HTTPDownloader.py
index 10831f1ba8bc9a47d217c27fd3b724c75165a924..2952f48cab295ab36381b26983f1c41a58984d2e 100644 (file)
--- a/apt_p2p/HTTPDownloader.py
+++ b/apt_p2p/HTTPDownloader.py
@@ -10,6 +10,7 @@ from twisted import version as twisted_version
 from twisted.python import log
 from twisted.web2.client.interfaces import IHTTPClientManager
 from twisted.web2.client.http import ProtocolError, ClientRequest, HTTPClientProtocol
+from twisted.web2.channel.http import PERSIST_NO_PIPELINE, PERSIST_PIPELINE
 from twisted.web2 import stream as stream_mod, http_headers
 from twisted.web2 import version as web2_version
 from twisted.trial import unittest
@@ -133,11 +134,26 @@ class Peer(ClientFactory):
             return
         if self.outstanding and not self.pipeline:
             return
+        if not ((self.proto.readPersistent is PERSIST_NO_PIPELINE
+                 and not self.proto.inRequests)
+                 or self.proto.readPersistent is PERSIST_PIPELINE):
+            log.msg('HTTP protocol is not ready though we were told to pipeline: %r, %r' %
+                    (self.proto.readPersistent, self.proto.inRequests))
+            return
 
         req, deferRequest, submissionTime = self.request_queue.pop(0)
+        try:
+            deferResponse = self.proto.submitRequest(req, False)
+        except:
+            # Try again later
+            log.msg('Got an error trying to submit a new HTTP request %s' % (request.uri, ))
+            log.err()
+            self.request_queue.insert(0, (request, deferRequest, submissionTime))
+            ractor.callLater(1, self.processQueue)
+            return
+            
         self.outstanding += 1
         self.rerank()
-        deferResponse = self.proto.submitRequest(req, False)
         deferResponse.addCallbacks(self.requestComplete, self.requestError,
                                    callbackArgs = (req, deferRequest, submissionTime),
                                    errbackArgs = (req, deferRequest))
@@ -147,7 +163,7 @@ class Peer(ClientFactory):
         self._processLastResponse()
         self.outstanding -= 1
         assert self.outstanding >= 0
-        log.msg('%s of %s completed with code %d' % (req.method, req.uri, resp.code))
+        log.msg('%s of %s completed with code %d (%r)' % (req.method, req.uri, resp.code, resp.headers))
         self._completed += 1
         now = datetime.now()
         self._responseTimes.append((now, now - submissionTime))