HTTPDownloader.py

   1
   2 from twisted.web2.client import http
   3 from twisted.internet.defer import Deferred
   4
   5 class HTTPClientManager(object):
   6     """A manager for all HTTP requests to a site.
   7
   8
   9     """
  10
  11     implements(IHTTPClientManager)
  12
  13     def __init__(self, host, port):
  14         self.host = host
  15         self.port = port
  16         self.client = http.HTTPClientProtocol(self)
  17         self.busy = False
  18         self.pipeline = False
  19         self.closed = False
  20         self.pending_requests = []
  21
  22     def get(self, path):
  23         uri = 'http://' + self.host + ':' + self.port + path
  24         request = http.ClientRequest('GET', uri, {}, None)
  25         request.responseDefer = Deferred()
  26         self.pending_requests.append(request)
  27         if not self.busy:
  28             self._submitRequest()
  29
  30         return request.responseDefer
  31
  32     def _submitRequest(self):
  33         assert self.pending_requests
  34         if self.closed:
  35             del self.client
  36             self.client = http.HTTPClientProtocol(self)
  37
  38         request = self.pending_requests.pop()
  39         d = self.client.submitRequest(request, False)
  40         d.addCallback(request.responseDefer.callback)
  41
  42     def clientBusy(self, proto):
  43         self.busy = True
  44
  45     def clientIdle(self, proto):
  46         self.busy = False
  47         if self.pending_requests:
  48             self._submitRequest()
  49
  50     def clientPipelining(self, proto):
  51         self.pipeline = True
  52
  53     def clientGone(self, proto):
  54         self.closed = True
  55         self.busy = False
  56         self.pipeline = False
  57         del self.client
  58         if self.pending_requests:
  59             self._submitRequest()