HTTPDownloader.py

   1
   2 import unittest
   3
   4 from twisted.web2.client import http
   5 from twisted.internet.defer import Deferred
   6 from twisted.internet.protocol import ClientFactory
   7 from twisted.web2.client.interfaces import IHTTPClientManager
   8 from zope.interface import implements
   9
  10 class HTTPClientManager(ClientFactory):
  11     """A manager for all HTTP requests to a site.
  12
  13
  14     """
  15
  16     implements(IHTTPClientManager)
  17
  18     protocol = HTTPClientProtocol
  19
  20     def __init__(self, host, port):
  21         self.host = host
  22         self.port = port
  23         self.client = http.HTTPClientProtocol(self)
  24         self.busy = False
  25         self.pipeline = False
  26         self.closed = False
  27         self.pending_requests = []
  28
  29     def get(self, path):
  30         uri = 'http://' + self.host + ':' + str(self.port) + path
  31         request = http.ClientRequest('GET', uri, {}, None)
  32         request.responseDefer = Deferred()
  33         self.pending_requests.append(request)
  34         if not self.busy:
  35             self._submitRequest()
  36
  37         return request.responseDefer
  38
  39     def _submitRequest(self):
  40         assert self.pending_requests
  41         if self.closed:
  42             del self.client
  43             self.client = http.HTTPClientProtocol(self)
  44
  45         request = self.pending_requests.pop()
  46         d = self.client.submitRequest(request, False)
  47         d.addCallback(request.responseDefer.callback)
  48
  49     def clientBusy(self, proto):
  50         self.busy = True
  51
  52     def clientIdle(self, proto):
  53         self.busy = False
  54         if self.pending_requests:
  55             self._submitRequest()
  56
  57     def clientPipelining(self, proto):
  58         self.pipeline = True
  59
  60     def clientGone(self, proto):
  61         self.closed = True
  62         self.busy = False
  63         self.pipeline = False
  64         del self.client
  65         if self.pending_requests:
  66             self._submitRequest()
  67
  68 class TestDownloader(unittest.TestCase):
  69
  70     def test_download(self):
  71         h = HTTPClientManager('www.google.ca', 80)
  72         def print_resutls(result):
  73             print result
  74
  75         d = h.get('/index.html')
  76         d.addCallback(print_results)
  77         reactor.run()
  78
  79 if __name__ == '__main__':
  80     unittest.main()