From 4efc0d25095680d57493fee290c2ee4b4a2f44bb Mon Sep 17 00:00:00 2001 From: Cameron Dale Date: Sun, 9 Dec 2007 23:20:59 -0800 Subject: [PATCH] First attempt (untested) at new HTTP downloader. --- HTTPDownloader.py | 59 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 HTTPDownloader.py diff --git a/HTTPDownloader.py b/HTTPDownloader.py new file mode 100644 index 0000000..7d96444 --- /dev/null +++ b/HTTPDownloader.py @@ -0,0 +1,59 @@ + +from twisted.web2.client import http +from twisted.internet.defer import Deferred + +class HTTPClientManager(object): + """A manager for all HTTP requests to a site. + + + """ + + implements(IHTTPClientManager) + + def __init__(self, host, port): + self.host = host + self.port = port + self.client = http.HTTPClientProtocol(self) + self.busy = False + self.pipeline = False + self.closed = False + self.pending_requests = [] + + def get(self, path): + uri = 'http://' + self.host + ':' + self.port + path + request = http.ClientRequest('GET', uri, {}, None) + request.responseDefer = Deferred() + self.pending_requests.append(request) + if not self.busy: + self._submitRequest() + + return request.responseDefer + + def _submitRequest(self): + assert self.pending_requests + if self.closed: + del self.client + self.client = http.HTTPClientProtocol(self) + + request = self.pending_requests.pop() + d = self.client.submitRequest(request, False) + d.addCallback(request.responseDefer.callback) + + def clientBusy(self, proto): + self.busy = True + + def clientIdle(self, proto): + self.busy = False + if self.pending_requests: + self._submitRequest() + + def clientPipelining(self, proto): + self.pipeline = True + + def clientGone(self, proto): + self.closed = True + self.busy = False + self.pipeline = False + del self.client + if self.pending_requests: + self._submitRequest() -- 2.39.2