from twisted.internet import reactor, defer, protocol
from twisted.internet.protocol import ClientFactory
from twisted import version as twisted_version
+from twisted.python import log
from twisted.web2.client.interfaces import IHTTPClientManager
from twisted.web2.client.http import ProtocolError, ClientRequest, HTTPClientProtocol
from twisted.web2 import stream as stream_mod, http_headers
self.connector = None
def connect(self):
- assert(self.closed and not self.connecting)
+ assert self.closed and not self.connecting
self.connecting = True
d = protocol.ClientCreator(reactor, HTTPClientProtocol, self).connectTCP(self.host, self.port)
d.addCallback(self.connected)
req = self.request_queue.pop(0)
self.response_queue.append(req)
req.deferResponse = self.proto.submitRequest(req, False)
- req.deferResponse.addCallback(self.requestComplete)
- req.deferResponse.addErrback(self.requestError)
+ req.deferResponse.addCallbacks(self.requestComplete, self.requestError)
def requestComplete(self, resp):
req = self.response_queue.pop(0)
+ log.msg('%s of %s completed with code %d' % (req.method, req.uri, resp.code))
req.deferRequest.callback(resp)
def requestError(self, error):
req = self.response_queue.pop(0)
+ log.msg('Download of %s generated error %r' % (req.uri, error))
req.deferRequest.errback(error)
def clientBusy(self, proto):
(version.short(), twisted_version.short(), web2_version.short()))
return headers
- def get(self, path, method="GET"):
+ def get(self, path, method="GET", modtime=None):
headers = self.setCommonHeaders()
+ if modtime:
+ headers.setHeader('If-Modified-Since', modtime)
return self.submitRequest(ClientRequest(method, path, headers, None))
def getRange(self, path, rangeStart, rangeEnd, method="GET"):
stream_mod.readStream(resp.stream, print_).addCallback(printdone)
def test_download(self):
- host = 'www.camrdale.org'
+ host = 'www.ietf.org'
self.client = HTTPClientManager(host, 80)
self.timeout = 10
- d = self.client.get('/robots.txt')
- d.addCallback(self.gotResp, 1, 309)
+ d = self.client.get('/rfc/rfc0013.txt')
+ d.addCallback(self.gotResp, 1, 1070)
return d
def test_head(self):
- host = 'www.camrdale.org'
+ host = 'www.ietf.org'
self.client = HTTPClientManager(host, 80)
self.timeout = 10
- d = self.client.get('/robots.txt', "HEAD")
+ d = self.client.get('/rfc/rfc0013.txt', "HEAD")
d.addCallback(self.gotResp, 1, 0)
return d
def test_multiple_downloads(self):
- host = 'www.camrdale.org'
+ host = 'www.ietf.org'
self.client = HTTPClientManager(host, 80)
self.timeout = 120
lastDefer = defer.Deferred()
d = self.client.get(path)
d.addCallback(self.gotResp, num, expect)
if last:
- d.addCallback(lastDefer.callback)
+ d.addBoth(lastDefer.callback)
- newRequest("/", 1, 3433)
- newRequest("/blog/", 2, 37121)
- newRequest("/camrdale.html", 3, 2234)
- self.pending_calls.append(reactor.callLater(1, newRequest, '/robots.txt', 4, 309))
- self.pending_calls.append(reactor.callLater(10, newRequest, '/wikilink.html', 5, 3084))
- self.pending_calls.append(reactor.callLater(30, newRequest, '/sitemap.html', 6, 4750))
- self.pending_calls.append(reactor.callLater(31, newRequest, '/PlanetLab.html', 7, 2783))
- self.pending_calls.append(reactor.callLater(32, newRequest, '/openid.html', 8, 2525))
- self.pending_calls.append(reactor.callLater(32, newRequest, '/subpage.html', 9, 2381))
- self.pending_calls.append(reactor.callLater(62, newRequest, '/sitemap2.rss', 0, 302362, True))
+ newRequest("/rfc/rfc0006.txt", 1, 1776)
+ newRequest("/rfc/rfc2362.txt", 2, 159833)
+ newRequest("/rfc/rfc0801.txt", 3, 40824)
+ self.pending_calls.append(reactor.callLater(1, newRequest, '/rfc/rfc0013.txt', 4, 1070))
+ self.pending_calls.append(reactor.callLater(10, newRequest, '/rfc/rfc0022.txt', 5, 4606))
+ self.pending_calls.append(reactor.callLater(30, newRequest, '/rfc/rfc0048.txt', 6, 41696))
+ self.pending_calls.append(reactor.callLater(31, newRequest, '/rfc/rfc3261.txt', 7, 647976))
+ self.pending_calls.append(reactor.callLater(32, newRequest, '/rfc/rfc0014.txt', 8, 27))
+ self.pending_calls.append(reactor.callLater(32, newRequest, '/rfc/rfc0001.txt', 9, 21088))
+ self.pending_calls.append(reactor.callLater(62, newRequest, '/rfc/rfc2801.txt', 0, 598794, True))
return lastDefer
def test_multiple_quick_downloads(self):
- host = 'www.camrdale.org'
+ host = 'www.ietf.org'
self.client = HTTPClientManager(host, 80)
self.timeout = 30
lastDefer = defer.Deferred()
d = self.client.get(path)
d.addCallback(self.gotResp, num, expect)
if last:
- d.addCallback(lastDefer.callback)
+ d.addBoth(lastDefer.callback)
- newRequest("/", 1, 3433)
- newRequest("/blog/", 2, 37121)
- newRequest("/camrdale.html", 3, 2234)
- self.pending_calls.append(reactor.callLater(0, newRequest, '/robots.txt', 4, 309))
- self.pending_calls.append(reactor.callLater(0, newRequest, '/wikilink.html', 5, 3084))
- self.pending_calls.append(reactor.callLater(0, newRequest, '/sitemap.html', 6, 4750))
- self.pending_calls.append(reactor.callLater(0, newRequest, '/PlanetLab.html', 7, 2783))
- self.pending_calls.append(reactor.callLater(0, newRequest, '/openid.html', 8, 2525))
- self.pending_calls.append(reactor.callLater(0, newRequest, '/subpage.html', 9, 2381))
- self.pending_calls.append(reactor.callLater(0, newRequest, '/sitemap2.rss', 0, 302362, True))
+ newRequest("/rfc/rfc0006.txt", 1, 1776)
+ newRequest("/rfc/rfc2362.txt", 2, 159833)
+ newRequest("/rfc/rfc0801.txt", 3, 40824)
+ self.pending_calls.append(reactor.callLater(0, newRequest, '/rfc/rfc0013.txt', 4, 1070))
+ self.pending_calls.append(reactor.callLater(0, newRequest, '/rfc/rfc0022.txt', 5, 4606))
+ self.pending_calls.append(reactor.callLater(0, newRequest, '/rfc/rfc0048.txt', 6, 41696))
+ self.pending_calls.append(reactor.callLater(0, newRequest, '/rfc/rfc3261.txt', 7, 647976))
+ self.pending_calls.append(reactor.callLater(0, newRequest, '/rfc/rfc0014.txt', 8, 27))
+ self.pending_calls.append(reactor.callLater(0, newRequest, '/rfc/rfc0001.txt', 9, 21088))
+ self.pending_calls.append(reactor.callLater(0, newRequest, '/rfc/rfc2801.txt', 0, 598794, True))
return lastDefer
def test_range(self):
- host = 'www.camrdale.org'
+ host = 'www.ietf.org'
self.client = HTTPClientManager(host, 80)
self.timeout = 10
- d = self.client.getRange('/robots.txt', 100, 199)
+ d = self.client.getRange('/rfc/rfc0013.txt', 100, 199)
d.addCallback(self.gotResp, 1, 100)
return d