2 from random import choice
3 from urlparse import urlparse, urlunparse
4 from urllib import quote_plus
6 from twisted.internet import reactor, defer
7 from twisted.python import log
8 from twisted.trial import unittest
9 from twisted.web2 import stream as stream_mod
10 from twisted.web2.http import splitHostPort
12 from HTTPDownloader import HTTPClientManager
13 from util import uncompact
19 def get(self, hash, mirror, peers = [], method="GET", modtime=None):
20 """Download from a list of peers or fallback to a mirror.
22 @type peers: C{list} of C{string}
23 @param peers: a list of the peers where the file can be found
26 compact_peer = choice(peers)
27 peer = uncompact(compact_peer['c'])
28 log.msg('Downloading from peer %r' % (peer, ))
30 path = '/~/' + quote_plus(hash.expected())
32 log.msg('Downloading (%s) from mirror %s' % (method, mirror))
33 parsed = urlparse(mirror)
34 assert parsed[0] == "http", "Only HTTP is supported, not '%s'" % parsed[0]
35 host, port = splitHostPort(parsed[0], parsed[1])
36 path = urlunparse(('', '') + parsed[2:])
38 return self.getPeer(host, port, path, method, modtime)
40 def getPeer(self, host, port, path, method="GET", modtime=None):
43 site = host + ":" + str(port)
44 if site not in self.clients:
45 self.clients[site] = HTTPClientManager(host, port)
46 return self.clients[site].get(path, method, modtime)
49 for site in self.clients:
50 self.clients[site].close()
53 class TestPeerManager(unittest.TestCase):
54 """Unit tests for the PeerManager."""
59 def gotResp(self, resp, num, expect):
60 self.failUnless(resp.code >= 200 and resp.code < 300, "Got a non-200 response: %r" % resp.code)
61 if expect is not None:
62 self.failUnless(resp.stream.length == expect, "Length was incorrect, got %r, expected %r" % (resp.stream.length, expect))
67 stream_mod.readStream(resp.stream, print_).addCallback(printdone)
69 def test_download(self):
70 self.manager = PeerManager()
74 d = self.manager.get('', 'http://' + host + '/rfc/rfc0013.txt')
75 d.addCallback(self.gotResp, 1, 1070)
79 self.manager = PeerManager()
83 d = self.manager.get('', 'http://' + host + '/rfc/rfc0013.txt', method = "HEAD")
84 d.addCallback(self.gotResp, 1, 0)
87 def test_multiple_downloads(self):
88 self.manager = PeerManager()
90 lastDefer = defer.Deferred()
92 def newRequest(host, path, num, expect, last=False):
93 d = self.manager.get('', 'http://' + host + ':' + str(80) + path)
94 d.addCallback(self.gotResp, num, expect)
96 d.addBoth(lastDefer.callback)
98 newRequest('www.ietf.org', "/rfc/rfc0006.txt", 1, 1776)
99 newRequest('www.ietf.org', "/rfc/rfc2362.txt", 2, 159833)
100 newRequest('www.google.ca', "/", 3, None)
101 self.pending_calls.append(reactor.callLater(1, newRequest, 'www.sfu.ca', '/', 4, None))
102 self.pending_calls.append(reactor.callLater(10, newRequest, 'www.ietf.org', '/rfc/rfc0048.txt', 5, 41696))
103 self.pending_calls.append(reactor.callLater(30, newRequest, 'www.ietf.org', '/rfc/rfc0022.txt', 6, 4606))
104 self.pending_calls.append(reactor.callLater(31, newRequest, 'www.sfu.ca', '/studentcentral/index.html', 7, None))
105 self.pending_calls.append(reactor.callLater(32, newRequest, 'www.ietf.org', '/rfc/rfc0014.txt', 8, 27))
106 self.pending_calls.append(reactor.callLater(32, newRequest, 'www.ietf.org', '/rfc/rfc0001.txt', 9, 21088))
107 self.pending_calls.append(reactor.callLater(62, newRequest, 'www.google.ca', '/intl/en/options/', 0, None, True))
111 for p in self.pending_calls:
114 self.pending_calls = []