2 from urlparse import urlparse
5 from twisted.python import log, filepath
6 from twisted.internet import defer
7 from twisted.trial import unittest
8 from twisted.web2 import stream
9 from twisted.web2.http import splitHostPort
11 from AptPackages import AptPackages
15 class MirrorError(Exception):
16 """Exception raised when there's a problem with the mirror."""
18 class ProxyFileStream(stream.SimpleStream):
19 """Saves a stream to a file while providing a new stream."""
21 def __init__(self, stream, outFile):
22 """Initializes the proxy.
24 @type stream: C{twisted.web2.stream.IByteStream}
25 @param stream: the input stream to read from
26 @type outFile: C{twisted.python.filepath.FilePath}
27 @param outFile: the file to write to
30 self.outFile = outFile.open('w')
31 self.length = self.stream.length
35 """Close the output file."""
39 """Read some data from the stream."""
40 if self.outFile.closed:
43 data = self.stream.read()
44 if isinstance(data, defer.Deferred):
45 data.addCallbacks(self._write, self._done)
51 def _write(self, data):
52 """Write the stream data to the file and return it for others to use."""
57 self.outFile.write(data)
61 """Clean everything up and return None to future reads."""
67 """Manages all requests for mirror objects."""
69 def __init__(self, cache_dir):
70 self.cache_dir = cache_dir
71 self.cache = filepath.FilePath(self.cache_dir)
74 def extractPath(self, url):
75 parsed = urlparse(url)
76 host, port = splitHostPort(parsed[0], parsed[1])
77 site = host + ":" + str(port)
80 i = max(path.rfind('/dists/'), path.rfind('/pool/'))
85 # Uh oh, this is not good
86 log.msg("Couldn't find a good base directory for path: %s" % (site + path))
88 if site in self.apt_caches:
90 for base in self.apt_caches[site]:
92 for dirs in path.split('/'):
93 if base.startswith(base_match + '/' + dirs):
94 base_match += '/' + dirs
97 if len(base_match) > longest_match:
98 longest_match = len(base_match)
100 log.msg("Settled on baseDir: %s" % baseDir)
102 log.msg("Parsing '%s' gave '%s', '%s', '%s'" % (url, site, baseDir, path))
103 return site, baseDir, path
105 def init(self, site, baseDir):
106 if site not in self.apt_caches:
107 self.apt_caches[site] = {}
109 if baseDir not in self.apt_caches[site]:
110 site_cache = os.path.join(self.cache_dir, aptpkg_dir, 'mirrors', site + baseDir.replace('/', '_'))
111 self.apt_caches[site][baseDir] = AptPackages(site_cache)
113 def updatedFile(self, url, file_path):
114 site, baseDir, path = self.extractPath(url)
115 self.init(site, baseDir)
116 self.apt_caches[site][baseDir].file_updated(path, file_path)
118 def findHash(self, url):
119 log.msg('Trying to find hash for %s' % url)
120 site, baseDir, path = self.extractPath(url)
121 if site in self.apt_caches and baseDir in self.apt_caches[site]:
122 return self.apt_caches[site][baseDir].findHash(path)
124 d.errback(MirrorError("Site Not Found"))
127 def save_file(self, response, hash, size, url):
128 """Save a downloaded file to the cache and stream it."""
129 log.msg('Returning file: %s' % url)
131 parsed = urlparse(url)
132 destFile = self.cache.preauthChild(parsed[1] + parsed[2])
133 log.msg('Cache file: %s' % destFile.path)
135 if destFile.exists():
136 log.err('File already exists: %s', destFile.path)
140 destFile.parent().makedirs()
141 log.msg('Saving returned %i byte file to: %s' % (response.stream.length, destFile.path))
143 orig_stream = response.stream
144 response.stream = ProxyFileStream(orig_stream, destFile)
147 def save_error(self, failure, url):
148 """An error has occurred in downloadign or saving the file."""
149 log.msg('Error occurred downloading %s' % url)
153 class TestMirrorManager(unittest.TestCase):
154 """Unit tests for the mirror manager."""
161 self.client = MirrorManager('/tmp')
163 def test_extractPath(self):
164 site, baseDir, path = self.client.extractPath('http://ftp.us.debian.org/debian/dists/unstable/Release')
165 self.failUnless(site == "ftp.us.debian.org:80", "no match: %s" % site)
166 self.failUnless(baseDir == "/debian", "no match: %s" % baseDir)
167 self.failUnless(path == "/dists/unstable/Release", "no match: %s" % path)
169 site, baseDir, path = self.client.extractPath('http://ftp.us.debian.org:16999/debian/pool/d/dpkg/dpkg_1.2.1-1.tar.gz')
170 self.failUnless(site == "ftp.us.debian.org:16999", "no match: %s" % site)
171 self.failUnless(baseDir == "/debian", "no match: %s" % baseDir)
172 self.failUnless(path == "/pool/d/dpkg/dpkg_1.2.1-1.tar.gz", "no match: %s" % path)
174 site, baseDir, path = self.client.extractPath('http://debian.camrdale.org/dists/unstable/Release')
175 self.failUnless(site == "debian.camrdale.org:80", "no match: %s" % site)
176 self.failUnless(baseDir == "", "no match: %s" % baseDir)
177 self.failUnless(path == "/dists/unstable/Release", "no match: %s" % path)
179 def verifyHash(self, found_hash, path, true_hash):
180 self.failUnless(found_hash[0] == true_hash,
181 "%s hashes don't match: %s != %s" % (path, found_hash[0], true_hash))
183 def test_findHash(self):
184 self.packagesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "_main_.*Packages$" | tail -n 1').read().rstrip('\n')
185 self.sourcesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "_main_.*Sources$" | tail -n 1').read().rstrip('\n')
186 for f in os.walk('/var/lib/apt/lists').next()[2]:
187 if f[-7:] == "Release" and self.packagesFile.startswith(f[:-7]):
191 self.client.updatedFile('http://' + self.releaseFile.replace('_','/'),
192 '/var/lib/apt/lists/' + self.releaseFile)
193 self.client.updatedFile('http://' + self.releaseFile[:self.releaseFile.find('_dists_')+1].replace('_','/') +
194 self.packagesFile[self.packagesFile.find('_dists_')+1:].replace('_','/'),
195 '/var/lib/apt/lists/' + self.packagesFile)
196 self.client.updatedFile('http://' + self.releaseFile[:self.releaseFile.find('_dists_')+1].replace('_','/') +
197 self.sourcesFile[self.sourcesFile.find('_dists_')+1:].replace('_','/'),
198 '/var/lib/apt/lists/' + self.sourcesFile)
200 lastDefer = defer.Deferred()
202 idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
203 '/var/lib/apt/lists/' + self.releaseFile +
204 ' | grep -E " main/binary-i386/Packages.bz2$"'
205 ' | head -n 1 | cut -d\ -f 2').read().rstrip('\n')
206 idx_path = 'http://' + self.releaseFile.replace('_','/')[:-7] + 'main/binary-i386/Packages.bz2'
208 d = self.client.findHash(idx_path)
209 d.addCallback(self.verifyHash, idx_path, idx_hash)
211 pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
212 '/var/lib/apt/lists/' + self.packagesFile +
213 ' | grep -E "^SHA1:" | head -n 1' +
214 ' | cut -d\ -f 2').read().rstrip('\n')
215 pkg_path = 'http://' + self.releaseFile[:self.releaseFile.find('_dists_')+1].replace('_','/') + \
216 os.popen('grep -A 30 -E "^Package: dpkg$" ' +
217 '/var/lib/apt/lists/' + self.packagesFile +
218 ' | grep -E "^Filename:" | head -n 1' +
219 ' | cut -d\ -f 2').read().rstrip('\n')
221 d = self.client.findHash(pkg_path)
222 d.addCallback(self.verifyHash, pkg_path, pkg_hash)
224 src_dir = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
225 '/var/lib/apt/lists/' + self.sourcesFile +
226 ' | grep -E "^Directory:" | head -n 1' +
227 ' | cut -d\ -f 2').read().rstrip('\n')
228 src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
229 '/var/lib/apt/lists/' + self.sourcesFile +
230 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
231 ' | cut -d\ -f 2').read().split('\n')[:-1]
232 src_paths = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
233 '/var/lib/apt/lists/' + self.sourcesFile +
234 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
235 ' | cut -d\ -f 4').read().split('\n')[:-1]
237 for i in range(len(src_hashes)):
238 src_path = 'http://' + self.releaseFile[:self.releaseFile.find('_dists_')+1].replace('_','/') + src_dir + '/' + src_paths[i]
239 d = self.client.findHash(src_path)
240 d.addCallback(self.verifyHash, src_path, src_hashes[i])
242 idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
243 '/var/lib/apt/lists/' + self.releaseFile +
244 ' | grep -E " main/source/Sources.bz2$"'
245 ' | head -n 1 | cut -d\ -f 2').read().rstrip('\n')
246 idx_path = 'http://' + self.releaseFile.replace('_','/')[:-7] + 'main/source/Sources.bz2'
248 d = self.client.findHash(idx_path)
249 d.addCallback(self.verifyHash, idx_path, idx_hash)
251 d.addBoth(lastDefer.callback)
255 for p in self.pending_calls: