X-Git-Url: https://git.mxchange.org/?p=quix0rs-apt-p2p.git;a=blobdiff_plain;f=apt_dht%2FMirrorManager.py;h=738fdebc4d7e668d6dee6e42ee7ce2860c68ceb7;hp=b4995e9445a1a9ca175399430b4196b5582a1e1e;hb=be126fc5cd40e26479c50fee4f64cc478f1eb55a;hpb=85d160514bda0e66058f0fb1ac70c8cf52b4435d diff --git a/apt_dht/MirrorManager.py b/apt_dht/MirrorManager.py index b4995e9..738fdeb 100644 --- a/apt_dht/MirrorManager.py +++ b/apt_dht/MirrorManager.py @@ -2,80 +2,25 @@ from urlparse import urlparse import os -from twisted.python import log, filepath +from twisted.python import log +from twisted.python.filepath import FilePath from twisted.internet import defer from twisted.trial import unittest -from twisted.web2 import stream from twisted.web2.http import splitHostPort from AptPackages import AptPackages -aptpkg_dir='.apt-dht' +aptpkg_dir='apt-packages' class MirrorError(Exception): """Exception raised when there's a problem with the mirror.""" -class ProxyFileStream(stream.SimpleStream): - """Saves a stream to a file while providing a new stream.""" - - def __init__(self, stream, outFile, modtime = None): - """Initializes the proxy. - - @type stream: C{twisted.web2.stream.IByteStream} - @param stream: the input stream to read from - @type outFile: C{twisted.python.filepath.FilePath} - @param outFile: the file to write to - @type modtime: C{int} - @param modtime: the modification time to set for the file - """ - self.stream = stream - self.outFile = outFile - self.openFile = outFile.open('w') - self.modtime = modtime - self.length = self.stream.length - self.start = 0 - - def _done(self): - """Close the output file.""" - if not self.openFile.closed: - self.openFile.close() - if self.modtime: - os.utime(self.outFile.path, (self.modtime, self.modtime)) - - def read(self): - """Read some data from the stream.""" - if self.openFile.closed: - return None - - data = self.stream.read() - if isinstance(data, defer.Deferred): - data.addCallbacks(self._write, self._done) - return data - - self._write(data) - return data - - def _write(self, data): - """Write the stream data to the file and return it for others to use.""" - if data is None: - self._done() - return data - - self.openFile.write(data) - return data - - def close(self): - """Clean everything up and return None to future reads.""" - self.length = 0 - self._done() - self.stream.close() - class MirrorManager: """Manages all requests for mirror objects.""" - def __init__(self, cache_dir): + def __init__(self, cache_dir, unload_delay): self.cache_dir = cache_dir - self.cache = filepath.FilePath(self.cache_dir) + self.unload_delay = unload_delay self.apt_caches = {} def extractPath(self, url): @@ -113,14 +58,15 @@ class MirrorManager: self.apt_caches[site] = {} if baseDir not in self.apt_caches[site]: - site_cache = os.path.join(self.cache_dir, aptpkg_dir, 'mirrors', site + baseDir.replace('/', '_')) - self.apt_caches[site][baseDir] = AptPackages(site_cache) + site_cache = self.cache_dir.child(aptpkg_dir).child('mirrors').child(site + baseDir.replace('/', '_')) + site_cache.makedirs + self.apt_caches[site][baseDir] = AptPackages(site_cache, self.unload_delay) def updatedFile(self, url, file_path): site, baseDir, path = self.extractPath(url) self.init(site, baseDir) self.apt_caches[site][baseDir].file_updated(path, file_path) - + def findHash(self, url): site, baseDir, path = self.extractPath(url) if site in self.apt_caches and baseDir in self.apt_caches[site]: @@ -128,31 +74,7 @@ class MirrorManager: d = defer.Deferred() d.errback(MirrorError("Site Not Found")) return d - - def save_file(self, response, hash, size, url): - """Save a downloaded file to the cache and stream it.""" - log.msg('Returning file: %s' % url) - - parsed = urlparse(url) - destFile = self.cache.preauthChild(parsed[1] + parsed[2]) - log.msg('Saving returned %r byte file to cache: %s' % (response.stream.length, destFile.path)) - - if destFile.exists(): - log.msg('File already exists, removing: %s' % destFile.path) - destFile.remove() - else: - destFile.parent().makedirs() - - orig_stream = response.stream - response.stream = ProxyFileStream(orig_stream, destFile, response.headers.getHeader('Last-Modified')) - return response - - def save_error(self, failure, url): - """An error has occurred in downloadign or saving the file.""" - log.msg('Error occurred downloading %s' % url) - log.err(failure) - return failure - + class TestMirrorManager(unittest.TestCase): """Unit tests for the mirror manager.""" @@ -161,7 +83,7 @@ class TestMirrorManager(unittest.TestCase): client = None def setUp(self): - self.client = MirrorManager('/tmp') + self.client = MirrorManager(FilePath('/tmp/.apt-dht'), 300) def test_extractPath(self): site, baseDir, path = self.client.extractPath('http://ftp.us.debian.org/debian/dists/unstable/Release') @@ -180,8 +102,8 @@ class TestMirrorManager(unittest.TestCase): self.failUnless(path == "/dists/unstable/Release", "no match: %s" % path) def verifyHash(self, found_hash, path, true_hash): - self.failUnless(found_hash[0] == true_hash, - "%s hashes don't match: %s != %s" % (path, found_hash[0], true_hash)) + self.failUnless(found_hash.hexexpected() == true_hash, + "%s hashes don't match: %s != %s" % (path, found_hash.hexexpected(), true_hash)) def test_findHash(self): self.packagesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "_main_.*Packages$" | tail -n 1').read().rstrip('\n') @@ -192,13 +114,13 @@ class TestMirrorManager(unittest.TestCase): break self.client.updatedFile('http://' + self.releaseFile.replace('_','/'), - '/var/lib/apt/lists/' + self.releaseFile) + FilePath('/var/lib/apt/lists/' + self.releaseFile)) self.client.updatedFile('http://' + self.releaseFile[:self.releaseFile.find('_dists_')+1].replace('_','/') + self.packagesFile[self.packagesFile.find('_dists_')+1:].replace('_','/'), - '/var/lib/apt/lists/' + self.packagesFile) + FilePath('/var/lib/apt/lists/' + self.packagesFile)) self.client.updatedFile('http://' + self.releaseFile[:self.releaseFile.find('_dists_')+1].replace('_','/') + self.sourcesFile[self.sourcesFile.find('_dists_')+1:].replace('_','/'), - '/var/lib/apt/lists/' + self.sourcesFile) + FilePath('/var/lib/apt/lists/' + self.sourcesFile)) lastDefer = defer.Deferred()