X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;f=apt_dht%2FAptPackages.py;h=48dd4480fd729dc3b31aeb5f7a213f0bffff712f;hb=53e3701b94b11cedbd57b5a6fc9c9f58e0a10932;hp=a2e743e60fde74f07c9c9bed7d6132b6e66f9d4a;hpb=48b7787f8a2c434d67eb92d1d8bc6e97e9ecb755;p=quix0rs-apt-p2p.git diff --git a/apt_dht/AptPackages.py b/apt_dht/AptPackages.py index a2e743e..48dd448 100644 --- a/apt_dht/AptPackages.py +++ b/apt_dht/AptPackages.py @@ -1,3 +1,20 @@ +# +# Copyright (C) 2002 Manuel Estrada Sainz +# Copyright (C) 2008 Cameron Dale +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of version 2.1 of the GNU General Public +# License as published by the Free Software Foundation. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + # Disable the FutureWarning from the apt module import warnings warnings.simplefilter("ignore", FutureWarning) @@ -8,12 +25,16 @@ from shutil import rmtree from copy import deepcopy from UserDict import DictMixin -from twisted.internet import threads, defer +from twisted.internet import threads, defer, reactor from twisted.python import log +from twisted.python.filepath import FilePath from twisted.trial import unittest import apt_pkg, apt_inst from apt import OpProgress +from debian_bundle import deb822 + +from Hash import HashObject apt_pkg.init() @@ -28,15 +49,16 @@ class PackageFileList(DictMixin): def __init__(self, cache_dir): self.cache_dir = cache_dir - if not os.path.exists(self.cache_dir): - os.makedirs(self.cache_dir) + self.cache_dir.restat(False) + if not self.cache_dir.exists(): + self.cache_dir.makedirs() self.packages = None self.open() def open(self): """Open the persistent dictionary of files in this backend.""" if self.packages is None: - self.packages = shelve.open(self.cache_dir+'/packages.db') + self.packages = shelve.open(self.cache_dir.child('packages.db').path) def close(self): """Close the persistent dictionary.""" @@ -60,7 +82,8 @@ class PackageFileList(DictMixin): """Check all files in the database to make sure they exist.""" files = self.packages.keys() for f in files: - if not os.path.exists(self.packages[f]): + self.packages[f].restat(False) + if not self.packages[f].exists(): log.msg("File in packages database has been deleted: "+f) del self.packages[f] @@ -112,32 +135,30 @@ class AptPackages: 'apt/lists/partial') essential_files = ('apt/dpkg/status', 'apt/etc/sources.list',) - def __init__(self, cache_dir): + def __init__(self, cache_dir, unload_delay): """Construct a new packages manager. - @ivar backendName: name of backend associated with this packages file - @ivar cache_dir: cache directory from config file + @param cache_dir: cache directory from config file """ self.cache_dir = cache_dir + self.unload_delay = unload_delay self.apt_config = deepcopy(self.DEFAULT_APT_CONFIG) for dir in self.essential_dirs: - path = os.path.join(self.cache_dir, dir) - if not os.path.exists(path): - os.makedirs(path) + path = self.cache_dir.preauthChild(dir) + if not path.exists(): + path.makedirs() for file in self.essential_files: - path = os.path.join(self.cache_dir, file) - if not os.path.exists(path): - f = open(path,'w') - f.close() - del f + path = self.cache_dir.preauthChild(file) + if not path.exists(): + path.touch() - self.apt_config['Dir'] = self.cache_dir - self.apt_config['Dir::State::status'] = os.path.join(self.cache_dir, - self.apt_config['Dir::State'], self.apt_config['Dir::State::status']) + self.apt_config['Dir'] = self.cache_dir.path + self.apt_config['Dir::State::status'] = self.cache_dir.preauthChild(self.apt_config['Dir::State']).preauthChild(self.apt_config['Dir::State::status']).path self.packages = PackageFileList(cache_dir) self.loaded = 0 self.loading = None + self.unload_later = None def __del__(self): self.cleanup() @@ -150,31 +171,13 @@ class AptPackages: self.indexrecords[cache_path] = {} read_packages = False - f = open(file_path, 'r') + f = file_path.open('r') - for line in f: - line = line.rstrip() - - if line[:1] != " ": - read_packages = False - try: - # Read the various headers from the file - h, v = line.split(":", 1) - if h == "MD5Sum" or h == "SHA1" or h == "SHA256": - read_packages = True - hash_type = h - except: - # Bad header line, just ignore it - log.msg("WARNING: Ignoring badly formatted Release line: %s" % line) - - # Skip to the next line - continue + rel = deb822.Release(f, fields = ['MD5Sum', 'SHA1', 'SHA256']) + for hash_type in rel: + for file in rel[hash_type]: + self.indexrecords[cache_path].setdefault(file['name'], {})[hash_type.upper()] = (file[hash_type], file['size']) - # Read file names from the multiple hash sections of the file - if read_packages: - p = line.split() - self.indexrecords[cache_path].setdefault(p[2], {})[hash_type] = (p[0], p[1]) - f.close() def file_updated(self, cache_path, file_path): @@ -187,7 +190,12 @@ class AptPackages: def load(self): """Make sure the package is initialized and loaded.""" + if self.unload_later and self.unload_later.active(): + self.unload_later.reset(self.unload_delay) + else: + self.unload_later = reactor.callLater(self.unload_delay, self.unload) if self.loading is None: + log.msg('Loading the packages cache') self.loading = threads.deferToThread(self._load) self.loading.addCallback(self.doneLoading) return self.loading @@ -202,13 +210,14 @@ class AptPackages: """Regenerates the fake configuration and load the packages cache.""" if self.loaded: return True apt_pkg.InitSystem() - rmtree(os.path.join(self.cache_dir, self.apt_config['Dir::State'], - self.apt_config['Dir::State::Lists'])) - os.makedirs(os.path.join(self.cache_dir, self.apt_config['Dir::State'], - self.apt_config['Dir::State::Lists'], 'partial')) - sources_filename = os.path.join(self.cache_dir, self.apt_config['Dir::Etc'], - self.apt_config['Dir::Etc::sourcelist']) - sources = open(sources_filename, 'w') + self.cache_dir.preauthChild(self.apt_config['Dir::State'] + ).preauthChild(self.apt_config['Dir::State::Lists']).remove() + self.cache_dir.preauthChild(self.apt_config['Dir::State'] + ).preauthChild(self.apt_config['Dir::State::Lists'] + ).child('partial').makedirs() + sources_file = self.cache_dir.preauthChild(self.apt_config['Dir::Etc'] + ).preauthChild(self.apt_config['Dir::Etc::sourcelist']) + sources = sources_file.open('w') sources_count = 0 deb_src_added = False self.packages.check_files() @@ -216,9 +225,9 @@ class AptPackages: for f in self.packages: # we should probably clear old entries from self.packages and # take into account the recorded mtime as optimization - filepath = self.packages[f] + file = self.packages[f] if f.split('/')[-1] == "Release": - self.addRelease(f, filepath) + self.addRelease(f, file) fake_uri='http://apt-dht'+f fake_dirname = '/'.join(fake_uri.split('/')[:-1]) if f.endswith('Sources'): @@ -226,26 +235,24 @@ class AptPackages: source_line='deb-src '+fake_dirname+'/ /' else: source_line='deb '+fake_dirname+'/ /' - listpath=(os.path.join(self.cache_dir, self.apt_config['Dir::State'], - self.apt_config['Dir::State::Lists'], - apt_pkg.URItoFileName(fake_uri))) + listpath = self.cache_dir.preauthChild(self.apt_config['Dir::State'] + ).preauthChild(self.apt_config['Dir::State::Lists'] + ).child(apt_pkg.URItoFileName(fake_uri)) sources.write(source_line+'\n') log.msg("Sources line: " + source_line) sources_count = sources_count + 1 - try: + if listpath.exists(): #we should empty the directory instead - os.unlink(listpath) - except: - pass - os.symlink(filepath, listpath) + listpath.remove() + os.symlink(file.path, listpath.path) sources.close() if sources_count == 0: - log.msg("No Packages files available for %s backend"%(self.cache_dir)) + log.msg("No Packages files available for %s backend"%(self.cache_dir.path)) return False - log.msg("Loading Packages database for "+self.cache_dir) + log.msg("Loading Packages database for "+self.cache_dir.path) for key, value in self.apt_config.items(): apt_pkg.Config[key] = value @@ -261,7 +268,11 @@ class AptPackages: def unload(self): """Tries to make the packages server quit.""" + if self.unload_later and self.unload_later.active(): + self.unload_later.cancel() + self.unload_later = None if self.loaded: + log.msg('Unloading the packages cache') del self.cache del self.records del self.srcrecords @@ -290,7 +301,7 @@ class AptPackages: """An error occurred while trying to find a hash.""" log.msg('An error occurred while looking up a hash for: %s' % path) log.err(failure) - d.callback((None, None)) + d.callback(HashObject()) def _findHash(self, loadResult, path, d): """Really find the hash for a path. @@ -299,7 +310,7 @@ class AptPackages: function are pending. """ if not loadResult: - d.callback((None, None)) + d.callback(HashObject()) return loadResult # First look for the path in the cache of index files @@ -307,7 +318,9 @@ class AptPackages: if path.startswith(release[:-7]): for indexFile in self.indexrecords[release]: if release[:-7] + indexFile == path: - d.callback(self.indexrecords[release][indexFile]['SHA1']) + h = HashObject() + h.setFromIndexRecord(self.indexrecords[release][indexFile]) + d.callback(h) return loadResult package = path.split('/')[-1].split('_')[0] @@ -319,7 +332,9 @@ class AptPackages: for verFile in version.FileList: if self.records.Lookup(verFile): if '/' + self.records.FileName == path: - d.callback((self.records.SHA1Hash, size)) + h = HashObject() + h.setFromPkgRecord(self.records, size) + d.callback(h) return loadResult except KeyError: pass @@ -330,10 +345,12 @@ class AptPackages: if self.srcrecords.Lookup(package): for f in self.srcrecords.Files: if path == '/' + f[2]: - d.callback((f[0], f[1])) + h = HashObject() + h.setFromSrcRecord(f) + d.callback(h) return loadResult - d.callback((None, None)) + d.callback(HashObject()) return loadResult class TestAptPackages(unittest.TestCase): @@ -347,7 +364,7 @@ class TestAptPackages(unittest.TestCase): releaseFile = '' def setUp(self): - self.client = AptPackages('/tmp/.apt-dht') + self.client = AptPackages(FilePath('/tmp/.apt-dht'), 300) self.packagesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "_main_.*Packages$" | tail -n 1').read().rstrip('\n') self.sourcesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "_main_.*Sources$" | tail -n 1').read().rstrip('\n') @@ -357,11 +374,11 @@ class TestAptPackages(unittest.TestCase): break self.client.file_updated(self.releaseFile[self.releaseFile.find('_dists_'):].replace('_','/'), - '/var/lib/apt/lists/' + self.releaseFile) + FilePath('/var/lib/apt/lists/' + self.releaseFile)) self.client.file_updated(self.packagesFile[self.packagesFile.find('_dists_'):].replace('_','/'), - '/var/lib/apt/lists/' + self.packagesFile) + FilePath('/var/lib/apt/lists/' + self.packagesFile)) self.client.file_updated(self.sourcesFile[self.sourcesFile.find('_dists_'):].replace('_','/'), - '/var/lib/apt/lists/' + self.sourcesFile) + FilePath('/var/lib/apt/lists/' + self.sourcesFile)) def test_pkg_hash(self): self.client._load() @@ -402,8 +419,8 @@ class TestAptPackages(unittest.TestCase): self.failUnless(indexhash == idx_hash, "Hashes don't match: %s != %s" % (indexhash, idx_hash)) def verifyHash(self, found_hash, path, true_hash): - self.failUnless(found_hash[0] == true_hash, - "%s hashes don't match: %s != %s" % (path, found_hash[0], true_hash)) + self.failUnless(found_hash.hexexpected() == true_hash, + "%s hashes don't match: %s != %s" % (path, found_hash.hexexpected(), true_hash)) def test_findIndexHash(self): lastDefer = defer.Deferred()