1 # Disable the FutureWarning from the apt module
3 warnings.simplefilter("ignore", FutureWarning)
6 from random import choice
7 from shutil import rmtree
8 from copy import deepcopy
9 from UserDict import DictMixin
11 from twisted.internet import threads, defer
12 from twisted.python import log
13 from twisted.python.filepath import FilePath
14 from twisted.trial import unittest
16 import apt_pkg, apt_inst
17 from apt import OpProgress
18 from debian_bundle import deb822
20 from Hash import HashObject
24 TRACKED_FILES = ['release', 'sources', 'packages']
26 class PackageFileList(DictMixin):
27 """Manages a list of package files belonging to a backend.
29 @type packages: C{shelve dictionary}
30 @ivar packages: the files stored for this backend
33 def __init__(self, cache_dir):
34 self.cache_dir = cache_dir
35 self.cache_dir.restat(False)
36 if not self.cache_dir.exists():
37 self.cache_dir.makedirs()
42 """Open the persistent dictionary of files in this backend."""
43 if self.packages is None:
44 self.packages = shelve.open(self.cache_dir.child('packages.db').path)
47 """Close the persistent dictionary."""
48 if self.packages is not None:
51 def update_file(self, cache_path, file_path):
52 """Check if an updated file needs to be tracked.
54 Called from the mirror manager when files get updated so we can update our
55 fake lists and sources.list.
57 filename = cache_path.split('/')[-1]
58 if filename.lower() in TRACKED_FILES:
59 log.msg("Registering package file: "+cache_path)
60 self.packages[cache_path] = file_path
64 def check_files(self):
65 """Check all files in the database to make sure they exist."""
66 files = self.packages.keys()
68 self.packages[f].restat(False)
69 if not self.packages[f].exists():
70 log.msg("File in packages database has been deleted: "+f)
73 # Standard dictionary implementation so this class can be used like a dictionary.
74 def __getitem__(self, key): return self.packages[key]
75 def __setitem__(self, key, item): self.packages[key] = item
76 def __delitem__(self, key): del self.packages[key]
77 def keys(self): return self.packages.keys()
80 """Uses python-apt to answer queries about packages.
82 Makes a fake configuration for python-apt for each backend.
85 DEFAULT_APT_CONFIG = {
87 #'APT::Architecture' : 'i386', # Commented so the machine's config will set this
88 #'APT::Default-Release' : 'unstable',
90 'Dir::State' : 'apt/', # var/lib/apt/
91 'Dir::State::Lists': 'lists/', # lists/
92 #'Dir::State::cdroms' : 'cdroms.list',
93 'Dir::State::userstatus' : 'status.user',
94 'Dir::State::status': 'dpkg/status', # '/var/lib/dpkg/status'
95 'Dir::Cache' : '.apt/cache/', # var/cache/apt/
96 #'Dir::Cache::archives' : 'archives/',
97 'Dir::Cache::srcpkgcache' : 'srcpkgcache.bin',
98 'Dir::Cache::pkgcache' : 'pkgcache.bin',
99 'Dir::Etc' : 'apt/etc/', # etc/apt/
100 'Dir::Etc::sourcelist' : 'sources.list',
101 'Dir::Etc::vendorlist' : 'vendors.list',
102 'Dir::Etc::vendorparts' : 'vendors.list.d',
103 #'Dir::Etc::main' : 'apt.conf',
104 #'Dir::Etc::parts' : 'apt.conf.d',
105 #'Dir::Etc::preferences' : 'preferences',
107 #'Dir::Bin::methods' : '', #'/usr/lib/apt/methods'
108 'Dir::Bin::dpkg' : '/usr/bin/dpkg',
110 #'DPkg::Pre-Install-Pkgs' : '',
112 #'DPkg::Tools::Options' : '',
113 #'DPkg::Tools::Options::/usr/bin/apt-listchanges' : '',
114 #'DPkg::Tools::Options::/usr/bin/apt-listchanges::Version' : '2',
115 #'DPkg::Post-Invoke' : '',
117 essential_dirs = ('apt', 'apt/cache', 'apt/dpkg', 'apt/etc', 'apt/lists',
119 essential_files = ('apt/dpkg/status', 'apt/etc/sources.list',)
121 def __init__(self, cache_dir):
122 """Construct a new packages manager.
124 @ivar backendName: name of backend associated with this packages file
125 @ivar cache_dir: cache directory from config file
127 self.cache_dir = cache_dir
128 self.apt_config = deepcopy(self.DEFAULT_APT_CONFIG)
130 for dir in self.essential_dirs:
131 path = self.cache_dir.preauthChild(dir)
132 if not path.exists():
134 for file in self.essential_files:
135 path = self.cache_dir.preauthChild(file)
136 if not path.exists():
139 self.apt_config['Dir'] = self.cache_dir.path
140 self.apt_config['Dir::State::status'] = self.cache_dir.preauthChild(self.apt_config['Dir::State']).preauthChild(self.apt_config['Dir::State::status']).path
141 self.packages = PackageFileList(cache_dir)
147 self.packages.close()
149 def addRelease(self, cache_path, file_path):
150 """Dirty hack until python-apt supports apt-pkg/indexrecords.h
153 self.indexrecords[cache_path] = {}
155 read_packages = False
156 f = file_path.open('r')
158 rel = deb822.Release(f, fields = ['MD5Sum', 'SHA1', 'SHA256'])
159 for hash_type in rel:
160 for file in rel[hash_type]:
161 self.indexrecords[cache_path].setdefault(file['name'], {})[hash_type.upper()] = (file['hash_type'], file['size'])
165 def file_updated(self, cache_path, file_path):
166 """A file in the backend has changed, manage it.
168 If this affects us, unload our apt database
170 if self.packages.update_file(cache_path, file_path):
174 """Make sure the package is initialized and loaded."""
175 if self.loading is None:
176 self.loading = threads.deferToThread(self._load)
177 self.loading.addCallback(self.doneLoading)
180 def doneLoading(self, loadResult):
181 """Cache is loaded."""
183 # Must pass on the result for the next callback
187 """Regenerates the fake configuration and load the packages cache."""
188 if self.loaded: return True
190 self.cache_dir.preauthChild(self.apt_config['Dir::State']
191 ).preauthChild(self.apt_config['Dir::State::Lists']).remove()
192 self.cache_dir.preauthChild(self.apt_config['Dir::State']
193 ).preauthChild(self.apt_config['Dir::State::Lists']
194 ).child('partial').makedirs()
195 sources_file = self.cache_dir.preauthChild(self.apt_config['Dir::Etc']
196 ).preauthChild(self.apt_config['Dir::Etc::sourcelist'])
197 sources = sources_file.open('w')
199 deb_src_added = False
200 self.packages.check_files()
201 self.indexrecords = {}
202 for f in self.packages:
203 # we should probably clear old entries from self.packages and
204 # take into account the recorded mtime as optimization
205 file = self.packages[f]
206 if f.split('/')[-1] == "Release":
207 self.addRelease(f, file)
208 fake_uri='http://apt-dht'+f
209 fake_dirname = '/'.join(fake_uri.split('/')[:-1])
210 if f.endswith('Sources'):
212 source_line='deb-src '+fake_dirname+'/ /'
214 source_line='deb '+fake_dirname+'/ /'
215 listpath = self.cache_dir.preauthChild(self.apt_config['Dir::State']
216 ).preauthChild(self.apt_config['Dir::State::Lists']
217 ).child(apt_pkg.URItoFileName(fake_uri))
218 sources.write(source_line+'\n')
219 log.msg("Sources line: " + source_line)
220 sources_count = sources_count + 1
222 if listpath.exists():
223 #we should empty the directory instead
225 os.symlink(file.path, listpath.path)
228 if sources_count == 0:
229 log.msg("No Packages files available for %s backend"%(self.cache_dir.path))
232 log.msg("Loading Packages database for "+self.cache_dir.path)
233 for key, value in self.apt_config.items():
234 apt_pkg.Config[key] = value
236 self.cache = apt_pkg.GetCache(OpProgress())
237 self.records = apt_pkg.GetPkgRecords(self.cache)
239 self.srcrecords = apt_pkg.GetPkgSrcRecords()
241 self.srcrecords = None
247 """Tries to make the packages server quit."""
252 del self.indexrecords
256 """Cleanup and close any loaded caches."""
258 self.packages.close()
260 def findHash(self, path):
261 """Find the hash for a given path in this mirror.
263 Returns a deferred so it can make sure the cache is loaded first.
267 deferLoad = self.load()
268 deferLoad.addCallback(self._findHash, path, d)
269 deferLoad.addErrback(self._findHash_error, path, d)
273 def _findHash_error(self, failure, path, d):
274 """An error occurred while trying to find a hash."""
275 log.msg('An error occurred while looking up a hash for: %s' % path)
277 d.callback(HashObject())
279 def _findHash(self, loadResult, path, d):
280 """Really find the hash for a path.
282 Have to pass the returned loadResult on in case other calls to this
283 function are pending.
286 d.callback(HashObject())
289 # First look for the path in the cache of index files
290 for release in self.indexrecords:
291 if path.startswith(release[:-7]):
292 for indexFile in self.indexrecords[release]:
293 if release[:-7] + indexFile == path:
295 h.setFromIndexRecord(self.indexrecords[release][indexFile])
299 package = path.split('/')[-1].split('_')[0]
301 # Check the binary packages
303 for version in self.cache[package].VersionList:
305 for verFile in version.FileList:
306 if self.records.Lookup(verFile):
307 if '/' + self.records.FileName == path:
309 h.setFromPkgRecord(self.records, size)
315 # Check the source packages' files
317 self.srcrecords.Restart()
318 if self.srcrecords.Lookup(package):
319 for f in self.srcrecords.Files:
320 if path == '/' + f[2]:
322 h.setFromSrcRecord(f)
326 d.callback(HashObject())
329 class TestAptPackages(unittest.TestCase):
330 """Unit tests for the AptPackages cache."""
340 self.client = AptPackages(FilePath('/tmp/.apt-dht'))
342 self.packagesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "_main_.*Packages$" | tail -n 1').read().rstrip('\n')
343 self.sourcesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "_main_.*Sources$" | tail -n 1').read().rstrip('\n')
344 for f in os.walk('/var/lib/apt/lists').next()[2]:
345 if f[-7:] == "Release" and self.packagesFile.startswith(f[:-7]):
349 self.client.file_updated(self.releaseFile[self.releaseFile.find('_dists_'):].replace('_','/'),
350 FilePath('/var/lib/apt/lists/' + self.releaseFile))
351 self.client.file_updated(self.packagesFile[self.packagesFile.find('_dists_'):].replace('_','/'),
352 FilePath('/var/lib/apt/lists/' + self.packagesFile))
353 self.client.file_updated(self.sourcesFile[self.sourcesFile.find('_dists_'):].replace('_','/'),
354 FilePath('/var/lib/apt/lists/' + self.sourcesFile))
356 def test_pkg_hash(self):
359 self.client.records.Lookup(self.client.cache['dpkg'].VersionList[0].FileList[0])
361 pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
362 '/var/lib/apt/lists/' + self.packagesFile +
363 ' | grep -E "^SHA1:" | head -n 1' +
364 ' | cut -d\ -f 2').read().rstrip('\n')
366 self.failUnless(self.client.records.SHA1Hash == pkg_hash,
367 "Hashes don't match: %s != %s" % (self.client.records.SHA1Hash, pkg_hash))
369 def test_src_hash(self):
372 self.client.srcrecords.Lookup('dpkg')
374 src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
375 '/var/lib/apt/lists/' + self.sourcesFile +
376 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
377 ' | cut -d\ -f 2').read().split('\n')[:-1]
379 for f in self.client.srcrecords.Files:
380 self.failUnless(f[0] in src_hashes, "Couldn't find %s in: %r" % (f[0], src_hashes))
382 def test_index_hash(self):
385 indexhash = self.client.indexrecords[self.releaseFile[self.releaseFile.find('_dists_'):].replace('_','/')]['main/binary-i386/Packages.bz2']['SHA1'][0]
387 idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
388 '/var/lib/apt/lists/' + self.releaseFile +
389 ' | grep -E " main/binary-i386/Packages.bz2$"'
390 ' | head -n 1 | cut -d\ -f 2').read().rstrip('\n')
392 self.failUnless(indexhash == idx_hash, "Hashes don't match: %s != %s" % (indexhash, idx_hash))
394 def verifyHash(self, found_hash, path, true_hash):
395 self.failUnless(found_hash.hexexpected() == true_hash,
396 "%s hashes don't match: %s != %s" % (path, found_hash.hexexpected(), true_hash))
398 def test_findIndexHash(self):
399 lastDefer = defer.Deferred()
401 idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
402 '/var/lib/apt/lists/' + self.releaseFile +
403 ' | grep -E " main/binary-i386/Packages.bz2$"'
404 ' | head -n 1 | cut -d\ -f 2').read().rstrip('\n')
405 idx_path = '/' + self.releaseFile[self.releaseFile.find('_dists_')+1:].replace('_','/')[:-7] + 'main/binary-i386/Packages.bz2'
407 d = self.client.findHash(idx_path)
408 d.addCallback(self.verifyHash, idx_path, idx_hash)
410 d.addBoth(lastDefer.callback)
413 def test_findPkgHash(self):
414 lastDefer = defer.Deferred()
416 pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
417 '/var/lib/apt/lists/' + self.packagesFile +
418 ' | grep -E "^SHA1:" | head -n 1' +
419 ' | cut -d\ -f 2').read().rstrip('\n')
420 pkg_path = '/' + os.popen('grep -A 30 -E "^Package: dpkg$" ' +
421 '/var/lib/apt/lists/' + self.packagesFile +
422 ' | grep -E "^Filename:" | head -n 1' +
423 ' | cut -d\ -f 2').read().rstrip('\n')
425 d = self.client.findHash(pkg_path)
426 d.addCallback(self.verifyHash, pkg_path, pkg_hash)
428 d.addBoth(lastDefer.callback)
431 def test_findSrcHash(self):
432 lastDefer = defer.Deferred()
434 src_dir = '/' + os.popen('grep -A 30 -E "^Package: dpkg$" ' +
435 '/var/lib/apt/lists/' + self.sourcesFile +
436 ' | grep -E "^Directory:" | head -n 1' +
437 ' | cut -d\ -f 2').read().rstrip('\n')
438 src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
439 '/var/lib/apt/lists/' + self.sourcesFile +
440 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
441 ' | cut -d\ -f 2').read().split('\n')[:-1]
442 src_paths = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
443 '/var/lib/apt/lists/' + self.sourcesFile +
444 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
445 ' | cut -d\ -f 4').read().split('\n')[:-1]
447 i = choice(range(len(src_hashes)))
448 d = self.client.findHash(src_dir + '/' + src_paths[i])
449 d.addCallback(self.verifyHash, src_dir + '/' + src_paths[i], src_hashes[i])
451 d.addBoth(lastDefer.callback)
454 def test_multipleFindHash(self):
455 lastDefer = defer.Deferred()
457 idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
458 '/var/lib/apt/lists/' + self.releaseFile +
459 ' | grep -E " main/binary-i386/Packages.bz2$"'
460 ' | head -n 1 | cut -d\ -f 2').read().rstrip('\n')
461 idx_path = '/' + self.releaseFile[self.releaseFile.find('_dists_')+1:].replace('_','/')[:-7] + 'main/binary-i386/Packages.bz2'
463 d = self.client.findHash(idx_path)
464 d.addCallback(self.verifyHash, idx_path, idx_hash)
466 pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
467 '/var/lib/apt/lists/' + self.packagesFile +
468 ' | grep -E "^SHA1:" | head -n 1' +
469 ' | cut -d\ -f 2').read().rstrip('\n')
470 pkg_path = '/' + os.popen('grep -A 30 -E "^Package: dpkg$" ' +
471 '/var/lib/apt/lists/' + self.packagesFile +
472 ' | grep -E "^Filename:" | head -n 1' +
473 ' | cut -d\ -f 2').read().rstrip('\n')
475 d = self.client.findHash(pkg_path)
476 d.addCallback(self.verifyHash, pkg_path, pkg_hash)
478 src_dir = '/' + os.popen('grep -A 30 -E "^Package: dpkg$" ' +
479 '/var/lib/apt/lists/' + self.sourcesFile +
480 ' | grep -E "^Directory:" | head -n 1' +
481 ' | cut -d\ -f 2').read().rstrip('\n')
482 src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
483 '/var/lib/apt/lists/' + self.sourcesFile +
484 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
485 ' | cut -d\ -f 2').read().split('\n')[:-1]
486 src_paths = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
487 '/var/lib/apt/lists/' + self.sourcesFile +
488 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
489 ' | cut -d\ -f 4').read().split('\n')[:-1]
491 for i in range(len(src_hashes)):
492 d = self.client.findHash(src_dir + '/' + src_paths[i])
493 d.addCallback(self.verifyHash, src_dir + '/' + src_paths[i], src_hashes[i])
495 idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
496 '/var/lib/apt/lists/' + self.releaseFile +
497 ' | grep -E " main/source/Sources.bz2$"'
498 ' | head -n 1 | cut -d\ -f 2').read().rstrip('\n')
499 idx_path = '/' + self.releaseFile[self.releaseFile.find('_dists_')+1:].replace('_','/')[:-7] + 'main/source/Sources.bz2'
501 d = self.client.findHash(idx_path)
502 d.addCallback(self.verifyHash, idx_path, idx_hash)
504 d.addBoth(lastDefer.callback)
508 for p in self.pending_calls:
511 self.pending_calls = []
512 self.client.cleanup()