1 # Disable the FutureWarning from the apt module
3 warnings.simplefilter("ignore", FutureWarning)
6 from random import choice
7 from shutil import rmtree
8 from copy import deepcopy
9 from UserDict import DictMixin
11 from twisted.internet import threads, defer
12 from twisted.python import log
13 from twisted.python.filepath import FilePath
14 from twisted.trial import unittest
16 import apt_pkg, apt_inst
17 from apt import OpProgress
19 from Hash import HashObject
23 TRACKED_FILES = ['release', 'sources', 'packages']
25 class PackageFileList(DictMixin):
26 """Manages a list of package files belonging to a backend.
28 @type packages: C{shelve dictionary}
29 @ivar packages: the files stored for this backend
32 def __init__(self, cache_dir):
33 self.cache_dir = cache_dir
34 self.cache_dir.restat(False)
35 if not self.cache_dir.exists():
36 self.cache_dir.makedirs()
41 """Open the persistent dictionary of files in this backend."""
42 if self.packages is None:
43 self.packages = shelve.open(self.cache_dir.child('packages.db').path)
46 """Close the persistent dictionary."""
47 if self.packages is not None:
50 def update_file(self, cache_path, file_path):
51 """Check if an updated file needs to be tracked.
53 Called from the mirror manager when files get updated so we can update our
54 fake lists and sources.list.
56 filename = cache_path.split('/')[-1]
57 if filename.lower() in TRACKED_FILES:
58 log.msg("Registering package file: "+cache_path)
59 self.packages[cache_path] = file_path
63 def check_files(self):
64 """Check all files in the database to make sure they exist."""
65 files = self.packages.keys()
67 self.packages[f].restat(False)
68 if not self.packages[f].exists():
69 log.msg("File in packages database has been deleted: "+f)
72 # Standard dictionary implementation so this class can be used like a dictionary.
73 def __getitem__(self, key): return self.packages[key]
74 def __setitem__(self, key, item): self.packages[key] = item
75 def __delitem__(self, key): del self.packages[key]
76 def keys(self): return self.packages.keys()
79 """Uses python-apt to answer queries about packages.
81 Makes a fake configuration for python-apt for each backend.
84 DEFAULT_APT_CONFIG = {
86 #'APT::Architecture' : 'i386', # Commented so the machine's config will set this
87 #'APT::Default-Release' : 'unstable',
89 'Dir::State' : 'apt/', # var/lib/apt/
90 'Dir::State::Lists': 'lists/', # lists/
91 #'Dir::State::cdroms' : 'cdroms.list',
92 'Dir::State::userstatus' : 'status.user',
93 'Dir::State::status': 'dpkg/status', # '/var/lib/dpkg/status'
94 'Dir::Cache' : '.apt/cache/', # var/cache/apt/
95 #'Dir::Cache::archives' : 'archives/',
96 'Dir::Cache::srcpkgcache' : 'srcpkgcache.bin',
97 'Dir::Cache::pkgcache' : 'pkgcache.bin',
98 'Dir::Etc' : 'apt/etc/', # etc/apt/
99 'Dir::Etc::sourcelist' : 'sources.list',
100 'Dir::Etc::vendorlist' : 'vendors.list',
101 'Dir::Etc::vendorparts' : 'vendors.list.d',
102 #'Dir::Etc::main' : 'apt.conf',
103 #'Dir::Etc::parts' : 'apt.conf.d',
104 #'Dir::Etc::preferences' : 'preferences',
106 #'Dir::Bin::methods' : '', #'/usr/lib/apt/methods'
107 'Dir::Bin::dpkg' : '/usr/bin/dpkg',
109 #'DPkg::Pre-Install-Pkgs' : '',
111 #'DPkg::Tools::Options' : '',
112 #'DPkg::Tools::Options::/usr/bin/apt-listchanges' : '',
113 #'DPkg::Tools::Options::/usr/bin/apt-listchanges::Version' : '2',
114 #'DPkg::Post-Invoke' : '',
116 essential_dirs = ('apt', 'apt/cache', 'apt/dpkg', 'apt/etc', 'apt/lists',
118 essential_files = ('apt/dpkg/status', 'apt/etc/sources.list',)
120 def __init__(self, cache_dir):
121 """Construct a new packages manager.
123 @ivar backendName: name of backend associated with this packages file
124 @ivar cache_dir: cache directory from config file
126 self.cache_dir = cache_dir
127 self.apt_config = deepcopy(self.DEFAULT_APT_CONFIG)
129 for dir in self.essential_dirs:
130 path = self.cache_dir.preauthChild(dir)
131 if not path.exists():
133 for file in self.essential_files:
134 path = self.cache_dir.preauthChild(file)
135 if not path.exists():
138 self.apt_config['Dir'] = self.cache_dir.path
139 self.apt_config['Dir::State::status'] = self.cache_dir.preauthChild(self.apt_config['Dir::State']).preauthChild(self.apt_config['Dir::State::status']).path
140 self.packages = PackageFileList(cache_dir)
146 self.packages.close()
148 def addRelease(self, cache_path, file_path):
149 """Dirty hack until python-apt supports apt-pkg/indexrecords.h
152 self.indexrecords[cache_path] = {}
154 read_packages = False
155 f = file_path.open('r')
161 read_packages = False
163 # Read the various headers from the file
164 h, v = line.split(":", 1)
165 if h == "MD5Sum" or h == "SHA1" or h == "SHA256":
169 # Bad header line, just ignore it
170 log.msg("WARNING: Ignoring badly formatted Release line: %s" % line)
172 # Skip to the next line
175 # Read file names from the multiple hash sections of the file
178 self.indexrecords[cache_path].setdefault(p[2], {})[hash_type] = (p[0], p[1])
182 def file_updated(self, cache_path, file_path):
183 """A file in the backend has changed, manage it.
185 If this affects us, unload our apt database
187 if self.packages.update_file(cache_path, file_path):
191 """Make sure the package is initialized and loaded."""
192 if self.loading is None:
193 self.loading = threads.deferToThread(self._load)
194 self.loading.addCallback(self.doneLoading)
197 def doneLoading(self, loadResult):
198 """Cache is loaded."""
200 # Must pass on the result for the next callback
204 """Regenerates the fake configuration and load the packages cache."""
205 if self.loaded: return True
207 self.cache_dir.preauthChild(self.apt_config['Dir::State']
208 ).preauthChild(self.apt_config['Dir::State::Lists']).remove()
209 self.cache_dir.preauthChild(self.apt_config['Dir::State']
210 ).preauthChild(self.apt_config['Dir::State::Lists']
211 ).child('partial').makedirs()
212 sources_file = self.cache_dir.preauthChild(self.apt_config['Dir::Etc']
213 ).preauthChild(self.apt_config['Dir::Etc::sourcelist'])
214 sources = sources_file.open('w')
216 deb_src_added = False
217 self.packages.check_files()
218 self.indexrecords = {}
219 for f in self.packages:
220 # we should probably clear old entries from self.packages and
221 # take into account the recorded mtime as optimization
222 file = self.packages[f]
223 if f.split('/')[-1] == "Release":
224 self.addRelease(f, file)
225 fake_uri='http://apt-dht'+f
226 fake_dirname = '/'.join(fake_uri.split('/')[:-1])
227 if f.endswith('Sources'):
229 source_line='deb-src '+fake_dirname+'/ /'
231 source_line='deb '+fake_dirname+'/ /'
232 listpath = self.cache_dir.preauthChild(self.apt_config['Dir::State']
233 ).preauthChild(self.apt_config['Dir::State::Lists']
234 ).child(apt_pkg.URItoFileName(fake_uri))
235 sources.write(source_line+'\n')
236 log.msg("Sources line: " + source_line)
237 sources_count = sources_count + 1
239 if listpath.exists():
240 #we should empty the directory instead
242 os.symlink(file.path, listpath.path)
245 if sources_count == 0:
246 log.msg("No Packages files available for %s backend"%(self.cache_dir.path))
249 log.msg("Loading Packages database for "+self.cache_dir.path)
250 for key, value in self.apt_config.items():
251 apt_pkg.Config[key] = value
253 self.cache = apt_pkg.GetCache(OpProgress())
254 self.records = apt_pkg.GetPkgRecords(self.cache)
256 self.srcrecords = apt_pkg.GetPkgSrcRecords()
258 self.srcrecords = None
264 """Tries to make the packages server quit."""
269 del self.indexrecords
273 """Cleanup and close any loaded caches."""
275 self.packages.close()
277 def findHash(self, path):
278 """Find the hash for a given path in this mirror.
280 Returns a deferred so it can make sure the cache is loaded first.
284 deferLoad = self.load()
285 deferLoad.addCallback(self._findHash, path, d)
286 deferLoad.addErrback(self._findHash_error, path, d)
290 def _findHash_error(self, failure, path, d):
291 """An error occurred while trying to find a hash."""
292 log.msg('An error occurred while looking up a hash for: %s' % path)
294 d.callback(HashObject())
296 def _findHash(self, loadResult, path, d):
297 """Really find the hash for a path.
299 Have to pass the returned loadResult on in case other calls to this
300 function are pending.
303 d.callback(HashObject())
306 # First look for the path in the cache of index files
307 for release in self.indexrecords:
308 if path.startswith(release[:-7]):
309 for indexFile in self.indexrecords[release]:
310 if release[:-7] + indexFile == path:
312 h.setFromIndexRecord(self.indexrecords[release][indexFile])
316 package = path.split('/')[-1].split('_')[0]
318 # Check the binary packages
320 for version in self.cache[package].VersionList:
322 for verFile in version.FileList:
323 if self.records.Lookup(verFile):
324 if '/' + self.records.FileName == path:
326 h.setFromPkgRecord(self.records, size)
332 # Check the source packages' files
334 self.srcrecords.Restart()
335 if self.srcrecords.Lookup(package):
336 for f in self.srcrecords.Files:
337 if path == '/' + f[2]:
339 h.setFromSrcRecord(f)
343 d.callback(HashObject())
346 class TestAptPackages(unittest.TestCase):
347 """Unit tests for the AptPackages cache."""
357 self.client = AptPackages(FilePath('/tmp/.apt-dht'))
359 self.packagesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "_main_.*Packages$" | tail -n 1').read().rstrip('\n')
360 self.sourcesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "_main_.*Sources$" | tail -n 1').read().rstrip('\n')
361 for f in os.walk('/var/lib/apt/lists').next()[2]:
362 if f[-7:] == "Release" and self.packagesFile.startswith(f[:-7]):
366 self.client.file_updated(self.releaseFile[self.releaseFile.find('_dists_'):].replace('_','/'),
367 FilePath('/var/lib/apt/lists/' + self.releaseFile))
368 self.client.file_updated(self.packagesFile[self.packagesFile.find('_dists_'):].replace('_','/'),
369 FilePath('/var/lib/apt/lists/' + self.packagesFile))
370 self.client.file_updated(self.sourcesFile[self.sourcesFile.find('_dists_'):].replace('_','/'),
371 FilePath('/var/lib/apt/lists/' + self.sourcesFile))
373 def test_pkg_hash(self):
376 self.client.records.Lookup(self.client.cache['dpkg'].VersionList[0].FileList[0])
378 pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
379 '/var/lib/apt/lists/' + self.packagesFile +
380 ' | grep -E "^SHA1:" | head -n 1' +
381 ' | cut -d\ -f 2').read().rstrip('\n')
383 self.failUnless(self.client.records.SHA1Hash == pkg_hash,
384 "Hashes don't match: %s != %s" % (self.client.records.SHA1Hash, pkg_hash))
386 def test_src_hash(self):
389 self.client.srcrecords.Lookup('dpkg')
391 src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
392 '/var/lib/apt/lists/' + self.sourcesFile +
393 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
394 ' | cut -d\ -f 2').read().split('\n')[:-1]
396 for f in self.client.srcrecords.Files:
397 self.failUnless(f[0] in src_hashes, "Couldn't find %s in: %r" % (f[0], src_hashes))
399 def test_index_hash(self):
402 indexhash = self.client.indexrecords[self.releaseFile[self.releaseFile.find('_dists_'):].replace('_','/')]['main/binary-i386/Packages.bz2']['SHA1'][0]
404 idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
405 '/var/lib/apt/lists/' + self.releaseFile +
406 ' | grep -E " main/binary-i386/Packages.bz2$"'
407 ' | head -n 1 | cut -d\ -f 2').read().rstrip('\n')
409 self.failUnless(indexhash == idx_hash, "Hashes don't match: %s != %s" % (indexhash, idx_hash))
411 def verifyHash(self, found_hash, path, true_hash):
412 self.failUnless(found_hash.hexexpected() == true_hash,
413 "%s hashes don't match: %s != %s" % (path, found_hash.hexexpected(), true_hash))
415 def test_findIndexHash(self):
416 lastDefer = defer.Deferred()
418 idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
419 '/var/lib/apt/lists/' + self.releaseFile +
420 ' | grep -E " main/binary-i386/Packages.bz2$"'
421 ' | head -n 1 | cut -d\ -f 2').read().rstrip('\n')
422 idx_path = '/' + self.releaseFile[self.releaseFile.find('_dists_')+1:].replace('_','/')[:-7] + 'main/binary-i386/Packages.bz2'
424 d = self.client.findHash(idx_path)
425 d.addCallback(self.verifyHash, idx_path, idx_hash)
427 d.addBoth(lastDefer.callback)
430 def test_findPkgHash(self):
431 lastDefer = defer.Deferred()
433 pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
434 '/var/lib/apt/lists/' + self.packagesFile +
435 ' | grep -E "^SHA1:" | head -n 1' +
436 ' | cut -d\ -f 2').read().rstrip('\n')
437 pkg_path = '/' + os.popen('grep -A 30 -E "^Package: dpkg$" ' +
438 '/var/lib/apt/lists/' + self.packagesFile +
439 ' | grep -E "^Filename:" | head -n 1' +
440 ' | cut -d\ -f 2').read().rstrip('\n')
442 d = self.client.findHash(pkg_path)
443 d.addCallback(self.verifyHash, pkg_path, pkg_hash)
445 d.addBoth(lastDefer.callback)
448 def test_findSrcHash(self):
449 lastDefer = defer.Deferred()
451 src_dir = '/' + os.popen('grep -A 30 -E "^Package: dpkg$" ' +
452 '/var/lib/apt/lists/' + self.sourcesFile +
453 ' | grep -E "^Directory:" | head -n 1' +
454 ' | cut -d\ -f 2').read().rstrip('\n')
455 src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
456 '/var/lib/apt/lists/' + self.sourcesFile +
457 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
458 ' | cut -d\ -f 2').read().split('\n')[:-1]
459 src_paths = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
460 '/var/lib/apt/lists/' + self.sourcesFile +
461 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
462 ' | cut -d\ -f 4').read().split('\n')[:-1]
464 i = choice(range(len(src_hashes)))
465 d = self.client.findHash(src_dir + '/' + src_paths[i])
466 d.addCallback(self.verifyHash, src_dir + '/' + src_paths[i], src_hashes[i])
468 d.addBoth(lastDefer.callback)
471 def test_multipleFindHash(self):
472 lastDefer = defer.Deferred()
474 idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
475 '/var/lib/apt/lists/' + self.releaseFile +
476 ' | grep -E " main/binary-i386/Packages.bz2$"'
477 ' | head -n 1 | cut -d\ -f 2').read().rstrip('\n')
478 idx_path = '/' + self.releaseFile[self.releaseFile.find('_dists_')+1:].replace('_','/')[:-7] + 'main/binary-i386/Packages.bz2'
480 d = self.client.findHash(idx_path)
481 d.addCallback(self.verifyHash, idx_path, idx_hash)
483 pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
484 '/var/lib/apt/lists/' + self.packagesFile +
485 ' | grep -E "^SHA1:" | head -n 1' +
486 ' | cut -d\ -f 2').read().rstrip('\n')
487 pkg_path = '/' + os.popen('grep -A 30 -E "^Package: dpkg$" ' +
488 '/var/lib/apt/lists/' + self.packagesFile +
489 ' | grep -E "^Filename:" | head -n 1' +
490 ' | cut -d\ -f 2').read().rstrip('\n')
492 d = self.client.findHash(pkg_path)
493 d.addCallback(self.verifyHash, pkg_path, pkg_hash)
495 src_dir = '/' + os.popen('grep -A 30 -E "^Package: dpkg$" ' +
496 '/var/lib/apt/lists/' + self.sourcesFile +
497 ' | grep -E "^Directory:" | head -n 1' +
498 ' | cut -d\ -f 2').read().rstrip('\n')
499 src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
500 '/var/lib/apt/lists/' + self.sourcesFile +
501 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
502 ' | cut -d\ -f 2').read().split('\n')[:-1]
503 src_paths = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
504 '/var/lib/apt/lists/' + self.sourcesFile +
505 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
506 ' | cut -d\ -f 4').read().split('\n')[:-1]
508 for i in range(len(src_hashes)):
509 d = self.client.findHash(src_dir + '/' + src_paths[i])
510 d.addCallback(self.verifyHash, src_dir + '/' + src_paths[i], src_hashes[i])
512 idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
513 '/var/lib/apt/lists/' + self.releaseFile +
514 ' | grep -E " main/source/Sources.bz2$"'
515 ' | head -n 1 | cut -d\ -f 2').read().rstrip('\n')
516 idx_path = '/' + self.releaseFile[self.releaseFile.find('_dists_')+1:].replace('_','/')[:-7] + 'main/source/Sources.bz2'
518 d = self.client.findHash(idx_path)
519 d.addCallback(self.verifyHash, idx_path, idx_hash)
521 d.addBoth(lastDefer.callback)
525 for p in self.pending_calls:
528 self.pending_calls = []
529 self.client.cleanup()