1 # Disable the FutureWarning from the apt module
3 warnings.simplefilter("ignore", FutureWarning)
6 from random import choice
7 from shutil import rmtree
8 from copy import deepcopy
9 from UserDict import DictMixin
11 from twisted.internet import threads, defer
12 from twisted.python import log
13 from twisted.trial import unittest
15 import apt_pkg, apt_inst
16 from apt import OpProgress
18 from Hash import HashObject
22 TRACKED_FILES = ['release', 'sources', 'packages']
24 class PackageFileList(DictMixin):
25 """Manages a list of package files belonging to a backend.
27 @type packages: C{shelve dictionary}
28 @ivar packages: the files stored for this backend
31 def __init__(self, cache_dir):
32 self.cache_dir = cache_dir
33 if not os.path.exists(self.cache_dir):
34 os.makedirs(self.cache_dir)
39 """Open the persistent dictionary of files in this backend."""
40 if self.packages is None:
41 self.packages = shelve.open(self.cache_dir+'/packages.db')
44 """Close the persistent dictionary."""
45 if self.packages is not None:
48 def update_file(self, cache_path, file_path):
49 """Check if an updated file needs to be tracked.
51 Called from the mirror manager when files get updated so we can update our
52 fake lists and sources.list.
54 filename = cache_path.split('/')[-1]
55 if filename.lower() in TRACKED_FILES:
56 log.msg("Registering package file: "+cache_path)
57 self.packages[cache_path] = file_path
61 def check_files(self):
62 """Check all files in the database to make sure they exist."""
63 files = self.packages.keys()
65 if not os.path.exists(self.packages[f]):
66 log.msg("File in packages database has been deleted: "+f)
69 # Standard dictionary implementation so this class can be used like a dictionary.
70 def __getitem__(self, key): return self.packages[key]
71 def __setitem__(self, key, item): self.packages[key] = item
72 def __delitem__(self, key): del self.packages[key]
73 def keys(self): return self.packages.keys()
76 """Uses python-apt to answer queries about packages.
78 Makes a fake configuration for python-apt for each backend.
81 DEFAULT_APT_CONFIG = {
83 #'APT::Architecture' : 'i386', # Commented so the machine's config will set this
84 #'APT::Default-Release' : 'unstable',
86 'Dir::State' : 'apt/', # var/lib/apt/
87 'Dir::State::Lists': 'lists/', # lists/
88 #'Dir::State::cdroms' : 'cdroms.list',
89 'Dir::State::userstatus' : 'status.user',
90 'Dir::State::status': 'dpkg/status', # '/var/lib/dpkg/status'
91 'Dir::Cache' : '.apt/cache/', # var/cache/apt/
92 #'Dir::Cache::archives' : 'archives/',
93 'Dir::Cache::srcpkgcache' : 'srcpkgcache.bin',
94 'Dir::Cache::pkgcache' : 'pkgcache.bin',
95 'Dir::Etc' : 'apt/etc/', # etc/apt/
96 'Dir::Etc::sourcelist' : 'sources.list',
97 'Dir::Etc::vendorlist' : 'vendors.list',
98 'Dir::Etc::vendorparts' : 'vendors.list.d',
99 #'Dir::Etc::main' : 'apt.conf',
100 #'Dir::Etc::parts' : 'apt.conf.d',
101 #'Dir::Etc::preferences' : 'preferences',
103 #'Dir::Bin::methods' : '', #'/usr/lib/apt/methods'
104 'Dir::Bin::dpkg' : '/usr/bin/dpkg',
106 #'DPkg::Pre-Install-Pkgs' : '',
108 #'DPkg::Tools::Options' : '',
109 #'DPkg::Tools::Options::/usr/bin/apt-listchanges' : '',
110 #'DPkg::Tools::Options::/usr/bin/apt-listchanges::Version' : '2',
111 #'DPkg::Post-Invoke' : '',
113 essential_dirs = ('apt', 'apt/cache', 'apt/dpkg', 'apt/etc', 'apt/lists',
115 essential_files = ('apt/dpkg/status', 'apt/etc/sources.list',)
117 def __init__(self, cache_dir):
118 """Construct a new packages manager.
120 @ivar backendName: name of backend associated with this packages file
121 @ivar cache_dir: cache directory from config file
123 self.cache_dir = cache_dir
124 self.apt_config = deepcopy(self.DEFAULT_APT_CONFIG)
126 for dir in self.essential_dirs:
127 path = os.path.join(self.cache_dir, dir)
128 if not os.path.exists(path):
130 for file in self.essential_files:
131 path = os.path.join(self.cache_dir, file)
132 if not os.path.exists(path):
137 self.apt_config['Dir'] = self.cache_dir
138 self.apt_config['Dir::State::status'] = os.path.join(self.cache_dir,
139 self.apt_config['Dir::State'], self.apt_config['Dir::State::status'])
140 self.packages = PackageFileList(cache_dir)
146 self.packages.close()
148 def addRelease(self, cache_path, file_path):
149 """Dirty hack until python-apt supports apt-pkg/indexrecords.h
152 self.indexrecords[cache_path] = {}
154 read_packages = False
155 f = open(file_path, 'r')
161 read_packages = False
163 # Read the various headers from the file
164 h, v = line.split(":", 1)
165 if h == "MD5Sum" or h == "SHA1" or h == "SHA256":
169 # Bad header line, just ignore it
170 log.msg("WARNING: Ignoring badly formatted Release line: %s" % line)
172 # Skip to the next line
175 # Read file names from the multiple hash sections of the file
178 self.indexrecords[cache_path].setdefault(p[2], {})[hash_type] = (p[0], p[1])
182 def file_updated(self, cache_path, file_path):
183 """A file in the backend has changed, manage it.
185 If this affects us, unload our apt database
187 if self.packages.update_file(cache_path, file_path):
191 """Make sure the package is initialized and loaded."""
192 if self.loading is None:
193 self.loading = threads.deferToThread(self._load)
194 self.loading.addCallback(self.doneLoading)
197 def doneLoading(self, loadResult):
198 """Cache is loaded."""
200 # Must pass on the result for the next callback
204 """Regenerates the fake configuration and load the packages cache."""
205 if self.loaded: return True
207 rmtree(os.path.join(self.cache_dir, self.apt_config['Dir::State'],
208 self.apt_config['Dir::State::Lists']))
209 os.makedirs(os.path.join(self.cache_dir, self.apt_config['Dir::State'],
210 self.apt_config['Dir::State::Lists'], 'partial'))
211 sources_filename = os.path.join(self.cache_dir, self.apt_config['Dir::Etc'],
212 self.apt_config['Dir::Etc::sourcelist'])
213 sources = open(sources_filename, 'w')
215 deb_src_added = False
216 self.packages.check_files()
217 self.indexrecords = {}
218 for f in self.packages:
219 # we should probably clear old entries from self.packages and
220 # take into account the recorded mtime as optimization
221 filepath = self.packages[f]
222 if f.split('/')[-1] == "Release":
223 self.addRelease(f, filepath)
224 fake_uri='http://apt-dht'+f
225 fake_dirname = '/'.join(fake_uri.split('/')[:-1])
226 if f.endswith('Sources'):
228 source_line='deb-src '+fake_dirname+'/ /'
230 source_line='deb '+fake_dirname+'/ /'
231 listpath=(os.path.join(self.cache_dir, self.apt_config['Dir::State'],
232 self.apt_config['Dir::State::Lists'],
233 apt_pkg.URItoFileName(fake_uri)))
234 sources.write(source_line+'\n')
235 log.msg("Sources line: " + source_line)
236 sources_count = sources_count + 1
239 #we should empty the directory instead
243 os.symlink(filepath, listpath)
246 if sources_count == 0:
247 log.msg("No Packages files available for %s backend"%(self.cache_dir))
250 log.msg("Loading Packages database for "+self.cache_dir)
251 for key, value in self.apt_config.items():
252 apt_pkg.Config[key] = value
254 self.cache = apt_pkg.GetCache(OpProgress())
255 self.records = apt_pkg.GetPkgRecords(self.cache)
257 self.srcrecords = apt_pkg.GetPkgSrcRecords()
259 self.srcrecords = None
265 """Tries to make the packages server quit."""
270 del self.indexrecords
274 """Cleanup and close any loaded caches."""
276 self.packages.close()
278 def findHash(self, path):
279 """Find the hash for a given path in this mirror.
281 Returns a deferred so it can make sure the cache is loaded first.
285 deferLoad = self.load()
286 deferLoad.addCallback(self._findHash, path, d)
287 deferLoad.addErrback(self._findHash_error, path, d)
291 def _findHash_error(self, failure, path, d):
292 """An error occurred while trying to find a hash."""
293 log.msg('An error occurred while looking up a hash for: %s' % path)
295 d.callback(HashObject())
297 def _findHash(self, loadResult, path, d):
298 """Really find the hash for a path.
300 Have to pass the returned loadResult on in case other calls to this
301 function are pending.
304 d.callback(HashObject())
307 # First look for the path in the cache of index files
308 for release in self.indexrecords:
309 if path.startswith(release[:-7]):
310 for indexFile in self.indexrecords[release]:
311 if release[:-7] + indexFile == path:
313 h.setFromIndexRecord(self.indexrecords[release][indexFile])
317 package = path.split('/')[-1].split('_')[0]
319 # Check the binary packages
321 for version in self.cache[package].VersionList:
323 for verFile in version.FileList:
324 if self.records.Lookup(verFile):
325 if '/' + self.records.FileName == path:
327 h.setFromPkgRecord(self.records, size)
333 # Check the source packages' files
335 self.srcrecords.Restart()
336 if self.srcrecords.Lookup(package):
337 for f in self.srcrecords.Files:
338 if path == '/' + f[2]:
340 h.setFromSrcRecord(f)
344 d.callback(HashObject())
347 class TestAptPackages(unittest.TestCase):
348 """Unit tests for the AptPackages cache."""
358 self.client = AptPackages('/tmp/.apt-dht')
360 self.packagesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "_main_.*Packages$" | tail -n 1').read().rstrip('\n')
361 self.sourcesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "_main_.*Sources$" | tail -n 1').read().rstrip('\n')
362 for f in os.walk('/var/lib/apt/lists').next()[2]:
363 if f[-7:] == "Release" and self.packagesFile.startswith(f[:-7]):
367 self.client.file_updated(self.releaseFile[self.releaseFile.find('_dists_'):].replace('_','/'),
368 '/var/lib/apt/lists/' + self.releaseFile)
369 self.client.file_updated(self.packagesFile[self.packagesFile.find('_dists_'):].replace('_','/'),
370 '/var/lib/apt/lists/' + self.packagesFile)
371 self.client.file_updated(self.sourcesFile[self.sourcesFile.find('_dists_'):].replace('_','/'),
372 '/var/lib/apt/lists/' + self.sourcesFile)
374 def test_pkg_hash(self):
377 self.client.records.Lookup(self.client.cache['dpkg'].VersionList[0].FileList[0])
379 pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
380 '/var/lib/apt/lists/' + self.packagesFile +
381 ' | grep -E "^SHA1:" | head -n 1' +
382 ' | cut -d\ -f 2').read().rstrip('\n')
384 self.failUnless(self.client.records.SHA1Hash == pkg_hash,
385 "Hashes don't match: %s != %s" % (self.client.records.SHA1Hash, pkg_hash))
387 def test_src_hash(self):
390 self.client.srcrecords.Lookup('dpkg')
392 src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
393 '/var/lib/apt/lists/' + self.sourcesFile +
394 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
395 ' | cut -d\ -f 2').read().split('\n')[:-1]
397 for f in self.client.srcrecords.Files:
398 self.failUnless(f[0] in src_hashes, "Couldn't find %s in: %r" % (f[0], src_hashes))
400 def test_index_hash(self):
403 indexhash = self.client.indexrecords[self.releaseFile[self.releaseFile.find('_dists_'):].replace('_','/')]['main/binary-i386/Packages.bz2']['SHA1'][0]
405 idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
406 '/var/lib/apt/lists/' + self.releaseFile +
407 ' | grep -E " main/binary-i386/Packages.bz2$"'
408 ' | head -n 1 | cut -d\ -f 2').read().rstrip('\n')
410 self.failUnless(indexhash == idx_hash, "Hashes don't match: %s != %s" % (indexhash, idx_hash))
412 def verifyHash(self, found_hash, path, true_hash):
413 self.failUnless(found_hash.hexexpected() == true_hash,
414 "%s hashes don't match: %s != %s" % (path, found_hash.hexexpected(), true_hash))
416 def test_findIndexHash(self):
417 lastDefer = defer.Deferred()
419 idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
420 '/var/lib/apt/lists/' + self.releaseFile +
421 ' | grep -E " main/binary-i386/Packages.bz2$"'
422 ' | head -n 1 | cut -d\ -f 2').read().rstrip('\n')
423 idx_path = '/' + self.releaseFile[self.releaseFile.find('_dists_')+1:].replace('_','/')[:-7] + 'main/binary-i386/Packages.bz2'
425 d = self.client.findHash(idx_path)
426 d.addCallback(self.verifyHash, idx_path, idx_hash)
428 d.addBoth(lastDefer.callback)
431 def test_findPkgHash(self):
432 lastDefer = defer.Deferred()
434 pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
435 '/var/lib/apt/lists/' + self.packagesFile +
436 ' | grep -E "^SHA1:" | head -n 1' +
437 ' | cut -d\ -f 2').read().rstrip('\n')
438 pkg_path = '/' + os.popen('grep -A 30 -E "^Package: dpkg$" ' +
439 '/var/lib/apt/lists/' + self.packagesFile +
440 ' | grep -E "^Filename:" | head -n 1' +
441 ' | cut -d\ -f 2').read().rstrip('\n')
443 d = self.client.findHash(pkg_path)
444 d.addCallback(self.verifyHash, pkg_path, pkg_hash)
446 d.addBoth(lastDefer.callback)
449 def test_findSrcHash(self):
450 lastDefer = defer.Deferred()
452 src_dir = '/' + os.popen('grep -A 30 -E "^Package: dpkg$" ' +
453 '/var/lib/apt/lists/' + self.sourcesFile +
454 ' | grep -E "^Directory:" | head -n 1' +
455 ' | cut -d\ -f 2').read().rstrip('\n')
456 src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
457 '/var/lib/apt/lists/' + self.sourcesFile +
458 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
459 ' | cut -d\ -f 2').read().split('\n')[:-1]
460 src_paths = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
461 '/var/lib/apt/lists/' + self.sourcesFile +
462 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
463 ' | cut -d\ -f 4').read().split('\n')[:-1]
465 i = choice(range(len(src_hashes)))
466 d = self.client.findHash(src_dir + '/' + src_paths[i])
467 d.addCallback(self.verifyHash, src_dir + '/' + src_paths[i], src_hashes[i])
469 d.addBoth(lastDefer.callback)
472 def test_multipleFindHash(self):
473 lastDefer = defer.Deferred()
475 idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
476 '/var/lib/apt/lists/' + self.releaseFile +
477 ' | grep -E " main/binary-i386/Packages.bz2$"'
478 ' | head -n 1 | cut -d\ -f 2').read().rstrip('\n')
479 idx_path = '/' + self.releaseFile[self.releaseFile.find('_dists_')+1:].replace('_','/')[:-7] + 'main/binary-i386/Packages.bz2'
481 d = self.client.findHash(idx_path)
482 d.addCallback(self.verifyHash, idx_path, idx_hash)
484 pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
485 '/var/lib/apt/lists/' + self.packagesFile +
486 ' | grep -E "^SHA1:" | head -n 1' +
487 ' | cut -d\ -f 2').read().rstrip('\n')
488 pkg_path = '/' + os.popen('grep -A 30 -E "^Package: dpkg$" ' +
489 '/var/lib/apt/lists/' + self.packagesFile +
490 ' | grep -E "^Filename:" | head -n 1' +
491 ' | cut -d\ -f 2').read().rstrip('\n')
493 d = self.client.findHash(pkg_path)
494 d.addCallback(self.verifyHash, pkg_path, pkg_hash)
496 src_dir = '/' + os.popen('grep -A 30 -E "^Package: dpkg$" ' +
497 '/var/lib/apt/lists/' + self.sourcesFile +
498 ' | grep -E "^Directory:" | head -n 1' +
499 ' | cut -d\ -f 2').read().rstrip('\n')
500 src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
501 '/var/lib/apt/lists/' + self.sourcesFile +
502 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
503 ' | cut -d\ -f 2').read().split('\n')[:-1]
504 src_paths = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
505 '/var/lib/apt/lists/' + self.sourcesFile +
506 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
507 ' | cut -d\ -f 4').read().split('\n')[:-1]
509 for i in range(len(src_hashes)):
510 d = self.client.findHash(src_dir + '/' + src_paths[i])
511 d.addCallback(self.verifyHash, src_dir + '/' + src_paths[i], src_hashes[i])
513 idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
514 '/var/lib/apt/lists/' + self.releaseFile +
515 ' | grep -E " main/source/Sources.bz2$"'
516 ' | head -n 1 | cut -d\ -f 2').read().rstrip('\n')
517 idx_path = '/' + self.releaseFile[self.releaseFile.find('_dists_')+1:].replace('_','/')[:-7] + 'main/source/Sources.bz2'
519 d = self.client.findHash(idx_path)
520 d.addCallback(self.verifyHash, idx_path, idx_hash)
522 d.addBoth(lastDefer.callback)
526 for p in self.pending_calls:
529 self.pending_calls = []
530 self.client.cleanup()