1 # Disable the FutureWarning from the apt module
3 warnings.simplefilter("ignore", FutureWarning)
6 from random import choice
7 from shutil import rmtree
8 from copy import deepcopy
9 from UserDict import DictMixin
11 from twisted.internet import threads, defer
12 from twisted.python import log
13 from twisted.trial import unittest
15 import apt_pkg, apt_inst
16 from apt import OpProgress
20 TRACKED_FILES = ['release', 'sources', 'packages']
22 class PackageFileList(DictMixin):
23 """Manages a list of package files belonging to a backend.
25 @type packages: C{shelve dictionary}
26 @ivar packages: the files stored for this backend
29 def __init__(self, cache_dir):
30 self.cache_dir = cache_dir
31 if not os.path.exists(self.cache_dir):
32 os.makedirs(self.cache_dir)
37 """Open the persistent dictionary of files in this backend."""
38 if self.packages is None:
39 self.packages = shelve.open(self.cache_dir+'/packages.db')
42 """Close the persistent dictionary."""
43 if self.packages is not None:
46 def update_file(self, cache_path, file_path):
47 """Check if an updated file needs to be tracked.
49 Called from the mirror manager when files get updated so we can update our
50 fake lists and sources.list.
52 filename = cache_path.split('/')[-1]
53 if filename.lower() in TRACKED_FILES:
54 log.msg("Registering package file: "+cache_path)
55 self.packages[cache_path] = file_path
59 def check_files(self):
60 """Check all files in the database to make sure they exist."""
61 files = self.packages.keys()
63 if not os.path.exists(self.packages[f]):
64 log.msg("File in packages database has been deleted: "+f)
67 # Standard dictionary implementation so this class can be used like a dictionary.
68 def __getitem__(self, key): return self.packages[key]
69 def __setitem__(self, key, item): self.packages[key] = item
70 def __delitem__(self, key): del self.packages[key]
71 def keys(self): return self.packages.keys()
74 """Uses python-apt to answer queries about packages.
76 Makes a fake configuration for python-apt for each backend.
79 DEFAULT_APT_CONFIG = {
81 #'APT::Architecture' : 'i386', # Commented so the machine's config will set this
82 #'APT::Default-Release' : 'unstable',
84 'Dir::State' : 'apt/', # var/lib/apt/
85 'Dir::State::Lists': 'lists/', # lists/
86 #'Dir::State::cdroms' : 'cdroms.list',
87 'Dir::State::userstatus' : 'status.user',
88 'Dir::State::status': 'dpkg/status', # '/var/lib/dpkg/status'
89 'Dir::Cache' : '.apt/cache/', # var/cache/apt/
90 #'Dir::Cache::archives' : 'archives/',
91 'Dir::Cache::srcpkgcache' : 'srcpkgcache.bin',
92 'Dir::Cache::pkgcache' : 'pkgcache.bin',
93 'Dir::Etc' : 'apt/etc/', # etc/apt/
94 'Dir::Etc::sourcelist' : 'sources.list',
95 'Dir::Etc::vendorlist' : 'vendors.list',
96 'Dir::Etc::vendorparts' : 'vendors.list.d',
97 #'Dir::Etc::main' : 'apt.conf',
98 #'Dir::Etc::parts' : 'apt.conf.d',
99 #'Dir::Etc::preferences' : 'preferences',
101 #'Dir::Bin::methods' : '', #'/usr/lib/apt/methods'
102 'Dir::Bin::dpkg' : '/usr/bin/dpkg',
104 #'DPkg::Pre-Install-Pkgs' : '',
106 #'DPkg::Tools::Options' : '',
107 #'DPkg::Tools::Options::/usr/bin/apt-listchanges' : '',
108 #'DPkg::Tools::Options::/usr/bin/apt-listchanges::Version' : '2',
109 #'DPkg::Post-Invoke' : '',
111 essential_dirs = ('apt', 'apt/cache', 'apt/dpkg', 'apt/etc', 'apt/lists',
113 essential_files = ('apt/dpkg/status', 'apt/etc/sources.list',)
115 def __init__(self, cache_dir):
116 """Construct a new packages manager.
118 @ivar backendName: name of backend associated with this packages file
119 @ivar cache_dir: cache directory from config file
121 self.cache_dir = cache_dir
122 self.apt_config = deepcopy(self.DEFAULT_APT_CONFIG)
124 for dir in self.essential_dirs:
125 path = os.path.join(self.cache_dir, dir)
126 if not os.path.exists(path):
128 for file in self.essential_files:
129 path = os.path.join(self.cache_dir, file)
130 if not os.path.exists(path):
135 self.apt_config['Dir'] = self.cache_dir
136 self.apt_config['Dir::State::status'] = os.path.join(self.cache_dir,
137 self.apt_config['Dir::State'], self.apt_config['Dir::State::status'])
138 self.packages = PackageFileList(cache_dir)
144 self.packages.close()
146 def addRelease(self, cache_path, file_path):
147 """Dirty hack until python-apt supports apt-pkg/indexrecords.h
150 self.indexrecords[cache_path] = {}
152 read_packages = False
153 f = open(file_path, 'r')
159 read_packages = False
161 # Read the various headers from the file
162 h, v = line.split(":", 1)
163 if h == "MD5Sum" or h == "SHA1" or h == "SHA256":
167 # Bad header line, just ignore it
168 log.msg("WARNING: Ignoring badly formatted Release line: %s" % line)
170 # Skip to the next line
173 # Read file names from the multiple hash sections of the file
176 self.indexrecords[cache_path].setdefault(p[2], {})[hash_type] = (p[0], p[1])
180 def file_updated(self, cache_path, file_path):
181 """A file in the backend has changed, manage it.
183 If this affects us, unload our apt database
185 if self.packages.update_file(cache_path, file_path):
189 """Make sure the package is initialized and loaded."""
190 if self.loading is None:
191 self.loading = threads.deferToThread(self._load)
192 self.loading.addCallback(self.doneLoading)
195 def doneLoading(self, loadResult):
196 """Cache is loaded."""
198 # Must pass on the result for the next callback
202 """Regenerates the fake configuration and load the packages cache."""
203 if self.loaded: return True
205 rmtree(os.path.join(self.cache_dir, self.apt_config['Dir::State'],
206 self.apt_config['Dir::State::Lists']))
207 os.makedirs(os.path.join(self.cache_dir, self.apt_config['Dir::State'],
208 self.apt_config['Dir::State::Lists'], 'partial'))
209 sources_filename = os.path.join(self.cache_dir, self.apt_config['Dir::Etc'],
210 self.apt_config['Dir::Etc::sourcelist'])
211 sources = open(sources_filename, 'w')
213 deb_src_added = False
214 self.packages.check_files()
215 self.indexrecords = {}
216 for f in self.packages:
217 # we should probably clear old entries from self.packages and
218 # take into account the recorded mtime as optimization
219 filepath = self.packages[f]
220 if f.split('/')[-1] == "Release":
221 self.addRelease(f, filepath)
222 fake_uri='http://apt-dht'+f
223 fake_dirname = '/'.join(fake_uri.split('/')[:-1])
224 if f.endswith('Sources'):
226 source_line='deb-src '+fake_dirname+'/ /'
228 source_line='deb '+fake_dirname+'/ /'
229 listpath=(os.path.join(self.cache_dir, self.apt_config['Dir::State'],
230 self.apt_config['Dir::State::Lists'],
231 apt_pkg.URItoFileName(fake_uri)))
232 sources.write(source_line+'\n')
233 log.msg("Sources line: " + source_line)
234 sources_count = sources_count + 1
237 #we should empty the directory instead
241 os.symlink(filepath, listpath)
244 if sources_count == 0:
245 log.msg("No Packages files available for %s backend"%(self.cache_dir))
248 log.msg("Loading Packages database for "+self.cache_dir)
249 for key, value in self.apt_config.items():
250 apt_pkg.Config[key] = value
252 self.cache = apt_pkg.GetCache(OpProgress())
253 self.records = apt_pkg.GetPkgRecords(self.cache)
255 self.srcrecords = apt_pkg.GetPkgSrcRecords()
257 self.srcrecords = None
263 """Tries to make the packages server quit."""
268 del self.indexrecords
272 """Cleanup and close any loaded caches."""
274 self.packages.close()
276 def findHash(self, path):
277 """Find the hash for a given path in this mirror.
279 Returns a deferred so it can make sure the cache is loaded first.
283 deferLoad = self.load()
284 deferLoad.addCallback(self._findHash, path, d)
285 deferLoad.addErrback(self._findHash_error, path, d)
289 def _findHash_error(self, failure, path, d):
290 """An error occurred while trying to find a hash."""
291 log.msg('An error occurred while looking up a hash for: %s' % path)
293 d.callback((None, None))
295 def _findHash(self, loadResult, path, d):
296 """Really find the hash for a path.
298 Have to pass the returned loadResult on in case other calls to this
299 function are pending.
302 d.callback((None, None))
305 # First look for the path in the cache of index files
306 for release in self.indexrecords:
307 if path.startswith(release[:-7]):
308 for indexFile in self.indexrecords[release]:
309 if release[:-7] + indexFile == path:
310 d.callback(self.indexrecords[release][indexFile]['SHA1'])
313 package = path.split('/')[-1].split('_')[0]
315 # Check the binary packages
317 for version in self.cache[package].VersionList:
319 for verFile in version.FileList:
320 if self.records.Lookup(verFile):
321 if '/' + self.records.FileName == path:
322 d.callback((self.records.SHA1Hash, size))
327 # Check the source packages' files
329 self.srcrecords.Restart()
330 if self.srcrecords.Lookup(package):
331 for f in self.srcrecords.Files:
332 if path == '/' + f[2]:
333 d.callback((f[0], f[1]))
336 d.callback((None, None))
339 class TestAptPackages(unittest.TestCase):
340 """Unit tests for the AptPackages cache."""
350 self.client = AptPackages('/tmp/.apt-dht')
352 self.packagesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "_main_.*Packages$" | tail -n 1').read().rstrip('\n')
353 self.sourcesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "_main_.*Sources$" | tail -n 1').read().rstrip('\n')
354 for f in os.walk('/var/lib/apt/lists').next()[2]:
355 if f[-7:] == "Release" and self.packagesFile.startswith(f[:-7]):
359 self.client.file_updated(self.releaseFile[self.releaseFile.find('_dists_'):].replace('_','/'),
360 '/var/lib/apt/lists/' + self.releaseFile)
361 self.client.file_updated(self.packagesFile[self.packagesFile.find('_dists_'):].replace('_','/'),
362 '/var/lib/apt/lists/' + self.packagesFile)
363 self.client.file_updated(self.sourcesFile[self.sourcesFile.find('_dists_'):].replace('_','/'),
364 '/var/lib/apt/lists/' + self.sourcesFile)
366 def test_pkg_hash(self):
369 self.client.records.Lookup(self.client.cache['dpkg'].VersionList[0].FileList[0])
371 pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
372 '/var/lib/apt/lists/' + self.packagesFile +
373 ' | grep -E "^SHA1:" | head -n 1' +
374 ' | cut -d\ -f 2').read().rstrip('\n')
376 self.failUnless(self.client.records.SHA1Hash == pkg_hash,
377 "Hashes don't match: %s != %s" % (self.client.records.SHA1Hash, pkg_hash))
379 def test_src_hash(self):
382 self.client.srcrecords.Lookup('dpkg')
384 src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
385 '/var/lib/apt/lists/' + self.sourcesFile +
386 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
387 ' | cut -d\ -f 2').read().split('\n')[:-1]
389 for f in self.client.srcrecords.Files:
390 self.failUnless(f[0] in src_hashes, "Couldn't find %s in: %r" % (f[0], src_hashes))
392 def test_index_hash(self):
395 indexhash = self.client.indexrecords[self.releaseFile[self.releaseFile.find('_dists_'):].replace('_','/')]['main/binary-i386/Packages.bz2']['SHA1'][0]
397 idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
398 '/var/lib/apt/lists/' + self.releaseFile +
399 ' | grep -E " main/binary-i386/Packages.bz2$"'
400 ' | head -n 1 | cut -d\ -f 2').read().rstrip('\n')
402 self.failUnless(indexhash == idx_hash, "Hashes don't match: %s != %s" % (indexhash, idx_hash))
404 def verifyHash(self, found_hash, path, true_hash):
405 self.failUnless(found_hash[0] == true_hash,
406 "%s hashes don't match: %s != %s" % (path, found_hash[0], true_hash))
408 def test_findIndexHash(self):
409 lastDefer = defer.Deferred()
411 idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
412 '/var/lib/apt/lists/' + self.releaseFile +
413 ' | grep -E " main/binary-i386/Packages.bz2$"'
414 ' | head -n 1 | cut -d\ -f 2').read().rstrip('\n')
415 idx_path = '/' + self.releaseFile[self.releaseFile.find('_dists_')+1:].replace('_','/')[:-7] + 'main/binary-i386/Packages.bz2'
417 d = self.client.findHash(idx_path)
418 d.addCallback(self.verifyHash, idx_path, idx_hash)
420 d.addBoth(lastDefer.callback)
423 def test_findPkgHash(self):
424 lastDefer = defer.Deferred()
426 pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
427 '/var/lib/apt/lists/' + self.packagesFile +
428 ' | grep -E "^SHA1:" | head -n 1' +
429 ' | cut -d\ -f 2').read().rstrip('\n')
430 pkg_path = '/' + os.popen('grep -A 30 -E "^Package: dpkg$" ' +
431 '/var/lib/apt/lists/' + self.packagesFile +
432 ' | grep -E "^Filename:" | head -n 1' +
433 ' | cut -d\ -f 2').read().rstrip('\n')
435 d = self.client.findHash(pkg_path)
436 d.addCallback(self.verifyHash, pkg_path, pkg_hash)
438 d.addBoth(lastDefer.callback)
441 def test_findSrcHash(self):
442 lastDefer = defer.Deferred()
444 src_dir = '/' + os.popen('grep -A 30 -E "^Package: dpkg$" ' +
445 '/var/lib/apt/lists/' + self.sourcesFile +
446 ' | grep -E "^Directory:" | head -n 1' +
447 ' | cut -d\ -f 2').read().rstrip('\n')
448 src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
449 '/var/lib/apt/lists/' + self.sourcesFile +
450 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
451 ' | cut -d\ -f 2').read().split('\n')[:-1]
452 src_paths = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
453 '/var/lib/apt/lists/' + self.sourcesFile +
454 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
455 ' | cut -d\ -f 4').read().split('\n')[:-1]
457 i = choice(range(len(src_hashes)))
458 d = self.client.findHash(src_dir + '/' + src_paths[i])
459 d.addCallback(self.verifyHash, src_dir + '/' + src_paths[i], src_hashes[i])
461 d.addBoth(lastDefer.callback)
464 def test_multipleFindHash(self):
465 lastDefer = defer.Deferred()
467 idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
468 '/var/lib/apt/lists/' + self.releaseFile +
469 ' | grep -E " main/binary-i386/Packages.bz2$"'
470 ' | head -n 1 | cut -d\ -f 2').read().rstrip('\n')
471 idx_path = '/' + self.releaseFile[self.releaseFile.find('_dists_')+1:].replace('_','/')[:-7] + 'main/binary-i386/Packages.bz2'
473 d = self.client.findHash(idx_path)
474 d.addCallback(self.verifyHash, idx_path, idx_hash)
476 pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
477 '/var/lib/apt/lists/' + self.packagesFile +
478 ' | grep -E "^SHA1:" | head -n 1' +
479 ' | cut -d\ -f 2').read().rstrip('\n')
480 pkg_path = '/' + os.popen('grep -A 30 -E "^Package: dpkg$" ' +
481 '/var/lib/apt/lists/' + self.packagesFile +
482 ' | grep -E "^Filename:" | head -n 1' +
483 ' | cut -d\ -f 2').read().rstrip('\n')
485 d = self.client.findHash(pkg_path)
486 d.addCallback(self.verifyHash, pkg_path, pkg_hash)
488 src_dir = '/' + os.popen('grep -A 30 -E "^Package: dpkg$" ' +
489 '/var/lib/apt/lists/' + self.sourcesFile +
490 ' | grep -E "^Directory:" | head -n 1' +
491 ' | cut -d\ -f 2').read().rstrip('\n')
492 src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
493 '/var/lib/apt/lists/' + self.sourcesFile +
494 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
495 ' | cut -d\ -f 2').read().split('\n')[:-1]
496 src_paths = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
497 '/var/lib/apt/lists/' + self.sourcesFile +
498 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
499 ' | cut -d\ -f 4').read().split('\n')[:-1]
501 for i in range(len(src_hashes)):
502 d = self.client.findHash(src_dir + '/' + src_paths[i])
503 d.addCallback(self.verifyHash, src_dir + '/' + src_paths[i], src_hashes[i])
505 idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
506 '/var/lib/apt/lists/' + self.releaseFile +
507 ' | grep -E " main/source/Sources.bz2$"'
508 ' | head -n 1 | cut -d\ -f 2').read().rstrip('\n')
509 idx_path = '/' + self.releaseFile[self.releaseFile.find('_dists_')+1:].replace('_','/')[:-7] + 'main/source/Sources.bz2'
511 d = self.client.findHash(idx_path)
512 d.addCallback(self.verifyHash, idx_path, idx_hash)
514 d.addBoth(lastDefer.callback)
518 for p in self.pending_calls:
521 self.pending_calls = []
522 self.client.cleanup()