1 # Disable the FutureWarning from the apt module
3 warnings.simplefilter("ignore", FutureWarning)
6 from random import choice
7 from shutil import rmtree
8 from copy import deepcopy
9 from UserDict import DictMixin
11 from twisted.internet import threads, defer
12 from twisted.python import log
13 from twisted.trial import unittest
15 import apt_pkg, apt_inst
16 from apt import OpProgress
20 class PackageFileList(DictMixin):
21 """Manages a list of package files belonging to a backend.
23 @type packages: C{shelve dictionary}
24 @ivar packages: the files stored for this backend
27 def __init__(self, cache_dir):
28 self.cache_dir = cache_dir
29 if not os.path.exists(self.cache_dir):
30 os.makedirs(self.cache_dir)
35 """Open the persistent dictionary of files in this backend."""
36 if self.packages is None:
37 self.packages = shelve.open(self.cache_dir+'/packages.db')
40 """Close the persistent dictionary."""
41 if self.packages is not None:
44 def update_file(self, cache_path, file_path):
45 """Check if an updated file needs to be tracked.
47 Called from the mirror manager when files get updated so we can update our
48 fake lists and sources.list.
50 filename = cache_path.split('/')[-1]
51 if filename=="Packages" or filename=="Release" or filename=="Sources":
52 log.msg("Registering package file: "+cache_path)
53 self.packages[cache_path] = file_path
57 def check_files(self):
58 """Check all files in the database to make sure they exist."""
59 files = self.packages.keys()
61 if not os.path.exists(self.packages[f]):
62 log.msg("File in packages database has been deleted: "+f)
65 # Standard dictionary implementation so this class can be used like a dictionary.
66 def __getitem__(self, key): return self.packages[key]
67 def __setitem__(self, key, item): self.packages[key] = item
68 def __delitem__(self, key): del self.packages[key]
69 def keys(self): return self.packages.keys()
72 """Uses python-apt to answer queries about packages.
74 Makes a fake configuration for python-apt for each backend.
77 DEFAULT_APT_CONFIG = {
79 #'APT::Architecture' : 'i386', # Commented so the machine's config will set this
80 #'APT::Default-Release' : 'unstable',
82 'Dir::State' : 'apt/', # var/lib/apt/
83 'Dir::State::Lists': 'lists/', # lists/
84 #'Dir::State::cdroms' : 'cdroms.list',
85 'Dir::State::userstatus' : 'status.user',
86 'Dir::State::status': 'dpkg/status', # '/var/lib/dpkg/status'
87 'Dir::Cache' : '.apt/cache/', # var/cache/apt/
88 #'Dir::Cache::archives' : 'archives/',
89 'Dir::Cache::srcpkgcache' : 'srcpkgcache.bin',
90 'Dir::Cache::pkgcache' : 'pkgcache.bin',
91 'Dir::Etc' : 'apt/etc/', # etc/apt/
92 'Dir::Etc::sourcelist' : 'sources.list',
93 'Dir::Etc::vendorlist' : 'vendors.list',
94 'Dir::Etc::vendorparts' : 'vendors.list.d',
95 #'Dir::Etc::main' : 'apt.conf',
96 #'Dir::Etc::parts' : 'apt.conf.d',
97 #'Dir::Etc::preferences' : 'preferences',
99 #'Dir::Bin::methods' : '', #'/usr/lib/apt/methods'
100 'Dir::Bin::dpkg' : '/usr/bin/dpkg',
102 #'DPkg::Pre-Install-Pkgs' : '',
104 #'DPkg::Tools::Options' : '',
105 #'DPkg::Tools::Options::/usr/bin/apt-listchanges' : '',
106 #'DPkg::Tools::Options::/usr/bin/apt-listchanges::Version' : '2',
107 #'DPkg::Post-Invoke' : '',
109 essential_dirs = ('apt', 'apt/cache', 'apt/dpkg', 'apt/etc', 'apt/lists',
111 essential_files = ('apt/dpkg/status', 'apt/etc/sources.list',)
113 def __init__(self, cache_dir):
114 """Construct a new packages manager.
116 @ivar backendName: name of backend associated with this packages file
117 @ivar cache_dir: cache directory from config file
119 self.cache_dir = cache_dir
120 self.apt_config = deepcopy(self.DEFAULT_APT_CONFIG)
122 for dir in self.essential_dirs:
123 path = os.path.join(self.cache_dir, dir)
124 if not os.path.exists(path):
126 for file in self.essential_files:
127 path = os.path.join(self.cache_dir, file)
128 if not os.path.exists(path):
133 self.apt_config['Dir'] = self.cache_dir
134 self.apt_config['Dir::State::status'] = os.path.join(self.cache_dir,
135 self.apt_config['Dir::State'], self.apt_config['Dir::State::status'])
136 self.packages = PackageFileList(cache_dir)
142 self.packages.close()
144 def addRelease(self, cache_path, file_path):
145 """Dirty hack until python-apt supports apt-pkg/indexrecords.h
148 self.indexrecords[cache_path] = {}
150 read_packages = False
151 f = open(file_path, 'r')
157 read_packages = False
159 # Read the various headers from the file
160 h, v = line.split(":", 1)
161 if h == "MD5Sum" or h == "SHA1" or h == "SHA256":
165 # Bad header line, just ignore it
166 log.msg("WARNING: Ignoring badly formatted Release line: %s" % line)
168 # Skip to the next line
171 # Read file names from the multiple hash sections of the file
174 self.indexrecords[cache_path].setdefault(p[2], {})[hash_type] = (p[0], p[1])
178 def file_updated(self, cache_path, file_path):
179 """A file in the backend has changed, manage it.
181 If this affects us, unload our apt database
183 if self.packages.update_file(cache_path, file_path):
187 """Make sure the package is initialized and loaded."""
188 if self.loading is None:
189 self.loading = threads.deferToThread(self._load)
190 self.loading.addCallback(self.doneLoading)
193 def doneLoading(self, loadResult):
194 """Cache is loaded."""
196 # Must pass on the result for the next callback
200 """Regenerates the fake configuration and load the packages cache."""
201 if self.loaded: return True
203 rmtree(os.path.join(self.cache_dir, self.apt_config['Dir::State'],
204 self.apt_config['Dir::State::Lists']))
205 os.makedirs(os.path.join(self.cache_dir, self.apt_config['Dir::State'],
206 self.apt_config['Dir::State::Lists'], 'partial'))
207 sources_filename = os.path.join(self.cache_dir, self.apt_config['Dir::Etc'],
208 self.apt_config['Dir::Etc::sourcelist'])
209 sources = open(sources_filename, 'w')
211 self.packages.check_files()
212 self.indexrecords = {}
213 for f in self.packages:
214 # we should probably clear old entries from self.packages and
215 # take into account the recorded mtime as optimization
216 filepath = self.packages[f]
217 if f.split('/')[-1] == "Release":
218 self.addRelease(f, filepath)
219 fake_uri='http://apt-dht'+f
220 fake_dirname = '/'.join(fake_uri.split('/')[:-1])
221 if f.endswith('Sources'):
222 source_line='deb-src '+fake_dirname+'/ /'
224 source_line='deb '+fake_dirname+'/ /'
225 listpath=(os.path.join(self.cache_dir, self.apt_config['Dir::State'],
226 self.apt_config['Dir::State::Lists'],
227 apt_pkg.URItoFileName(fake_uri)))
228 sources.write(source_line+'\n')
229 log.msg("Sources line: " + source_line)
230 sources_count = sources_count + 1
233 #we should empty the directory instead
237 os.symlink(filepath, listpath)
240 if sources_count == 0:
241 log.msg("No Packages files available for %s backend"%(self.cache_dir))
244 log.msg("Loading Packages database for "+self.cache_dir)
245 for key, value in self.apt_config.items():
246 apt_pkg.Config[key] = value
248 self.cache = apt_pkg.GetCache(OpProgress())
249 self.records = apt_pkg.GetPkgRecords(self.cache)
250 self.srcrecords = apt_pkg.GetPkgSrcRecords()
256 """Tries to make the packages server quit."""
261 del self.indexrecords
265 """Cleanup and close any loaded caches."""
267 self.packages.close()
269 def findHash(self, path):
270 """Find the hash for a given path in this mirror.
272 Returns a deferred so it can make sure the cache is loaded first.
276 deferLoad = self.load()
277 deferLoad.addCallback(self._findHash, path, d)
281 def _findHash(self, loadResult, path, d):
282 """Really find the hash for a path.
284 Have to pass the returned loadResult on in case other calls to this
285 function are pending.
288 d.callback((None, None))
291 # First look for the path in the cache of index files
292 for release in self.indexrecords:
293 if path.startswith(release[:-7]):
294 for indexFile in self.indexrecords[release]:
295 if release[:-7] + indexFile == path:
296 d.callback(self.indexrecords[release][indexFile]['SHA1'])
299 package = path.split('/')[-1].split('_')[0]
301 # Check the binary packages
303 for version in self.cache[package].VersionList:
305 for verFile in version.FileList:
306 if self.records.Lookup(verFile):
307 if '/' + self.records.FileName == path:
308 d.callback((self.records.SHA1Hash, size))
313 # Check the source packages' files
314 self.srcrecords.Restart()
315 if self.srcrecords.Lookup(package):
316 for f in self.srcrecords.Files:
317 if path == '/' + f[2]:
318 d.callback((f[0], f[1]))
321 d.callback((None, None))
324 class TestAptPackages(unittest.TestCase):
325 """Unit tests for the AptPackages cache."""
334 self.client = AptPackages('/tmp/.apt-dht')
336 self.packagesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "Packages$" | tail -n 1').read().rstrip('\n')
337 self.sourcesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "Sources$" | tail -n 1').read().rstrip('\n')
338 for f in os.walk('/var/lib/apt/lists').next()[2]:
339 if f[-7:] == "Release" and self.packagesFile.startswith(f[:-7]):
343 self.client.file_updated(self.releaseFile[self.releaseFile.find('_debian_')+1:].replace('_','/'),
344 '/var/lib/apt/lists/' + self.releaseFile)
345 self.client.file_updated(self.packagesFile[self.packagesFile.find('_debian_')+1:].replace('_','/'),
346 '/var/lib/apt/lists/' + self.packagesFile)
347 self.client.file_updated(self.sourcesFile[self.sourcesFile.find('_debian_')+1:].replace('_','/'),
348 '/var/lib/apt/lists/' + self.sourcesFile)
350 def test_pkg_hash(self):
353 self.client.records.Lookup(self.client.cache['dpkg'].VersionList[0].FileList[0])
355 pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
356 '/var/lib/apt/lists/' + self.packagesFile +
357 ' | grep -E "^SHA1:" | head -n 1' +
358 ' | cut -d\ -f 2').read().rstrip('\n')
360 self.failUnless(self.client.records.SHA1Hash == pkg_hash,
361 "Hashes don't match: %s != %s" % (self.client.records.SHA1Hash, pkg_hash))
363 def test_src_hash(self):
366 self.client.srcrecords.Lookup('dpkg')
368 src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
369 '/var/lib/apt/lists/' + self.sourcesFile +
370 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
371 ' | cut -d\ -f 2').read().split('\n')[:-1]
373 for f in self.client.srcrecords.Files:
374 self.failUnless(f[0] in src_hashes, "Couldn't find %s in: %r" % (f[0], src_hashes))
376 def test_index_hash(self):
379 indexhash = self.client.indexrecords[self.releaseFile[self.releaseFile.find('_debian_')+1:].replace('_','/')]['main/binary-i386/Packages.bz2']['SHA1'][0]
381 idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
382 '/var/lib/apt/lists/' + self.releaseFile +
383 ' | grep -E " main/binary-i386/Packages.bz2$"'
384 ' | head -n 1 | cut -d\ -f 2').read().rstrip('\n')
386 self.failUnless(indexhash == idx_hash, "Hashes don't match: %s != %s" % (indexhash, idx_hash))
388 def verifyHash(self, found_hash, path, true_hash):
389 self.failUnless(found_hash[0] == true_hash,
390 "%s hashes don't match: %s != %s" % (path, found_hash[0], true_hash))
392 def test_findIndexHash(self):
393 lastDefer = defer.Deferred()
395 idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
396 '/var/lib/apt/lists/' + self.releaseFile +
397 ' | grep -E " main/binary-i386/Packages.bz2$"'
398 ' | head -n 1 | cut -d\ -f 2').read().rstrip('\n')
399 idx_path = self.releaseFile[self.releaseFile.find('_debian_')+1:].replace('_','/')[:-7] + 'main/binary-i386/Packages.bz2'
401 d = self.client.findHash(idx_path)
402 d.addCallback(self.verifyHash, idx_path, idx_hash)
404 d.addCallback(lastDefer.callback)
407 def test_findPkgHash(self):
408 lastDefer = defer.Deferred()
410 pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
411 '/var/lib/apt/lists/' + self.packagesFile +
412 ' | grep -E "^SHA1:" | head -n 1' +
413 ' | cut -d\ -f 2').read().rstrip('\n')
414 pkg_path = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
415 '/var/lib/apt/lists/' + self.packagesFile +
416 ' | grep -E "^Filename:" | head -n 1' +
417 ' | cut -d\ -f 2').read().rstrip('\n')
419 d = self.client.findHash(pkg_path)
420 d.addCallback(self.verifyHash, pkg_path, pkg_hash)
422 d.addCallback(lastDefer.callback)
425 def test_findSrcHash(self):
426 lastDefer = defer.Deferred()
428 src_dir = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
429 '/var/lib/apt/lists/' + self.sourcesFile +
430 ' | grep -E "^Directory:" | head -n 1' +
431 ' | cut -d\ -f 2').read().rstrip('\n')
432 src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
433 '/var/lib/apt/lists/' + self.sourcesFile +
434 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
435 ' | cut -d\ -f 2').read().split('\n')[:-1]
436 src_paths = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
437 '/var/lib/apt/lists/' + self.sourcesFile +
438 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
439 ' | cut -d\ -f 4').read().split('\n')[:-1]
441 i = choice(range(len(src_hashes)))
442 d = self.client.findHash(src_dir + '/' + src_paths[i])
443 d.addCallback(self.verifyHash, src_dir + '/' + src_paths[i], src_hashes[i])
445 d.addCallback(lastDefer.callback)
448 def test_multipleFindHash(self):
449 lastDefer = defer.Deferred()
451 idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
452 '/var/lib/apt/lists/' + self.releaseFile +
453 ' | grep -E " main/binary-i386/Packages.bz2$"'
454 ' | head -n 1 | cut -d\ -f 2').read().rstrip('\n')
455 idx_path = self.releaseFile[self.releaseFile.find('_debian_')+1:].replace('_','/')[:-7] + 'main/binary-i386/Packages.bz2'
457 d = self.client.findHash(idx_path)
458 d.addCallback(self.verifyHash, idx_path, idx_hash)
460 pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
461 '/var/lib/apt/lists/' + self.packagesFile +
462 ' | grep -E "^SHA1:" | head -n 1' +
463 ' | cut -d\ -f 2').read().rstrip('\n')
464 pkg_path = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
465 '/var/lib/apt/lists/' + self.packagesFile +
466 ' | grep -E "^Filename:" | head -n 1' +
467 ' | cut -d\ -f 2').read().rstrip('\n')
469 d = self.client.findHash(pkg_path)
470 d.addCallback(self.verifyHash, pkg_path, pkg_hash)
472 src_dir = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
473 '/var/lib/apt/lists/' + self.sourcesFile +
474 ' | grep -E "^Directory:" | head -n 1' +
475 ' | cut -d\ -f 2').read().rstrip('\n')
476 src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
477 '/var/lib/apt/lists/' + self.sourcesFile +
478 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
479 ' | cut -d\ -f 2').read().split('\n')[:-1]
480 src_paths = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
481 '/var/lib/apt/lists/' + self.sourcesFile +
482 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
483 ' | cut -d\ -f 4').read().split('\n')[:-1]
485 for i in range(len(src_hashes)):
486 d = self.client.findHash(src_dir + '/' + src_paths[i])
487 d.addCallback(self.verifyHash, src_dir + '/' + src_paths[i], src_hashes[i])
489 idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
490 '/var/lib/apt/lists/' + self.releaseFile +
491 ' | grep -E " main/source/Sources.bz2$"'
492 ' | head -n 1 | cut -d\ -f 2').read().rstrip('\n')
493 idx_path = self.releaseFile[self.releaseFile.find('_debian_')+1:].replace('_','/')[:-7] + 'main/source/Sources.bz2'
495 d = self.client.findHash(idx_path)
496 d.addCallback(self.verifyHash, idx_path, idx_hash)
498 d.addCallback(lastDefer.callback)
502 for p in self.pending_calls:
505 self.pending_calls = []
506 self.client.cleanup()