1 # Disable the FutureWarning from the apt module
3 warnings.simplefilter("ignore", FutureWarning)
6 from random import choice
7 from shutil import rmtree
8 from copy import deepcopy
9 from UserDict import DictMixin
11 from twisted.internet import threads, defer
12 from twisted.python import log
13 from twisted.trial import unittest
15 import apt_pkg, apt_inst
16 from apt import OpProgress
20 class PackageFileList(DictMixin):
21 """Manages a list of package files belonging to a backend.
23 @type packages: C{shelve dictionary}
24 @ivar packages: the files stored for this backend
27 def __init__(self, cache_dir):
28 self.cache_dir = cache_dir
29 if not os.path.exists(self.cache_dir):
30 os.makedirs(self.cache_dir)
35 """Open the persistent dictionary of files in this backend."""
36 if self.packages is None:
37 self.packages = shelve.open(self.cache_dir+'/packages.db')
40 """Close the persistent dictionary."""
41 if self.packages is not None:
44 def update_file(self, cache_path, file_path):
45 """Check if an updated file needs to be tracked.
47 Called from the mirror manager when files get updated so we can update our
48 fake lists and sources.list.
50 filename = cache_path.split('/')[-1]
51 if filename=="Packages" or filename=="Release" or filename=="Sources":
52 log.msg("Registering package file: "+cache_path)
53 self.packages[cache_path] = file_path
57 def check_files(self):
58 """Check all files in the database to make sure they exist."""
59 files = self.packages.keys()
61 if not os.path.exists(self.packages[f]):
62 log.msg("File in packages database has been deleted: "+f)
65 # Standard dictionary implementation so this class can be used like a dictionary.
66 def __getitem__(self, key): return self.packages[key]
67 def __setitem__(self, key, item): self.packages[key] = item
68 def __delitem__(self, key): del self.packages[key]
69 def keys(self): return self.packages.keys()
72 """Uses python-apt to answer queries about packages.
74 Makes a fake configuration for python-apt for each backend.
77 DEFAULT_APT_CONFIG = {
79 #'APT::Architecture' : 'i386', # Commented so the machine's config will set this
80 #'APT::Default-Release' : 'unstable',
82 'Dir::State' : 'apt/', # var/lib/apt/
83 'Dir::State::Lists': 'lists/', # lists/
84 #'Dir::State::cdroms' : 'cdroms.list',
85 'Dir::State::userstatus' : 'status.user',
86 'Dir::State::status': 'dpkg/status', # '/var/lib/dpkg/status'
87 'Dir::Cache' : '.apt/cache/', # var/cache/apt/
88 #'Dir::Cache::archives' : 'archives/',
89 'Dir::Cache::srcpkgcache' : 'srcpkgcache.bin',
90 'Dir::Cache::pkgcache' : 'pkgcache.bin',
91 'Dir::Etc' : 'apt/etc/', # etc/apt/
92 'Dir::Etc::sourcelist' : 'sources.list',
93 'Dir::Etc::vendorlist' : 'vendors.list',
94 'Dir::Etc::vendorparts' : 'vendors.list.d',
95 #'Dir::Etc::main' : 'apt.conf',
96 #'Dir::Etc::parts' : 'apt.conf.d',
97 #'Dir::Etc::preferences' : 'preferences',
99 #'Dir::Bin::methods' : '', #'/usr/lib/apt/methods'
100 'Dir::Bin::dpkg' : '/usr/bin/dpkg',
102 #'DPkg::Pre-Install-Pkgs' : '',
104 #'DPkg::Tools::Options' : '',
105 #'DPkg::Tools::Options::/usr/bin/apt-listchanges' : '',
106 #'DPkg::Tools::Options::/usr/bin/apt-listchanges::Version' : '2',
107 #'DPkg::Post-Invoke' : '',
109 essential_dirs = ('apt', 'apt/cache', 'apt/dpkg', 'apt/etc', 'apt/lists',
111 essential_files = ('apt/dpkg/status', 'apt/etc/sources.list',)
113 def __init__(self, cache_dir):
114 """Construct a new packages manager.
116 @ivar backendName: name of backend associated with this packages file
117 @ivar cache_dir: cache directory from config file
119 self.cache_dir = cache_dir
120 self.apt_config = deepcopy(self.DEFAULT_APT_CONFIG)
122 for dir in self.essential_dirs:
123 path = os.path.join(self.cache_dir, dir)
124 if not os.path.exists(path):
126 for file in self.essential_files:
127 path = os.path.join(self.cache_dir, file)
128 if not os.path.exists(path):
133 self.apt_config['Dir'] = self.cache_dir
134 self.apt_config['Dir::State::status'] = os.path.join(self.cache_dir,
135 self.apt_config['Dir::State'], self.apt_config['Dir::State::status'])
136 self.packages = PackageFileList(cache_dir)
142 self.packages.close()
144 def addRelease(self, cache_path, file_path):
145 """Dirty hack until python-apt supports apt-pkg/indexrecords.h
148 self.indexrecords[cache_path] = {}
150 read_packages = False
151 f = open(file_path, 'r')
157 read_packages = False
159 # Read the various headers from the file
160 h, v = line.split(":", 1)
161 if h == "MD5Sum" or h == "SHA1" or h == "SHA256":
165 # Bad header line, just ignore it
166 log.msg("WARNING: Ignoring badly formatted Release line: %s" % line)
168 # Skip to the next line
171 # Read file names from the multiple hash sections of the file
174 self.indexrecords[cache_path].setdefault(p[2], {})[hash_type] = (p[0], p[1])
178 def file_updated(self, cache_path, file_path):
179 """A file in the backend has changed, manage it.
181 If this affects us, unload our apt database
183 if self.packages.update_file(cache_path, file_path):
187 """Make sure the package is initialized and loaded."""
188 if self.loading is None:
189 self.loading = threads.deferToThread(self._load)
190 self.loading.addCallback(self.doneLoading)
193 def doneLoading(self, loadResult):
194 """Cache is loaded."""
196 # Must pass on the result for the next callback
200 """Regenerates the fake configuration and load the packages cache."""
201 if self.loaded: return True
203 rmtree(os.path.join(self.cache_dir, self.apt_config['Dir::State'],
204 self.apt_config['Dir::State::Lists']))
205 os.makedirs(os.path.join(self.cache_dir, self.apt_config['Dir::State'],
206 self.apt_config['Dir::State::Lists'], 'partial'))
207 sources_filename = os.path.join(self.cache_dir, self.apt_config['Dir::Etc'],
208 self.apt_config['Dir::Etc::sourcelist'])
209 sources = open(sources_filename, 'w')
211 self.packages.check_files()
212 self.indexrecords = {}
213 for f in self.packages:
214 # we should probably clear old entries from self.packages and
215 # take into account the recorded mtime as optimization
216 filepath = self.packages[f]
217 if f.split('/')[-1] == "Release":
218 self.addRelease(f, filepath)
219 fake_uri='http://apt-dht'+f
220 fake_dirname = '/'.join(fake_uri.split('/')[:-1])
221 if f.endswith('Sources'):
222 source_line='deb-src '+fake_dirname+'/ /'
224 source_line='deb '+fake_dirname+'/ /'
225 listpath=(os.path.join(self.cache_dir, self.apt_config['Dir::State'],
226 self.apt_config['Dir::State::Lists'],
227 apt_pkg.URItoFileName(fake_uri)))
228 sources.write(source_line+'\n')
229 log.msg("Sources line: " + source_line)
230 sources_count = sources_count + 1
233 #we should empty the directory instead
237 os.symlink(filepath, listpath)
240 if sources_count == 0:
241 log.msg("No Packages files available for %s backend"%(self.cache_dir))
244 log.msg("Loading Packages database for "+self.cache_dir)
245 for key, value in self.apt_config.items():
246 apt_pkg.Config[key] = value
248 self.cache = apt_pkg.GetCache(OpProgress())
249 self.records = apt_pkg.GetPkgRecords(self.cache)
250 self.srcrecords = apt_pkg.GetPkgSrcRecords()
256 """Tries to make the packages server quit."""
261 del self.indexrecords
265 """Cleanup and close any loaded caches."""
267 self.packages.close()
269 def findHash(self, path):
270 """Find the hash for a given path in this mirror.
272 Returns a deferred so it can make sure the cache is loaded first.
276 deferLoad = self.load()
277 deferLoad.addCallback(self._findHash, path, d)
281 def _findHash(self, loadResult, path, d):
282 """Really find the hash for a path.
284 Have to pass the returned loadResult on in case other calls to this
285 function are pending.
288 d.callback((None, None))
291 # First look for the path in the cache of index files
292 for release in self.indexrecords:
293 if path.startswith(release[:-7]):
294 for indexFile in self.indexrecords[release]:
295 if release[:-7] + indexFile == path:
296 d.callback(self.indexrecords[release][indexFile]['SHA1'])
299 package = path.split('/')[-1].split('_')[0]
301 # Check the binary packages
303 for version in self.cache[package].VersionList:
305 for verFile in version.FileList:
306 if self.records.Lookup(verFile):
307 if '/' + self.records.FileName == path:
308 d.callback((self.records.SHA1Hash, size))
313 # Check the source packages' files
314 self.srcrecords.Restart()
315 if self.srcrecords.Lookup(package):
316 for f in self.srcrecords.Files:
317 if path == '/' + f[2]:
318 d.callback((f[0], f[1]))
321 d.callback((None, None))
324 class TestAptPackages(unittest.TestCase):
325 """Unit tests for the AptPackages cache."""
335 self.client = AptPackages('/tmp/.apt-dht')
337 self.packagesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "_main_.*Packages$" | tail -n 1').read().rstrip('\n')
338 self.sourcesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "_main_.*Sources$" | tail -n 1').read().rstrip('\n')
339 for f in os.walk('/var/lib/apt/lists').next()[2]:
340 if f[-7:] == "Release" and self.packagesFile.startswith(f[:-7]):
344 self.client.file_updated(self.releaseFile[self.releaseFile.find('_dists_'):].replace('_','/'),
345 '/var/lib/apt/lists/' + self.releaseFile)
346 self.client.file_updated(self.packagesFile[self.packagesFile.find('_dists_'):].replace('_','/'),
347 '/var/lib/apt/lists/' + self.packagesFile)
348 self.client.file_updated(self.sourcesFile[self.sourcesFile.find('_dists_'):].replace('_','/'),
349 '/var/lib/apt/lists/' + self.sourcesFile)
351 def test_pkg_hash(self):
354 self.client.records.Lookup(self.client.cache['dpkg'].VersionList[0].FileList[0])
356 pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
357 '/var/lib/apt/lists/' + self.packagesFile +
358 ' | grep -E "^SHA1:" | head -n 1' +
359 ' | cut -d\ -f 2').read().rstrip('\n')
361 self.failUnless(self.client.records.SHA1Hash == pkg_hash,
362 "Hashes don't match: %s != %s" % (self.client.records.SHA1Hash, pkg_hash))
364 def test_src_hash(self):
367 self.client.srcrecords.Lookup('dpkg')
369 src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
370 '/var/lib/apt/lists/' + self.sourcesFile +
371 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
372 ' | cut -d\ -f 2').read().split('\n')[:-1]
374 for f in self.client.srcrecords.Files:
375 self.failUnless(f[0] in src_hashes, "Couldn't find %s in: %r" % (f[0], src_hashes))
377 def test_index_hash(self):
380 indexhash = self.client.indexrecords[self.releaseFile[self.releaseFile.find('_dists_'):].replace('_','/')]['main/binary-i386/Packages.bz2']['SHA1'][0]
382 idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
383 '/var/lib/apt/lists/' + self.releaseFile +
384 ' | grep -E " main/binary-i386/Packages.bz2$"'
385 ' | head -n 1 | cut -d\ -f 2').read().rstrip('\n')
387 self.failUnless(indexhash == idx_hash, "Hashes don't match: %s != %s" % (indexhash, idx_hash))
389 def verifyHash(self, found_hash, path, true_hash):
390 self.failUnless(found_hash[0] == true_hash,
391 "%s hashes don't match: %s != %s" % (path, found_hash[0], true_hash))
393 def test_findIndexHash(self):
394 lastDefer = defer.Deferred()
396 idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
397 '/var/lib/apt/lists/' + self.releaseFile +
398 ' | grep -E " main/binary-i386/Packages.bz2$"'
399 ' | head -n 1 | cut -d\ -f 2').read().rstrip('\n')
400 idx_path = '/' + self.releaseFile[self.releaseFile.find('_dists_')+1:].replace('_','/')[:-7] + 'main/binary-i386/Packages.bz2'
402 d = self.client.findHash(idx_path)
403 d.addCallback(self.verifyHash, idx_path, idx_hash)
405 d.addBoth(lastDefer.callback)
408 def test_findPkgHash(self):
409 lastDefer = defer.Deferred()
411 pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
412 '/var/lib/apt/lists/' + self.packagesFile +
413 ' | grep -E "^SHA1:" | head -n 1' +
414 ' | cut -d\ -f 2').read().rstrip('\n')
415 pkg_path = '/' + os.popen('grep -A 30 -E "^Package: dpkg$" ' +
416 '/var/lib/apt/lists/' + self.packagesFile +
417 ' | grep -E "^Filename:" | head -n 1' +
418 ' | cut -d\ -f 2').read().rstrip('\n')
420 d = self.client.findHash(pkg_path)
421 d.addCallback(self.verifyHash, pkg_path, pkg_hash)
423 d.addBoth(lastDefer.callback)
426 def test_findSrcHash(self):
427 lastDefer = defer.Deferred()
429 src_dir = '/' + os.popen('grep -A 30 -E "^Package: dpkg$" ' +
430 '/var/lib/apt/lists/' + self.sourcesFile +
431 ' | grep -E "^Directory:" | head -n 1' +
432 ' | cut -d\ -f 2').read().rstrip('\n')
433 src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
434 '/var/lib/apt/lists/' + self.sourcesFile +
435 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
436 ' | cut -d\ -f 2').read().split('\n')[:-1]
437 src_paths = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
438 '/var/lib/apt/lists/' + self.sourcesFile +
439 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
440 ' | cut -d\ -f 4').read().split('\n')[:-1]
442 i = choice(range(len(src_hashes)))
443 d = self.client.findHash(src_dir + '/' + src_paths[i])
444 d.addCallback(self.verifyHash, src_dir + '/' + src_paths[i], src_hashes[i])
446 d.addBoth(lastDefer.callback)
449 def test_multipleFindHash(self):
450 lastDefer = defer.Deferred()
452 idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
453 '/var/lib/apt/lists/' + self.releaseFile +
454 ' | grep -E " main/binary-i386/Packages.bz2$"'
455 ' | head -n 1 | cut -d\ -f 2').read().rstrip('\n')
456 idx_path = '/' + self.releaseFile[self.releaseFile.find('_dists_')+1:].replace('_','/')[:-7] + 'main/binary-i386/Packages.bz2'
458 d = self.client.findHash(idx_path)
459 d.addCallback(self.verifyHash, idx_path, idx_hash)
461 pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
462 '/var/lib/apt/lists/' + self.packagesFile +
463 ' | grep -E "^SHA1:" | head -n 1' +
464 ' | cut -d\ -f 2').read().rstrip('\n')
465 pkg_path = '/' + os.popen('grep -A 30 -E "^Package: dpkg$" ' +
466 '/var/lib/apt/lists/' + self.packagesFile +
467 ' | grep -E "^Filename:" | head -n 1' +
468 ' | cut -d\ -f 2').read().rstrip('\n')
470 d = self.client.findHash(pkg_path)
471 d.addCallback(self.verifyHash, pkg_path, pkg_hash)
473 src_dir = '/' + os.popen('grep -A 30 -E "^Package: dpkg$" ' +
474 '/var/lib/apt/lists/' + self.sourcesFile +
475 ' | grep -E "^Directory:" | head -n 1' +
476 ' | cut -d\ -f 2').read().rstrip('\n')
477 src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
478 '/var/lib/apt/lists/' + self.sourcesFile +
479 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
480 ' | cut -d\ -f 2').read().split('\n')[:-1]
481 src_paths = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
482 '/var/lib/apt/lists/' + self.sourcesFile +
483 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
484 ' | cut -d\ -f 4').read().split('\n')[:-1]
486 for i in range(len(src_hashes)):
487 d = self.client.findHash(src_dir + '/' + src_paths[i])
488 d.addCallback(self.verifyHash, src_dir + '/' + src_paths[i], src_hashes[i])
490 idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
491 '/var/lib/apt/lists/' + self.releaseFile +
492 ' | grep -E " main/source/Sources.bz2$"'
493 ' | head -n 1 | cut -d\ -f 2').read().rstrip('\n')
494 idx_path = '/' + self.releaseFile[self.releaseFile.find('_dists_')+1:].replace('_','/')[:-7] + 'main/source/Sources.bz2'
496 d = self.client.findHash(idx_path)
497 d.addCallback(self.verifyHash, idx_path, idx_hash)
499 d.addBoth(lastDefer.callback)
503 for p in self.pending_calls:
506 self.pending_calls = []
507 self.client.cleanup()