1 # Disable the FutureWarning from the apt module
3 warnings.simplefilter("ignore", FutureWarning)
5 import os, os.path, stat, random, re, shelve, shutil, fcntl, copy, UserDict
7 from twisted.internet import threads, defer
8 from twisted.python import log
9 from twisted.trial import unittest
11 import apt_pkg, apt_inst
12 from apt import OpProgress
16 class PackageFileList(UserDict.DictMixin):
17 """Manages a list of package files belonging to a backend.
19 @type packages: C{shelve dictionary}
20 @ivar packages: the files stored for this backend
23 def __init__(self, cache_dir):
24 self.cache_dir = cache_dir
25 if not os.path.exists(self.cache_dir):
26 os.makedirs(self.cache_dir)
31 """Open the persistent dictionary of files in this backend."""
32 if self.packages is None:
33 self.packages = shelve.open(self.cache_dir+'/packages.db')
36 """Close the persistent dictionary."""
37 if self.packages is not None:
40 def update_file(self, cache_path, file_path):
41 """Check if an updated file needs to be tracked.
43 Called from the mirror manager when files get updated so we can update our
44 fake lists and sources.list.
46 filename = cache_path.split('/')[-1]
47 if filename=="Packages" or filename=="Release" or filename=="Sources":
48 log.msg("Registering package file: "+cache_path)
49 self.packages[cache_path] = file_path
53 def check_files(self):
54 """Check all files in the database to make sure they exist."""
55 files = self.packages.keys()
57 if not os.path.exists(self.packages[f]):
58 log.msg("File in packages database has been deleted: "+f)
61 # Standard dictionary implementation so this class can be used like a dictionary.
62 def __getitem__(self, key): return self.packages[key]
63 def __setitem__(self, key, item): self.packages[key] = item
64 def __delitem__(self, key): del self.packages[key]
65 def keys(self): return self.packages.keys()
68 """Uses python-apt to answer queries about packages.
70 Makes a fake configuration for python-apt for each backend.
73 DEFAULT_APT_CONFIG = {
75 #'APT::Architecture' : 'i386', # Commented so the machine's config will set this
76 #'APT::Default-Release' : 'unstable',
78 'Dir::State' : 'apt/', # var/lib/apt/
79 'Dir::State::Lists': 'lists/', # lists/
80 #'Dir::State::cdroms' : 'cdroms.list',
81 'Dir::State::userstatus' : 'status.user',
82 'Dir::State::status': 'dpkg/status', # '/var/lib/dpkg/status'
83 'Dir::Cache' : '.apt/cache/', # var/cache/apt/
84 #'Dir::Cache::archives' : 'archives/',
85 'Dir::Cache::srcpkgcache' : 'srcpkgcache.bin',
86 'Dir::Cache::pkgcache' : 'pkgcache.bin',
87 'Dir::Etc' : 'apt/etc/', # etc/apt/
88 'Dir::Etc::sourcelist' : 'sources.list',
89 'Dir::Etc::vendorlist' : 'vendors.list',
90 'Dir::Etc::vendorparts' : 'vendors.list.d',
91 #'Dir::Etc::main' : 'apt.conf',
92 #'Dir::Etc::parts' : 'apt.conf.d',
93 #'Dir::Etc::preferences' : 'preferences',
95 #'Dir::Bin::methods' : '', #'/usr/lib/apt/methods'
96 'Dir::Bin::dpkg' : '/usr/bin/dpkg',
98 #'DPkg::Pre-Install-Pkgs' : '',
100 #'DPkg::Tools::Options' : '',
101 #'DPkg::Tools::Options::/usr/bin/apt-listchanges' : '',
102 #'DPkg::Tools::Options::/usr/bin/apt-listchanges::Version' : '2',
103 #'DPkg::Post-Invoke' : '',
105 essential_dirs = ('apt', 'apt/cache', 'apt/dpkg', 'apt/etc', 'apt/lists',
107 essential_files = ('apt/dpkg/status', 'apt/etc/sources.list',)
109 def __init__(self, cache_dir):
110 """Construct a new packages manager.
112 @ivar backendName: name of backend associated with this packages file
113 @ivar cache_dir: cache directory from config file
115 self.cache_dir = cache_dir
116 self.apt_config = copy.deepcopy(self.DEFAULT_APT_CONFIG)
118 for dir in self.essential_dirs:
119 path = os.path.join(self.cache_dir, dir)
120 if not os.path.exists(path):
122 for file in self.essential_files:
123 path = os.path.join(self.cache_dir, file)
124 if not os.path.exists(path):
129 self.apt_config['Dir'] = self.cache_dir
130 self.apt_config['Dir::State::status'] = os.path.join(self.cache_dir,
131 self.apt_config['Dir::State'], self.apt_config['Dir::State::status'])
132 self.packages = PackageFileList(cache_dir)
138 self.packages.close()
140 def addRelease(self, cache_path, file_path):
141 """Dirty hack until python-apt supports apt-pkg/indexrecords.h
144 self.indexrecords[cache_path] = {}
146 read_packages = False
147 f = open(file_path, 'r')
153 read_packages = False
155 # Read the various headers from the file
156 h, v = line.split(":", 1)
157 if h == "MD5Sum" or h == "SHA1" or h == "SHA256":
161 # Bad header line, just ignore it
162 log.msg("WARNING: Ignoring badly formatted Release line: %s" % line)
164 # Skip to the next line
167 # Read file names from the multiple hash sections of the file
170 self.indexrecords[cache_path].setdefault(p[2], {})[hash_type] = (p[0], p[1])
174 def file_updated(self, cache_path, file_path):
175 """A file in the backend has changed, manage it.
177 If this affects us, unload our apt database
179 if self.packages.update_file(cache_path, file_path):
183 """Make sure the package is initialized and loaded."""
184 if self.loading is None:
185 self.loading = threads.deferToThread(self._load)
186 self.loading.addCallback(self.doneLoading)
189 def doneLoading(self, loadResult):
190 """Cache is loaded."""
192 # Must pass on the result for the next callback
196 """Regenerates the fake configuration and load the packages cache."""
197 if self.loaded: return True
199 shutil.rmtree(os.path.join(self.cache_dir, self.apt_config['Dir::State'],
200 self.apt_config['Dir::State::Lists']))
201 os.makedirs(os.path.join(self.cache_dir, self.apt_config['Dir::State'],
202 self.apt_config['Dir::State::Lists'], 'partial'))
203 sources_filename = os.path.join(self.cache_dir, self.apt_config['Dir::Etc'],
204 self.apt_config['Dir::Etc::sourcelist'])
205 sources = open(sources_filename, 'w')
207 self.packages.check_files()
208 self.indexrecords = {}
209 for f in self.packages:
210 # we should probably clear old entries from self.packages and
211 # take into account the recorded mtime as optimization
212 filepath = self.packages[f]
213 if f.split('/')[-1] == "Release":
214 self.addRelease(f, filepath)
215 fake_uri='http://apt-dht'+f
216 fake_dirname = '/'.join(fake_uri.split('/')[:-1])
217 if f.endswith('Sources'):
218 source_line='deb-src '+fake_dirname+'/ /'
220 source_line='deb '+fake_dirname+'/ /'
221 listpath=(os.path.join(self.cache_dir, self.apt_config['Dir::State'],
222 self.apt_config['Dir::State::Lists'],
223 apt_pkg.URItoFileName(fake_uri)))
224 sources.write(source_line+'\n')
225 log.msg("Sources line: " + source_line)
226 sources_count = sources_count + 1
229 #we should empty the directory instead
233 os.symlink(filepath, listpath)
236 if sources_count == 0:
237 log.msg("No Packages files available for %s backend"%(self.cache_dir))
240 log.msg("Loading Packages database for "+self.cache_dir)
241 for key, value in self.apt_config.items():
242 apt_pkg.Config[key] = value
244 self.cache = apt_pkg.GetCache(OpProgress())
245 self.records = apt_pkg.GetPkgRecords(self.cache)
246 self.srcrecords = apt_pkg.GetPkgSrcRecords()
252 """Tries to make the packages server quit."""
257 del self.indexrecords
261 """Cleanup and close any loaded caches."""
263 self.packages.close()
265 def findHash(self, path):
266 """Find the hash for a given path in this mirror.
268 Returns a deferred so it can make sure the cache is loaded first.
272 deferLoad = self.load()
273 deferLoad.addCallback(self._findHash, path, d)
277 def _findHash(self, loadResult, path, d):
278 """Really find the hash for a path.
280 Have to pass the returned loadResult on in case other calls to this
281 function are pending.
284 d.callback((None, None))
287 # First look for the path in the cache of index files
288 for release in self.indexrecords:
289 if path.startswith(release[:-7]):
290 for indexFile in self.indexrecords[release]:
291 if release[:-7] + indexFile == path:
292 d.callback(self.indexrecords[release][indexFile]['SHA1'])
295 package = path.split('/')[-1].split('_')[0]
297 # Check the binary packages
299 for version in self.cache[package].VersionList:
301 for verFile in version.FileList:
302 if self.records.Lookup(verFile):
303 if self.records.FileName == path:
304 d.callback((self.records.SHA1Hash, size))
309 # Check the source packages' files
310 self.srcrecords.Restart()
311 if self.srcrecords.Lookup(package):
312 for f in self.srcrecords.Files:
314 d.callback((f[0], f[1]))
317 d.callback((None, None))
320 class TestAptPackages(unittest.TestCase):
321 """Unit tests for the AptPackages cache."""
330 self.client = AptPackages('/tmp/.apt-dht')
332 self.packagesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "Packages$" | tail -n 1').read().rstrip('\n')
333 self.sourcesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "Sources$" | tail -n 1').read().rstrip('\n')
334 for f in os.walk('/var/lib/apt/lists').next()[2]:
335 if f[-7:] == "Release" and self.packagesFile.startswith(f[:-7]):
339 self.client.file_updated(self.releaseFile[self.releaseFile.find('_debian_')+1:].replace('_','/'),
340 '/var/lib/apt/lists/' + self.releaseFile)
341 self.client.file_updated(self.packagesFile[self.packagesFile.find('_debian_')+1:].replace('_','/'),
342 '/var/lib/apt/lists/' + self.packagesFile)
343 self.client.file_updated(self.sourcesFile[self.sourcesFile.find('_debian_')+1:].replace('_','/'),
344 '/var/lib/apt/lists/' + self.sourcesFile)
346 def test_pkg_hash(self):
349 self.client.records.Lookup(self.client.cache['dpkg'].VersionList[0].FileList[0])
351 pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
352 '/var/lib/apt/lists/' + self.packagesFile +
353 ' | grep -E "^SHA1:" | head -n 1' +
354 ' | cut -d\ -f 2').read().rstrip('\n')
356 self.failUnless(self.client.records.SHA1Hash == pkg_hash,
357 "Hashes don't match: %s != %s" % (self.client.records.SHA1Hash, pkg_hash))
359 def test_src_hash(self):
362 self.client.srcrecords.Lookup('dpkg')
364 src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
365 '/var/lib/apt/lists/' + self.sourcesFile +
366 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
367 ' | cut -d\ -f 2').read().split('\n')[:-1]
369 for f in self.client.srcrecords.Files:
370 self.failUnless(f[0] in src_hashes, "Couldn't find %s in: %r" % (f[0], src_hashes))
372 def test_index_hash(self):
375 indexhash = self.client.indexrecords[self.releaseFile[self.releaseFile.find('_debian_')+1:].replace('_','/')]['main/binary-i386/Packages.bz2']['SHA1'][0]
377 idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
378 '/var/lib/apt/lists/' + self.releaseFile +
379 ' | grep -E " main/binary-i386/Packages.bz2$"'
380 ' | head -n 1 | cut -d\ -f 2').read().rstrip('\n')
382 self.failUnless(indexhash == idx_hash, "Hashes don't match: %s != %s" % (indexhash, idx_hash))
384 def verifyHash(self, found_hash, path, true_hash):
385 self.failUnless(found_hash[0] == true_hash,
386 "%s hashes don't match: %s != %s" % (path, found_hash[0], true_hash))
388 def test_findIndexHash(self):
389 lastDefer = defer.Deferred()
391 idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
392 '/var/lib/apt/lists/' + self.releaseFile +
393 ' | grep -E " main/binary-i386/Packages.bz2$"'
394 ' | head -n 1 | cut -d\ -f 2').read().rstrip('\n')
395 idx_path = self.releaseFile[self.releaseFile.find('_debian_')+1:].replace('_','/')[:-7] + 'main/binary-i386/Packages.bz2'
397 d = self.client.findHash(idx_path)
398 d.addCallback(self.verifyHash, idx_path, idx_hash)
400 d.addCallback(lastDefer.callback)
403 def test_findPkgHash(self):
404 lastDefer = defer.Deferred()
406 pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
407 '/var/lib/apt/lists/' + self.packagesFile +
408 ' | grep -E "^SHA1:" | head -n 1' +
409 ' | cut -d\ -f 2').read().rstrip('\n')
410 pkg_path = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
411 '/var/lib/apt/lists/' + self.packagesFile +
412 ' | grep -E "^Filename:" | head -n 1' +
413 ' | cut -d\ -f 2').read().rstrip('\n')
415 d = self.client.findHash(pkg_path)
416 d.addCallback(self.verifyHash, pkg_path, pkg_hash)
418 d.addCallback(lastDefer.callback)
421 def test_findSrcHash(self):
422 lastDefer = defer.Deferred()
424 src_dir = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
425 '/var/lib/apt/lists/' + self.sourcesFile +
426 ' | grep -E "^Directory:" | head -n 1' +
427 ' | cut -d\ -f 2').read().rstrip('\n')
428 src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
429 '/var/lib/apt/lists/' + self.sourcesFile +
430 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
431 ' | cut -d\ -f 2').read().split('\n')[:-1]
432 src_paths = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
433 '/var/lib/apt/lists/' + self.sourcesFile +
434 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
435 ' | cut -d\ -f 4').read().split('\n')[:-1]
437 i = random.choice(range(len(src_hashes)))
438 d = self.client.findHash(src_dir + '/' + src_paths[i])
439 d.addCallback(self.verifyHash, src_dir + '/' + src_paths[i], src_hashes[i])
441 d.addCallback(lastDefer.callback)
444 def test_multipleFindHash(self):
445 lastDefer = defer.Deferred()
447 idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
448 '/var/lib/apt/lists/' + self.releaseFile +
449 ' | grep -E " main/binary-i386/Packages.bz2$"'
450 ' | head -n 1 | cut -d\ -f 2').read().rstrip('\n')
451 idx_path = self.releaseFile[self.releaseFile.find('_debian_')+1:].replace('_','/')[:-7] + 'main/binary-i386/Packages.bz2'
453 d = self.client.findHash(idx_path)
454 d.addCallback(self.verifyHash, idx_path, idx_hash)
456 pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
457 '/var/lib/apt/lists/' + self.packagesFile +
458 ' | grep -E "^SHA1:" | head -n 1' +
459 ' | cut -d\ -f 2').read().rstrip('\n')
460 pkg_path = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
461 '/var/lib/apt/lists/' + self.packagesFile +
462 ' | grep -E "^Filename:" | head -n 1' +
463 ' | cut -d\ -f 2').read().rstrip('\n')
465 d = self.client.findHash(pkg_path)
466 d.addCallback(self.verifyHash, pkg_path, pkg_hash)
468 src_dir = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
469 '/var/lib/apt/lists/' + self.sourcesFile +
470 ' | grep -E "^Directory:" | head -n 1' +
471 ' | cut -d\ -f 2').read().rstrip('\n')
472 src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
473 '/var/lib/apt/lists/' + self.sourcesFile +
474 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
475 ' | cut -d\ -f 2').read().split('\n')[:-1]
476 src_paths = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
477 '/var/lib/apt/lists/' + self.sourcesFile +
478 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
479 ' | cut -d\ -f 4').read().split('\n')[:-1]
481 for i in range(len(src_hashes)):
482 d = self.client.findHash(src_dir + '/' + src_paths[i])
483 d.addCallback(self.verifyHash, src_dir + '/' + src_paths[i], src_hashes[i])
485 idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
486 '/var/lib/apt/lists/' + self.releaseFile +
487 ' | grep -E " main/source/Sources.bz2$"'
488 ' | head -n 1 | cut -d\ -f 2').read().rstrip('\n')
489 idx_path = self.releaseFile[self.releaseFile.find('_debian_')+1:].replace('_','/')[:-7] + 'main/source/Sources.bz2'
491 d = self.client.findHash(idx_path)
492 d.addCallback(self.verifyHash, idx_path, idx_hash)
494 d.addCallback(lastDefer.callback)
498 for p in self.pending_calls:
501 self.pending_calls = []
502 self.client.cleanup()