2 # Copyright (C) 2002 Manuel Estrada Sainz <ranty@debian.org>
3 # Copyright (C) 2008 Cameron Dale <camrdale@gmail.com>
5 # This library is free software; you can redistribute it and/or
6 # modify it under the terms of version 2.1 of the GNU General Public
7 # License as published by the Free Software Foundation.
9 # This library is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 # General Public License for more details.
14 # You should have received a copy of the GNU General Public
15 # License along with this library; if not, write to the Free Software
16 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18 # Disable the FutureWarning from the apt module
20 warnings.simplefilter("ignore", FutureWarning)
23 from random import choice
24 from shutil import rmtree
25 from copy import deepcopy
26 from UserDict import DictMixin
28 from twisted.internet import threads, defer
29 from twisted.python import log
30 from twisted.python.filepath import FilePath
31 from twisted.trial import unittest
33 import apt_pkg, apt_inst
34 from apt import OpProgress
35 from debian_bundle import deb822
37 from Hash import HashObject
41 TRACKED_FILES = ['release', 'sources', 'packages']
43 class PackageFileList(DictMixin):
44 """Manages a list of package files belonging to a backend.
46 @type packages: C{shelve dictionary}
47 @ivar packages: the files stored for this backend
50 def __init__(self, cache_dir):
51 self.cache_dir = cache_dir
52 self.cache_dir.restat(False)
53 if not self.cache_dir.exists():
54 self.cache_dir.makedirs()
59 """Open the persistent dictionary of files in this backend."""
60 if self.packages is None:
61 self.packages = shelve.open(self.cache_dir.child('packages.db').path)
64 """Close the persistent dictionary."""
65 if self.packages is not None:
68 def update_file(self, cache_path, file_path):
69 """Check if an updated file needs to be tracked.
71 Called from the mirror manager when files get updated so we can update our
72 fake lists and sources.list.
74 filename = cache_path.split('/')[-1]
75 if filename.lower() in TRACKED_FILES:
76 log.msg("Registering package file: "+cache_path)
77 self.packages[cache_path] = file_path
81 def check_files(self):
82 """Check all files in the database to make sure they exist."""
83 files = self.packages.keys()
85 self.packages[f].restat(False)
86 if not self.packages[f].exists():
87 log.msg("File in packages database has been deleted: "+f)
90 # Standard dictionary implementation so this class can be used like a dictionary.
91 def __getitem__(self, key): return self.packages[key]
92 def __setitem__(self, key, item): self.packages[key] = item
93 def __delitem__(self, key): del self.packages[key]
94 def keys(self): return self.packages.keys()
97 """Uses python-apt to answer queries about packages.
99 Makes a fake configuration for python-apt for each backend.
102 DEFAULT_APT_CONFIG = {
104 #'APT::Architecture' : 'i386', # Commented so the machine's config will set this
105 #'APT::Default-Release' : 'unstable',
107 'Dir::State' : 'apt/', # var/lib/apt/
108 'Dir::State::Lists': 'lists/', # lists/
109 #'Dir::State::cdroms' : 'cdroms.list',
110 'Dir::State::userstatus' : 'status.user',
111 'Dir::State::status': 'dpkg/status', # '/var/lib/dpkg/status'
112 'Dir::Cache' : '.apt/cache/', # var/cache/apt/
113 #'Dir::Cache::archives' : 'archives/',
114 'Dir::Cache::srcpkgcache' : 'srcpkgcache.bin',
115 'Dir::Cache::pkgcache' : 'pkgcache.bin',
116 'Dir::Etc' : 'apt/etc/', # etc/apt/
117 'Dir::Etc::sourcelist' : 'sources.list',
118 'Dir::Etc::vendorlist' : 'vendors.list',
119 'Dir::Etc::vendorparts' : 'vendors.list.d',
120 #'Dir::Etc::main' : 'apt.conf',
121 #'Dir::Etc::parts' : 'apt.conf.d',
122 #'Dir::Etc::preferences' : 'preferences',
124 #'Dir::Bin::methods' : '', #'/usr/lib/apt/methods'
125 'Dir::Bin::dpkg' : '/usr/bin/dpkg',
127 #'DPkg::Pre-Install-Pkgs' : '',
129 #'DPkg::Tools::Options' : '',
130 #'DPkg::Tools::Options::/usr/bin/apt-listchanges' : '',
131 #'DPkg::Tools::Options::/usr/bin/apt-listchanges::Version' : '2',
132 #'DPkg::Post-Invoke' : '',
134 essential_dirs = ('apt', 'apt/cache', 'apt/dpkg', 'apt/etc', 'apt/lists',
136 essential_files = ('apt/dpkg/status', 'apt/etc/sources.list',)
138 def __init__(self, cache_dir):
139 """Construct a new packages manager.
141 @param cache_dir: cache directory from config file
143 self.cache_dir = cache_dir
144 self.apt_config = deepcopy(self.DEFAULT_APT_CONFIG)
146 for dir in self.essential_dirs:
147 path = self.cache_dir.preauthChild(dir)
148 if not path.exists():
150 for file in self.essential_files:
151 path = self.cache_dir.preauthChild(file)
152 if not path.exists():
155 self.apt_config['Dir'] = self.cache_dir.path
156 self.apt_config['Dir::State::status'] = self.cache_dir.preauthChild(self.apt_config['Dir::State']).preauthChild(self.apt_config['Dir::State::status']).path
157 self.packages = PackageFileList(cache_dir)
163 self.packages.close()
165 def addRelease(self, cache_path, file_path):
166 """Dirty hack until python-apt supports apt-pkg/indexrecords.h
169 self.indexrecords[cache_path] = {}
171 read_packages = False
172 f = file_path.open('r')
174 rel = deb822.Release(f, fields = ['MD5Sum', 'SHA1', 'SHA256'])
175 for hash_type in rel:
176 for file in rel[hash_type]:
177 self.indexrecords[cache_path].setdefault(file['name'], {})[hash_type.upper()] = (file[hash_type], file['size'])
181 def file_updated(self, cache_path, file_path):
182 """A file in the backend has changed, manage it.
184 If this affects us, unload our apt database
186 if self.packages.update_file(cache_path, file_path):
190 """Make sure the package is initialized and loaded."""
191 if self.loading is None:
192 self.loading = threads.deferToThread(self._load)
193 self.loading.addCallback(self.doneLoading)
196 def doneLoading(self, loadResult):
197 """Cache is loaded."""
199 # Must pass on the result for the next callback
203 """Regenerates the fake configuration and load the packages cache."""
204 if self.loaded: return True
206 self.cache_dir.preauthChild(self.apt_config['Dir::State']
207 ).preauthChild(self.apt_config['Dir::State::Lists']).remove()
208 self.cache_dir.preauthChild(self.apt_config['Dir::State']
209 ).preauthChild(self.apt_config['Dir::State::Lists']
210 ).child('partial').makedirs()
211 sources_file = self.cache_dir.preauthChild(self.apt_config['Dir::Etc']
212 ).preauthChild(self.apt_config['Dir::Etc::sourcelist'])
213 sources = sources_file.open('w')
215 deb_src_added = False
216 self.packages.check_files()
217 self.indexrecords = {}
218 for f in self.packages:
219 # we should probably clear old entries from self.packages and
220 # take into account the recorded mtime as optimization
221 file = self.packages[f]
222 if f.split('/')[-1] == "Release":
223 self.addRelease(f, file)
224 fake_uri='http://apt-dht'+f
225 fake_dirname = '/'.join(fake_uri.split('/')[:-1])
226 if f.endswith('Sources'):
228 source_line='deb-src '+fake_dirname+'/ /'
230 source_line='deb '+fake_dirname+'/ /'
231 listpath = self.cache_dir.preauthChild(self.apt_config['Dir::State']
232 ).preauthChild(self.apt_config['Dir::State::Lists']
233 ).child(apt_pkg.URItoFileName(fake_uri))
234 sources.write(source_line+'\n')
235 log.msg("Sources line: " + source_line)
236 sources_count = sources_count + 1
238 if listpath.exists():
239 #we should empty the directory instead
241 os.symlink(file.path, listpath.path)
244 if sources_count == 0:
245 log.msg("No Packages files available for %s backend"%(self.cache_dir.path))
248 log.msg("Loading Packages database for "+self.cache_dir.path)
249 for key, value in self.apt_config.items():
250 apt_pkg.Config[key] = value
252 self.cache = apt_pkg.GetCache(OpProgress())
253 self.records = apt_pkg.GetPkgRecords(self.cache)
255 self.srcrecords = apt_pkg.GetPkgSrcRecords()
257 self.srcrecords = None
263 """Tries to make the packages server quit."""
268 del self.indexrecords
272 """Cleanup and close any loaded caches."""
274 self.packages.close()
276 def findHash(self, path):
277 """Find the hash for a given path in this mirror.
279 Returns a deferred so it can make sure the cache is loaded first.
283 deferLoad = self.load()
284 deferLoad.addCallback(self._findHash, path, d)
285 deferLoad.addErrback(self._findHash_error, path, d)
289 def _findHash_error(self, failure, path, d):
290 """An error occurred while trying to find a hash."""
291 log.msg('An error occurred while looking up a hash for: %s' % path)
293 d.callback(HashObject())
295 def _findHash(self, loadResult, path, d):
296 """Really find the hash for a path.
298 Have to pass the returned loadResult on in case other calls to this
299 function are pending.
302 d.callback(HashObject())
305 # First look for the path in the cache of index files
306 for release in self.indexrecords:
307 if path.startswith(release[:-7]):
308 for indexFile in self.indexrecords[release]:
309 if release[:-7] + indexFile == path:
311 h.setFromIndexRecord(self.indexrecords[release][indexFile])
315 package = path.split('/')[-1].split('_')[0]
317 # Check the binary packages
319 for version in self.cache[package].VersionList:
321 for verFile in version.FileList:
322 if self.records.Lookup(verFile):
323 if '/' + self.records.FileName == path:
325 h.setFromPkgRecord(self.records, size)
331 # Check the source packages' files
333 self.srcrecords.Restart()
334 if self.srcrecords.Lookup(package):
335 for f in self.srcrecords.Files:
336 if path == '/' + f[2]:
338 h.setFromSrcRecord(f)
342 d.callback(HashObject())
345 class TestAptPackages(unittest.TestCase):
346 """Unit tests for the AptPackages cache."""
356 self.client = AptPackages(FilePath('/tmp/.apt-dht'))
358 self.packagesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "_main_.*Packages$" | tail -n 1').read().rstrip('\n')
359 self.sourcesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "_main_.*Sources$" | tail -n 1').read().rstrip('\n')
360 for f in os.walk('/var/lib/apt/lists').next()[2]:
361 if f[-7:] == "Release" and self.packagesFile.startswith(f[:-7]):
365 self.client.file_updated(self.releaseFile[self.releaseFile.find('_dists_'):].replace('_','/'),
366 FilePath('/var/lib/apt/lists/' + self.releaseFile))
367 self.client.file_updated(self.packagesFile[self.packagesFile.find('_dists_'):].replace('_','/'),
368 FilePath('/var/lib/apt/lists/' + self.packagesFile))
369 self.client.file_updated(self.sourcesFile[self.sourcesFile.find('_dists_'):].replace('_','/'),
370 FilePath('/var/lib/apt/lists/' + self.sourcesFile))
372 def test_pkg_hash(self):
375 self.client.records.Lookup(self.client.cache['dpkg'].VersionList[0].FileList[0])
377 pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
378 '/var/lib/apt/lists/' + self.packagesFile +
379 ' | grep -E "^SHA1:" | head -n 1' +
380 ' | cut -d\ -f 2').read().rstrip('\n')
382 self.failUnless(self.client.records.SHA1Hash == pkg_hash,
383 "Hashes don't match: %s != %s" % (self.client.records.SHA1Hash, pkg_hash))
385 def test_src_hash(self):
388 self.client.srcrecords.Lookup('dpkg')
390 src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
391 '/var/lib/apt/lists/' + self.sourcesFile +
392 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
393 ' | cut -d\ -f 2').read().split('\n')[:-1]
395 for f in self.client.srcrecords.Files:
396 self.failUnless(f[0] in src_hashes, "Couldn't find %s in: %r" % (f[0], src_hashes))
398 def test_index_hash(self):
401 indexhash = self.client.indexrecords[self.releaseFile[self.releaseFile.find('_dists_'):].replace('_','/')]['main/binary-i386/Packages.bz2']['SHA1'][0]
403 idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
404 '/var/lib/apt/lists/' + self.releaseFile +
405 ' | grep -E " main/binary-i386/Packages.bz2$"'
406 ' | head -n 1 | cut -d\ -f 2').read().rstrip('\n')
408 self.failUnless(indexhash == idx_hash, "Hashes don't match: %s != %s" % (indexhash, idx_hash))
410 def verifyHash(self, found_hash, path, true_hash):
411 self.failUnless(found_hash.hexexpected() == true_hash,
412 "%s hashes don't match: %s != %s" % (path, found_hash.hexexpected(), true_hash))
414 def test_findIndexHash(self):
415 lastDefer = defer.Deferred()
417 idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
418 '/var/lib/apt/lists/' + self.releaseFile +
419 ' | grep -E " main/binary-i386/Packages.bz2$"'
420 ' | head -n 1 | cut -d\ -f 2').read().rstrip('\n')
421 idx_path = '/' + self.releaseFile[self.releaseFile.find('_dists_')+1:].replace('_','/')[:-7] + 'main/binary-i386/Packages.bz2'
423 d = self.client.findHash(idx_path)
424 d.addCallback(self.verifyHash, idx_path, idx_hash)
426 d.addBoth(lastDefer.callback)
429 def test_findPkgHash(self):
430 lastDefer = defer.Deferred()
432 pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
433 '/var/lib/apt/lists/' + self.packagesFile +
434 ' | grep -E "^SHA1:" | head -n 1' +
435 ' | cut -d\ -f 2').read().rstrip('\n')
436 pkg_path = '/' + os.popen('grep -A 30 -E "^Package: dpkg$" ' +
437 '/var/lib/apt/lists/' + self.packagesFile +
438 ' | grep -E "^Filename:" | head -n 1' +
439 ' | cut -d\ -f 2').read().rstrip('\n')
441 d = self.client.findHash(pkg_path)
442 d.addCallback(self.verifyHash, pkg_path, pkg_hash)
444 d.addBoth(lastDefer.callback)
447 def test_findSrcHash(self):
448 lastDefer = defer.Deferred()
450 src_dir = '/' + os.popen('grep -A 30 -E "^Package: dpkg$" ' +
451 '/var/lib/apt/lists/' + self.sourcesFile +
452 ' | grep -E "^Directory:" | head -n 1' +
453 ' | cut -d\ -f 2').read().rstrip('\n')
454 src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
455 '/var/lib/apt/lists/' + self.sourcesFile +
456 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
457 ' | cut -d\ -f 2').read().split('\n')[:-1]
458 src_paths = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
459 '/var/lib/apt/lists/' + self.sourcesFile +
460 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
461 ' | cut -d\ -f 4').read().split('\n')[:-1]
463 i = choice(range(len(src_hashes)))
464 d = self.client.findHash(src_dir + '/' + src_paths[i])
465 d.addCallback(self.verifyHash, src_dir + '/' + src_paths[i], src_hashes[i])
467 d.addBoth(lastDefer.callback)
470 def test_multipleFindHash(self):
471 lastDefer = defer.Deferred()
473 idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
474 '/var/lib/apt/lists/' + self.releaseFile +
475 ' | grep -E " main/binary-i386/Packages.bz2$"'
476 ' | head -n 1 | cut -d\ -f 2').read().rstrip('\n')
477 idx_path = '/' + self.releaseFile[self.releaseFile.find('_dists_')+1:].replace('_','/')[:-7] + 'main/binary-i386/Packages.bz2'
479 d = self.client.findHash(idx_path)
480 d.addCallback(self.verifyHash, idx_path, idx_hash)
482 pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
483 '/var/lib/apt/lists/' + self.packagesFile +
484 ' | grep -E "^SHA1:" | head -n 1' +
485 ' | cut -d\ -f 2').read().rstrip('\n')
486 pkg_path = '/' + os.popen('grep -A 30 -E "^Package: dpkg$" ' +
487 '/var/lib/apt/lists/' + self.packagesFile +
488 ' | grep -E "^Filename:" | head -n 1' +
489 ' | cut -d\ -f 2').read().rstrip('\n')
491 d = self.client.findHash(pkg_path)
492 d.addCallback(self.verifyHash, pkg_path, pkg_hash)
494 src_dir = '/' + os.popen('grep -A 30 -E "^Package: dpkg$" ' +
495 '/var/lib/apt/lists/' + self.sourcesFile +
496 ' | grep -E "^Directory:" | head -n 1' +
497 ' | cut -d\ -f 2').read().rstrip('\n')
498 src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
499 '/var/lib/apt/lists/' + self.sourcesFile +
500 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
501 ' | cut -d\ -f 2').read().split('\n')[:-1]
502 src_paths = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
503 '/var/lib/apt/lists/' + self.sourcesFile +
504 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
505 ' | cut -d\ -f 4').read().split('\n')[:-1]
507 for i in range(len(src_hashes)):
508 d = self.client.findHash(src_dir + '/' + src_paths[i])
509 d.addCallback(self.verifyHash, src_dir + '/' + src_paths[i], src_hashes[i])
511 idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
512 '/var/lib/apt/lists/' + self.releaseFile +
513 ' | grep -E " main/source/Sources.bz2$"'
514 ' | head -n 1 | cut -d\ -f 2').read().rstrip('\n')
515 idx_path = '/' + self.releaseFile[self.releaseFile.find('_dists_')+1:].replace('_','/')[:-7] + 'main/source/Sources.bz2'
517 d = self.client.findHash(idx_path)
518 d.addCallback(self.verifyHash, idx_path, idx_hash)
520 d.addBoth(lastDefer.callback)
524 for p in self.pending_calls:
527 self.pending_calls = []
528 self.client.cleanup()