1 # Disable the FutureWarning from the apt module
3 warnings.simplefilter("ignore", FutureWarning)
5 import os, stat, random, re, shelve, shutil, fcntl, copy, UserDict
6 from os.path import dirname
8 from twisted.internet import threads, defer
9 from twisted.python import log
10 from twisted.trial import unittest
12 import apt_pkg, apt_inst
13 from apt import OpProgress
18 class PackageFileList(UserDict.DictMixin):
19 """Manages a list of package files belonging to a backend.
21 @type packages: C{shelve dictionary}
22 @ivar packages: the files stored for this backend
25 def __init__(self, backendName, cache_dir):
26 self.cache_dir = cache_dir
27 self.packagedb_dir = cache_dir+'/'+ aptpkg_dir + \
28 '/backends/' + backendName
29 if not os.path.exists(self.packagedb_dir):
30 os.makedirs(self.packagedb_dir)
35 """Open the persistent dictionary of files in this backend."""
36 if self.packages is None:
37 self.packages = shelve.open(self.packagedb_dir+'/packages.db')
40 """Close the persistent dictionary."""
41 if self.packages is not None:
44 def update_file(self, filename, cache_path, file_path):
45 """Check if an updated file needs to be tracked.
47 Called from the mirror manager when files get updated so we can update our
48 fake lists and sources.list.
50 if filename=="Packages" or filename=="Release" or filename=="Sources":
51 log.msg("Registering package file: "+cache_path)
52 self.packages[cache_path] = file_path
56 def check_files(self):
57 """Check all files in the database to make sure they exist."""
58 files = self.packages.keys()
60 if not os.path.exists(self.packages[f]):
61 log.msg("File in packages database has been deleted: "+f)
64 # Standard dictionary implementation so this class can be used like a dictionary.
65 def __getitem__(self, key): return self.packages[key]
66 def __setitem__(self, key, item): self.packages[key] = item
67 def __delitem__(self, key): del self.packages[key]
68 def keys(self): return self.packages.keys()
71 """Uses python-apt to answer queries about packages.
73 Makes a fake configuration for python-apt for each backend.
76 DEFAULT_APT_CONFIG = {
78 #'APT::Architecture' : 'i386', # Commented so the machine's config will set this
79 #'APT::Default-Release' : 'unstable',
81 'Dir::State' : 'apt/', # var/lib/apt/
82 'Dir::State::Lists': 'lists/', # lists/
83 #'Dir::State::cdroms' : 'cdroms.list',
84 'Dir::State::userstatus' : 'status.user',
85 'Dir::State::status': 'dpkg/status', # '/var/lib/dpkg/status'
86 'Dir::Cache' : '.apt/cache/', # var/cache/apt/
87 #'Dir::Cache::archives' : 'archives/',
88 'Dir::Cache::srcpkgcache' : 'srcpkgcache.bin',
89 'Dir::Cache::pkgcache' : 'pkgcache.bin',
90 'Dir::Etc' : 'apt/etc/', # etc/apt/
91 'Dir::Etc::sourcelist' : 'sources.list',
92 'Dir::Etc::vendorlist' : 'vendors.list',
93 'Dir::Etc::vendorparts' : 'vendors.list.d',
94 #'Dir::Etc::main' : 'apt.conf',
95 #'Dir::Etc::parts' : 'apt.conf.d',
96 #'Dir::Etc::preferences' : 'preferences',
98 #'Dir::Bin::methods' : '', #'/usr/lib/apt/methods'
99 'Dir::Bin::dpkg' : '/usr/bin/dpkg',
101 #'DPkg::Pre-Install-Pkgs' : '',
103 #'DPkg::Tools::Options' : '',
104 #'DPkg::Tools::Options::/usr/bin/apt-listchanges' : '',
105 #'DPkg::Tools::Options::/usr/bin/apt-listchanges::Version' : '2',
106 #'DPkg::Post-Invoke' : '',
108 essential_dirs = ('apt', 'apt/cache', 'apt/dpkg', 'apt/etc', 'apt/lists',
110 essential_files = ('apt/dpkg/status', 'apt/etc/sources.list',)
112 def __init__(self, backendName, cache_dir):
113 """Construct a new packages manager.
115 @ivar backendName: name of backend associated with this packages file
116 @ivar cache_dir: cache directory from config file
118 self.backendName = backendName
119 self.cache_dir = cache_dir
120 self.apt_config = copy.deepcopy(self.DEFAULT_APT_CONFIG)
122 self.status_dir = (cache_dir+'/'+ aptpkg_dir
123 +'/backends/'+backendName)
124 for dir in self.essential_dirs:
125 path = self.status_dir+'/'+dir
126 if not os.path.exists(path):
128 for file in self.essential_files:
129 path = self.status_dir+'/'+file
130 if not os.path.exists(path):
135 self.apt_config['Dir'] = self.status_dir
136 self.apt_config['Dir::State::status'] = self.status_dir + '/apt/dpkg/status'
137 self.packages = PackageFileList(backendName, cache_dir)
138 self.indexrecords = {}
144 self.packages.close()
146 def addRelease(self, cache_path, file_path):
147 """Dirty hack until python-apt supports apt-pkg/indexrecords.h
150 self.indexrecords[cache_path] = {}
152 read_packages = False
153 f = open(file_path, 'r')
159 read_packages = False
161 # Read the various headers from the file
162 h, v = line.split(":", 1)
163 if h == "MD5Sum" or h == "SHA1" or h == "SHA256":
167 # Bad header line, just ignore it
168 log.msg("WARNING: Ignoring badly formatted Release line: %s" % line)
170 # Skip to the next line
173 # Read file names from the multiple hash sections of the file
176 self.indexrecords[cache_path].setdefault(p[2], {})[hash_type] = (p[0], p[1])
180 def file_updated(self, filename, cache_path, file_path):
181 """A file in the backend has changed, manage it.
183 If this affects us, unload our apt database
185 if filename == "Release":
186 self.addRelease(cache_path, file_path)
187 if self.packages.update_file(filename, cache_path, file_path):
191 """Make sure the package is initialized and loaded."""
192 if self.loading is None:
193 self.loading = threads.deferToThread(self._load)
194 self.loading.addCallback(self.doneLoading)
197 def doneLoading(self, loadResult):
198 """Cache is loaded."""
200 # Must pass on the result for the next callback
204 """Regenerates the fake configuration and load the packages cache."""
205 if self.loaded: return True
207 shutil.rmtree(self.status_dir+'/apt/lists/')
208 os.makedirs(self.status_dir+'/apt/lists/partial')
209 sources_filename = self.status_dir+'/'+'apt/etc/sources.list'
210 sources = open(sources_filename, 'w')
212 self.packages.check_files()
213 for f in self.packages:
214 # we should probably clear old entries from self.packages and
215 # take into account the recorded mtime as optimization
216 filepath = self.packages[f]
217 fake_uri='http://apt-dht/'+f
218 if f.endswith('Sources'):
219 source_line='deb-src '+dirname(fake_uri)+'/ /'
221 source_line='deb '+dirname(fake_uri)+'/ /'
222 listpath=(self.status_dir+'/apt/lists/'
223 +apt_pkg.URItoFileName(fake_uri))
224 sources.write(source_line+'\n')
225 log.msg("Sources line: " + source_line)
226 sources_count = sources_count + 1
229 #we should empty the directory instead
233 os.symlink(filepath, listpath)
236 if sources_count == 0:
237 log.msg("No Packages files available for %s backend"%(self.backendName))
240 log.msg("Loading Packages database for "+self.status_dir)
241 for key, value in self.apt_config.items():
242 apt_pkg.Config[key] = value
244 self.cache = apt_pkg.GetCache(OpProgress())
245 self.records = apt_pkg.GetPkgRecords(self.cache)
246 self.srcrecords = apt_pkg.GetPkgSrcRecords()
252 """Tries to make the packages server quit."""
260 """Cleanup and close any loaded caches."""
262 self.packages.close()
264 def findHash(self, path):
265 """Find the hash for a given path in this mirror.
267 Returns a deferred so it can make sure the cache is loaded first.
271 # First look for the path in the cache of index files
272 for release in self.indexrecords:
273 if path.startswith(release[:-7]):
274 for indexFile in self.indexrecords[release]:
275 if release[:-7] + indexFile == path:
276 d.callback(self.indexrecords[release][indexFile]['SHA1'])
279 deferLoad = self.load()
280 deferLoad.addCallback(self._findHash, path, d)
284 def _findHash(self, loadResult, path, d):
285 """Really find the hash for a path.
287 Have to pass the returned loadResult on in case other calls to this
288 function are pending.
291 d.callback((None, None))
294 package = path.split('/')[-1].split('_')[0]
296 # Check the binary packages
298 for version in self.cache[package].VersionList:
300 for verFile in version.FileList:
301 if self.records.Lookup(verFile):
302 if self.records.FileName == path:
303 d.callback((self.records.SHA1Hash, size))
308 # Check the source packages' files
309 self.srcrecords.Restart()
310 if self.srcrecords.Lookup(package):
311 for f in self.srcrecords.Files:
313 d.callback((f[0], f[1]))
316 d.callback((None, None))
319 class TestAptPackages(unittest.TestCase):
320 """Unit tests for the AptPackages cache."""
329 self.client = AptPackages('whatever', '/tmp')
331 self.packagesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "Packages$" | tail -n 1').read().rstrip('\n')
332 self.sourcesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "Sources$" | tail -n 1').read().rstrip('\n')
333 for f in os.walk('/var/lib/apt/lists').next()[2]:
334 if f[-7:] == "Release" and self.packagesFile.startswith(f[:-7]):
338 self.client.file_updated('Release',
339 self.releaseFile[self.releaseFile.find('_debian_')+1:].replace('_','/'),
340 '/var/lib/apt/lists/' + self.releaseFile)
341 self.client.file_updated('Packages',
342 self.packagesFile[self.packagesFile.find('_debian_')+1:].replace('_','/'),
343 '/var/lib/apt/lists/' + self.packagesFile)
344 self.client.file_updated('Sources',
345 self.sourcesFile[self.sourcesFile.find('_debian_')+1:].replace('_','/'),
346 '/var/lib/apt/lists/' + self.sourcesFile)
348 def test_pkg_hash(self):
351 self.client.records.Lookup(self.client.cache['dpkg'].VersionList[0].FileList[0])
353 pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
354 '/var/lib/apt/lists/' + self.packagesFile +
355 ' | grep -E "^SHA1:" | head -n 1' +
356 ' | cut -d\ -f 2').read().rstrip('\n')
358 self.failUnless(self.client.records.SHA1Hash == pkg_hash,
359 "Hashes don't match: %s != %s" % (self.client.records.SHA1Hash, pkg_hash))
361 def test_src_hash(self):
364 self.client.srcrecords.Lookup('dpkg')
366 src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
367 '/var/lib/apt/lists/' + self.sourcesFile +
368 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
369 ' | cut -d\ -f 2').read().split('\n')[:-1]
371 for f in self.client.srcrecords.Files:
372 self.failUnless(f[0] in src_hashes, "Couldn't find %s in: %r" % (f[0], src_hashes))
374 def test_index_hash(self):
377 indexhash = self.client.indexrecords[self.releaseFile[self.releaseFile.find('_debian_')+1:].replace('_','/')]['main/binary-i386/Packages.bz2']['SHA1'][0]
379 idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
380 '/var/lib/apt/lists/' + self.releaseFile +
381 ' | grep -E " main/binary-i386/Packages.bz2$"'
382 ' | head -n 1 | cut -d\ -f 2').read().rstrip('\n')
384 self.failUnless(indexhash == idx_hash, "Hashes don't match: %s != %s" % (indexhash, idx_hash))
386 def verifyHash(self, found_hash, path, true_hash):
387 self.failUnless(found_hash[0] == true_hash,
388 "%s hashes don't match: %s != %s" % (path, found_hash[0], true_hash))
390 def test_findIndexHash(self):
391 lastDefer = defer.Deferred()
393 idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
394 '/var/lib/apt/lists/' + self.releaseFile +
395 ' | grep -E " main/binary-i386/Packages.bz2$"'
396 ' | head -n 1 | cut -d\ -f 2').read().rstrip('\n')
397 idx_path = self.releaseFile[self.releaseFile.find('_debian_')+1:].replace('_','/')[:-7] + 'main/binary-i386/Packages.bz2'
399 d = self.client.findHash(idx_path)
400 d.addCallback(self.verifyHash, idx_path, idx_hash)
402 d.addCallback(lastDefer.callback)
405 def test_findPkgHash(self):
406 lastDefer = defer.Deferred()
408 pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
409 '/var/lib/apt/lists/' + self.packagesFile +
410 ' | grep -E "^SHA1:" | head -n 1' +
411 ' | cut -d\ -f 2').read().rstrip('\n')
412 pkg_path = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
413 '/var/lib/apt/lists/' + self.packagesFile +
414 ' | grep -E "^Filename:" | head -n 1' +
415 ' | cut -d\ -f 2').read().rstrip('\n')
417 d = self.client.findHash(pkg_path)
418 d.addCallback(self.verifyHash, pkg_path, pkg_hash)
420 d.addCallback(lastDefer.callback)
423 def test_findSrcHash(self):
424 lastDefer = defer.Deferred()
426 src_dir = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
427 '/var/lib/apt/lists/' + self.sourcesFile +
428 ' | grep -E "^Directory:" | head -n 1' +
429 ' | cut -d\ -f 2').read().rstrip('\n')
430 src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
431 '/var/lib/apt/lists/' + self.sourcesFile +
432 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
433 ' | cut -d\ -f 2').read().split('\n')[:-1]
434 src_paths = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
435 '/var/lib/apt/lists/' + self.sourcesFile +
436 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
437 ' | cut -d\ -f 4').read().split('\n')[:-1]
439 i = random.choice(range(len(src_hashes)))
440 d = self.client.findHash(src_dir + '/' + src_paths[i])
441 d.addCallback(self.verifyHash, src_dir + '/' + src_paths[i], src_hashes[i])
443 d.addCallback(lastDefer.callback)
446 def test_multipleFindHash(self):
447 lastDefer = defer.Deferred()
449 idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
450 '/var/lib/apt/lists/' + self.releaseFile +
451 ' | grep -E " main/binary-i386/Packages.bz2$"'
452 ' | head -n 1 | cut -d\ -f 2').read().rstrip('\n')
453 idx_path = self.releaseFile[self.releaseFile.find('_debian_')+1:].replace('_','/')[:-7] + 'main/binary-i386/Packages.bz2'
455 d = self.client.findHash(idx_path)
456 d.addCallback(self.verifyHash, idx_path, idx_hash)
458 pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
459 '/var/lib/apt/lists/' + self.packagesFile +
460 ' | grep -E "^SHA1:" | head -n 1' +
461 ' | cut -d\ -f 2').read().rstrip('\n')
462 pkg_path = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
463 '/var/lib/apt/lists/' + self.packagesFile +
464 ' | grep -E "^Filename:" | head -n 1' +
465 ' | cut -d\ -f 2').read().rstrip('\n')
467 d = self.client.findHash(pkg_path)
468 d.addCallback(self.verifyHash, pkg_path, pkg_hash)
470 src_dir = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
471 '/var/lib/apt/lists/' + self.sourcesFile +
472 ' | grep -E "^Directory:" | head -n 1' +
473 ' | cut -d\ -f 2').read().rstrip('\n')
474 src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
475 '/var/lib/apt/lists/' + self.sourcesFile +
476 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
477 ' | cut -d\ -f 2').read().split('\n')[:-1]
478 src_paths = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
479 '/var/lib/apt/lists/' + self.sourcesFile +
480 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
481 ' | cut -d\ -f 4').read().split('\n')[:-1]
483 for i in range(len(src_hashes)):
484 d = self.client.findHash(src_dir + '/' + src_paths[i])
485 d.addCallback(self.verifyHash, src_dir + '/' + src_paths[i], src_hashes[i])
487 d.addCallback(lastDefer.callback)
491 for p in self.pending_calls:
494 self.pending_calls = []
495 self.client.cleanup()