apt_dht/AptPackages.py

   1 #
   2 # Copyright (C) 2002 Manuel Estrada Sainz <ranty@debian.org>
   3 # Copyright (C) 2008 Cameron Dale <camrdale@gmail.com>
   4 #
   5 # This library is free software; you can redistribute it and/or
   6 # modify it under the terms of version 2.1 of the GNU General Public
   7 # License as published by the Free Software Foundation.
   8 #
   9 # This library is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12 # General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public
  15 # License along with this library; if not, write to the Free Software
  16 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  17
  18 # Disable the FutureWarning from the apt module
  19 import warnings
  20 warnings.simplefilter("ignore", FutureWarning)
  21
  22 import os, shelve
  23 from random import choice
  24 from shutil import rmtree
  25 from copy import deepcopy
  26 from UserDict import DictMixin
  27
  28 from twisted.internet import threads, defer, reactor
  29 from twisted.python import log
  30 from twisted.python.filepath import FilePath
  31 from twisted.trial import unittest
  32
  33 import apt_pkg, apt_inst
  34 from apt import OpProgress
  35 from debian_bundle import deb822
  36
  37 from Hash import HashObject
  38
  39 apt_pkg.init()
  40
  41 TRACKED_FILES = ['release', 'sources', 'packages']
  42
  43 class PackageFileList(DictMixin):
  44     """Manages a list of package files belonging to a backend.
  45
  46     @type packages: C{shelve dictionary}
  47     @ivar packages: the files stored for this backend
  48     """
  49
  50     def __init__(self, cache_dir):
  51         self.cache_dir = cache_dir
  52         self.cache_dir.restat(False)
  53         if not self.cache_dir.exists():
  54             self.cache_dir.makedirs()
  55         self.packages = None
  56         self.open()
  57
  58     def open(self):
  59         """Open the persistent dictionary of files in this backend."""
  60         if self.packages is None:
  61             self.packages = shelve.open(self.cache_dir.child('packages.db').path)
  62
  63     def close(self):
  64         """Close the persistent dictionary."""
  65         if self.packages is not None:
  66             self.packages.close()
  67
  68     def update_file(self, cache_path, file_path):
  69         """Check if an updated file needs to be tracked.
  70
  71         Called from the mirror manager when files get updated so we can update our
  72         fake lists and sources.list.
  73         """
  74         filename = cache_path.split('/')[-1]
  75         if filename.lower() in TRACKED_FILES:
  76             log.msg("Registering package file: "+cache_path)
  77             self.packages[cache_path] = file_path
  78             return True
  79         return False
  80
  81     def check_files(self):
  82         """Check all files in the database to make sure they exist."""
  83         files = self.packages.keys()
  84         for f in files:
  85             self.packages[f].restat(False)
  86             if not self.packages[f].exists():
  87                 log.msg("File in packages database has been deleted: "+f)
  88                 del self.packages[f]
  89
  90     # Standard dictionary implementation so this class can be used like a dictionary.
  91     def __getitem__(self, key): return self.packages[key]
  92     def __setitem__(self, key, item): self.packages[key] = item
  93     def __delitem__(self, key): del self.packages[key]
  94     def keys(self): return self.packages.keys()
  95
  96 class AptPackages:
  97     """Uses python-apt to answer queries about packages.
  98
  99     Makes a fake configuration for python-apt for each backend.
 100     """
 101
 102     DEFAULT_APT_CONFIG = {
 103         #'APT' : '',
 104         #'APT::Architecture' : 'i386',  # Commented so the machine's config will set this
 105         #'APT::Default-Release' : 'unstable',
 106         'Dir':'.', # /
 107         'Dir::State' : 'apt/', # var/lib/apt/
 108         'Dir::State::Lists': 'lists/', # lists/
 109         #'Dir::State::cdroms' : 'cdroms.list',
 110         'Dir::State::userstatus' : 'status.user',
 111         'Dir::State::status': 'dpkg/status', # '/var/lib/dpkg/status'
 112         'Dir::Cache' : '.apt/cache/', # var/cache/apt/
 113         #'Dir::Cache::archives' : 'archives/',
 114         'Dir::Cache::srcpkgcache' : 'srcpkgcache.bin',
 115         'Dir::Cache::pkgcache' : 'pkgcache.bin',
 116         'Dir::Etc' : 'apt/etc/', # etc/apt/
 117         'Dir::Etc::sourcelist' : 'sources.list',
 118         'Dir::Etc::vendorlist' : 'vendors.list',
 119         'Dir::Etc::vendorparts' : 'vendors.list.d',
 120         #'Dir::Etc::main' : 'apt.conf',
 121         #'Dir::Etc::parts' : 'apt.conf.d',
 122         #'Dir::Etc::preferences' : 'preferences',
 123         'Dir::Bin' : '',
 124         #'Dir::Bin::methods' : '', #'/usr/lib/apt/methods'
 125         'Dir::Bin::dpkg' : '/usr/bin/dpkg',
 126         #'DPkg' : '',
 127         #'DPkg::Pre-Install-Pkgs' : '',
 128         #'DPkg::Tools' : '',
 129         #'DPkg::Tools::Options' : '',
 130         #'DPkg::Tools::Options::/usr/bin/apt-listchanges' : '',
 131         #'DPkg::Tools::Options::/usr/bin/apt-listchanges::Version' : '2',
 132         #'DPkg::Post-Invoke' : '',
 133         }
 134     essential_dirs = ('apt', 'apt/cache', 'apt/dpkg', 'apt/etc', 'apt/lists',
 135                       'apt/lists/partial')
 136     essential_files = ('apt/dpkg/status', 'apt/etc/sources.list',)
 137
 138     def __init__(self, cache_dir, unload_delay):
 139         """Construct a new packages manager.
 140
 141         @param cache_dir: cache directory from config file
 142         """
 143         self.cache_dir = cache_dir
 144         self.unload_delay = unload_delay
 145         self.apt_config = deepcopy(self.DEFAULT_APT_CONFIG)
 146
 147         for dir in self.essential_dirs:
 148             path = self.cache_dir.preauthChild(dir)
 149             if not path.exists():
 150                 path.makedirs()
 151         for file in self.essential_files:
 152             path = self.cache_dir.preauthChild(file)
 153             if not path.exists():
 154                 path.touch()
 155
 156         self.apt_config['Dir'] = self.cache_dir.path
 157         self.apt_config['Dir::State::status'] = self.cache_dir.preauthChild(self.apt_config['Dir::State']).preauthChild(self.apt_config['Dir::State::status']).path
 158         self.packages = PackageFileList(cache_dir)
 159         self.loaded = 0
 160         self.loading = None
 161         self.unload_later = None
 162
 163     def __del__(self):
 164         self.cleanup()
 165
 166     def addRelease(self, cache_path, file_path):
 167         """Dirty hack until python-apt supports apt-pkg/indexrecords.h
 168         (see Bug #456141)
 169         """
 170         self.indexrecords[cache_path] = {}
 171
 172         read_packages = False
 173         f = file_path.open('r')
 174
 175         rel = deb822.Release(f, fields = ['MD5Sum', 'SHA1', 'SHA256'])
 176         for hash_type in rel:
 177             for file in rel[hash_type]:
 178                 self.indexrecords[cache_path].setdefault(file['name'], {})[hash_type.upper()] = (file[hash_type], file['size'])
 179
 180         f.close()
 181
 182     def file_updated(self, cache_path, file_path):
 183         """A file in the backend has changed, manage it.
 184
 185         If this affects us, unload our apt database
 186         """
 187         if self.packages.update_file(cache_path, file_path):
 188             self.unload()
 189
 190     def load(self):
 191         """Make sure the package is initialized and loaded."""
 192         if self.unload_later and self.unload_later.active():
 193             self.unload_later.reset(self.unload_delay)
 194         else:
 195             self.unload_later = reactor.callLater(self.unload_delay, self.unload)
 196         if self.loading is None:
 197             log.msg('Loading the packages cache')
 198             self.loading = threads.deferToThread(self._load)
 199             self.loading.addCallback(self.doneLoading)
 200         return self.loading
 201
 202     def doneLoading(self, loadResult):
 203         """Cache is loaded."""
 204         self.loading = None
 205         # Must pass on the result for the next callback
 206         return loadResult
 207
 208     def _load(self):
 209         """Regenerates the fake configuration and load the packages cache."""
 210         if self.loaded: return True
 211         apt_pkg.InitSystem()
 212         self.cache_dir.preauthChild(self.apt_config['Dir::State']
 213                      ).preauthChild(self.apt_config['Dir::State::Lists']).remove()
 214         self.cache_dir.preauthChild(self.apt_config['Dir::State']
 215                      ).preauthChild(self.apt_config['Dir::State::Lists']
 216                      ).child('partial').makedirs()
 217         sources_file = self.cache_dir.preauthChild(self.apt_config['Dir::Etc']
 218                                ).preauthChild(self.apt_config['Dir::Etc::sourcelist'])
 219         sources = sources_file.open('w')
 220         sources_count = 0
 221         deb_src_added = False
 222         self.packages.check_files()
 223         self.indexrecords = {}
 224         for f in self.packages:
 225             # we should probably clear old entries from self.packages and
 226             # take into account the recorded mtime as optimization
 227             file = self.packages[f]
 228             if f.split('/')[-1] == "Release":
 229                 self.addRelease(f, file)
 230             fake_uri='http://apt-dht'+f
 231             fake_dirname = '/'.join(fake_uri.split('/')[:-1])
 232             if f.endswith('Sources'):
 233                 deb_src_added = True
 234                 source_line='deb-src '+fake_dirname+'/ /'
 235             else:
 236                 source_line='deb '+fake_dirname+'/ /'
 237             listpath = self.cache_dir.preauthChild(self.apt_config['Dir::State']
 238                                     ).preauthChild(self.apt_config['Dir::State::Lists']
 239                                     ).child(apt_pkg.URItoFileName(fake_uri))
 240             sources.write(source_line+'\n')
 241             log.msg("Sources line: " + source_line)
 242             sources_count = sources_count + 1
 243
 244             if listpath.exists():
 245                 #we should empty the directory instead
 246                 listpath.remove()
 247             os.symlink(file.path, listpath.path)
 248         sources.close()
 249
 250         if sources_count == 0:
 251             log.msg("No Packages files available for %s backend"%(self.cache_dir.path))
 252             return False
 253
 254         log.msg("Loading Packages database for "+self.cache_dir.path)
 255         for key, value in self.apt_config.items():
 256             apt_pkg.Config[key] = value
 257
 258         self.cache = apt_pkg.GetCache(OpProgress())
 259         self.records = apt_pkg.GetPkgRecords(self.cache)
 260         if deb_src_added:
 261             self.srcrecords = apt_pkg.GetPkgSrcRecords()
 262         else:
 263             self.srcrecords = None
 264
 265         self.loaded = 1
 266         return True
 267
 268     def unload(self):
 269         """Tries to make the packages server quit."""
 270         if self.unload_later and self.unload_later.active():
 271             self.unload_later.cancel()
 272         self.unload_later = None
 273         if self.loaded:
 274             log.msg('Unloading the packages cache')
 275             del self.cache
 276             del self.records
 277             del self.srcrecords
 278             del self.indexrecords
 279             self.loaded = 0
 280
 281     def cleanup(self):
 282         """Cleanup and close any loaded caches."""
 283         self.unload()
 284         if self.unload_later and self.unload_later.active():
 285             self.unload_later.cancel()
 286         self.packages.close()
 287
 288     def findHash(self, path):
 289         """Find the hash for a given path in this mirror.
 290
 291         Returns a deferred so it can make sure the cache is loaded first.
 292         """
 293         d = defer.Deferred()
 294
 295         deferLoad = self.load()
 296         deferLoad.addCallback(self._findHash, path, d)
 297         deferLoad.addErrback(self._findHash_error, path, d)
 298
 299         return d
 300
 301     def _findHash_error(self, failure, path, d):
 302         """An error occurred while trying to find a hash."""
 303         log.msg('An error occurred while looking up a hash for: %s' % path)
 304         log.err(failure)
 305         d.callback(HashObject())
 306
 307     def _findHash(self, loadResult, path, d):
 308         """Really find the hash for a path.
 309
 310         Have to pass the returned loadResult on in case other calls to this
 311         function are pending.
 312         """
 313         if not loadResult:
 314             d.callback(HashObject())
 315             return loadResult
 316
 317         # First look for the path in the cache of index files
 318         for release in self.indexrecords:
 319             if path.startswith(release[:-7]):
 320                 for indexFile in self.indexrecords[release]:
 321                     if release[:-7] + indexFile == path:
 322                         h = HashObject()
 323                         h.setFromIndexRecord(self.indexrecords[release][indexFile])
 324                         d.callback(h)
 325                         return loadResult
 326
 327         package = path.split('/')[-1].split('_')[0]
 328
 329         # Check the binary packages
 330         try:
 331             for version in self.cache[package].VersionList:
 332                 size = version.Size
 333                 for verFile in version.FileList:
 334                     if self.records.Lookup(verFile):
 335                         if '/' + self.records.FileName == path:
 336                             h = HashObject()
 337                             h.setFromPkgRecord(self.records, size)
 338                             d.callback(h)
 339                             return loadResult
 340         except KeyError:
 341             pass
 342
 343         # Check the source packages' files
 344         if self.srcrecords:
 345             self.srcrecords.Restart()
 346             if self.srcrecords.Lookup(package):
 347                 for f in self.srcrecords.Files:
 348                     if path == '/' + f[2]:
 349                         h = HashObject()
 350                         h.setFromSrcRecord(f)
 351                         d.callback(h)
 352                         return loadResult
 353
 354         d.callback(HashObject())
 355         return loadResult
 356
 357 class TestAptPackages(unittest.TestCase):
 358     """Unit tests for the AptPackages cache."""
 359
 360     pending_calls = []
 361     client = None
 362     timeout = 10
 363     packagesFile = ''
 364     sourcesFile = ''
 365     releaseFile = ''
 366
 367     def setUp(self):
 368         self.client = AptPackages(FilePath('/tmp/.apt-dht'), 300)
 369
 370         self.packagesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "_main_.*Packages$" | tail -n 1').read().rstrip('\n')
 371         self.sourcesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "_main_.*Sources$" | tail -n 1').read().rstrip('\n')
 372         for f in os.walk('/var/lib/apt/lists').next()[2]:
 373             if f[-7:] == "Release" and self.packagesFile.startswith(f[:-7]):
 374                 self.releaseFile = f
 375                 break
 376
 377         self.client.file_updated(self.releaseFile[self.releaseFile.find('_dists_'):].replace('_','/'),
 378                                  FilePath('/var/lib/apt/lists/' + self.releaseFile))
 379         self.client.file_updated(self.packagesFile[self.packagesFile.find('_dists_'):].replace('_','/'),
 380                                  FilePath('/var/lib/apt/lists/' + self.packagesFile))
 381         self.client.file_updated(self.sourcesFile[self.sourcesFile.find('_dists_'):].replace('_','/'),
 382                                  FilePath('/var/lib/apt/lists/' + self.sourcesFile))
 383
 384     def test_pkg_hash(self):
 385         self.client._load()
 386
 387         self.client.records.Lookup(self.client.cache['dpkg'].VersionList[0].FileList[0])
 388
 389         pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 390                             '/var/lib/apt/lists/' + self.packagesFile +
 391                             ' | grep -E "^SHA1:" | head -n 1' +
 392                             ' | cut -d\  -f 2').read().rstrip('\n')
 393
 394         self.failUnless(self.client.records.SHA1Hash == pkg_hash,
 395                         "Hashes don't match: %s != %s" % (self.client.records.SHA1Hash, pkg_hash))
 396
 397     def test_src_hash(self):
 398         self.client._load()
 399
 400         self.client.srcrecords.Lookup('dpkg')
 401
 402         src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
 403                             '/var/lib/apt/lists/' + self.sourcesFile +
 404                             ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
 405                             ' | cut -d\  -f 2').read().split('\n')[:-1]
 406
 407         for f in self.client.srcrecords.Files:
 408             self.failUnless(f[0] in src_hashes, "Couldn't find %s in: %r" % (f[0], src_hashes))
 409
 410     def test_index_hash(self):
 411         self.client._load()
 412
 413         indexhash = self.client.indexrecords[self.releaseFile[self.releaseFile.find('_dists_'):].replace('_','/')]['main/binary-i386/Packages.bz2']['SHA1'][0]
 414
 415         idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
 416                             '/var/lib/apt/lists/' + self.releaseFile +
 417                             ' | grep -E " main/binary-i386/Packages.bz2$"'
 418                             ' | head -n 1 | cut -d\  -f 2').read().rstrip('\n')
 419
 420         self.failUnless(indexhash == idx_hash, "Hashes don't match: %s != %s" % (indexhash, idx_hash))
 421
 422     def verifyHash(self, found_hash, path, true_hash):
 423         self.failUnless(found_hash.hexexpected() == true_hash,
 424                     "%s hashes don't match: %s != %s" % (path, found_hash.hexexpected(), true_hash))
 425
 426     def test_findIndexHash(self):
 427         lastDefer = defer.Deferred()
 428
 429         idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
 430                             '/var/lib/apt/lists/' + self.releaseFile +
 431                             ' | grep -E " main/binary-i386/Packages.bz2$"'
 432                             ' | head -n 1 | cut -d\  -f 2').read().rstrip('\n')
 433         idx_path = '/' + self.releaseFile[self.releaseFile.find('_dists_')+1:].replace('_','/')[:-7] + 'main/binary-i386/Packages.bz2'
 434
 435         d = self.client.findHash(idx_path)
 436         d.addCallback(self.verifyHash, idx_path, idx_hash)
 437
 438         d.addBoth(lastDefer.callback)
 439         return lastDefer
 440
 441     def test_findPkgHash(self):
 442         lastDefer = defer.Deferred()
 443
 444         pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 445                             '/var/lib/apt/lists/' + self.packagesFile +
 446                             ' | grep -E "^SHA1:" | head -n 1' +
 447                             ' | cut -d\  -f 2').read().rstrip('\n')
 448         pkg_path = '/' + os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 449                             '/var/lib/apt/lists/' + self.packagesFile +
 450                             ' | grep -E "^Filename:" | head -n 1' +
 451                             ' | cut -d\  -f 2').read().rstrip('\n')
 452
 453         d = self.client.findHash(pkg_path)
 454         d.addCallback(self.verifyHash, pkg_path, pkg_hash)
 455
 456         d.addBoth(lastDefer.callback)
 457         return lastDefer
 458
 459     def test_findSrcHash(self):
 460         lastDefer = defer.Deferred()
 461
 462         src_dir = '/' + os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 463                             '/var/lib/apt/lists/' + self.sourcesFile +
 464                             ' | grep -E "^Directory:" | head -n 1' +
 465                             ' | cut -d\  -f 2').read().rstrip('\n')
 466         src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
 467                             '/var/lib/apt/lists/' + self.sourcesFile +
 468                             ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
 469                             ' | cut -d\  -f 2').read().split('\n')[:-1]
 470         src_paths = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
 471                             '/var/lib/apt/lists/' + self.sourcesFile +
 472                             ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
 473                             ' | cut -d\  -f 4').read().split('\n')[:-1]
 474
 475         i = choice(range(len(src_hashes)))
 476         d = self.client.findHash(src_dir + '/' + src_paths[i])
 477         d.addCallback(self.verifyHash, src_dir + '/' + src_paths[i], src_hashes[i])
 478
 479         d.addBoth(lastDefer.callback)
 480         return lastDefer
 481
 482     def test_multipleFindHash(self):
 483         lastDefer = defer.Deferred()
 484
 485         idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
 486                             '/var/lib/apt/lists/' + self.releaseFile +
 487                             ' | grep -E " main/binary-i386/Packages.bz2$"'
 488                             ' | head -n 1 | cut -d\  -f 2').read().rstrip('\n')
 489         idx_path = '/' + self.releaseFile[self.releaseFile.find('_dists_')+1:].replace('_','/')[:-7] + 'main/binary-i386/Packages.bz2'
 490
 491         d = self.client.findHash(idx_path)
 492         d.addCallback(self.verifyHash, idx_path, idx_hash)
 493
 494         pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 495                             '/var/lib/apt/lists/' + self.packagesFile +
 496                             ' | grep -E "^SHA1:" | head -n 1' +
 497                             ' | cut -d\  -f 2').read().rstrip('\n')
 498         pkg_path = '/' + os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 499                             '/var/lib/apt/lists/' + self.packagesFile +
 500                             ' | grep -E "^Filename:" | head -n 1' +
 501                             ' | cut -d\  -f 2').read().rstrip('\n')
 502
 503         d = self.client.findHash(pkg_path)
 504         d.addCallback(self.verifyHash, pkg_path, pkg_hash)
 505
 506         src_dir = '/' + os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 507                             '/var/lib/apt/lists/' + self.sourcesFile +
 508                             ' | grep -E "^Directory:" | head -n 1' +
 509                             ' | cut -d\  -f 2').read().rstrip('\n')
 510         src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
 511                             '/var/lib/apt/lists/' + self.sourcesFile +
 512                             ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
 513                             ' | cut -d\  -f 2').read().split('\n')[:-1]
 514         src_paths = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
 515                             '/var/lib/apt/lists/' + self.sourcesFile +
 516                             ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
 517                             ' | cut -d\  -f 4').read().split('\n')[:-1]
 518
 519         for i in range(len(src_hashes)):
 520             d = self.client.findHash(src_dir + '/' + src_paths[i])
 521             d.addCallback(self.verifyHash, src_dir + '/' + src_paths[i], src_hashes[i])
 522
 523         idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
 524                             '/var/lib/apt/lists/' + self.releaseFile +
 525                             ' | grep -E " main/source/Sources.bz2$"'
 526                             ' | head -n 1 | cut -d\  -f 2').read().rstrip('\n')
 527         idx_path = '/' + self.releaseFile[self.releaseFile.find('_dists_')+1:].replace('_','/')[:-7] + 'main/source/Sources.bz2'
 528
 529         d = self.client.findHash(idx_path)
 530         d.addCallback(self.verifyHash, idx_path, idx_hash)
 531
 532         d.addBoth(lastDefer.callback)
 533         return lastDefer
 534
 535     def tearDown(self):
 536         for p in self.pending_calls:
 537             if p.active():
 538                 p.cancel()
 539         self.pending_calls = []
 540         self.client.cleanup()
 541         self.client = None