AptPackages.py

   1 # Disable the FutureWarning from the apt module
   2 import warnings
   3 warnings.simplefilter("ignore", FutureWarning)
   4
   5 import os, os.path, stat, random, re, shelve, shutil, fcntl, copy, UserDict
   6
   7 from twisted.internet import threads, defer
   8 from twisted.python import log
   9 from twisted.trial import unittest
  10
  11 import apt_pkg, apt_inst
  12 from apt import OpProgress
  13
  14 apt_pkg.init()
  15
  16 class PackageFileList(UserDict.DictMixin):
  17     """Manages a list of package files belonging to a backend.
  18
  19     @type packages: C{shelve dictionary}
  20     @ivar packages: the files stored for this backend
  21     """
  22
  23     def __init__(self, cache_dir):
  24         self.cache_dir = cache_dir
  25         if not os.path.exists(self.cache_dir):
  26             os.makedirs(self.cache_dir)
  27         self.packages = None
  28         self.open()
  29
  30     def open(self):
  31         """Open the persistent dictionary of files in this backend."""
  32         if self.packages is None:
  33             self.packages = shelve.open(self.cache_dir+'/packages.db')
  34
  35     def close(self):
  36         """Close the persistent dictionary."""
  37         if self.packages is not None:
  38             self.packages.close()
  39
  40     def update_file(self, cache_path, file_path):
  41         """Check if an updated file needs to be tracked.
  42
  43         Called from the mirror manager when files get updated so we can update our
  44         fake lists and sources.list.
  45         """
  46         filename = cache_path.split('/')[-1]
  47         if filename=="Packages" or filename=="Release" or filename=="Sources":
  48             log.msg("Registering package file: "+cache_path)
  49             self.packages[cache_path] = file_path
  50             return True
  51         return False
  52
  53     def check_files(self):
  54         """Check all files in the database to make sure they exist."""
  55         files = self.packages.keys()
  56         for f in files:
  57             if not os.path.exists(self.packages[f]):
  58                 log.msg("File in packages database has been deleted: "+f)
  59                 del self.packages[f]
  60
  61     # Standard dictionary implementation so this class can be used like a dictionary.
  62     def __getitem__(self, key): return self.packages[key]
  63     def __setitem__(self, key, item): self.packages[key] = item
  64     def __delitem__(self, key): del self.packages[key]
  65     def keys(self): return self.packages.keys()
  66
  67 class AptPackages:
  68     """Uses python-apt to answer queries about packages.
  69
  70     Makes a fake configuration for python-apt for each backend.
  71     """
  72
  73     DEFAULT_APT_CONFIG = {
  74         #'APT' : '',
  75         #'APT::Architecture' : 'i386',  # Commented so the machine's config will set this
  76         #'APT::Default-Release' : 'unstable',
  77         'Dir':'.', # /
  78         'Dir::State' : 'apt/', # var/lib/apt/
  79         'Dir::State::Lists': 'lists/', # lists/
  80         #'Dir::State::cdroms' : 'cdroms.list',
  81         'Dir::State::userstatus' : 'status.user',
  82         'Dir::State::status': 'dpkg/status', # '/var/lib/dpkg/status'
  83         'Dir::Cache' : '.apt/cache/', # var/cache/apt/
  84         #'Dir::Cache::archives' : 'archives/',
  85         'Dir::Cache::srcpkgcache' : 'srcpkgcache.bin',
  86         'Dir::Cache::pkgcache' : 'pkgcache.bin',
  87         'Dir::Etc' : 'apt/etc/', # etc/apt/
  88         'Dir::Etc::sourcelist' : 'sources.list',
  89         'Dir::Etc::vendorlist' : 'vendors.list',
  90         'Dir::Etc::vendorparts' : 'vendors.list.d',
  91         #'Dir::Etc::main' : 'apt.conf',
  92         #'Dir::Etc::parts' : 'apt.conf.d',
  93         #'Dir::Etc::preferences' : 'preferences',
  94         'Dir::Bin' : '',
  95         #'Dir::Bin::methods' : '', #'/usr/lib/apt/methods'
  96         'Dir::Bin::dpkg' : '/usr/bin/dpkg',
  97         #'DPkg' : '',
  98         #'DPkg::Pre-Install-Pkgs' : '',
  99         #'DPkg::Tools' : '',
 100         #'DPkg::Tools::Options' : '',
 101         #'DPkg::Tools::Options::/usr/bin/apt-listchanges' : '',
 102         #'DPkg::Tools::Options::/usr/bin/apt-listchanges::Version' : '2',
 103         #'DPkg::Post-Invoke' : '',
 104         }
 105     essential_dirs = ('apt', 'apt/cache', 'apt/dpkg', 'apt/etc', 'apt/lists',
 106                       'apt/lists/partial')
 107     essential_files = ('apt/dpkg/status', 'apt/etc/sources.list',)
 108
 109     def __init__(self, cache_dir):
 110         """Construct a new packages manager.
 111
 112         @ivar backendName: name of backend associated with this packages file
 113         @ivar cache_dir: cache directory from config file
 114         """
 115         self.cache_dir = cache_dir
 116         self.apt_config = copy.deepcopy(self.DEFAULT_APT_CONFIG)
 117
 118         for dir in self.essential_dirs:
 119             path = os.path.join(self.cache_dir, dir)
 120             if not os.path.exists(path):
 121                 os.makedirs(path)
 122         for file in self.essential_files:
 123             path = os.path.join(self.cache_dir, file)
 124             if not os.path.exists(path):
 125                 f = open(path,'w')
 126                 f.close()
 127                 del f
 128
 129         self.apt_config['Dir'] = self.cache_dir
 130         self.apt_config['Dir::State::status'] = os.path.join(self.cache_dir,
 131                       self.apt_config['Dir::State'], self.apt_config['Dir::State::status'])
 132         self.packages = PackageFileList(cache_dir)
 133         self.loaded = 0
 134         self.loading = None
 135
 136     def __del__(self):
 137         self.cleanup()
 138         self.packages.close()
 139
 140     def addRelease(self, cache_path, file_path):
 141         """Dirty hack until python-apt supports apt-pkg/indexrecords.h
 142         (see Bug #456141)
 143         """
 144         self.indexrecords[cache_path] = {}
 145
 146         read_packages = False
 147         f = open(file_path, 'r')
 148
 149         for line in f:
 150             line = line.rstrip()
 151
 152             if line[:1] != " ":
 153                 read_packages = False
 154                 try:
 155                     # Read the various headers from the file
 156                     h, v = line.split(":", 1)
 157                     if h == "MD5Sum" or h == "SHA1" or h == "SHA256":
 158                         read_packages = True
 159                         hash_type = h
 160                 except:
 161                     # Bad header line, just ignore it
 162                     log.msg("WARNING: Ignoring badly formatted Release line: %s" % line)
 163
 164                 # Skip to the next line
 165                 continue
 166
 167             # Read file names from the multiple hash sections of the file
 168             if read_packages:
 169                 p = line.split()
 170                 self.indexrecords[cache_path].setdefault(p[2], {})[hash_type] = (p[0], p[1])
 171
 172         f.close()
 173
 174     def file_updated(self, cache_path, file_path):
 175         """A file in the backend has changed, manage it.
 176
 177         If this affects us, unload our apt database
 178         """
 179         if self.packages.update_file(cache_path, file_path):
 180             self.unload()
 181
 182     def load(self):
 183         """Make sure the package is initialized and loaded."""
 184         if self.loading is None:
 185             self.loading = threads.deferToThread(self._load)
 186             self.loading.addCallback(self.doneLoading)
 187         return self.loading
 188
 189     def doneLoading(self, loadResult):
 190         """Cache is loaded."""
 191         self.loading = None
 192         # Must pass on the result for the next callback
 193         return loadResult
 194
 195     def _load(self):
 196         """Regenerates the fake configuration and load the packages cache."""
 197         if self.loaded: return True
 198         apt_pkg.InitSystem()
 199         shutil.rmtree(os.path.join(self.cache_dir, self.apt_config['Dir::State'],
 200                                    self.apt_config['Dir::State::Lists']))
 201         os.makedirs(os.path.join(self.cache_dir, self.apt_config['Dir::State'],
 202                                  self.apt_config['Dir::State::Lists'], 'partial'))
 203         sources_filename = os.path.join(self.cache_dir, self.apt_config['Dir::Etc'],
 204                                         self.apt_config['Dir::Etc::sourcelist'])
 205         sources = open(sources_filename, 'w')
 206         sources_count = 0
 207         self.packages.check_files()
 208         self.indexrecords = {}
 209         for f in self.packages:
 210             # we should probably clear old entries from self.packages and
 211             # take into account the recorded mtime as optimization
 212             filepath = self.packages[f]
 213             if f.split('/')[-1] == "Release":
 214                 self.addRelease(f, filepath)
 215             fake_uri='http://apt-dht'+f
 216             fake_dirname = '/'.join(fake_uri.split('/')[:-1])
 217             if f.endswith('Sources'):
 218                 source_line='deb-src '+fake_dirname+'/ /'
 219             else:
 220                 source_line='deb '+fake_dirname+'/ /'
 221             listpath=(os.path.join(self.cache_dir, self.apt_config['Dir::State'],
 222                                    self.apt_config['Dir::State::Lists'],
 223                                    apt_pkg.URItoFileName(fake_uri)))
 224             sources.write(source_line+'\n')
 225             log.msg("Sources line: " + source_line)
 226             sources_count = sources_count + 1
 227
 228             try:
 229                 #we should empty the directory instead
 230                 os.unlink(listpath)
 231             except:
 232                 pass
 233             os.symlink(filepath, listpath)
 234         sources.close()
 235
 236         if sources_count == 0:
 237             log.msg("No Packages files available for %s backend"%(self.cache_dir))
 238             return False
 239
 240         log.msg("Loading Packages database for "+self.cache_dir)
 241         for key, value in self.apt_config.items():
 242             apt_pkg.Config[key] = value
 243
 244         self.cache = apt_pkg.GetCache(OpProgress())
 245         self.records = apt_pkg.GetPkgRecords(self.cache)
 246         self.srcrecords = apt_pkg.GetPkgSrcRecords()
 247
 248         self.loaded = 1
 249         return True
 250
 251     def unload(self):
 252         """Tries to make the packages server quit."""
 253         if self.loaded:
 254             del self.cache
 255             del self.records
 256             del self.srcrecords
 257             del self.indexrecords
 258             self.loaded = 0
 259
 260     def cleanup(self):
 261         """Cleanup and close any loaded caches."""
 262         self.unload()
 263         self.packages.close()
 264
 265     def findHash(self, path):
 266         """Find the hash for a given path in this mirror.
 267
 268         Returns a deferred so it can make sure the cache is loaded first.
 269         """
 270         d = defer.Deferred()
 271
 272         deferLoad = self.load()
 273         deferLoad.addCallback(self._findHash, path, d)
 274
 275         return d
 276
 277     def _findHash(self, loadResult, path, d):
 278         """Really find the hash for a path.
 279
 280         Have to pass the returned loadResult on in case other calls to this
 281         function are pending.
 282         """
 283         if not loadResult:
 284             d.callback((None, None))
 285             return loadResult
 286
 287         # First look for the path in the cache of index files
 288         for release in self.indexrecords:
 289             if path.startswith(release[:-7]):
 290                 for indexFile in self.indexrecords[release]:
 291                     if release[:-7] + indexFile == path:
 292                         d.callback(self.indexrecords[release][indexFile]['SHA1'])
 293                         return loadResult
 294
 295         package = path.split('/')[-1].split('_')[0]
 296
 297         # Check the binary packages
 298         try:
 299             for version in self.cache[package].VersionList:
 300                 size = version.Size
 301                 for verFile in version.FileList:
 302                     if self.records.Lookup(verFile):
 303                         if self.records.FileName == path:
 304                             d.callback((self.records.SHA1Hash, size))
 305                             return loadResult
 306         except KeyError:
 307             pass
 308
 309         # Check the source packages' files
 310         self.srcrecords.Restart()
 311         if self.srcrecords.Lookup(package):
 312             for f in self.srcrecords.Files:
 313                 if path == f[2]:
 314                     d.callback((f[0], f[1]))
 315                     return loadResult
 316
 317         d.callback((None, None))
 318         return loadResult
 319
 320 class TestAptPackages(unittest.TestCase):
 321     """Unit tests for the AptPackages cache."""
 322
 323     pending_calls = []
 324     client = None
 325     packagesFile = ''
 326     sourcesFile = ''
 327     releaseFile = ''
 328
 329     def setUp(self):
 330         self.client = AptPackages('/tmp/.apt-dht')
 331
 332         self.packagesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "Packages$" | tail -n 1').read().rstrip('\n')
 333         self.sourcesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "Sources$" | tail -n 1').read().rstrip('\n')
 334         for f in os.walk('/var/lib/apt/lists').next()[2]:
 335             if f[-7:] == "Release" and self.packagesFile.startswith(f[:-7]):
 336                 self.releaseFile = f
 337                 break
 338
 339         self.client.file_updated(self.releaseFile[self.releaseFile.find('_debian_')+1:].replace('_','/'),
 340                                  '/var/lib/apt/lists/' + self.releaseFile)
 341         self.client.file_updated(self.packagesFile[self.packagesFile.find('_debian_')+1:].replace('_','/'),
 342                                  '/var/lib/apt/lists/' + self.packagesFile)
 343         self.client.file_updated(self.sourcesFile[self.sourcesFile.find('_debian_')+1:].replace('_','/'),
 344                                  '/var/lib/apt/lists/' + self.sourcesFile)
 345
 346     def test_pkg_hash(self):
 347         self.client._load()
 348
 349         self.client.records.Lookup(self.client.cache['dpkg'].VersionList[0].FileList[0])
 350
 351         pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 352                             '/var/lib/apt/lists/' + self.packagesFile +
 353                             ' | grep -E "^SHA1:" | head -n 1' +
 354                             ' | cut -d\  -f 2').read().rstrip('\n')
 355
 356         self.failUnless(self.client.records.SHA1Hash == pkg_hash,
 357                         "Hashes don't match: %s != %s" % (self.client.records.SHA1Hash, pkg_hash))
 358
 359     def test_src_hash(self):
 360         self.client._load()
 361
 362         self.client.srcrecords.Lookup('dpkg')
 363
 364         src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
 365                             '/var/lib/apt/lists/' + self.sourcesFile +
 366                             ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
 367                             ' | cut -d\  -f 2').read().split('\n')[:-1]
 368
 369         for f in self.client.srcrecords.Files:
 370             self.failUnless(f[0] in src_hashes, "Couldn't find %s in: %r" % (f[0], src_hashes))
 371
 372     def test_index_hash(self):
 373         self.client._load()
 374
 375         indexhash = self.client.indexrecords[self.releaseFile[self.releaseFile.find('_debian_')+1:].replace('_','/')]['main/binary-i386/Packages.bz2']['SHA1'][0]
 376
 377         idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
 378                             '/var/lib/apt/lists/' + self.releaseFile +
 379                             ' | grep -E " main/binary-i386/Packages.bz2$"'
 380                             ' | head -n 1 | cut -d\  -f 2').read().rstrip('\n')
 381
 382         self.failUnless(indexhash == idx_hash, "Hashes don't match: %s != %s" % (indexhash, idx_hash))
 383
 384     def verifyHash(self, found_hash, path, true_hash):
 385         self.failUnless(found_hash[0] == true_hash,
 386                     "%s hashes don't match: %s != %s" % (path, found_hash[0], true_hash))
 387
 388     def test_findIndexHash(self):
 389         lastDefer = defer.Deferred()
 390
 391         idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
 392                             '/var/lib/apt/lists/' + self.releaseFile +
 393                             ' | grep -E " main/binary-i386/Packages.bz2$"'
 394                             ' | head -n 1 | cut -d\  -f 2').read().rstrip('\n')
 395         idx_path = self.releaseFile[self.releaseFile.find('_debian_')+1:].replace('_','/')[:-7] + 'main/binary-i386/Packages.bz2'
 396
 397         d = self.client.findHash(idx_path)
 398         d.addCallback(self.verifyHash, idx_path, idx_hash)
 399
 400         d.addCallback(lastDefer.callback)
 401         return lastDefer
 402
 403     def test_findPkgHash(self):
 404         lastDefer = defer.Deferred()
 405
 406         pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 407                             '/var/lib/apt/lists/' + self.packagesFile +
 408                             ' | grep -E "^SHA1:" | head -n 1' +
 409                             ' | cut -d\  -f 2').read().rstrip('\n')
 410         pkg_path = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 411                             '/var/lib/apt/lists/' + self.packagesFile +
 412                             ' | grep -E "^Filename:" | head -n 1' +
 413                             ' | cut -d\  -f 2').read().rstrip('\n')
 414
 415         d = self.client.findHash(pkg_path)
 416         d.addCallback(self.verifyHash, pkg_path, pkg_hash)
 417
 418         d.addCallback(lastDefer.callback)
 419         return lastDefer
 420
 421     def test_findSrcHash(self):
 422         lastDefer = defer.Deferred()
 423
 424         src_dir = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 425                             '/var/lib/apt/lists/' + self.sourcesFile +
 426                             ' | grep -E "^Directory:" | head -n 1' +
 427                             ' | cut -d\  -f 2').read().rstrip('\n')
 428         src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
 429                             '/var/lib/apt/lists/' + self.sourcesFile +
 430                             ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
 431                             ' | cut -d\  -f 2').read().split('\n')[:-1]
 432         src_paths = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
 433                             '/var/lib/apt/lists/' + self.sourcesFile +
 434                             ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
 435                             ' | cut -d\  -f 4').read().split('\n')[:-1]
 436
 437         i = random.choice(range(len(src_hashes)))
 438         d = self.client.findHash(src_dir + '/' + src_paths[i])
 439         d.addCallback(self.verifyHash, src_dir + '/' + src_paths[i], src_hashes[i])
 440
 441         d.addCallback(lastDefer.callback)
 442         return lastDefer
 443
 444     def test_multipleFindHash(self):
 445         lastDefer = defer.Deferred()
 446
 447         idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
 448                             '/var/lib/apt/lists/' + self.releaseFile +
 449                             ' | grep -E " main/binary-i386/Packages.bz2$"'
 450                             ' | head -n 1 | cut -d\  -f 2').read().rstrip('\n')
 451         idx_path = self.releaseFile[self.releaseFile.find('_debian_')+1:].replace('_','/')[:-7] + 'main/binary-i386/Packages.bz2'
 452
 453         d = self.client.findHash(idx_path)
 454         d.addCallback(self.verifyHash, idx_path, idx_hash)
 455
 456         pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 457                             '/var/lib/apt/lists/' + self.packagesFile +
 458                             ' | grep -E "^SHA1:" | head -n 1' +
 459                             ' | cut -d\  -f 2').read().rstrip('\n')
 460         pkg_path = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 461                             '/var/lib/apt/lists/' + self.packagesFile +
 462                             ' | grep -E "^Filename:" | head -n 1' +
 463                             ' | cut -d\  -f 2').read().rstrip('\n')
 464
 465         d = self.client.findHash(pkg_path)
 466         d.addCallback(self.verifyHash, pkg_path, pkg_hash)
 467
 468         src_dir = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 469                             '/var/lib/apt/lists/' + self.sourcesFile +
 470                             ' | grep -E "^Directory:" | head -n 1' +
 471                             ' | cut -d\  -f 2').read().rstrip('\n')
 472         src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
 473                             '/var/lib/apt/lists/' + self.sourcesFile +
 474                             ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
 475                             ' | cut -d\  -f 2').read().split('\n')[:-1]
 476         src_paths = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
 477                             '/var/lib/apt/lists/' + self.sourcesFile +
 478                             ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
 479                             ' | cut -d\  -f 4').read().split('\n')[:-1]
 480
 481         for i in range(len(src_hashes)):
 482             d = self.client.findHash(src_dir + '/' + src_paths[i])
 483             d.addCallback(self.verifyHash, src_dir + '/' + src_paths[i], src_hashes[i])
 484
 485         idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
 486                             '/var/lib/apt/lists/' + self.releaseFile +
 487                             ' | grep -E " main/source/Sources.bz2$"'
 488                             ' | head -n 1 | cut -d\  -f 2').read().rstrip('\n')
 489         idx_path = self.releaseFile[self.releaseFile.find('_debian_')+1:].replace('_','/')[:-7] + 'main/source/Sources.bz2'
 490
 491         d = self.client.findHash(idx_path)
 492         d.addCallback(self.verifyHash, idx_path, idx_hash)
 493
 494         d.addCallback(lastDefer.callback)
 495         return lastDefer
 496
 497     def tearDown(self):
 498         for p in self.pending_calls:
 499             if p.active():
 500                 p.cancel()
 501         self.pending_calls = []
 502         self.client.cleanup()
 503         self.client = None