AptPackages.py

   1 # Disable the FutureWarning from the apt module
   2 import warnings
   3 warnings.simplefilter("ignore", FutureWarning)
   4
   5 import os, stat, random, re, shelve, shutil, fcntl, copy, UserDict
   6 from os.path import dirname
   7
   8 from twisted.internet import threads, defer
   9 from twisted.python import log
  10 from twisted.trial import unittest
  11
  12 import apt_pkg, apt_inst
  13 from apt import OpProgress
  14
  15 aptpkg_dir='.apt-dht'
  16 apt_pkg.init()
  17
  18 class PackageFileList(UserDict.DictMixin):
  19     """Manages a list of package files belonging to a backend.
  20
  21     @type packages: C{shelve dictionary}
  22     @ivar packages: the files stored for this backend
  23     """
  24
  25     def __init__(self, backendName, cache_dir):
  26         self.cache_dir = cache_dir
  27         self.packagedb_dir = cache_dir+'/'+ aptpkg_dir + \
  28                            '/backends/' + backendName
  29         if not os.path.exists(self.packagedb_dir):
  30             os.makedirs(self.packagedb_dir)
  31         self.packages = None
  32         self.open()
  33
  34     def open(self):
  35         """Open the persistent dictionary of files in this backend."""
  36         if self.packages is None:
  37             self.packages = shelve.open(self.packagedb_dir+'/packages.db')
  38
  39     def close(self):
  40         """Close the persistent dictionary."""
  41         if self.packages is not None:
  42             self.packages.close()
  43
  44     def update_file(self, filename, cache_path, file_path):
  45         """Check if an updated file needs to be tracked.
  46
  47         Called from the mirror manager when files get updated so we can update our
  48         fake lists and sources.list.
  49         """
  50         if filename=="Packages" or filename=="Release" or filename=="Sources":
  51             log.msg("Registering package file: "+cache_path)
  52             self.packages[cache_path] = file_path
  53             return True
  54         return False
  55
  56     def check_files(self):
  57         """Check all files in the database to make sure they exist."""
  58         files = self.packages.keys()
  59         for f in files:
  60             if not os.path.exists(self.packages[f]):
  61                 log.msg("File in packages database has been deleted: "+f)
  62                 del self.packages[f]
  63
  64     # Standard dictionary implementation so this class can be used like a dictionary.
  65     def __getitem__(self, key): return self.packages[key]
  66     def __setitem__(self, key, item): self.packages[key] = item
  67     def __delitem__(self, key): del self.packages[key]
  68     def keys(self): return self.packages.keys()
  69
  70 class AptPackages:
  71     """Uses python-apt to answer queries about packages.
  72
  73     Makes a fake configuration for python-apt for each backend.
  74     """
  75
  76     DEFAULT_APT_CONFIG = {
  77         #'APT' : '',
  78         #'APT::Architecture' : 'i386',  # Commented so the machine's config will set this
  79         #'APT::Default-Release' : 'unstable',
  80         'Dir':'.', # /
  81         'Dir::State' : 'apt/', # var/lib/apt/
  82         'Dir::State::Lists': 'lists/', # lists/
  83         #'Dir::State::cdroms' : 'cdroms.list',
  84         'Dir::State::userstatus' : 'status.user',
  85         'Dir::State::status': 'dpkg/status', # '/var/lib/dpkg/status'
  86         'Dir::Cache' : '.apt/cache/', # var/cache/apt/
  87         #'Dir::Cache::archives' : 'archives/',
  88         'Dir::Cache::srcpkgcache' : 'srcpkgcache.bin',
  89         'Dir::Cache::pkgcache' : 'pkgcache.bin',
  90         'Dir::Etc' : 'apt/etc/', # etc/apt/
  91         'Dir::Etc::sourcelist' : 'sources.list',
  92         'Dir::Etc::vendorlist' : 'vendors.list',
  93         'Dir::Etc::vendorparts' : 'vendors.list.d',
  94         #'Dir::Etc::main' : 'apt.conf',
  95         #'Dir::Etc::parts' : 'apt.conf.d',
  96         #'Dir::Etc::preferences' : 'preferences',
  97         'Dir::Bin' : '',
  98         #'Dir::Bin::methods' : '', #'/usr/lib/apt/methods'
  99         'Dir::Bin::dpkg' : '/usr/bin/dpkg',
 100         #'DPkg' : '',
 101         #'DPkg::Pre-Install-Pkgs' : '',
 102         #'DPkg::Tools' : '',
 103         #'DPkg::Tools::Options' : '',
 104         #'DPkg::Tools::Options::/usr/bin/apt-listchanges' : '',
 105         #'DPkg::Tools::Options::/usr/bin/apt-listchanges::Version' : '2',
 106         #'DPkg::Post-Invoke' : '',
 107         }
 108     essential_dirs = ('apt', 'apt/cache', 'apt/dpkg', 'apt/etc', 'apt/lists',
 109                       'apt/lists/partial')
 110     essential_files = ('apt/dpkg/status', 'apt/etc/sources.list',)
 111
 112     def __init__(self, backendName, cache_dir):
 113         """Construct a new packages manager.
 114
 115         @ivar backendName: name of backend associated with this packages file
 116         @ivar cache_dir: cache directory from config file
 117         """
 118         self.backendName = backendName
 119         self.cache_dir = cache_dir
 120         self.apt_config = copy.deepcopy(self.DEFAULT_APT_CONFIG)
 121
 122         self.status_dir = (cache_dir+'/'+ aptpkg_dir
 123                            +'/backends/'+backendName)
 124         for dir in self.essential_dirs:
 125             path = self.status_dir+'/'+dir
 126             if not os.path.exists(path):
 127                 os.makedirs(path)
 128         for file in self.essential_files:
 129             path = self.status_dir+'/'+file
 130             if not os.path.exists(path):
 131                 f = open(path,'w')
 132                 f.close()
 133                 del f
 134
 135         self.apt_config['Dir'] = self.status_dir
 136         self.apt_config['Dir::State::status'] = self.status_dir + '/apt/dpkg/status'
 137         self.packages = PackageFileList(backendName, cache_dir)
 138         self.indexrecords = {}
 139         self.loaded = 0
 140         self.loading = None
 141
 142     def __del__(self):
 143         self.cleanup()
 144         self.packages.close()
 145
 146     def addRelease(self, cache_path, file_path):
 147         """Dirty hack until python-apt supports apt-pkg/indexrecords.h
 148         (see Bug #456141)
 149         """
 150         self.indexrecords[cache_path] = {}
 151
 152         read_packages = False
 153         f = open(file_path, 'r')
 154
 155         for line in f:
 156             line = line.rstrip()
 157
 158             if line[:1] != " ":
 159                 read_packages = False
 160                 try:
 161                     # Read the various headers from the file
 162                     h, v = line.split(":", 1)
 163                     if h == "MD5Sum" or h == "SHA1" or h == "SHA256":
 164                         read_packages = True
 165                         hash_type = h
 166                 except:
 167                     # Bad header line, just ignore it
 168                     log.msg("WARNING: Ignoring badly formatted Release line: %s" % line)
 169
 170                 # Skip to the next line
 171                 continue
 172
 173             # Read file names from the multiple hash sections of the file
 174             if read_packages:
 175                 p = line.split()
 176                 self.indexrecords[cache_path].setdefault(p[2], {})[hash_type] = (p[0], p[1])
 177
 178         f.close()
 179
 180     def file_updated(self, filename, cache_path, file_path):
 181         """A file in the backend has changed, manage it.
 182
 183         If this affects us, unload our apt database
 184         """
 185         if filename == "Release":
 186             self.addRelease(cache_path, file_path)
 187         if self.packages.update_file(filename, cache_path, file_path):
 188             self.unload()
 189
 190     def load(self):
 191         """Make sure the package is initialized and loaded."""
 192         if self.loading is None:
 193             self.loading = threads.deferToThread(self._load)
 194             self.loading.addCallback(self.doneLoading)
 195         return self.loading
 196
 197     def doneLoading(self, loadResult):
 198         """Cache is loaded."""
 199         self.loading = None
 200         # Must pass on the result for the next callback
 201         return loadResult
 202
 203     def _load(self):
 204         """Regenerates the fake configuration and load the packages cache."""
 205         if self.loaded: return True
 206         apt_pkg.InitSystem()
 207         shutil.rmtree(self.status_dir+'/apt/lists/')
 208         os.makedirs(self.status_dir+'/apt/lists/partial')
 209         sources_filename = self.status_dir+'/'+'apt/etc/sources.list'
 210         sources = open(sources_filename, 'w')
 211         sources_count = 0
 212         self.packages.check_files()
 213         for f in self.packages:
 214             # we should probably clear old entries from self.packages and
 215             # take into account the recorded mtime as optimization
 216             filepath = self.packages[f]
 217             fake_uri='http://apt-dht/'+f
 218             if f.endswith('Sources'):
 219                 source_line='deb-src '+dirname(fake_uri)+'/ /'
 220             else:
 221                 source_line='deb '+dirname(fake_uri)+'/ /'
 222             listpath=(self.status_dir+'/apt/lists/'
 223                     +apt_pkg.URItoFileName(fake_uri))
 224             sources.write(source_line+'\n')
 225             log.msg("Sources line: " + source_line)
 226             sources_count = sources_count + 1
 227
 228             try:
 229                 #we should empty the directory instead
 230                 os.unlink(listpath)
 231             except:
 232                 pass
 233             os.symlink(filepath, listpath)
 234         sources.close()
 235
 236         if sources_count == 0:
 237             log.msg("No Packages files available for %s backend"%(self.backendName))
 238             return False
 239
 240         log.msg("Loading Packages database for "+self.status_dir)
 241         for key, value in self.apt_config.items():
 242             apt_pkg.Config[key] = value
 243
 244         self.cache = apt_pkg.GetCache(OpProgress())
 245         self.records = apt_pkg.GetPkgRecords(self.cache)
 246         self.srcrecords = apt_pkg.GetPkgSrcRecords()
 247
 248         self.loaded = 1
 249         return True
 250
 251     def unload(self):
 252         """Tries to make the packages server quit."""
 253         if self.loaded:
 254             del self.cache
 255             del self.records
 256             del self.srcrecords
 257             self.loaded = 0
 258
 259     def cleanup(self):
 260         """Cleanup and close any loaded caches."""
 261         self.unload()
 262         self.packages.close()
 263
 264     def findHash(self, path):
 265         """Find the hash for a given path in this mirror.
 266
 267         Returns a deferred so it can make sure the cache is loaded first.
 268         """
 269         d = defer.Deferred()
 270
 271         # First look for the path in the cache of index files
 272         for release in self.indexrecords:
 273             if path.startswith(release[:-7]):
 274                 for indexFile in self.indexrecords[release]:
 275                     if release[:-7] + indexFile == path:
 276                         d.callback(self.indexrecords[release][indexFile]['SHA1'])
 277                         return d
 278
 279         deferLoad = self.load()
 280         deferLoad.addCallback(self._findHash, path, d)
 281
 282         return d
 283
 284     def _findHash(self, loadResult, path, d):
 285         """Really find the hash for a path.
 286
 287         Have to pass the returned loadResult on in case other calls to this
 288         function are pending.
 289         """
 290         if not loadResult:
 291             d.callback((None, None))
 292             return loadResult
 293
 294         package = path.split('/')[-1].split('_')[0]
 295
 296         # Check the binary packages
 297         try:
 298             for version in self.cache[package].VersionList:
 299                 size = version.Size
 300                 for verFile in version.FileList:
 301                     if self.records.Lookup(verFile):
 302                         if self.records.FileName == path:
 303                             d.callback((self.records.SHA1Hash, size))
 304                             return loadResult
 305         except KeyError:
 306             pass
 307
 308         # Check the source packages' files
 309         self.srcrecords.Restart()
 310         if self.srcrecords.Lookup(package):
 311             for f in self.srcrecords.Files:
 312                 if path == f[2]:
 313                     d.callback((f[0], f[1]))
 314                     return loadResult
 315
 316         d.callback((None, None))
 317         return loadResult
 318
 319 class TestAptPackages(unittest.TestCase):
 320     """Unit tests for the AptPackages cache."""
 321
 322     pending_calls = []
 323     client = None
 324     packagesFile = ''
 325     sourcesFile = ''
 326     releaseFile = ''
 327
 328     def setUp(self):
 329         self.client = AptPackages('whatever', '/tmp')
 330
 331         self.packagesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "Packages$" | tail -n 1').read().rstrip('\n')
 332         self.sourcesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "Sources$" | tail -n 1').read().rstrip('\n')
 333         for f in os.walk('/var/lib/apt/lists').next()[2]:
 334             if f[-7:] == "Release" and self.packagesFile.startswith(f[:-7]):
 335                 self.releaseFile = f
 336                 break
 337
 338         self.client.file_updated('Release',
 339                                  self.releaseFile[self.releaseFile.find('_debian_')+1:].replace('_','/'),
 340                                  '/var/lib/apt/lists/' + self.releaseFile)
 341         self.client.file_updated('Packages',
 342                                  self.packagesFile[self.packagesFile.find('_debian_')+1:].replace('_','/'),
 343                                  '/var/lib/apt/lists/' + self.packagesFile)
 344         self.client.file_updated('Sources',
 345                                  self.sourcesFile[self.sourcesFile.find('_debian_')+1:].replace('_','/'),
 346                                  '/var/lib/apt/lists/' + self.sourcesFile)
 347
 348     def test_pkg_hash(self):
 349         self.client._load()
 350
 351         self.client.records.Lookup(self.client.cache['dpkg'].VersionList[0].FileList[0])
 352
 353         pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 354                             '/var/lib/apt/lists/' + self.packagesFile +
 355                             ' | grep -E "^SHA1:" | head -n 1' +
 356                             ' | cut -d\  -f 2').read().rstrip('\n')
 357
 358         self.failUnless(self.client.records.SHA1Hash == pkg_hash,
 359                         "Hashes don't match: %s != %s" % (self.client.records.SHA1Hash, pkg_hash))
 360
 361     def test_src_hash(self):
 362         self.client._load()
 363
 364         self.client.srcrecords.Lookup('dpkg')
 365
 366         src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
 367                             '/var/lib/apt/lists/' + self.sourcesFile +
 368                             ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
 369                             ' | cut -d\  -f 2').read().split('\n')[:-1]
 370
 371         for f in self.client.srcrecords.Files:
 372             self.failUnless(f[0] in src_hashes, "Couldn't find %s in: %r" % (f[0], src_hashes))
 373
 374     def test_index_hash(self):
 375         self.client._load()
 376
 377         indexhash = self.client.indexrecords[self.releaseFile[self.releaseFile.find('_debian_')+1:].replace('_','/')]['main/binary-i386/Packages.bz2']['SHA1'][0]
 378
 379         idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
 380                             '/var/lib/apt/lists/' + self.releaseFile +
 381                             ' | grep -E " main/binary-i386/Packages.bz2$"'
 382                             ' | head -n 1 | cut -d\  -f 2').read().rstrip('\n')
 383
 384         self.failUnless(indexhash == idx_hash, "Hashes don't match: %s != %s" % (indexhash, idx_hash))
 385
 386     def verifyHash(self, found_hash, path, true_hash):
 387         self.failUnless(found_hash[0] == true_hash,
 388                     "%s hashes don't match: %s != %s" % (path, found_hash[0], true_hash))
 389
 390     def test_findIndexHash(self):
 391         lastDefer = defer.Deferred()
 392
 393         idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
 394                             '/var/lib/apt/lists/' + self.releaseFile +
 395                             ' | grep -E " main/binary-i386/Packages.bz2$"'
 396                             ' | head -n 1 | cut -d\  -f 2').read().rstrip('\n')
 397         idx_path = self.releaseFile[self.releaseFile.find('_debian_')+1:].replace('_','/')[:-7] + 'main/binary-i386/Packages.bz2'
 398
 399         d = self.client.findHash(idx_path)
 400         d.addCallback(self.verifyHash, idx_path, idx_hash)
 401
 402         d.addCallback(lastDefer.callback)
 403         return lastDefer
 404
 405     def test_findPkgHash(self):
 406         lastDefer = defer.Deferred()
 407
 408         pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 409                             '/var/lib/apt/lists/' + self.packagesFile +
 410                             ' | grep -E "^SHA1:" | head -n 1' +
 411                             ' | cut -d\  -f 2').read().rstrip('\n')
 412         pkg_path = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 413                             '/var/lib/apt/lists/' + self.packagesFile +
 414                             ' | grep -E "^Filename:" | head -n 1' +
 415                             ' | cut -d\  -f 2').read().rstrip('\n')
 416
 417         d = self.client.findHash(pkg_path)
 418         d.addCallback(self.verifyHash, pkg_path, pkg_hash)
 419
 420         d.addCallback(lastDefer.callback)
 421         return lastDefer
 422
 423     def test_findSrcHash(self):
 424         lastDefer = defer.Deferred()
 425
 426         src_dir = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 427                             '/var/lib/apt/lists/' + self.sourcesFile +
 428                             ' | grep -E "^Directory:" | head -n 1' +
 429                             ' | cut -d\  -f 2').read().rstrip('\n')
 430         src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
 431                             '/var/lib/apt/lists/' + self.sourcesFile +
 432                             ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
 433                             ' | cut -d\  -f 2').read().split('\n')[:-1]
 434         src_paths = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
 435                             '/var/lib/apt/lists/' + self.sourcesFile +
 436                             ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
 437                             ' | cut -d\  -f 4').read().split('\n')[:-1]
 438
 439         i = random.choice(range(len(src_hashes)))
 440         d = self.client.findHash(src_dir + '/' + src_paths[i])
 441         d.addCallback(self.verifyHash, src_dir + '/' + src_paths[i], src_hashes[i])
 442
 443         d.addCallback(lastDefer.callback)
 444         return lastDefer
 445
 446     def test_multipleFindHash(self):
 447         lastDefer = defer.Deferred()
 448
 449         idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
 450                             '/var/lib/apt/lists/' + self.releaseFile +
 451                             ' | grep -E " main/binary-i386/Packages.bz2$"'
 452                             ' | head -n 1 | cut -d\  -f 2').read().rstrip('\n')
 453         idx_path = self.releaseFile[self.releaseFile.find('_debian_')+1:].replace('_','/')[:-7] + 'main/binary-i386/Packages.bz2'
 454
 455         d = self.client.findHash(idx_path)
 456         d.addCallback(self.verifyHash, idx_path, idx_hash)
 457
 458         pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 459                             '/var/lib/apt/lists/' + self.packagesFile +
 460                             ' | grep -E "^SHA1:" | head -n 1' +
 461                             ' | cut -d\  -f 2').read().rstrip('\n')
 462         pkg_path = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 463                             '/var/lib/apt/lists/' + self.packagesFile +
 464                             ' | grep -E "^Filename:" | head -n 1' +
 465                             ' | cut -d\  -f 2').read().rstrip('\n')
 466
 467         d = self.client.findHash(pkg_path)
 468         d.addCallback(self.verifyHash, pkg_path, pkg_hash)
 469
 470         src_dir = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 471                             '/var/lib/apt/lists/' + self.sourcesFile +
 472                             ' | grep -E "^Directory:" | head -n 1' +
 473                             ' | cut -d\  -f 2').read().rstrip('\n')
 474         src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
 475                             '/var/lib/apt/lists/' + self.sourcesFile +
 476                             ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
 477                             ' | cut -d\  -f 2').read().split('\n')[:-1]
 478         src_paths = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
 479                             '/var/lib/apt/lists/' + self.sourcesFile +
 480                             ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
 481                             ' | cut -d\  -f 4').read().split('\n')[:-1]
 482
 483         for i in range(len(src_hashes)):
 484             d = self.client.findHash(src_dir + '/' + src_paths[i])
 485             d.addCallback(self.verifyHash, src_dir + '/' + src_paths[i], src_hashes[i])
 486
 487         d.addCallback(lastDefer.callback)
 488         return lastDefer
 489
 490     def tearDown(self):
 491         for p in self.pending_calls:
 492             if p.active():
 493                 p.cancel()
 494         self.pending_calls = []
 495         self.client.cleanup()
 496         self.client = None