AptPackages.py

   1 # Disable the FutureWarning from the apt module
   2 import warnings
   3 warnings.simplefilter("ignore", FutureWarning)
   4
   5 import os, stat, random, re, shelve, shutil, fcntl, copy, UserDict
   6 from os.path import dirname, basename
   7
   8 from twisted.internet import threads, defer
   9 from twisted.python import log
  10 from twisted.trial import unittest
  11
  12 import apt_pkg, apt_inst
  13 from apt import OpProgress
  14
  15 aptpkg_dir='.apt-dht'
  16 apt_pkg.init()
  17
  18 class PackageFileList(UserDict.DictMixin):
  19     """Manages a list of package files belonging to a backend.
  20
  21     @type packages: C{shelve dictionary}
  22     @ivar packages: the files stored for this backend
  23     """
  24
  25     def __init__(self, backendName, cache_dir):
  26         self.cache_dir = cache_dir
  27         self.packagedb_dir = cache_dir+'/'+ aptpkg_dir + \
  28                            '/backends/' + backendName
  29         if not os.path.exists(self.packagedb_dir):
  30             os.makedirs(self.packagedb_dir)
  31         self.packages = None
  32         self.open()
  33
  34     def open(self):
  35         """Open the persistent dictionary of files in this backend."""
  36         if self.packages is None:
  37             self.packages = shelve.open(self.packagedb_dir+'/packages.db')
  38
  39     def close(self):
  40         """Close the persistent dictionary."""
  41         if self.packages is not None:
  42             self.packages.close()
  43
  44     def update_file(self, filename, cache_path, file_path):
  45         """Check if an updated file needs to be tracked.
  46
  47         Called from the mirror manager when files get updated so we can update our
  48         fake lists and sources.list.
  49         """
  50         if filename=="Packages" or filename=="Release" or filename=="Sources":
  51             log.msg("Registering package file: "+cache_path)
  52             self.packages[cache_path] = file_path
  53             return True
  54         return False
  55
  56     def check_files(self):
  57         """Check all files in the database to make sure they exist."""
  58         files = self.packages.keys()
  59         for f in files:
  60             if not os.path.exists(self.packages[f]):
  61                 log.msg("File in packages database has been deleted: "+f)
  62                 del self.packages[f]
  63
  64     # Standard dictionary implementation so this class can be used like a dictionary.
  65     def __getitem__(self, key): return self.packages[key]
  66     def __setitem__(self, key, item): self.packages[key] = item
  67     def __delitem__(self, key): del self.packages[key]
  68     def keys(self): return self.packages.keys()
  69
  70 class AptPackages:
  71     """Uses python-apt to answer queries about packages.
  72
  73     Makes a fake configuration for python-apt for each backend.
  74     """
  75
  76     DEFAULT_APT_CONFIG = {
  77         #'APT' : '',
  78         #'APT::Architecture' : 'i386',  # Commented so the machine's config will set this
  79         #'APT::Default-Release' : 'unstable',
  80         'Dir':'.', # /
  81         'Dir::State' : 'apt/', # var/lib/apt/
  82         'Dir::State::Lists': 'lists/', # lists/
  83         #'Dir::State::cdroms' : 'cdroms.list',
  84         'Dir::State::userstatus' : 'status.user',
  85         'Dir::State::status': 'dpkg/status', # '/var/lib/dpkg/status'
  86         'Dir::Cache' : '.apt/cache/', # var/cache/apt/
  87         #'Dir::Cache::archives' : 'archives/',
  88         'Dir::Cache::srcpkgcache' : 'srcpkgcache.bin',
  89         'Dir::Cache::pkgcache' : 'pkgcache.bin',
  90         'Dir::Etc' : 'apt/etc/', # etc/apt/
  91         'Dir::Etc::sourcelist' : 'sources.list',
  92         'Dir::Etc::vendorlist' : 'vendors.list',
  93         'Dir::Etc::vendorparts' : 'vendors.list.d',
  94         #'Dir::Etc::main' : 'apt.conf',
  95         #'Dir::Etc::parts' : 'apt.conf.d',
  96         #'Dir::Etc::preferences' : 'preferences',
  97         'Dir::Bin' : '',
  98         #'Dir::Bin::methods' : '', #'/usr/lib/apt/methods'
  99         'Dir::Bin::dpkg' : '/usr/bin/dpkg',
 100         #'DPkg' : '',
 101         #'DPkg::Pre-Install-Pkgs' : '',
 102         #'DPkg::Tools' : '',
 103         #'DPkg::Tools::Options' : '',
 104         #'DPkg::Tools::Options::/usr/bin/apt-listchanges' : '',
 105         #'DPkg::Tools::Options::/usr/bin/apt-listchanges::Version' : '2',
 106         #'DPkg::Post-Invoke' : '',
 107         }
 108     essential_dirs = ('apt', 'apt/cache', 'apt/dpkg', 'apt/etc', 'apt/lists',
 109                       'apt/lists/partial')
 110     essential_files = ('apt/dpkg/status', 'apt/etc/sources.list',)
 111
 112     def __init__(self, backendName, cache_dir):
 113         """Construct a new packages manager.
 114
 115         @ivar backendName: name of backend associated with this packages file
 116         @ivar cache_dir: cache directory from config file
 117         """
 118         self.backendName = backendName
 119         self.cache_dir = cache_dir
 120         self.apt_config = copy.deepcopy(self.DEFAULT_APT_CONFIG)
 121
 122         self.status_dir = (cache_dir+'/'+ aptpkg_dir
 123                            +'/backends/'+backendName)
 124         for dir in self.essential_dirs:
 125             path = self.status_dir+'/'+dir
 126             if not os.path.exists(path):
 127                 os.makedirs(path)
 128         for file in self.essential_files:
 129             path = self.status_dir+'/'+file
 130             if not os.path.exists(path):
 131                 f = open(path,'w')
 132                 f.close()
 133                 del f
 134
 135         self.apt_config['Dir'] = self.status_dir
 136         self.apt_config['Dir::State::status'] = self.status_dir + '/apt/dpkg/status'
 137         self.packages = PackageFileList(backendName, cache_dir)
 138         self.loaded = 0
 139         self.loading = None
 140
 141     def __del__(self):
 142         self.cleanup()
 143         self.packages.close()
 144
 145     def addRelease(self, cache_path, file_path):
 146         """Dirty hack until python-apt supports apt-pkg/indexrecords.h
 147         (see Bug #456141)
 148         """
 149         self.indexrecords[cache_path] = {}
 150
 151         read_packages = False
 152         f = open(file_path, 'r')
 153
 154         for line in f:
 155             line = line.rstrip()
 156
 157             if line[:1] != " ":
 158                 read_packages = False
 159                 try:
 160                     # Read the various headers from the file
 161                     h, v = line.split(":", 1)
 162                     if h == "MD5Sum" or h == "SHA1" or h == "SHA256":
 163                         read_packages = True
 164                         hash_type = h
 165                 except:
 166                     # Bad header line, just ignore it
 167                     log.msg("WARNING: Ignoring badly formatted Release line: %s" % line)
 168
 169                 # Skip to the next line
 170                 continue
 171
 172             # Read file names from the multiple hash sections of the file
 173             if read_packages:
 174                 p = line.split()
 175                 self.indexrecords[cache_path].setdefault(p[2], {})[hash_type] = (p[0], p[1])
 176
 177         f.close()
 178
 179     def file_updated(self, filename, cache_path, file_path):
 180         """A file in the backend has changed, manage it.
 181
 182         If this affects us, unload our apt database
 183         """
 184         if self.packages.update_file(filename, cache_path, file_path):
 185             self.unload()
 186
 187     def load(self):
 188         """Make sure the package is initialized and loaded."""
 189         if self.loading is None:
 190             self.loading = threads.deferToThread(self._load)
 191             self.loading.addCallback(self.doneLoading)
 192         return self.loading
 193
 194     def doneLoading(self, loadResult):
 195         """Cache is loaded."""
 196         self.loading = None
 197         # Must pass on the result for the next callback
 198         return loadResult
 199
 200     def _load(self):
 201         """Regenerates the fake configuration and load the packages cache."""
 202         if self.loaded: return True
 203         apt_pkg.InitSystem()
 204         shutil.rmtree(self.status_dir+'/apt/lists/')
 205         os.makedirs(self.status_dir+'/apt/lists/partial')
 206         sources_filename = self.status_dir+'/'+'apt/etc/sources.list'
 207         sources = open(sources_filename, 'w')
 208         sources_count = 0
 209         self.packages.check_files()
 210         self.indexrecords = {}
 211         for f in self.packages:
 212             # we should probably clear old entries from self.packages and
 213             # take into account the recorded mtime as optimization
 214             filepath = self.packages[f]
 215             if basename(f) == "Release":
 216                 self.addRelease(f, filepath)
 217             fake_uri='http://apt-dht/'+f
 218             if f.endswith('Sources'):
 219                 source_line='deb-src '+dirname(fake_uri)+'/ /'
 220             else:
 221                 source_line='deb '+dirname(fake_uri)+'/ /'
 222             listpath=(self.status_dir+'/apt/lists/'
 223                     +apt_pkg.URItoFileName(fake_uri))
 224             sources.write(source_line+'\n')
 225             log.msg("Sources line: " + source_line)
 226             sources_count = sources_count + 1
 227
 228             try:
 229                 #we should empty the directory instead
 230                 os.unlink(listpath)
 231             except:
 232                 pass
 233             os.symlink(filepath, listpath)
 234         sources.close()
 235
 236         if sources_count == 0:
 237             log.msg("No Packages files available for %s backend"%(self.backendName))
 238             return False
 239
 240         log.msg("Loading Packages database for "+self.status_dir)
 241         for key, value in self.apt_config.items():
 242             apt_pkg.Config[key] = value
 243
 244         self.cache = apt_pkg.GetCache(OpProgress())
 245         self.records = apt_pkg.GetPkgRecords(self.cache)
 246         self.srcrecords = apt_pkg.GetPkgSrcRecords()
 247
 248         self.loaded = 1
 249         return True
 250
 251     def unload(self):
 252         """Tries to make the packages server quit."""
 253         if self.loaded:
 254             del self.cache
 255             del self.records
 256             del self.srcrecords
 257             del self.indexrecords
 258             self.loaded = 0
 259
 260     def cleanup(self):
 261         """Cleanup and close any loaded caches."""
 262         self.unload()
 263         self.packages.close()
 264
 265     def findHash(self, path):
 266         """Find the hash for a given path in this mirror.
 267
 268         Returns a deferred so it can make sure the cache is loaded first.
 269         """
 270         d = defer.Deferred()
 271
 272         deferLoad = self.load()
 273         deferLoad.addCallback(self._findHash, path, d)
 274
 275         return d
 276
 277     def _findHash(self, loadResult, path, d):
 278         """Really find the hash for a path.
 279
 280         Have to pass the returned loadResult on in case other calls to this
 281         function are pending.
 282         """
 283         if not loadResult:
 284             d.callback((None, None))
 285             return loadResult
 286
 287         # First look for the path in the cache of index files
 288         for release in self.indexrecords:
 289             if path.startswith(release[:-7]):
 290                 for indexFile in self.indexrecords[release]:
 291                     if release[:-7] + indexFile == path:
 292                         d.callback(self.indexrecords[release][indexFile]['SHA1'])
 293                         return loadResult
 294
 295         package = path.split('/')[-1].split('_')[0]
 296
 297         # Check the binary packages
 298         try:
 299             for version in self.cache[package].VersionList:
 300                 size = version.Size
 301                 for verFile in version.FileList:
 302                     if self.records.Lookup(verFile):
 303                         if self.records.FileName == path:
 304                             d.callback((self.records.SHA1Hash, size))
 305                             return loadResult
 306         except KeyError:
 307             pass
 308
 309         # Check the source packages' files
 310         self.srcrecords.Restart()
 311         if self.srcrecords.Lookup(package):
 312             for f in self.srcrecords.Files:
 313                 if path == f[2]:
 314                     d.callback((f[0], f[1]))
 315                     return loadResult
 316
 317         d.callback((None, None))
 318         return loadResult
 319
 320 class TestAptPackages(unittest.TestCase):
 321     """Unit tests for the AptPackages cache."""
 322
 323     pending_calls = []
 324     client = None
 325     packagesFile = ''
 326     sourcesFile = ''
 327     releaseFile = ''
 328
 329     def setUp(self):
 330         self.client = AptPackages('whatever', '/tmp')
 331
 332         self.packagesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "Packages$" | tail -n 1').read().rstrip('\n')
 333         self.sourcesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "Sources$" | tail -n 1').read().rstrip('\n')
 334         for f in os.walk('/var/lib/apt/lists').next()[2]:
 335             if f[-7:] == "Release" and self.packagesFile.startswith(f[:-7]):
 336                 self.releaseFile = f
 337                 break
 338
 339         self.client.file_updated('Release',
 340                                  self.releaseFile[self.releaseFile.find('_debian_')+1:].replace('_','/'),
 341                                  '/var/lib/apt/lists/' + self.releaseFile)
 342         self.client.file_updated('Packages',
 343                                  self.packagesFile[self.packagesFile.find('_debian_')+1:].replace('_','/'),
 344                                  '/var/lib/apt/lists/' + self.packagesFile)
 345         self.client.file_updated('Sources',
 346                                  self.sourcesFile[self.sourcesFile.find('_debian_')+1:].replace('_','/'),
 347                                  '/var/lib/apt/lists/' + self.sourcesFile)
 348
 349     def test_pkg_hash(self):
 350         self.client._load()
 351
 352         self.client.records.Lookup(self.client.cache['dpkg'].VersionList[0].FileList[0])
 353
 354         pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 355                             '/var/lib/apt/lists/' + self.packagesFile +
 356                             ' | grep -E "^SHA1:" | head -n 1' +
 357                             ' | cut -d\  -f 2').read().rstrip('\n')
 358
 359         self.failUnless(self.client.records.SHA1Hash == pkg_hash,
 360                         "Hashes don't match: %s != %s" % (self.client.records.SHA1Hash, pkg_hash))
 361
 362     def test_src_hash(self):
 363         self.client._load()
 364
 365         self.client.srcrecords.Lookup('dpkg')
 366
 367         src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
 368                             '/var/lib/apt/lists/' + self.sourcesFile +
 369                             ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
 370                             ' | cut -d\  -f 2').read().split('\n')[:-1]
 371
 372         for f in self.client.srcrecords.Files:
 373             self.failUnless(f[0] in src_hashes, "Couldn't find %s in: %r" % (f[0], src_hashes))
 374
 375     def test_index_hash(self):
 376         self.client._load()
 377
 378         indexhash = self.client.indexrecords[self.releaseFile[self.releaseFile.find('_debian_')+1:].replace('_','/')]['main/binary-i386/Packages.bz2']['SHA1'][0]
 379
 380         idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
 381                             '/var/lib/apt/lists/' + self.releaseFile +
 382                             ' | grep -E " main/binary-i386/Packages.bz2$"'
 383                             ' | head -n 1 | cut -d\  -f 2').read().rstrip('\n')
 384
 385         self.failUnless(indexhash == idx_hash, "Hashes don't match: %s != %s" % (indexhash, idx_hash))
 386
 387     def verifyHash(self, found_hash, path, true_hash):
 388         self.failUnless(found_hash[0] == true_hash,
 389                     "%s hashes don't match: %s != %s" % (path, found_hash[0], true_hash))
 390
 391     def test_findIndexHash(self):
 392         lastDefer = defer.Deferred()
 393
 394         idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
 395                             '/var/lib/apt/lists/' + self.releaseFile +
 396                             ' | grep -E " main/binary-i386/Packages.bz2$"'
 397                             ' | head -n 1 | cut -d\  -f 2').read().rstrip('\n')
 398         idx_path = self.releaseFile[self.releaseFile.find('_debian_')+1:].replace('_','/')[:-7] + 'main/binary-i386/Packages.bz2'
 399
 400         d = self.client.findHash(idx_path)
 401         d.addCallback(self.verifyHash, idx_path, idx_hash)
 402
 403         d.addCallback(lastDefer.callback)
 404         return lastDefer
 405
 406     def test_findPkgHash(self):
 407         lastDefer = defer.Deferred()
 408
 409         pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 410                             '/var/lib/apt/lists/' + self.packagesFile +
 411                             ' | grep -E "^SHA1:" | head -n 1' +
 412                             ' | cut -d\  -f 2').read().rstrip('\n')
 413         pkg_path = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 414                             '/var/lib/apt/lists/' + self.packagesFile +
 415                             ' | grep -E "^Filename:" | head -n 1' +
 416                             ' | cut -d\  -f 2').read().rstrip('\n')
 417
 418         d = self.client.findHash(pkg_path)
 419         d.addCallback(self.verifyHash, pkg_path, pkg_hash)
 420
 421         d.addCallback(lastDefer.callback)
 422         return lastDefer
 423
 424     def test_findSrcHash(self):
 425         lastDefer = defer.Deferred()
 426
 427         src_dir = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 428                             '/var/lib/apt/lists/' + self.sourcesFile +
 429                             ' | grep -E "^Directory:" | head -n 1' +
 430                             ' | cut -d\  -f 2').read().rstrip('\n')
 431         src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
 432                             '/var/lib/apt/lists/' + self.sourcesFile +
 433                             ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
 434                             ' | cut -d\  -f 2').read().split('\n')[:-1]
 435         src_paths = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
 436                             '/var/lib/apt/lists/' + self.sourcesFile +
 437                             ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
 438                             ' | cut -d\  -f 4').read().split('\n')[:-1]
 439
 440         i = random.choice(range(len(src_hashes)))
 441         d = self.client.findHash(src_dir + '/' + src_paths[i])
 442         d.addCallback(self.verifyHash, src_dir + '/' + src_paths[i], src_hashes[i])
 443
 444         d.addCallback(lastDefer.callback)
 445         return lastDefer
 446
 447     def test_multipleFindHash(self):
 448         lastDefer = defer.Deferred()
 449
 450         idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
 451                             '/var/lib/apt/lists/' + self.releaseFile +
 452                             ' | grep -E " main/binary-i386/Packages.bz2$"'
 453                             ' | head -n 1 | cut -d\  -f 2').read().rstrip('\n')
 454         idx_path = self.releaseFile[self.releaseFile.find('_debian_')+1:].replace('_','/')[:-7] + 'main/binary-i386/Packages.bz2'
 455
 456         d = self.client.findHash(idx_path)
 457         d.addCallback(self.verifyHash, idx_path, idx_hash)
 458
 459         pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 460                             '/var/lib/apt/lists/' + self.packagesFile +
 461                             ' | grep -E "^SHA1:" | head -n 1' +
 462                             ' | cut -d\  -f 2').read().rstrip('\n')
 463         pkg_path = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 464                             '/var/lib/apt/lists/' + self.packagesFile +
 465                             ' | grep -E "^Filename:" | head -n 1' +
 466                             ' | cut -d\  -f 2').read().rstrip('\n')
 467
 468         d = self.client.findHash(pkg_path)
 469         d.addCallback(self.verifyHash, pkg_path, pkg_hash)
 470
 471         src_dir = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 472                             '/var/lib/apt/lists/' + self.sourcesFile +
 473                             ' | grep -E "^Directory:" | head -n 1' +
 474                             ' | cut -d\  -f 2').read().rstrip('\n')
 475         src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
 476                             '/var/lib/apt/lists/' + self.sourcesFile +
 477                             ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
 478                             ' | cut -d\  -f 2').read().split('\n')[:-1]
 479         src_paths = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
 480                             '/var/lib/apt/lists/' + self.sourcesFile +
 481                             ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
 482                             ' | cut -d\  -f 4').read().split('\n')[:-1]
 483
 484         for i in range(len(src_hashes)):
 485             d = self.client.findHash(src_dir + '/' + src_paths[i])
 486             d.addCallback(self.verifyHash, src_dir + '/' + src_paths[i], src_hashes[i])
 487
 488         idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
 489                             '/var/lib/apt/lists/' + self.releaseFile +
 490                             ' | grep -E " main/source/Sources.bz2$"'
 491                             ' | head -n 1 | cut -d\  -f 2').read().rstrip('\n')
 492         idx_path = self.releaseFile[self.releaseFile.find('_debian_')+1:].replace('_','/')[:-7] + 'main/source/Sources.bz2'
 493
 494         d = self.client.findHash(idx_path)
 495         d.addCallback(self.verifyHash, idx_path, idx_hash)
 496
 497         d.addCallback(lastDefer.callback)
 498         return lastDefer
 499
 500     def tearDown(self):
 501         for p in self.pending_calls:
 502             if p.active():
 503                 p.cancel()
 504         self.pending_calls = []
 505         self.client.cleanup()
 506         self.client = None