apt_dht/AptPackages.py

   1 # Disable the FutureWarning from the apt module
   2 import warnings
   3 warnings.simplefilter("ignore", FutureWarning)
   4
   5 import os, shelve
   6 from random import choice
   7 from shutil import rmtree
   8 from copy import deepcopy
   9 from UserDict import DictMixin
  10
  11 from twisted.internet import threads, defer
  12 from twisted.python import log
  13 from twisted.trial import unittest
  14
  15 import apt_pkg, apt_inst
  16 from apt import OpProgress
  17
  18 apt_pkg.init()
  19
  20 TRACKED_FILES = ['release', 'sources', 'packages']
  21
  22 class PackageFileList(DictMixin):
  23     """Manages a list of package files belonging to a backend.
  24
  25     @type packages: C{shelve dictionary}
  26     @ivar packages: the files stored for this backend
  27     """
  28
  29     def __init__(self, cache_dir):
  30         self.cache_dir = cache_dir
  31         if not os.path.exists(self.cache_dir):
  32             os.makedirs(self.cache_dir)
  33         self.packages = None
  34         self.open()
  35
  36     def open(self):
  37         """Open the persistent dictionary of files in this backend."""
  38         if self.packages is None:
  39             self.packages = shelve.open(self.cache_dir+'/packages.db')
  40
  41     def close(self):
  42         """Close the persistent dictionary."""
  43         if self.packages is not None:
  44             self.packages.close()
  45
  46     def update_file(self, cache_path, file_path):
  47         """Check if an updated file needs to be tracked.
  48
  49         Called from the mirror manager when files get updated so we can update our
  50         fake lists and sources.list.
  51         """
  52         filename = cache_path.split('/')[-1]
  53         if filename.lower() in TRACKED_FILES:
  54             log.msg("Registering package file: "+cache_path)
  55             self.packages[cache_path] = file_path
  56             return True
  57         return False
  58
  59     def check_files(self):
  60         """Check all files in the database to make sure they exist."""
  61         files = self.packages.keys()
  62         for f in files:
  63             if not os.path.exists(self.packages[f]):
  64                 log.msg("File in packages database has been deleted: "+f)
  65                 del self.packages[f]
  66
  67     # Standard dictionary implementation so this class can be used like a dictionary.
  68     def __getitem__(self, key): return self.packages[key]
  69     def __setitem__(self, key, item): self.packages[key] = item
  70     def __delitem__(self, key): del self.packages[key]
  71     def keys(self): return self.packages.keys()
  72
  73 class AptPackages:
  74     """Uses python-apt to answer queries about packages.
  75
  76     Makes a fake configuration for python-apt for each backend.
  77     """
  78
  79     DEFAULT_APT_CONFIG = {
  80         #'APT' : '',
  81         #'APT::Architecture' : 'i386',  # Commented so the machine's config will set this
  82         #'APT::Default-Release' : 'unstable',
  83         'Dir':'.', # /
  84         'Dir::State' : 'apt/', # var/lib/apt/
  85         'Dir::State::Lists': 'lists/', # lists/
  86         #'Dir::State::cdroms' : 'cdroms.list',
  87         'Dir::State::userstatus' : 'status.user',
  88         'Dir::State::status': 'dpkg/status', # '/var/lib/dpkg/status'
  89         'Dir::Cache' : '.apt/cache/', # var/cache/apt/
  90         #'Dir::Cache::archives' : 'archives/',
  91         'Dir::Cache::srcpkgcache' : 'srcpkgcache.bin',
  92         'Dir::Cache::pkgcache' : 'pkgcache.bin',
  93         'Dir::Etc' : 'apt/etc/', # etc/apt/
  94         'Dir::Etc::sourcelist' : 'sources.list',
  95         'Dir::Etc::vendorlist' : 'vendors.list',
  96         'Dir::Etc::vendorparts' : 'vendors.list.d',
  97         #'Dir::Etc::main' : 'apt.conf',
  98         #'Dir::Etc::parts' : 'apt.conf.d',
  99         #'Dir::Etc::preferences' : 'preferences',
 100         'Dir::Bin' : '',
 101         #'Dir::Bin::methods' : '', #'/usr/lib/apt/methods'
 102         'Dir::Bin::dpkg' : '/usr/bin/dpkg',
 103         #'DPkg' : '',
 104         #'DPkg::Pre-Install-Pkgs' : '',
 105         #'DPkg::Tools' : '',
 106         #'DPkg::Tools::Options' : '',
 107         #'DPkg::Tools::Options::/usr/bin/apt-listchanges' : '',
 108         #'DPkg::Tools::Options::/usr/bin/apt-listchanges::Version' : '2',
 109         #'DPkg::Post-Invoke' : '',
 110         }
 111     essential_dirs = ('apt', 'apt/cache', 'apt/dpkg', 'apt/etc', 'apt/lists',
 112                       'apt/lists/partial')
 113     essential_files = ('apt/dpkg/status', 'apt/etc/sources.list',)
 114
 115     def __init__(self, cache_dir):
 116         """Construct a new packages manager.
 117
 118         @ivar backendName: name of backend associated with this packages file
 119         @ivar cache_dir: cache directory from config file
 120         """
 121         self.cache_dir = cache_dir
 122         self.apt_config = deepcopy(self.DEFAULT_APT_CONFIG)
 123
 124         for dir in self.essential_dirs:
 125             path = os.path.join(self.cache_dir, dir)
 126             if not os.path.exists(path):
 127                 os.makedirs(path)
 128         for file in self.essential_files:
 129             path = os.path.join(self.cache_dir, file)
 130             if not os.path.exists(path):
 131                 f = open(path,'w')
 132                 f.close()
 133                 del f
 134
 135         self.apt_config['Dir'] = self.cache_dir
 136         self.apt_config['Dir::State::status'] = os.path.join(self.cache_dir,
 137                       self.apt_config['Dir::State'], self.apt_config['Dir::State::status'])
 138         self.packages = PackageFileList(cache_dir)
 139         self.loaded = 0
 140         self.loading = None
 141
 142     def __del__(self):
 143         self.cleanup()
 144         self.packages.close()
 145
 146     def addRelease(self, cache_path, file_path):
 147         """Dirty hack until python-apt supports apt-pkg/indexrecords.h
 148         (see Bug #456141)
 149         """
 150         self.indexrecords[cache_path] = {}
 151
 152         read_packages = False
 153         f = open(file_path, 'r')
 154
 155         for line in f:
 156             line = line.rstrip()
 157
 158             if line[:1] != " ":
 159                 read_packages = False
 160                 try:
 161                     # Read the various headers from the file
 162                     h, v = line.split(":", 1)
 163                     if h == "MD5Sum" or h == "SHA1" or h == "SHA256":
 164                         read_packages = True
 165                         hash_type = h
 166                 except:
 167                     # Bad header line, just ignore it
 168                     log.msg("WARNING: Ignoring badly formatted Release line: %s" % line)
 169
 170                 # Skip to the next line
 171                 continue
 172
 173             # Read file names from the multiple hash sections of the file
 174             if read_packages:
 175                 p = line.split()
 176                 self.indexrecords[cache_path].setdefault(p[2], {})[hash_type] = (p[0], p[1])
 177
 178         f.close()
 179
 180     def file_updated(self, cache_path, file_path):
 181         """A file in the backend has changed, manage it.
 182
 183         If this affects us, unload our apt database
 184         """
 185         if self.packages.update_file(cache_path, file_path):
 186             self.unload()
 187
 188     def load(self):
 189         """Make sure the package is initialized and loaded."""
 190         if self.loading is None:
 191             self.loading = threads.deferToThread(self._load)
 192             self.loading.addCallback(self.doneLoading)
 193         return self.loading
 194
 195     def doneLoading(self, loadResult):
 196         """Cache is loaded."""
 197         self.loading = None
 198         # Must pass on the result for the next callback
 199         return loadResult
 200
 201     def _load(self):
 202         """Regenerates the fake configuration and load the packages cache."""
 203         if self.loaded: return True
 204         apt_pkg.InitSystem()
 205         rmtree(os.path.join(self.cache_dir, self.apt_config['Dir::State'],
 206                             self.apt_config['Dir::State::Lists']))
 207         os.makedirs(os.path.join(self.cache_dir, self.apt_config['Dir::State'],
 208                                  self.apt_config['Dir::State::Lists'], 'partial'))
 209         sources_filename = os.path.join(self.cache_dir, self.apt_config['Dir::Etc'],
 210                                         self.apt_config['Dir::Etc::sourcelist'])
 211         sources = open(sources_filename, 'w')
 212         sources_count = 0
 213         deb_src_added = False
 214         self.packages.check_files()
 215         self.indexrecords = {}
 216         for f in self.packages:
 217             # we should probably clear old entries from self.packages and
 218             # take into account the recorded mtime as optimization
 219             filepath = self.packages[f]
 220             if f.split('/')[-1] == "Release":
 221                 self.addRelease(f, filepath)
 222             fake_uri='http://apt-dht'+f
 223             fake_dirname = '/'.join(fake_uri.split('/')[:-1])
 224             if f.endswith('Sources'):
 225                 deb_src_added = True
 226                 source_line='deb-src '+fake_dirname+'/ /'
 227             else:
 228                 source_line='deb '+fake_dirname+'/ /'
 229             listpath=(os.path.join(self.cache_dir, self.apt_config['Dir::State'],
 230                                    self.apt_config['Dir::State::Lists'],
 231                                    apt_pkg.URItoFileName(fake_uri)))
 232             sources.write(source_line+'\n')
 233             log.msg("Sources line: " + source_line)
 234             sources_count = sources_count + 1
 235
 236             try:
 237                 #we should empty the directory instead
 238                 os.unlink(listpath)
 239             except:
 240                 pass
 241             os.symlink(filepath, listpath)
 242         sources.close()
 243
 244         if sources_count == 0:
 245             log.msg("No Packages files available for %s backend"%(self.cache_dir))
 246             return False
 247
 248         log.msg("Loading Packages database for "+self.cache_dir)
 249         for key, value in self.apt_config.items():
 250             apt_pkg.Config[key] = value
 251
 252         self.cache = apt_pkg.GetCache(OpProgress())
 253         self.records = apt_pkg.GetPkgRecords(self.cache)
 254         if deb_src_added:
 255             self.srcrecords = apt_pkg.GetPkgSrcRecords()
 256         else:
 257             self.srcrecords = None
 258
 259         self.loaded = 1
 260         return True
 261
 262     def unload(self):
 263         """Tries to make the packages server quit."""
 264         if self.loaded:
 265             del self.cache
 266             del self.records
 267             del self.srcrecords
 268             del self.indexrecords
 269             self.loaded = 0
 270
 271     def cleanup(self):
 272         """Cleanup and close any loaded caches."""
 273         self.unload()
 274         self.packages.close()
 275
 276     def findHash(self, path):
 277         """Find the hash for a given path in this mirror.
 278
 279         Returns a deferred so it can make sure the cache is loaded first.
 280         """
 281         d = defer.Deferred()
 282
 283         deferLoad = self.load()
 284         deferLoad.addCallback(self._findHash, path, d)
 285         deferLoad.addErrback(self._findHash_error, path, d)
 286
 287         return d
 288
 289     def _findHash_error(self, failure, path, d):
 290         """An error occurred while trying to find a hash."""
 291         log.msg('An error occurred while looking up a hash for: %s' % path)
 292         log.err(failure)
 293         d.callback((None, None))
 294
 295     def _findHash(self, loadResult, path, d):
 296         """Really find the hash for a path.
 297
 298         Have to pass the returned loadResult on in case other calls to this
 299         function are pending.
 300         """
 301         if not loadResult:
 302             d.callback((None, None))
 303             return loadResult
 304
 305         # First look for the path in the cache of index files
 306         for release in self.indexrecords:
 307             if path.startswith(release[:-7]):
 308                 for indexFile in self.indexrecords[release]:
 309                     if release[:-7] + indexFile == path:
 310                         d.callback(self.indexrecords[release][indexFile]['SHA1'])
 311                         return loadResult
 312
 313         package = path.split('/')[-1].split('_')[0]
 314
 315         # Check the binary packages
 316         try:
 317             for version in self.cache[package].VersionList:
 318                 size = version.Size
 319                 for verFile in version.FileList:
 320                     if self.records.Lookup(verFile):
 321                         if '/' + self.records.FileName == path:
 322                             d.callback((self.records.SHA1Hash, size))
 323                             return loadResult
 324         except KeyError:
 325             pass
 326
 327         # Check the source packages' files
 328         if self.srcrecords:
 329             self.srcrecords.Restart()
 330             if self.srcrecords.Lookup(package):
 331                 for f in self.srcrecords.Files:
 332                     if path == '/' + f[2]:
 333                         d.callback((f[0], f[1]))
 334                         return loadResult
 335
 336         d.callback((None, None))
 337         return loadResult
 338
 339 class TestAptPackages(unittest.TestCase):
 340     """Unit tests for the AptPackages cache."""
 341
 342     pending_calls = []
 343     client = None
 344     timeout = 10
 345     packagesFile = ''
 346     sourcesFile = ''
 347     releaseFile = ''
 348
 349     def setUp(self):
 350         self.client = AptPackages('/tmp/.apt-dht')
 351
 352         self.packagesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "_main_.*Packages$" | tail -n 1').read().rstrip('\n')
 353         self.sourcesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "_main_.*Sources$" | tail -n 1').read().rstrip('\n')
 354         for f in os.walk('/var/lib/apt/lists').next()[2]:
 355             if f[-7:] == "Release" and self.packagesFile.startswith(f[:-7]):
 356                 self.releaseFile = f
 357                 break
 358
 359         self.client.file_updated(self.releaseFile[self.releaseFile.find('_dists_'):].replace('_','/'),
 360                                  '/var/lib/apt/lists/' + self.releaseFile)
 361         self.client.file_updated(self.packagesFile[self.packagesFile.find('_dists_'):].replace('_','/'),
 362                                  '/var/lib/apt/lists/' + self.packagesFile)
 363         self.client.file_updated(self.sourcesFile[self.sourcesFile.find('_dists_'):].replace('_','/'),
 364                                  '/var/lib/apt/lists/' + self.sourcesFile)
 365
 366     def test_pkg_hash(self):
 367         self.client._load()
 368
 369         self.client.records.Lookup(self.client.cache['dpkg'].VersionList[0].FileList[0])
 370
 371         pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 372                             '/var/lib/apt/lists/' + self.packagesFile +
 373                             ' | grep -E "^SHA1:" | head -n 1' +
 374                             ' | cut -d\  -f 2').read().rstrip('\n')
 375
 376         self.failUnless(self.client.records.SHA1Hash == pkg_hash,
 377                         "Hashes don't match: %s != %s" % (self.client.records.SHA1Hash, pkg_hash))
 378
 379     def test_src_hash(self):
 380         self.client._load()
 381
 382         self.client.srcrecords.Lookup('dpkg')
 383
 384         src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
 385                             '/var/lib/apt/lists/' + self.sourcesFile +
 386                             ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
 387                             ' | cut -d\  -f 2').read().split('\n')[:-1]
 388
 389         for f in self.client.srcrecords.Files:
 390             self.failUnless(f[0] in src_hashes, "Couldn't find %s in: %r" % (f[0], src_hashes))
 391
 392     def test_index_hash(self):
 393         self.client._load()
 394
 395         indexhash = self.client.indexrecords[self.releaseFile[self.releaseFile.find('_dists_'):].replace('_','/')]['main/binary-i386/Packages.bz2']['SHA1'][0]
 396
 397         idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
 398                             '/var/lib/apt/lists/' + self.releaseFile +
 399                             ' | grep -E " main/binary-i386/Packages.bz2$"'
 400                             ' | head -n 1 | cut -d\  -f 2').read().rstrip('\n')
 401
 402         self.failUnless(indexhash == idx_hash, "Hashes don't match: %s != %s" % (indexhash, idx_hash))
 403
 404     def verifyHash(self, found_hash, path, true_hash):
 405         self.failUnless(found_hash[0] == true_hash,
 406                     "%s hashes don't match: %s != %s" % (path, found_hash[0], true_hash))
 407
 408     def test_findIndexHash(self):
 409         lastDefer = defer.Deferred()
 410
 411         idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
 412                             '/var/lib/apt/lists/' + self.releaseFile +
 413                             ' | grep -E " main/binary-i386/Packages.bz2$"'
 414                             ' | head -n 1 | cut -d\  -f 2').read().rstrip('\n')
 415         idx_path = '/' + self.releaseFile[self.releaseFile.find('_dists_')+1:].replace('_','/')[:-7] + 'main/binary-i386/Packages.bz2'
 416
 417         d = self.client.findHash(idx_path)
 418         d.addCallback(self.verifyHash, idx_path, idx_hash)
 419
 420         d.addBoth(lastDefer.callback)
 421         return lastDefer
 422
 423     def test_findPkgHash(self):
 424         lastDefer = defer.Deferred()
 425
 426         pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 427                             '/var/lib/apt/lists/' + self.packagesFile +
 428                             ' | grep -E "^SHA1:" | head -n 1' +
 429                             ' | cut -d\  -f 2').read().rstrip('\n')
 430         pkg_path = '/' + os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 431                             '/var/lib/apt/lists/' + self.packagesFile +
 432                             ' | grep -E "^Filename:" | head -n 1' +
 433                             ' | cut -d\  -f 2').read().rstrip('\n')
 434
 435         d = self.client.findHash(pkg_path)
 436         d.addCallback(self.verifyHash, pkg_path, pkg_hash)
 437
 438         d.addBoth(lastDefer.callback)
 439         return lastDefer
 440
 441     def test_findSrcHash(self):
 442         lastDefer = defer.Deferred()
 443
 444         src_dir = '/' + os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 445                             '/var/lib/apt/lists/' + self.sourcesFile +
 446                             ' | grep -E "^Directory:" | head -n 1' +
 447                             ' | cut -d\  -f 2').read().rstrip('\n')
 448         src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
 449                             '/var/lib/apt/lists/' + self.sourcesFile +
 450                             ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
 451                             ' | cut -d\  -f 2').read().split('\n')[:-1]
 452         src_paths = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
 453                             '/var/lib/apt/lists/' + self.sourcesFile +
 454                             ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
 455                             ' | cut -d\  -f 4').read().split('\n')[:-1]
 456
 457         i = choice(range(len(src_hashes)))
 458         d = self.client.findHash(src_dir + '/' + src_paths[i])
 459         d.addCallback(self.verifyHash, src_dir + '/' + src_paths[i], src_hashes[i])
 460
 461         d.addBoth(lastDefer.callback)
 462         return lastDefer
 463
 464     def test_multipleFindHash(self):
 465         lastDefer = defer.Deferred()
 466
 467         idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
 468                             '/var/lib/apt/lists/' + self.releaseFile +
 469                             ' | grep -E " main/binary-i386/Packages.bz2$"'
 470                             ' | head -n 1 | cut -d\  -f 2').read().rstrip('\n')
 471         idx_path = '/' + self.releaseFile[self.releaseFile.find('_dists_')+1:].replace('_','/')[:-7] + 'main/binary-i386/Packages.bz2'
 472
 473         d = self.client.findHash(idx_path)
 474         d.addCallback(self.verifyHash, idx_path, idx_hash)
 475
 476         pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 477                             '/var/lib/apt/lists/' + self.packagesFile +
 478                             ' | grep -E "^SHA1:" | head -n 1' +
 479                             ' | cut -d\  -f 2').read().rstrip('\n')
 480         pkg_path = '/' + os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 481                             '/var/lib/apt/lists/' + self.packagesFile +
 482                             ' | grep -E "^Filename:" | head -n 1' +
 483                             ' | cut -d\  -f 2').read().rstrip('\n')
 484
 485         d = self.client.findHash(pkg_path)
 486         d.addCallback(self.verifyHash, pkg_path, pkg_hash)
 487
 488         src_dir = '/' + os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 489                             '/var/lib/apt/lists/' + self.sourcesFile +
 490                             ' | grep -E "^Directory:" | head -n 1' +
 491                             ' | cut -d\  -f 2').read().rstrip('\n')
 492         src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
 493                             '/var/lib/apt/lists/' + self.sourcesFile +
 494                             ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
 495                             ' | cut -d\  -f 2').read().split('\n')[:-1]
 496         src_paths = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
 497                             '/var/lib/apt/lists/' + self.sourcesFile +
 498                             ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
 499                             ' | cut -d\  -f 4').read().split('\n')[:-1]
 500
 501         for i in range(len(src_hashes)):
 502             d = self.client.findHash(src_dir + '/' + src_paths[i])
 503             d.addCallback(self.verifyHash, src_dir + '/' + src_paths[i], src_hashes[i])
 504
 505         idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
 506                             '/var/lib/apt/lists/' + self.releaseFile +
 507                             ' | grep -E " main/source/Sources.bz2$"'
 508                             ' | head -n 1 | cut -d\  -f 2').read().rstrip('\n')
 509         idx_path = '/' + self.releaseFile[self.releaseFile.find('_dists_')+1:].replace('_','/')[:-7] + 'main/source/Sources.bz2'
 510
 511         d = self.client.findHash(idx_path)
 512         d.addCallback(self.verifyHash, idx_path, idx_hash)
 513
 514         d.addBoth(lastDefer.callback)
 515         return lastDefer
 516
 517     def tearDown(self):
 518         for p in self.pending_calls:
 519             if p.active():
 520                 p.cancel()
 521         self.pending_calls = []
 522         self.client.cleanup()
 523         self.client = None