apt_dht/AptPackages.py

   1 # Disable the FutureWarning from the apt module
   2 import warnings
   3 warnings.simplefilter("ignore", FutureWarning)
   4
   5 import os, shelve
   6 from random import choice
   7 from shutil import rmtree
   8 from copy import deepcopy
   9 from UserDict import DictMixin
  10
  11 from twisted.internet import threads, defer
  12 from twisted.python import log
  13 from twisted.trial import unittest
  14
  15 import apt_pkg, apt_inst
  16 from apt import OpProgress
  17
  18 apt_pkg.init()
  19
  20 class PackageFileList(DictMixin):
  21     """Manages a list of package files belonging to a backend.
  22
  23     @type packages: C{shelve dictionary}
  24     @ivar packages: the files stored for this backend
  25     """
  26
  27     def __init__(self, cache_dir):
  28         self.cache_dir = cache_dir
  29         if not os.path.exists(self.cache_dir):
  30             os.makedirs(self.cache_dir)
  31         self.packages = None
  32         self.open()
  33
  34     def open(self):
  35         """Open the persistent dictionary of files in this backend."""
  36         if self.packages is None:
  37             self.packages = shelve.open(self.cache_dir+'/packages.db')
  38
  39     def close(self):
  40         """Close the persistent dictionary."""
  41         if self.packages is not None:
  42             self.packages.close()
  43
  44     def update_file(self, cache_path, file_path):
  45         """Check if an updated file needs to be tracked.
  46
  47         Called from the mirror manager when files get updated so we can update our
  48         fake lists and sources.list.
  49         """
  50         filename = cache_path.split('/')[-1]
  51         if filename=="Packages" or filename=="Release" or filename=="Sources":
  52             log.msg("Registering package file: "+cache_path)
  53             self.packages[cache_path] = file_path
  54             return True
  55         return False
  56
  57     def check_files(self):
  58         """Check all files in the database to make sure they exist."""
  59         files = self.packages.keys()
  60         for f in files:
  61             if not os.path.exists(self.packages[f]):
  62                 log.msg("File in packages database has been deleted: "+f)
  63                 del self.packages[f]
  64
  65     # Standard dictionary implementation so this class can be used like a dictionary.
  66     def __getitem__(self, key): return self.packages[key]
  67     def __setitem__(self, key, item): self.packages[key] = item
  68     def __delitem__(self, key): del self.packages[key]
  69     def keys(self): return self.packages.keys()
  70
  71 class AptPackages:
  72     """Uses python-apt to answer queries about packages.
  73
  74     Makes a fake configuration for python-apt for each backend.
  75     """
  76
  77     DEFAULT_APT_CONFIG = {
  78         #'APT' : '',
  79         #'APT::Architecture' : 'i386',  # Commented so the machine's config will set this
  80         #'APT::Default-Release' : 'unstable',
  81         'Dir':'.', # /
  82         'Dir::State' : 'apt/', # var/lib/apt/
  83         'Dir::State::Lists': 'lists/', # lists/
  84         #'Dir::State::cdroms' : 'cdroms.list',
  85         'Dir::State::userstatus' : 'status.user',
  86         'Dir::State::status': 'dpkg/status', # '/var/lib/dpkg/status'
  87         'Dir::Cache' : '.apt/cache/', # var/cache/apt/
  88         #'Dir::Cache::archives' : 'archives/',
  89         'Dir::Cache::srcpkgcache' : 'srcpkgcache.bin',
  90         'Dir::Cache::pkgcache' : 'pkgcache.bin',
  91         'Dir::Etc' : 'apt/etc/', # etc/apt/
  92         'Dir::Etc::sourcelist' : 'sources.list',
  93         'Dir::Etc::vendorlist' : 'vendors.list',
  94         'Dir::Etc::vendorparts' : 'vendors.list.d',
  95         #'Dir::Etc::main' : 'apt.conf',
  96         #'Dir::Etc::parts' : 'apt.conf.d',
  97         #'Dir::Etc::preferences' : 'preferences',
  98         'Dir::Bin' : '',
  99         #'Dir::Bin::methods' : '', #'/usr/lib/apt/methods'
 100         'Dir::Bin::dpkg' : '/usr/bin/dpkg',
 101         #'DPkg' : '',
 102         #'DPkg::Pre-Install-Pkgs' : '',
 103         #'DPkg::Tools' : '',
 104         #'DPkg::Tools::Options' : '',
 105         #'DPkg::Tools::Options::/usr/bin/apt-listchanges' : '',
 106         #'DPkg::Tools::Options::/usr/bin/apt-listchanges::Version' : '2',
 107         #'DPkg::Post-Invoke' : '',
 108         }
 109     essential_dirs = ('apt', 'apt/cache', 'apt/dpkg', 'apt/etc', 'apt/lists',
 110                       'apt/lists/partial')
 111     essential_files = ('apt/dpkg/status', 'apt/etc/sources.list',)
 112
 113     def __init__(self, cache_dir):
 114         """Construct a new packages manager.
 115
 116         @ivar backendName: name of backend associated with this packages file
 117         @ivar cache_dir: cache directory from config file
 118         """
 119         self.cache_dir = cache_dir
 120         self.apt_config = deepcopy(self.DEFAULT_APT_CONFIG)
 121
 122         for dir in self.essential_dirs:
 123             path = os.path.join(self.cache_dir, dir)
 124             if not os.path.exists(path):
 125                 os.makedirs(path)
 126         for file in self.essential_files:
 127             path = os.path.join(self.cache_dir, file)
 128             if not os.path.exists(path):
 129                 f = open(path,'w')
 130                 f.close()
 131                 del f
 132
 133         self.apt_config['Dir'] = self.cache_dir
 134         self.apt_config['Dir::State::status'] = os.path.join(self.cache_dir,
 135                       self.apt_config['Dir::State'], self.apt_config['Dir::State::status'])
 136         self.packages = PackageFileList(cache_dir)
 137         self.loaded = 0
 138         self.loading = None
 139
 140     def __del__(self):
 141         self.cleanup()
 142         self.packages.close()
 143
 144     def addRelease(self, cache_path, file_path):
 145         """Dirty hack until python-apt supports apt-pkg/indexrecords.h
 146         (see Bug #456141)
 147         """
 148         self.indexrecords[cache_path] = {}
 149
 150         read_packages = False
 151         f = open(file_path, 'r')
 152
 153         for line in f:
 154             line = line.rstrip()
 155
 156             if line[:1] != " ":
 157                 read_packages = False
 158                 try:
 159                     # Read the various headers from the file
 160                     h, v = line.split(":", 1)
 161                     if h == "MD5Sum" or h == "SHA1" or h == "SHA256":
 162                         read_packages = True
 163                         hash_type = h
 164                 except:
 165                     # Bad header line, just ignore it
 166                     log.msg("WARNING: Ignoring badly formatted Release line: %s" % line)
 167
 168                 # Skip to the next line
 169                 continue
 170
 171             # Read file names from the multiple hash sections of the file
 172             if read_packages:
 173                 p = line.split()
 174                 self.indexrecords[cache_path].setdefault(p[2], {})[hash_type] = (p[0], p[1])
 175
 176         f.close()
 177
 178     def file_updated(self, cache_path, file_path):
 179         """A file in the backend has changed, manage it.
 180
 181         If this affects us, unload our apt database
 182         """
 183         if self.packages.update_file(cache_path, file_path):
 184             self.unload()
 185
 186     def load(self):
 187         """Make sure the package is initialized and loaded."""
 188         if self.loading is None:
 189             self.loading = threads.deferToThread(self._load)
 190             self.loading.addCallback(self.doneLoading)
 191         return self.loading
 192
 193     def doneLoading(self, loadResult):
 194         """Cache is loaded."""
 195         self.loading = None
 196         # Must pass on the result for the next callback
 197         return loadResult
 198
 199     def _load(self):
 200         """Regenerates the fake configuration and load the packages cache."""
 201         if self.loaded: return True
 202         apt_pkg.InitSystem()
 203         rmtree(os.path.join(self.cache_dir, self.apt_config['Dir::State'],
 204                             self.apt_config['Dir::State::Lists']))
 205         os.makedirs(os.path.join(self.cache_dir, self.apt_config['Dir::State'],
 206                                  self.apt_config['Dir::State::Lists'], 'partial'))
 207         sources_filename = os.path.join(self.cache_dir, self.apt_config['Dir::Etc'],
 208                                         self.apt_config['Dir::Etc::sourcelist'])
 209         sources = open(sources_filename, 'w')
 210         sources_count = 0
 211         self.packages.check_files()
 212         self.indexrecords = {}
 213         for f in self.packages:
 214             # we should probably clear old entries from self.packages and
 215             # take into account the recorded mtime as optimization
 216             filepath = self.packages[f]
 217             if f.split('/')[-1] == "Release":
 218                 self.addRelease(f, filepath)
 219             fake_uri='http://apt-dht'+f
 220             fake_dirname = '/'.join(fake_uri.split('/')[:-1])
 221             if f.endswith('Sources'):
 222                 source_line='deb-src '+fake_dirname+'/ /'
 223             else:
 224                 source_line='deb '+fake_dirname+'/ /'
 225             listpath=(os.path.join(self.cache_dir, self.apt_config['Dir::State'],
 226                                    self.apt_config['Dir::State::Lists'],
 227                                    apt_pkg.URItoFileName(fake_uri)))
 228             sources.write(source_line+'\n')
 229             log.msg("Sources line: " + source_line)
 230             sources_count = sources_count + 1
 231
 232             try:
 233                 #we should empty the directory instead
 234                 os.unlink(listpath)
 235             except:
 236                 pass
 237             os.symlink(filepath, listpath)
 238         sources.close()
 239
 240         if sources_count == 0:
 241             log.msg("No Packages files available for %s backend"%(self.cache_dir))
 242             return False
 243
 244         log.msg("Loading Packages database for "+self.cache_dir)
 245         for key, value in self.apt_config.items():
 246             apt_pkg.Config[key] = value
 247
 248         self.cache = apt_pkg.GetCache(OpProgress())
 249         self.records = apt_pkg.GetPkgRecords(self.cache)
 250         self.srcrecords = apt_pkg.GetPkgSrcRecords()
 251
 252         self.loaded = 1
 253         return True
 254
 255     def unload(self):
 256         """Tries to make the packages server quit."""
 257         if self.loaded:
 258             del self.cache
 259             del self.records
 260             del self.srcrecords
 261             del self.indexrecords
 262             self.loaded = 0
 263
 264     def cleanup(self):
 265         """Cleanup and close any loaded caches."""
 266         self.unload()
 267         self.packages.close()
 268
 269     def findHash(self, path):
 270         """Find the hash for a given path in this mirror.
 271
 272         Returns a deferred so it can make sure the cache is loaded first.
 273         """
 274         d = defer.Deferred()
 275
 276         deferLoad = self.load()
 277         deferLoad.addCallback(self._findHash, path, d)
 278
 279         return d
 280
 281     def _findHash(self, loadResult, path, d):
 282         """Really find the hash for a path.
 283
 284         Have to pass the returned loadResult on in case other calls to this
 285         function are pending.
 286         """
 287         if not loadResult:
 288             d.callback((None, None))
 289             return loadResult
 290
 291         # First look for the path in the cache of index files
 292         for release in self.indexrecords:
 293             if path.startswith(release[:-7]):
 294                 for indexFile in self.indexrecords[release]:
 295                     if release[:-7] + indexFile == path:
 296                         d.callback(self.indexrecords[release][indexFile]['SHA1'])
 297                         return loadResult
 298
 299         package = path.split('/')[-1].split('_')[0]
 300
 301         # Check the binary packages
 302         try:
 303             for version in self.cache[package].VersionList:
 304                 size = version.Size
 305                 for verFile in version.FileList:
 306                     if self.records.Lookup(verFile):
 307                         if '/' + self.records.FileName == path:
 308                             d.callback((self.records.SHA1Hash, size))
 309                             return loadResult
 310         except KeyError:
 311             pass
 312
 313         # Check the source packages' files
 314         self.srcrecords.Restart()
 315         if self.srcrecords.Lookup(package):
 316             for f in self.srcrecords.Files:
 317                 if path == '/' + f[2]:
 318                     d.callback((f[0], f[1]))
 319                     return loadResult
 320
 321         d.callback((None, None))
 322         return loadResult
 323
 324 class TestAptPackages(unittest.TestCase):
 325     """Unit tests for the AptPackages cache."""
 326
 327     pending_calls = []
 328     client = None
 329     timeout = 10
 330     packagesFile = ''
 331     sourcesFile = ''
 332     releaseFile = ''
 333
 334     def setUp(self):
 335         self.client = AptPackages('/tmp/.apt-dht')
 336
 337         self.packagesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "_main_.*Packages$" | tail -n 1').read().rstrip('\n')
 338         self.sourcesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "_main_.*Sources$" | tail -n 1').read().rstrip('\n')
 339         for f in os.walk('/var/lib/apt/lists').next()[2]:
 340             if f[-7:] == "Release" and self.packagesFile.startswith(f[:-7]):
 341                 self.releaseFile = f
 342                 break
 343
 344         self.client.file_updated(self.releaseFile[self.releaseFile.find('_dists_')-6:].replace('_','/'),
 345                                  '/var/lib/apt/lists/' + self.releaseFile)
 346         self.client.file_updated(self.packagesFile[self.packagesFile.find('_dists_')-6:].replace('_','/'),
 347                                  '/var/lib/apt/lists/' + self.packagesFile)
 348         self.client.file_updated(self.sourcesFile[self.sourcesFile.find('_dists_')-6:].replace('_','/'),
 349                                  '/var/lib/apt/lists/' + self.sourcesFile)
 350
 351     def test_pkg_hash(self):
 352         self.client._load()
 353
 354         self.client.records.Lookup(self.client.cache['dpkg'].VersionList[0].FileList[0])
 355
 356         pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 357                             '/var/lib/apt/lists/' + self.packagesFile +
 358                             ' | grep -E "^SHA1:" | head -n 1' +
 359                             ' | cut -d\  -f 2').read().rstrip('\n')
 360
 361         self.failUnless(self.client.records.SHA1Hash == pkg_hash,
 362                         "Hashes don't match: %s != %s" % (self.client.records.SHA1Hash, pkg_hash))
 363
 364     def test_src_hash(self):
 365         self.client._load()
 366
 367         self.client.srcrecords.Lookup('dpkg')
 368
 369         src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
 370                             '/var/lib/apt/lists/' + self.sourcesFile +
 371                             ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
 372                             ' | cut -d\  -f 2').read().split('\n')[:-1]
 373
 374         for f in self.client.srcrecords.Files:
 375             self.failUnless(f[0] in src_hashes, "Couldn't find %s in: %r" % (f[0], src_hashes))
 376
 377     def test_index_hash(self):
 378         self.client._load()
 379
 380         indexhash = self.client.indexrecords[self.releaseFile[self.releaseFile.find('_dists_')-6:].replace('_','/')]['main/binary-i386/Packages.bz2']['SHA1'][0]
 381
 382         idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
 383                             '/var/lib/apt/lists/' + self.releaseFile +
 384                             ' | grep -E " main/binary-i386/Packages.bz2$"'
 385                             ' | head -n 1 | cut -d\  -f 2').read().rstrip('\n')
 386
 387         self.failUnless(indexhash == idx_hash, "Hashes don't match: %s != %s" % (indexhash, idx_hash))
 388
 389     def verifyHash(self, found_hash, path, true_hash):
 390         self.failUnless(found_hash[0] == true_hash,
 391                     "%s hashes don't match: %s != %s" % (path, found_hash[0], true_hash))
 392
 393     def test_findIndexHash(self):
 394         lastDefer = defer.Deferred()
 395
 396         idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
 397                             '/var/lib/apt/lists/' + self.releaseFile +
 398                             ' | grep -E " main/binary-i386/Packages.bz2$"'
 399                             ' | head -n 1 | cut -d\  -f 2').read().rstrip('\n')
 400         idx_path = self.releaseFile[self.releaseFile.find('_dists_')-6:].replace('_','/')[:-7] + 'main/binary-i386/Packages.bz2'
 401
 402         d = self.client.findHash(idx_path)
 403         d.addCallback(self.verifyHash, idx_path, idx_hash)
 404
 405         d.addCallback(lastDefer.callback)
 406         return lastDefer
 407
 408     def test_findPkgHash(self):
 409         lastDefer = defer.Deferred()
 410
 411         pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 412                             '/var/lib/apt/lists/' + self.packagesFile +
 413                             ' | grep -E "^SHA1:" | head -n 1' +
 414                             ' | cut -d\  -f 2').read().rstrip('\n')
 415         pkg_path = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 416                             '/var/lib/apt/lists/' + self.packagesFile +
 417                             ' | grep -E "^Filename:" | head -n 1' +
 418                             ' | cut -d\  -f 2').read().rstrip('\n')
 419
 420         d = self.client.findHash(pkg_path)
 421         d.addCallback(self.verifyHash, pkg_path, pkg_hash)
 422
 423         d.addCallback(lastDefer.callback)
 424         return lastDefer
 425
 426     def test_findSrcHash(self):
 427         lastDefer = defer.Deferred()
 428
 429         src_dir = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 430                             '/var/lib/apt/lists/' + self.sourcesFile +
 431                             ' | grep -E "^Directory:" | head -n 1' +
 432                             ' | cut -d\  -f 2').read().rstrip('\n')
 433         src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
 434                             '/var/lib/apt/lists/' + self.sourcesFile +
 435                             ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
 436                             ' | cut -d\  -f 2').read().split('\n')[:-1]
 437         src_paths = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
 438                             '/var/lib/apt/lists/' + self.sourcesFile +
 439                             ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
 440                             ' | cut -d\  -f 4').read().split('\n')[:-1]
 441
 442         i = choice(range(len(src_hashes)))
 443         d = self.client.findHash(src_dir + '/' + src_paths[i])
 444         d.addCallback(self.verifyHash, src_dir + '/' + src_paths[i], src_hashes[i])
 445
 446         d.addCallback(lastDefer.callback)
 447         return lastDefer
 448
 449     def test_multipleFindHash(self):
 450         lastDefer = defer.Deferred()
 451
 452         idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
 453                             '/var/lib/apt/lists/' + self.releaseFile +
 454                             ' | grep -E " main/binary-i386/Packages.bz2$"'
 455                             ' | head -n 1 | cut -d\  -f 2').read().rstrip('\n')
 456         idx_path = self.releaseFile[self.releaseFile.find('_dists_')-6:].replace('_','/')[:-7] + 'main/binary-i386/Packages.bz2'
 457
 458         d = self.client.findHash(idx_path)
 459         d.addCallback(self.verifyHash, idx_path, idx_hash)
 460
 461         pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 462                             '/var/lib/apt/lists/' + self.packagesFile +
 463                             ' | grep -E "^SHA1:" | head -n 1' +
 464                             ' | cut -d\  -f 2').read().rstrip('\n')
 465         pkg_path = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 466                             '/var/lib/apt/lists/' + self.packagesFile +
 467                             ' | grep -E "^Filename:" | head -n 1' +
 468                             ' | cut -d\  -f 2').read().rstrip('\n')
 469
 470         d = self.client.findHash(pkg_path)
 471         d.addCallback(self.verifyHash, pkg_path, pkg_hash)
 472
 473         src_dir = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 474                             '/var/lib/apt/lists/' + self.sourcesFile +
 475                             ' | grep -E "^Directory:" | head -n 1' +
 476                             ' | cut -d\  -f 2').read().rstrip('\n')
 477         src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
 478                             '/var/lib/apt/lists/' + self.sourcesFile +
 479                             ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
 480                             ' | cut -d\  -f 2').read().split('\n')[:-1]
 481         src_paths = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
 482                             '/var/lib/apt/lists/' + self.sourcesFile +
 483                             ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
 484                             ' | cut -d\  -f 4').read().split('\n')[:-1]
 485
 486         for i in range(len(src_hashes)):
 487             d = self.client.findHash(src_dir + '/' + src_paths[i])
 488             d.addCallback(self.verifyHash, src_dir + '/' + src_paths[i], src_hashes[i])
 489
 490         idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
 491                             '/var/lib/apt/lists/' + self.releaseFile +
 492                             ' | grep -E " main/source/Sources.bz2$"'
 493                             ' | head -n 1 | cut -d\  -f 2').read().rstrip('\n')
 494         idx_path = self.releaseFile[self.releaseFile.find('_dists_')-6:].replace('_','/')[:-7] + 'main/source/Sources.bz2'
 495
 496         d = self.client.findHash(idx_path)
 497         d.addCallback(self.verifyHash, idx_path, idx_hash)
 498
 499         d.addCallback(lastDefer.callback)
 500         return lastDefer
 501
 502     def tearDown(self):
 503         for p in self.pending_calls:
 504             if p.active():
 505                 p.cancel()
 506         self.pending_calls = []
 507         self.client.cleanup()
 508         self.client = None