AptPackages.py

   1 #
   2 # Copyright (C) 2002 Manuel Estrada Sainz <ranty@debian.org>
   3 #
   4 # This library is free software; you can redistribute it and/or
   5 # modify it under the terms of version 2.1 of the GNU Lesser General Public
   6 # License as published by the Free Software Foundation.
   7 #
   8 # This library is distributed in the hope that it will be useful,
   9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  11 # Lesser General Public License for more details.
  12 #
  13 # You should have received a copy of the GNU Lesser General Public
  14 # License along with this library; if not, write to the Free Software
  15 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  16
  17 import warnings
  18 warnings.simplefilter("ignore", FutureWarning)
  19 import apt_pkg, apt_inst, sys, os, stat, random
  20 from os.path import dirname, basename
  21 import re, shelve, shutil, fcntl
  22 from twisted.internet import process, threads, defer
  23 from twisted.python import log
  24 import copy, UserDict
  25 from twisted.trial import unittest
  26 from apt import OpProgress
  27
  28 aptpkg_dir='.apt-dht'
  29 apt_pkg.init()
  30
  31 class AptDpkgInfo(UserDict.UserDict):
  32     """
  33     Gets control fields from a .deb file.
  34
  35     And then behaves like a regular python dictionary.
  36
  37     See AptPackages.get_mirror_path
  38     """
  39
  40     def __init__(self, filename):
  41         UserDict.UserDict.__init__(self)
  42         try:
  43             filehandle = open(filename);
  44             try:
  45                 self.control = apt_inst.debExtractControl(filehandle)
  46             finally:
  47                 # Make sure that file is always closed.
  48                 filehandle.close()
  49         except SystemError:
  50             log.msg("Had problems reading: %s"%(filename))
  51             raise
  52         for line in self.control.split('\n'):
  53             if line.find(': ') != -1:
  54                 key, value = line.split(': ', 1)
  55                 self.data[key] = value
  56
  57 class PackageFileList(UserDict.DictMixin):
  58     """
  59     Manages a list of package files belonging to a backend
  60     """
  61     def __init__(self, backendName, cache_dir):
  62         self.cache_dir = cache_dir
  63         self.packagedb_dir = cache_dir+'/'+ aptpkg_dir + \
  64                            '/backends/' + backendName
  65         if not os.path.exists(self.packagedb_dir):
  66             os.makedirs(self.packagedb_dir)
  67         self.packages = None
  68         self.open()
  69
  70     def open(self):
  71         if self.packages is None:
  72             self.packages = shelve.open(self.packagedb_dir+'/packages.db')
  73
  74     def close(self):
  75         if self.packages is not None:
  76             self.packages.close()
  77
  78     def update_file(self, filename, cache_path, file_path):
  79         """
  80         Called from apt_proxy.py when files get updated so we can update our
  81         fake lists/ directory and sources.list.
  82         """
  83         if filename=="Packages" or filename=="Release" or filename=="Sources":
  84             log.msg("Registering package file: "+cache_path)
  85             self.packages[cache_path] = file_path
  86             return True
  87         return False
  88
  89     def check_files(self):
  90         """
  91         Check all files in the database to make sure it exists.
  92         """
  93         files = self.packages.keys()
  94         #print self.packages.keys()
  95         for f in files:
  96             if not os.path.exists(self.packages[f]):
  97                 log.msg("File in packages database has been deleted: "+f)
  98                 del self.packages[f]
  99
 100     def __getitem__(self, key): return self.packages[key]
 101     def __setitem__(self, key, item): self.packages[key] = item
 102     def __delitem__(self, key): del self.packages[key]
 103     def keys(self): return self.packages.keys()
 104
 105 class AptPackages:
 106     """
 107     Uses AptPackagesServer to answer queries about packages.
 108
 109     Makes a fake configuration for python-apt for each backend.
 110     """
 111     DEFAULT_APT_CONFIG = {
 112         #'APT' : '',
 113         #'APT::Architecture' : 'amd64',  # TODO: Fix this, see bug #436011 and #285360
 114         #'APT::Default-Release' : 'unstable',
 115
 116         'Dir':'.', # /
 117         'Dir::State' : 'apt/', # var/lib/apt/
 118         'Dir::State::Lists': 'lists/', # lists/
 119         #'Dir::State::cdroms' : 'cdroms.list',
 120         'Dir::State::userstatus' : 'status.user',
 121         'Dir::State::status': 'dpkg/status', # '/var/lib/dpkg/status'
 122         'Dir::Cache' : '.apt/cache/', # var/cache/apt/
 123         #'Dir::Cache::archives' : 'archives/',
 124         'Dir::Cache::srcpkgcache' : 'srcpkgcache.bin',
 125         'Dir::Cache::pkgcache' : 'pkgcache.bin',
 126         'Dir::Etc' : 'apt/etc/', # etc/apt/
 127         'Dir::Etc::sourcelist' : 'sources.list',
 128         'Dir::Etc::vendorlist' : 'vendors.list',
 129         'Dir::Etc::vendorparts' : 'vendors.list.d',
 130         #'Dir::Etc::main' : 'apt.conf',
 131         #'Dir::Etc::parts' : 'apt.conf.d',
 132         #'Dir::Etc::preferences' : 'preferences',
 133         'Dir::Bin' : '',
 134         #'Dir::Bin::methods' : '', #'/usr/lib/apt/methods'
 135         'Dir::Bin::dpkg' : '/usr/bin/dpkg',
 136         #'DPkg' : '',
 137         #'DPkg::Pre-Install-Pkgs' : '',
 138         #'DPkg::Tools' : '',
 139         #'DPkg::Tools::Options' : '',
 140         #'DPkg::Tools::Options::/usr/bin/apt-listchanges' : '',
 141         #'DPkg::Tools::Options::/usr/bin/apt-listchanges::Version' : '2',
 142         #'DPkg::Post-Invoke' : '',
 143         }
 144     essential_dirs = ('apt', 'apt/cache', 'apt/dpkg', 'apt/etc', 'apt/lists',
 145                       'apt/lists/partial')
 146     essential_files = ('apt/dpkg/status', 'apt/etc/sources.list',)
 147
 148     def __init__(self, backendName, cache_dir):
 149         """
 150         Construct new packages manager
 151         backend: Name of backend associated with this packages file
 152         cache_dir: cache directory from config file
 153         """
 154         self.backendName = backendName
 155         self.cache_dir = cache_dir
 156         self.apt_config = copy.deepcopy(self.DEFAULT_APT_CONFIG)
 157
 158         self.status_dir = (cache_dir+'/'+ aptpkg_dir
 159                            +'/backends/'+backendName)
 160         for dir in self.essential_dirs:
 161             path = self.status_dir+'/'+dir
 162             if not os.path.exists(path):
 163                 os.makedirs(path)
 164         for file in self.essential_files:
 165             path = self.status_dir+'/'+file
 166             if not os.path.exists(path):
 167                 f = open(path,'w')
 168                 f.close()
 169                 del f
 170
 171         self.apt_config['Dir'] = self.status_dir
 172         self.apt_config['Dir::State::status'] = self.status_dir + '/apt/dpkg/status'
 173         #os.system('find '+self.status_dir+' -ls ')
 174         #print "status:"+self.apt_config['Dir::State::status']
 175         self.packages = PackageFileList(backendName, cache_dir)
 176         self.indexrecords = {}
 177         self.loaded = 0
 178         self.loading = None
 179         #print "Loaded aptPackages [%s] %s " % (self.backendName, self.cache_dir)
 180
 181     def __del__(self):
 182         self.cleanup()
 183         #print "start aptPackages [%s] %s " % (self.backendName, self.cache_dir)
 184         self.packages.close()
 185         #print "Deleted aptPackages [%s] %s " % (self.backendName, self.cache_dir)
 186
 187     def addRelease(self, cache_path, file_path):
 188         """
 189         Dirty hack until python-apt supports apt-pkg/indexrecords.h
 190         (see Bug #456141)
 191         """
 192         self.indexrecords[cache_path] = {}
 193
 194         read_packages = False
 195         f = open(file_path, 'r')
 196
 197         for line in f:
 198             line = line.rstrip()
 199
 200             if line[:1] != " ":
 201                 read_packages = False
 202                 try:
 203                     # Read the various headers from the file
 204                     h, v = line.split(":", 1)
 205                     if h == "MD5Sum" or h == "SHA1" or h == "SHA256":
 206                         read_packages = True
 207                         hash_type = h
 208                 except:
 209                     # Bad header line, just ignore it
 210                     log.msg("WARNING: Ignoring badly formatted Release line: %s" % line)
 211
 212                 # Skip to the next line
 213                 continue
 214
 215             # Read file names from the multiple hash sections of the file
 216             if read_packages:
 217                 p = line.split()
 218                 self.indexrecords[cache_path].setdefault(p[2], {})[hash_type] = (p[0], p[1])
 219
 220         f.close()
 221
 222     def file_updated(self, filename, cache_path, file_path):
 223         """
 224         A file in the backend has changed.  If this affects us, unload our apt database
 225         """
 226         if filename == "Release":
 227             self.addRelease(cache_path, file_path)
 228         if self.packages.update_file(filename, cache_path, file_path):
 229             self.unload()
 230
 231     def load(self):
 232         if self.loading is None:
 233             self.loading = threads.deferToThread(self._load)
 234             self.loading.addCallback(self.doneLoading)
 235         return self.loading
 236
 237     def doneLoading(self, loadResult):
 238         self.loading = None
 239         return loadResult
 240
 241     def _load(self):
 242         """
 243         Regenerates the fake configuration and load the packages server.
 244         """
 245         if self.loaded: return True
 246         apt_pkg.InitSystem()
 247         #print "Load:", self.status_dir
 248         shutil.rmtree(self.status_dir+'/apt/lists/')
 249         os.makedirs(self.status_dir+'/apt/lists/partial')
 250         sources_filename = self.status_dir+'/'+'apt/etc/sources.list'
 251         sources = open(sources_filename, 'w')
 252         sources_count = 0
 253         self.packages.check_files()
 254         for f in self.packages:
 255             # we should probably clear old entries from self.packages and
 256             # take into account the recorded mtime as optimization
 257             filepath = self.packages[f]
 258             fake_uri='http://apt-dht/'+f
 259             if f.endswith('Sources'):
 260                 source_line='deb-src '+dirname(fake_uri)+'/ /'
 261             else:
 262                 source_line='deb '+dirname(fake_uri)+'/ /'
 263             listpath=(self.status_dir+'/apt/lists/'
 264                     +apt_pkg.URItoFileName(fake_uri))
 265             sources.write(source_line+'\n')
 266             log.msg("Sources line: " + source_line)
 267             sources_count = sources_count + 1
 268
 269             try:
 270                 #we should empty the directory instead
 271                 os.unlink(listpath)
 272             except:
 273                 pass
 274             os.symlink(self.packages[f], listpath)
 275         sources.close()
 276
 277         if sources_count == 0:
 278             log.msg("No Packages files available for %s backend"%(self.backendName))
 279             return False
 280
 281         log.msg("Loading Packages database for "+self.status_dir)
 282         #apt_pkg.Config = apt_pkg.newConfiguration(); #-- this causes unit tests to fail!
 283         for key, value in self.apt_config.items():
 284             apt_pkg.Config[key] = value
 285 #         print "apt_pkg config:"
 286 #         for I in apt_pkg.Config.keys():
 287 #            print "%s \"%s\";"%(I,apt_pkg.Config[I]);
 288
 289         self.cache = apt_pkg.GetCache(OpProgress())
 290         self.records = apt_pkg.GetPkgRecords(self.cache)
 291         self.srcrecords = apt_pkg.GetPkgSrcRecords()
 292         #for p in self.cache.Packages:
 293         #    print p
 294         #log.debug("%s packages found" % (len(self.cache)),'apt_pkg')
 295         self.loaded = 1
 296         return True
 297
 298     def unload(self):
 299         "Tries to make the packages server quit."
 300         if self.loaded:
 301             del self.cache
 302             del self.records
 303             del self.srcrecords
 304             self.loaded = 0
 305
 306     def cleanup(self):
 307         self.unload()
 308         self.packages.close()
 309
 310     def findHash(self, path):
 311         d = defer.Deferred()
 312
 313         for release in self.indexrecords:
 314             if path.startswith(release[:-7]):
 315                 for indexFile in self.indexrecords[release]:
 316                     if release[:-7] + indexFile == path:
 317                         d.callback(self.indexrecords[release][indexFile]['SHA1'])
 318                         return d
 319
 320         deferLoad = self.load()
 321         deferLoad.addCallback(self._findHash, path, d)
 322
 323         return d
 324
 325     def _findHash(self, loadResult, path, d):
 326         if not loadResult:
 327             d.callback((None, None))
 328             return loadResult
 329
 330         package = path.split('/')[-1].split('_')[0]
 331
 332         try:
 333             for version in self.cache[package].VersionList:
 334                 size = version.Size
 335                 for verFile in version.FileList:
 336                     if self.records.Lookup(verFile):
 337                         if self.records.FileName == path:
 338                             d.callback((self.records.SHA1Hash, size))
 339                             return loadResult
 340         except KeyError:
 341             pass
 342
 343         self.srcrecords.Restart()
 344         if self.srcrecords.Lookup(package):
 345             for f in self.srcrecords.Files:
 346                 if path == f[2]:
 347                     d.callback((f[0], f[1]))
 348                     return loadResult
 349
 350         d.callback((None, None))
 351         return loadResult
 352
 353     def get_mirror_path(self, name, version):
 354         "Find the path for version 'version' of package 'name'"
 355         if not self.load(): return None
 356         try:
 357             for pack_vers in self.cache[name].VersionList:
 358                 if(pack_vers.VerStr == version):
 359                     file, index = pack_vers.FileList[0]
 360                     self.records.Lookup((file,index))
 361                     path = self.records.FileName
 362                     if len(path)>2 and path[0:2] == './':
 363                         path = path[2:] # Remove any leading './'
 364                     return path
 365
 366         except KeyError:
 367             pass
 368         return None
 369
 370
 371     def get_mirror_versions(self, package_name):
 372         """
 373         Find the available versions of the package name given
 374         @type package_name: string
 375         @param package_name: package name to search for e.g. ;apt'
 376         @return: A list of mirror versions available
 377
 378         """
 379         vers = []
 380         if not self.load(): return vers
 381         try:
 382             for pack_vers in self.cache[package_name].VersionList:
 383                 vers.append(pack_vers.VerStr)
 384         except KeyError:
 385             pass
 386         return vers
 387
 388
 389 def cleanup(factory):
 390     for backend in factory.backends.values():
 391         backend.get_packages_db().cleanup()
 392
 393 def get_mirror_path(factory, file):
 394     """
 395     Look for the path of 'file' in all backends.
 396     """
 397     info = AptDpkgInfo(file)
 398     paths = []
 399     for backend in factory.backends.values():
 400         path = backend.get_packages_db().get_mirror_path(info['Package'],
 401                                                 info['Version'])
 402         if path:
 403             paths.append('/'+backend.base+'/'+path)
 404     return paths
 405
 406 def get_mirror_versions(factory, package):
 407     """
 408     Look for the available version of a package in all backends, given
 409     an existing package name
 410     """
 411     all_vers = []
 412     for backend in factory.backends.values():
 413         vers = backend.get_packages_db().get_mirror_versions(package)
 414         for ver in vers:
 415             path = backend.get_packages_db().get_mirror_path(package, ver)
 416             all_vers.append((ver, "%s/%s"%(backend.base,path)))
 417     return all_vers
 418
 419 def closest_match(info, others):
 420     def compare(a, b):
 421         return apt_pkg.VersionCompare(a[0], b[0])
 422
 423     others.sort(compare)
 424     version = info['Version']
 425     match = None
 426     for ver,path in others:
 427         if version <= ver:
 428             match = path
 429             break
 430     if not match:
 431         if not others:
 432             return None
 433         match = others[-1][1]
 434
 435     dirname=re.sub(r'/[^/]*$', '', match)
 436     version=re.sub(r'^[^:]*:', '', info['Version'])
 437     if dirname.find('/pool/') != -1:
 438         return "/%s/%s_%s_%s.deb"%(dirname, info['Package'],
 439                                   version, info['Architecture'])
 440     else:
 441         return "/%s/%s_%s.deb"%(dirname, info['Package'], version)
 442
 443 def import_directory(factory, dir, recursive=0):
 444     """
 445     Import all files in a given directory into the cache
 446     This is used by apt-proxy-import to import new files
 447     into the cache
 448     """
 449     imported_count  = 0
 450
 451     if not os.path.exists(dir):
 452         log.err('Directory ' + dir + ' does not exist')
 453         return
 454
 455     if recursive:
 456         log.msg("Importing packages from directory tree: " + dir)
 457         for root, dirs, files in os.walk(dir):
 458             for file in files:
 459                 imported_count += import_file(factory, root, file)
 460     else:
 461         log.msg("Importing packages from directory: " + dir)
 462         for file in os.listdir(dir):
 463             mode = os.stat(dir + '/' + file)[stat.ST_MODE]
 464             if not stat.S_ISDIR(mode):
 465                 imported_count += import_file(factory, dir, file)
 466
 467     for backend in factory.backends.values():
 468         backend.get_packages_db().unload()
 469
 470     log.msg("Imported %s files" % (imported_count))
 471     return imported_count
 472
 473 def import_file(factory, dir, file):
 474     """
 475     Import a .deb or .udeb into cache from given filename
 476     """
 477     if file[-4:]!='.deb' and file[-5:]!='.udeb':
 478         log.msg("Ignoring (unknown file type):"+ file)
 479         return 0
 480
 481     log.msg("considering: " + dir + '/' + file)
 482     try:
 483         paths = get_mirror_path(factory, dir+'/'+file)
 484     except SystemError:
 485         log.msg(file + ' skipped - wrong format or corrupted')
 486         return 0
 487     if paths:
 488         if len(paths) != 1:
 489             log.msg("WARNING: multiple ocurrences")
 490             log.msg(str(paths), 'import')
 491         cache_path = paths[0]
 492     else:
 493         log.msg("Not found, trying to guess")
 494         info = AptDpkgInfo(dir+'/'+file)
 495         cache_path = closest_match(info,
 496                                 get_mirror_versions(factory, info['Package']))
 497     if cache_path:
 498         log.msg("MIRROR_PATH:"+ cache_path)
 499         src_path = dir+'/'+file
 500         dest_path = factory.config.cache_dir+cache_path
 501
 502         if not os.path.exists(dest_path):
 503             log.msg("IMPORTING:" + src_path)
 504             dest_path = re.sub(r'/\./', '/', dest_path)
 505             if not os.path.exists(dirname(dest_path)):
 506                 os.makedirs(dirname(dest_path))
 507             f = open(dest_path, 'w')
 508             fcntl.lockf(f.fileno(), fcntl.LOCK_EX)
 509             f.truncate(0)
 510             shutil.copy2(src_path, dest_path)
 511             f.close()
 512             if hasattr(factory, 'access_times'):
 513                 atime = os.stat(src_path)[stat.ST_ATIME]
 514                 factory.access_times[cache_path] = atime
 515             log.msg(file + ' imported')
 516             return 1
 517         else:
 518             log.msg(file + ' skipped - already in cache')
 519             return 0
 520
 521     else:
 522         log.msg(file + ' skipped - no suitable backend found')
 523         return 0
 524
 525 class TestAptPackages(unittest.TestCase):
 526     """Unit tests for the AptPackages cache."""
 527
 528     pending_calls = []
 529     client = None
 530     packagesFile = ''
 531     sourcesFile = ''
 532     releaseFile = ''
 533
 534     def setUp(self):
 535         self.client = AptPackages('whatever', '/tmp')
 536
 537         self.packagesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "Packages$" | tail -n 1').read().rstrip('\n')
 538         self.sourcesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "Sources$" | tail -n 1').read().rstrip('\n')
 539         for f in os.walk('/var/lib/apt/lists').next()[2]:
 540             if f[-7:] == "Release" and self.packagesFile.startswith(f[:-7]):
 541                 self.releaseFile = f
 542                 break
 543
 544         self.client.file_updated('Release',
 545                                  self.releaseFile[self.releaseFile.find('_debian_')+1:].replace('_','/'),
 546                                  '/var/lib/apt/lists/' + self.releaseFile)
 547         self.client.file_updated('Packages',
 548                                  self.packagesFile[self.packagesFile.find('_debian_')+1:].replace('_','/'),
 549                                  '/var/lib/apt/lists/' + self.packagesFile)
 550         self.client.file_updated('Sources',
 551                                  self.sourcesFile[self.sourcesFile.find('_debian_')+1:].replace('_','/'),
 552                                  '/var/lib/apt/lists/' + self.sourcesFile)
 553
 554     def test_pkg_hash(self):
 555         self.client._load()
 556
 557         self.client.records.Lookup(self.client.cache['dpkg'].VersionList[0].FileList[0])
 558
 559         pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 560                             '/var/lib/apt/lists/' + self.packagesFile +
 561                             ' | grep -E "^SHA1:" | head -n 1' +
 562                             ' | cut -d\  -f 2').read().rstrip('\n')
 563
 564         self.failUnless(self.client.records.SHA1Hash == pkg_hash,
 565                         "Hashes don't match: %s != %s" % (self.client.records.SHA1Hash, pkg_hash))
 566
 567     def test_src_hash(self):
 568         self.client._load()
 569
 570         self.client.srcrecords.Lookup('dpkg')
 571
 572         src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
 573                             '/var/lib/apt/lists/' + self.sourcesFile +
 574                             ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
 575                             ' | cut -d\  -f 2').read().split('\n')[:-1]
 576
 577         for f in self.client.srcrecords.Files:
 578             self.failUnless(f[0] in src_hashes, "Couldn't find %s in: %r" % (f[0], src_hashes))
 579
 580     def test_index_hash(self):
 581         self.client._load()
 582
 583         indexhash = self.client.indexrecords[self.releaseFile[self.releaseFile.find('_debian_')+1:].replace('_','/')]['main/binary-i386/Packages.bz2']['SHA1'][0]
 584
 585         idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
 586                             '/var/lib/apt/lists/' + self.releaseFile +
 587                             ' | grep -E " main/binary-i386/Packages.bz2$"'
 588                             ' | head -n 1 | cut -d\  -f 2').read().rstrip('\n')
 589
 590         self.failUnless(indexhash == idx_hash, "Hashes don't match: %s != %s" % (indexhash, idx_hash))
 591
 592     def verifyHash(self, found_hash, path, true_hash):
 593         self.failUnless(found_hash[0] == true_hash,
 594                     "%s hashes don't match: %s != %s" % (path, found_hash[0], true_hash))
 595
 596     def test_findIndexHash(self):
 597         lastDefer = defer.Deferred()
 598
 599         idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
 600                             '/var/lib/apt/lists/' + self.releaseFile +
 601                             ' | grep -E " main/binary-i386/Packages.bz2$"'
 602                             ' | head -n 1 | cut -d\  -f 2').read().rstrip('\n')
 603         idx_path = self.releaseFile[self.releaseFile.find('_debian_')+1:].replace('_','/')[:-7] + 'main/binary-i386/Packages.bz2'
 604
 605         d = self.client.findHash(idx_path)
 606         d.addCallback(self.verifyHash, idx_path, idx_hash)
 607
 608         d.addCallback(lastDefer.callback)
 609         return lastDefer
 610
 611     def test_findPkgHash(self):
 612         lastDefer = defer.Deferred()
 613
 614         pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 615                             '/var/lib/apt/lists/' + self.packagesFile +
 616                             ' | grep -E "^SHA1:" | head -n 1' +
 617                             ' | cut -d\  -f 2').read().rstrip('\n')
 618         pkg_path = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 619                             '/var/lib/apt/lists/' + self.packagesFile +
 620                             ' | grep -E "^Filename:" | head -n 1' +
 621                             ' | cut -d\  -f 2').read().rstrip('\n')
 622
 623         d = self.client.findHash(pkg_path)
 624         d.addCallback(self.verifyHash, pkg_path, pkg_hash)
 625
 626         d.addCallback(lastDefer.callback)
 627         return lastDefer
 628
 629     def test_findSrcHash(self):
 630         lastDefer = defer.Deferred()
 631
 632         src_dir = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 633                             '/var/lib/apt/lists/' + self.sourcesFile +
 634                             ' | grep -E "^Directory:" | head -n 1' +
 635                             ' | cut -d\  -f 2').read().rstrip('\n')
 636         src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
 637                             '/var/lib/apt/lists/' + self.sourcesFile +
 638                             ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
 639                             ' | cut -d\  -f 2').read().split('\n')[:-1]
 640         src_paths = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
 641                             '/var/lib/apt/lists/' + self.sourcesFile +
 642                             ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
 643                             ' | cut -d\  -f 4').read().split('\n')[:-1]
 644
 645         i = random.choice(range(len(src_hashes)))
 646         d = self.client.findHash(src_dir + '/' + src_paths[i])
 647         d.addCallback(self.verifyHash, src_dir + '/' + src_paths[i], src_hashes[i])
 648
 649         d.addCallback(lastDefer.callback)
 650         return lastDefer
 651
 652     def test_multipleFindHash(self):
 653         lastDefer = defer.Deferred()
 654
 655         idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
 656                             '/var/lib/apt/lists/' + self.releaseFile +
 657                             ' | grep -E " main/binary-i386/Packages.bz2$"'
 658                             ' | head -n 1 | cut -d\  -f 2').read().rstrip('\n')
 659         idx_path = self.releaseFile[self.releaseFile.find('_debian_')+1:].replace('_','/')[:-7] + 'main/binary-i386/Packages.bz2'
 660
 661         d = self.client.findHash(idx_path)
 662         d.addCallback(self.verifyHash, idx_path, idx_hash)
 663
 664         pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 665                             '/var/lib/apt/lists/' + self.packagesFile +
 666                             ' | grep -E "^SHA1:" | head -n 1' +
 667                             ' | cut -d\  -f 2').read().rstrip('\n')
 668         pkg_path = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 669                             '/var/lib/apt/lists/' + self.packagesFile +
 670                             ' | grep -E "^Filename:" | head -n 1' +
 671                             ' | cut -d\  -f 2').read().rstrip('\n')
 672
 673         d = self.client.findHash(pkg_path)
 674         d.addCallback(self.verifyHash, pkg_path, pkg_hash)
 675
 676         src_dir = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
 677                             '/var/lib/apt/lists/' + self.sourcesFile +
 678                             ' | grep -E "^Directory:" | head -n 1' +
 679                             ' | cut -d\  -f 2').read().rstrip('\n')
 680         src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
 681                             '/var/lib/apt/lists/' + self.sourcesFile +
 682                             ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
 683                             ' | cut -d\  -f 2').read().split('\n')[:-1]
 684         src_paths = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
 685                             '/var/lib/apt/lists/' + self.sourcesFile +
 686                             ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
 687                             ' | cut -d\  -f 4').read().split('\n')[:-1]
 688
 689         for i in range(len(src_hashes)):
 690             d = self.client.findHash(src_dir + '/' + src_paths[i])
 691             d.addCallback(self.verifyHash, src_dir + '/' + src_paths[i], src_hashes[i])
 692
 693         d.addCallback(lastDefer.callback)
 694         return lastDefer
 695
 696     def tearDown(self):
 697         for p in self.pending_calls:
 698             if p.active():
 699                 p.cancel()
 700         self.pending_calls = []
 701         self.client.cleanup()
 702         self.client = None