Make the MultiDHT test_store work for large number of nodes.

[quix0rs-apt-p2p.git] / apt_dht / AptPackages.py

diff --git a/apt_dht/AptPackages.py b/apt_dht/AptPackages.py
index 580849cc7db386b716c69f94246d81f6f862d3e4..a2e743e60fde74f07c9c9bed7d6132b6e66f9d4a 100644 (file)
--- a/apt_dht/AptPackages.py
+++ b/apt_dht/AptPackages.py
@@ -17,6 +17,8 @@ from apt import OpProgress
 
 apt_pkg.init()
 
+TRACKED_FILES = ['release', 'sources', 'packages']
+
 class PackageFileList(DictMixin):
     """Manages a list of package files belonging to a backend.
     
@@ -48,7 +50,7 @@ class PackageFileList(DictMixin):
         fake lists and sources.list.
         """
         filename = cache_path.split('/')[-1]
-        if filename=="Packages" or filename=="Release" or filename=="Sources":
+        if filename.lower() in TRACKED_FILES:
             log.msg("Registering package file: "+cache_path)
             self.packages[cache_path] = file_path
             return True
@@ -208,6 +210,7 @@ class AptPackages:
                                         self.apt_config['Dir::Etc::sourcelist'])
         sources = open(sources_filename, 'w')
         sources_count = 0
+        deb_src_added = False
         self.packages.check_files()
         self.indexrecords = {}
         for f in self.packages:
@@ -219,6 +222,7 @@ class AptPackages:
             fake_uri='http://apt-dht'+f
             fake_dirname = '/'.join(fake_uri.split('/')[:-1])
             if f.endswith('Sources'):
+                deb_src_added = True
                 source_line='deb-src '+fake_dirname+'/ /'
             else:
                 source_line='deb '+fake_dirname+'/ /'
@@ -247,7 +251,10 @@ class AptPackages:
 
         self.cache = apt_pkg.GetCache(OpProgress())
         self.records = apt_pkg.GetPkgRecords(self.cache)
-        self.srcrecords = apt_pkg.GetPkgSrcRecords()
+        if deb_src_added:
+            self.srcrecords = apt_pkg.GetPkgSrcRecords()
+        else:
+            self.srcrecords = None
 
         self.loaded = 1
         return True
@@ -275,9 +282,16 @@ class AptPackages:
 
         deferLoad = self.load()
         deferLoad.addCallback(self._findHash, path, d)
+        deferLoad.addErrback(self._findHash_error, path, d)
         
         return d
 
+    def _findHash_error(self, failure, path, d):
+        """An error occurred while trying to find a hash."""
+        log.msg('An error occurred while looking up a hash for: %s' % path)
+        log.err(failure)
+        d.callback((None, None))
+
     def _findHash(self, loadResult, path, d):
         """Really find the hash for a path.
         
@@ -311,12 +325,13 @@ class AptPackages:
             pass
 
         # Check the source packages' files
-        self.srcrecords.Restart()
-        if self.srcrecords.Lookup(package):
-            for f in self.srcrecords.Files:
-                if path == '/' + f[2]:
-                    d.callback((f[0], f[1]))
-                    return loadResult
+        if self.srcrecords:
+            self.srcrecords.Restart()
+            if self.srcrecords.Lookup(package):
+                for f in self.srcrecords.Files:
+                    if path == '/' + f[2]:
+                        d.callback((f[0], f[1]))
+                        return loadResult
         
         d.callback((None, None))
         return loadResult
@@ -377,7 +392,7 @@ class TestAptPackages(unittest.TestCase):
     def test_index_hash(self):
         self.client._load()
 
-        indexhash = self.client.indexrecords[self.releaseFile[self.releaseFile.find('_dists_')-6:].replace('_','/')]['main/binary-i386/Packages.bz2']['SHA1'][0]
+        indexhash = self.client.indexrecords[self.releaseFile[self.releaseFile.find('_dists_'):].replace('_','/')]['main/binary-i386/Packages.bz2']['SHA1'][0]
 
         idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' + 
                             '/var/lib/apt/lists/' + self.releaseFile + 
@@ -402,7 +417,7 @@ class TestAptPackages(unittest.TestCase):
         d = self.client.findHash(idx_path)
         d.addCallback(self.verifyHash, idx_path, idx_hash)
 
-        d.addCallback(lastDefer.callback)
+        d.addBoth(lastDefer.callback)
         return lastDefer
 
     def test_findPkgHash(self):
@@ -420,7 +435,7 @@ class TestAptPackages(unittest.TestCase):
         d = self.client.findHash(pkg_path)
         d.addCallback(self.verifyHash, pkg_path, pkg_hash)
 
-        d.addCallback(lastDefer.callback)
+        d.addBoth(lastDefer.callback)
         return lastDefer
 
     def test_findSrcHash(self):
@@ -443,7 +458,7 @@ class TestAptPackages(unittest.TestCase):
         d = self.client.findHash(src_dir + '/' + src_paths[i])
         d.addCallback(self.verifyHash, src_dir + '/' + src_paths[i], src_hashes[i])
             
-        d.addCallback(lastDefer.callback)
+        d.addBoth(lastDefer.callback)
         return lastDefer
 
     def test_multipleFindHash(self):
@@ -491,12 +506,12 @@ class TestAptPackages(unittest.TestCase):
                             '/var/lib/apt/lists/' + self.releaseFile + 
                             ' | grep -E " main/source/Sources.bz2$"'
                             ' | head -n 1 | cut -d\  -f 2').read().rstrip('\n')
-        idx_path = self.releaseFile[self.releaseFile.find('_dists_')-6:].replace('_','/')[:-7] + 'main/source/Sources.bz2'
+        idx_path = '/' + self.releaseFile[self.releaseFile.find('_dists_')+1:].replace('_','/')[:-7] + 'main/source/Sources.bz2'
 
         d = self.client.findHash(idx_path)
         d.addCallback(self.verifyHash, idx_path, idx_hash)
 
-        d.addCallback(lastDefer.callback)
+        d.addBoth(lastDefer.callback)
         return lastDefer
 
     def tearDown(self):