Scanning cache directories on startup waits for DHT storeValue to return.

This slows down the scanning a lot, but is more consistent for adding
values to the DHT (prevents flooding).

diff --git a/apt_dht/CacheManager.py b/apt_dht/CacheManager.py
index c6aae89c416091fcceb1d32fba52c7f510a74b5f..9820ae165d7e46b549f5c1dcf043dbf9799da91b 100644 (file)
--- a/apt_dht/CacheManager.py
+++ b/apt_dht/CacheManager.py
@@ -168,7 +168,7 @@ class CacheManager:
         self.scanning = self.all_dirs[:]
         self._scanDirectories()
 
-    def _scanDirectories(self, walker = None):
+    def _scanDirectories(self, result = None, walker = None):
         # Need to start waling a new directory
         if walker is None:
             # If there are any left, get them
@@ -198,7 +198,7 @@ class CacheManager:
                 log.msg('entering directory: %s' % file.path)
             else:
                 log.msg('file is unchanged: %s' % file.path)
-            reactor.callLater(0, self._scanDirectories, walker)
+            reactor.callLater(0, self._scanDirectories, None, walker)
             return
 
         # Otherwise hash it
@@ -209,7 +209,6 @@ class CacheManager:
         df.addErrback(log.err)
     
     def _doneHashing(self, result, file, walker):
-        reactor.callLater(0, self._scanDirectories, walker)
     
         if isinstance(result, HashObject):
             log.msg('hash check of %s completed with hash: %s' % (file.path, result.hexdigest()))
@@ -222,10 +221,15 @@ class CacheManager:
                 url = None
             if newdir:
                 self.manager.setDirectories(self.db.getAllDirectories())
-            self.manager.new_cached_file(file, result, urlpath, url)
+            df = self.manager.new_cached_file(file, result, urlpath, url)
+            if df is None:
+                reactor.callLater(0, self._scanDirectories, None, walker)
+            else:
+                df.addBoth(self._scanDirectories, walker)
         else:
             log.msg('hash check of %s failed' % file.path)
             log.err(result)
+            reactor.callLater(0, self._scanDirectories, None, walker)
 
     def save_file(self, response, hash, url):
         """Save a downloaded file to the cache and stream it."""

diff --git a/apt_dht/apt_dht.py b/apt_dht/apt_dht.py
index d212729a5f30e34ff081eb21eab0a69fdcbd56ab..047d1c2b815dbf7c3d3643244c5f04fa9b8d0fad 100644 (file)
--- a/apt_dht/apt_dht.py
+++ b/apt_dht/apt_dht.py
@@ -168,7 +168,8 @@ class AptDHT:
             key = hash.norm(bits = config.getint(config.get('DEFAULT', 'DHT'), 'HASH_LENGTH'))
             storeDefer = self.dht.storeValue(key, full_path)
             storeDefer.addCallback(self.store_done, full_path)
-            storeDefer.addErrback(log.err)
+            return storeDefer
+        return None
 
     def store_done(self, result, path):
         log.msg('Added %s to the DHT: %r' % (path, result))