From 056a62b7b55e95a3a50d64d2e769d29b9eede4a9 Mon Sep 17 00:00:00 2001 From: Cameron Dale Date: Tue, 15 Jan 2008 14:08:14 -0800 Subject: [PATCH] Scanning cache directories on startup waits for DHT storeValue to return. This slows down the scanning a lot, but is more consistent for adding values to the DHT (prevents flooding). --- apt_dht/CacheManager.py | 12 ++++++++---- apt_dht/apt_dht.py | 3 ++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/apt_dht/CacheManager.py b/apt_dht/CacheManager.py index c6aae89..9820ae1 100644 --- a/apt_dht/CacheManager.py +++ b/apt_dht/CacheManager.py @@ -168,7 +168,7 @@ class CacheManager: self.scanning = self.all_dirs[:] self._scanDirectories() - def _scanDirectories(self, walker = None): + def _scanDirectories(self, result = None, walker = None): # Need to start waling a new directory if walker is None: # If there are any left, get them @@ -198,7 +198,7 @@ class CacheManager: log.msg('entering directory: %s' % file.path) else: log.msg('file is unchanged: %s' % file.path) - reactor.callLater(0, self._scanDirectories, walker) + reactor.callLater(0, self._scanDirectories, None, walker) return # Otherwise hash it @@ -209,7 +209,6 @@ class CacheManager: df.addErrback(log.err) def _doneHashing(self, result, file, walker): - reactor.callLater(0, self._scanDirectories, walker) if isinstance(result, HashObject): log.msg('hash check of %s completed with hash: %s' % (file.path, result.hexdigest())) @@ -222,10 +221,15 @@ class CacheManager: url = None if newdir: self.manager.setDirectories(self.db.getAllDirectories()) - self.manager.new_cached_file(file, result, urlpath, url) + df = self.manager.new_cached_file(file, result, urlpath, url) + if df is None: + reactor.callLater(0, self._scanDirectories, None, walker) + else: + df.addBoth(self._scanDirectories, walker) else: log.msg('hash check of %s failed' % file.path) log.err(result) + reactor.callLater(0, self._scanDirectories, None, walker) def save_file(self, response, hash, url): """Save a downloaded file to the cache and stream it.""" diff --git a/apt_dht/apt_dht.py b/apt_dht/apt_dht.py index d212729..047d1c2 100644 --- a/apt_dht/apt_dht.py +++ b/apt_dht/apt_dht.py @@ -168,7 +168,8 @@ class AptDHT: key = hash.norm(bits = config.getint(config.get('DEFAULT', 'DHT'), 'HASH_LENGTH')) storeDefer = self.dht.storeValue(key, full_path) storeDefer.addCallback(self.store_done, full_path) - storeDefer.addErrback(log.err) + return storeDefer + return None def store_done(self, result, path): log.msg('Added %s to the DHT: %r' % (path, result)) -- 2.39.5