Use compact encoding of peer info for downloads.

[quix0rs-apt-p2p.git] / apt_dht_Khashmir / DHT.py

diff --git a/apt_dht_Khashmir/DHT.py b/apt_dht_Khashmir/DHT.py
index 590a77cb224a59026e41ca41631340daf6362902..23b2755313f0d1f91e7f08de7829674bb37ae614 100644 (file)
--- a/apt_dht_Khashmir/DHT.py
+++ b/apt_dht_Khashmir/DHT.py
@@ -1,13 +1,18 @@
 
+from datetime import datetime
 import os, sha, random
 
 from twisted.internet import defer, reactor
 from twisted.internet.abstract import isIPAddress
+from twisted.python import log
 from twisted.trial import unittest
 from zope.interface import implements
 
 from apt_dht.interfaces import IDHT
 from khashmir import Khashmir
+from bencode import bencode, bdecode
+
+khashmir_dir = 'apt-dht-Khashmir'
 
 class DHTError(Exception):
     """Represents errors that occur in the DHT."""
@@ -23,6 +28,8 @@ class DHT:
         self.bootstrap_node = False
         self.joining = None
         self.joined = False
+        self.outstandingJoins = 0
+        self.foundAddrs = []
         self.storing = {}
         self.retrieving = {}
         self.retrieved = {}
@@ -32,7 +39,9 @@ class DHT:
         self.config_parser = config
         self.section = section
         self.config = {}
-        self.cache_dir = self.config_parser.get(section, 'cache_dir')
+        self.cache_dir = os.path.join(self.config_parser.get(section, 'cache_dir'), khashmir_dir)
+        if not os.path.exists(self.cache_dir):
+            os.makedirs(self.cache_dir)
         self.bootstrap = self.config_parser.getstringlist(section, 'BOOTSTRAP')
         self.bootstrap_node = self.config_parser.getboolean(section, 'BOOTSTRAP_NODE')
         for k in self.config_parser.options(section):
@@ -42,6 +51,8 @@ class DHT:
             elif k in ['CHECKPOINT_INTERVAL', 'MIN_PING_INTERVAL', 
                        'BUCKET_STALENESS', 'KEINITIAL_DELAY', 'KE_DELAY', 'KE_AGE']:
                 self.config[k] = self.config_parser.gettime(section, k)
+            elif k in ['SPEW']:
+                self.config[k] = self.config_parser.getboolean(section, k)
             else:
                 self.config[k] = self.config_parser.get(section, k)
     
@@ -67,25 +78,41 @@ class DHT:
 
     def _join_gotIP(self, ip, port):
         """Called after an IP address has been found for a single bootstrap node."""
-        self.khashmir.addContact(ip, port, self._join_single)
+        self.outstandingJoins += 1
+        self.khashmir.addContact(ip, port, self._join_single, self._join_error)
     
-    def _join_single(self):
+    def _join_single(self, addr):
         """Called when a single bootstrap node has been added."""
-        self.khashmir.findCloseNodes(self._join_complete)
+        self.outstandingJoins -= 1
+        if addr:
+            self.foundAddrs.append(addr)
+        if addr or self.outstandingJoins <= 0:
+            self.khashmir.findCloseNodes(self._join_complete, self._join_complete)
+        log.msg('Got back from bootstrap node: %r' % (addr,))
     
+    def _join_error(self, failure = None):
+        """Called when a single bootstrap node has failed."""
+        self.outstandingJoins -= 1
+        log.msg("bootstrap node could not be reached")
+        if self.outstandingJoins <= 0:
+            self.khashmir.findCloseNodes(self._join_complete, self._join_complete)
+
     def _join_complete(self, result):
         """Called when the tables have been initialized with nodes."""
-        if not self.joined:
+        if not self.joined and len(result) > 0:
             self.joined = True
-            if len(result) > 0 or self.bootstrap_node:
-                df = self.joining
-                self.joining = None
-                df.callback(result)
+        if self.joining and self.outstandingJoins <= 0:
+            df = self.joining
+            self.joining = None
+            if self.joined or self.bootstrap_node:
+                self.joined = True
+                df.callback(self.foundAddrs)
             else:
-                df = self.joining
-                self.joining = None
                 df.errback(DHTError('could not find any nodes to bootstrap to'))
         
+    def getAddrs(self):
+        return self.foundAddrs
+        
     def leave(self):
         """See L{apt_dht.interfaces.IDHT}."""
         if self.config is None:
@@ -113,7 +140,7 @@ class DHT:
         
     def _getValue(self, key, result):
         if result:
-            self.retrieved.setdefault(key, []).extend(result)
+            self.retrieved.setdefault(key, []).extend([bdecode(r) for r in result])
         else:
             final_result = []
             if key in self.retrieved:
@@ -131,21 +158,23 @@ class DHT:
         if not self.joined:
             raise DHTError, "have not joined a network yet"
 
-        if key in self.storing and value in self.storing[key]:
+        bvalue = bencode(value)
+
+        if key in self.storing and bvalue in self.storing[key]:
             raise DHTError, "already storing that key with the same value"
 
         d = defer.Deferred()
-        self.khashmir.storeValueForKey(key, value, self._storeValue)
-        self.storing.setdefault(key, {})[value] = d
+        self.khashmir.storeValueForKey(key, bvalue, self._storeValue)
+        self.storing.setdefault(key, {})[bvalue] = d
         return d
     
-    def _storeValue(self, key, value, result):
-        if key in self.storing and value in self.storing[key]:
+    def _storeValue(self, key, bvalue, result):
+        if key in self.storing and bvalue in self.storing[key]:
             if len(result) > 0:
-                self.storing[key][value].callback(result)
+                self.storing[key][bvalue].callback(result)
             else:
-                self.storing[key][value].errback(DHTError('could not store value %s in key %s' % (value, key)))
-            del self.storing[key][value]
+                self.storing[key][bvalue].errback(DHTError('could not store value %s in key %s' % (bvalue, key)))
+            del self.storing[key][bvalue]
             if len(self.storing[key].keys()) == 0:
                 del self.storing[key]
 
@@ -158,7 +187,7 @@ class TestSimpleDHT(unittest.TestCase):
                     'STORE_REDUNDANCY': 3, 'MAX_FAILURES': 3,
                     'MIN_PING_INTERVAL': 900,'BUCKET_STALENESS': 3600,
                     'KEINITIAL_DELAY': 15, 'KE_DELAY': 1200,
-                    'KE_AGE': 3600, }
+                    'KE_AGE': 3600, 'SPEW': False, }
 
     def setUp(self):
         self.a = DHT()
@@ -251,7 +280,7 @@ class TestMultiDHT(unittest.TestCase):
                     'STORE_REDUNDANCY': 3, 'MAX_FAILURES': 3,
                     'MIN_PING_INTERVAL': 900,'BUCKET_STALENESS': 3600,
                     'KEINITIAL_DELAY': 15, 'KE_DELAY': 1200,
-                    'KE_AGE': 3600, }
+                    'KE_AGE': 3600, 'SPEW': False, }
 
     def setUp(self):
         self.l = []