]> git.mxchange.org Git - quix0rs-apt-p2p.git/commitdiff
Added complicated testing to find our IP address.
authorCameron Dale <camrdale@gmail.com>
Fri, 11 Jan 2008 03:31:06 +0000 (19:31 -0800)
committerCameron Dale <camrdale@gmail.com>
Fri, 11 Jan 2008 03:31:06 +0000 (19:31 -0800)
apt-dht.py
apt_dht/MirrorManager.py
apt_dht/apt_dht.py
apt_dht_Khashmir/DHT.py
apt_dht_Khashmir/khashmir.py

index 3cc5d94f0ef502fc8fc416e44d4b24a918ff8feb..70a12640b7c7f5481bd86cdb327c80587a1d9853 100644 (file)
@@ -59,8 +59,6 @@ log.msg('Starting DHT')
 DHT = __import__(config.get('DEFAULT', 'DHT')+'.DHT', globals(), locals(), ['DHT'])
 assert IDHT.implementedBy(DHT.DHT), "You must provide a DHT implementation that implements the IDHT interface."
 myDHT = DHT.DHT()
-myDHT.loadConfig(config, config.get('DEFAULT', 'DHT'))
-myDHT.join()
 
 if not config.getboolean('DEFAULT', 'DHT-only'):
     log.msg('Starting main application server')
@@ -69,6 +67,9 @@ if not config.getboolean('DEFAULT', 'DHT-only'):
     site = myapp.getSite()
     s = strports.service('tcp:'+config.get('DEFAULT', 'port'), channel.HTTPFactory(site))
     s.setServiceParent(application)
+else:
+    myDHT.loadConfig(config, config.get('DEFAULT', 'DHT'))
+    myDHT.join()
 
 if __name__ == '__main__':
     # Run on command line
index dd1b163bf848b24693df4c05f2648f8c437cb469..70138e69e22ed6823f64e8d9ffa53651f28f2993 100644 (file)
@@ -149,7 +149,7 @@ class ProxyFileStream(stream.SimpleStream):
 class MirrorManager:
     """Manages all requests for mirror objects."""
     
-    def __init__(self, manager, cache_dir):
+    def __init__(self, cache_dir, manager = None):
         self.manager = manager
         self.cache_dir = cache_dir
         self.cache = filepath.FilePath(self.cache_dir)
@@ -257,8 +257,9 @@ class MirrorManager:
             self.updatedFile(url, destFile.path)
             if ext:
                 self.updatedFile(url[:-len(ext)], decFile.path)
-                
-            self.manager.download_complete(hash, url, destFile.path)
+            
+            if self.manager:
+                self.manager.download_complete(hash, url, destFile.path)
         else:
             log.msg("Hashes don't match %s != %s: %s" % (hash.hexexpected(), hash.hexdigest(), url))
 
index c3f1a768c5b2b85db6d90cab6c6927e2e78b5244..9fc72ffedafe871cef7faed2967395adb2ce6de2 100644 (file)
@@ -1,6 +1,6 @@
 
 from binascii import b2a_hex
-import os.path
+import os, re
 
 from twisted.internet import defer
 from twisted.web2 import server, http, http_headers
@@ -16,13 +16,113 @@ class AptDHT:
     def __init__(self, dht):
         log.msg('Initializing the main apt_dht application')
         self.dht = dht
+        self.dht.loadConfig(config, config.get('DEFAULT', 'DHT'))
+        self.dht.join().addCallbacks(self.joinComplete, self.joinError)
         self.http_server = TopLevel(config.get('DEFAULT', 'cache_dir'), self)
         self.http_site = server.Site(self.http_server)
         self.peers = PeerManager()
-        self.mirrors = MirrorManager(self, config.get('DEFAULT', 'cache_dir'))
+        self.mirrors = MirrorManager(config.get('DEFAULT', 'cache_dir'), self)
+        self.my_addr = None
+        self.isLocal = re.compile('^(192\.168\.[0-9]{1,3}\.[0-9]{1,3})|'+
+                                  '(10\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})|'+
+                                  '(172\.0?([1][6-9])|([2][0-9])|([3][0-1])\.[0-9]{1,3}\.[0-9]{1,3})|'+
+                                  '(127\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})$')
     
     def getSite(self):
         return self.http_site
+
+    def joinComplete(self, addrs):
+        log.msg("got addrs: %r" % (addrs,))
+        
+        try:
+            ifconfig = os.popen("/sbin/ifconfig |/bin/grep inet|"+
+                                "/usr/bin/awk '{print $2}' | "+
+                                "sed -e s/.*://", "r").read().strip().split('\n')
+        except:
+            ifconfig = []
+
+        # Get counts for all the non-local addresses returned
+        addr_count = {}
+        for addr in ifconfig:
+            if not self.isLocal.match(addr):
+                addr_count.setdefault(addr, 0)
+                addr_count[addr] += 1
+        
+        local_addrs = addr_count.keys()    
+        if len(local_addrs) == 1:
+            self.my_addr = local_addrs[0]
+            log.msg('Found remote address from ifconfig: %r' % (self.my_addr,))
+        
+        # Get counts for all the non-local addresses returned
+        addr_count = {}
+        port_count = {}
+        for addr in addrs:
+            if not self.isLocal.match(addr[0]):
+                addr_count.setdefault(addr[0], 0)
+                addr_count[addr[0]] += 1
+                port_count.setdefault(addr[1], 0)
+                port_count[addr[1]] += 1
+        
+        # Find the most popular address
+        popular_addr = []
+        popular_count = 0
+        for addr in addr_count:
+            if addr_count[addr] > popular_count:
+                popular_addr = [addr]
+                popular_count = addr_count[addr]
+            elif addr_count[addr] == popular_count:
+                popular_addr.append(addr)
+        
+        # Find the most popular port
+        popular_port = []
+        popular_count = 0
+        for port in port_count:
+            if port_count[port] > popular_count:
+                popular_port = [port]
+                popular_count = port_count[port]
+            elif port_count[port] == popular_count:
+                popular_port.append(port)
+                
+        port = config.getint(config.get('DEFAULT', 'DHT'), 'PORT')
+        if len(port_count.keys()) > 1:
+            log.msg('Problem, multiple ports have been found: %r' % (port_count,))
+            if port not in port_count.keys():
+                log.msg('And none of the ports found match the intended one')
+        elif len(port_count.keys()) == 1:
+            port = port_count.keys()[0]
+        else:
+            log.msg('Port was not found')
+
+        if len(popular_addr) == 1:
+            log.msg('Found popular address: %r' % (popular_addr[0],))
+            if self.my_addr and self.my_addr != popular_addr[0]:
+                log.msg('But the popular address does not match: %s != %s' % (popular_addr[0], self.my_addr))
+            self.my_addr = popular_addr[0]
+        elif len(popular_addr) > 1:
+            log.msg('Found multiple popular addresses: %r' % (popular_addr,))
+            if self.my_addr and self.my_addr not in popular_addr:
+                log.msg('And none of the addresses found match the ifconfig one')
+        else:
+            log.msg('No non-local addresses found: %r' % (popular_addr,))
+            
+        if not self.my_addr:
+            log.err(RuntimeError("Remote IP Address could not be found for this machine"))
+
+    def ipAddrFromChicken(self):
+        import urllib
+        ip_search = re.compile('\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}')
+        try:
+             f = urllib.urlopen("http://www.ipchicken.com")
+             data = f.read()
+             f.close()
+             current_ip = ip_search.findall(data)
+             return current_ip
+        except Exception:
+             return []
+
+    def joinError(self, failure):
+        log.msg("joining DHT failed miserably")
+        log.err(failure)
     
     def check_freshness(self, path, modtime, resp):
         log.msg('Checking if %s is still fresh' % path)
index 933c35300a58b93a77de0574fbffa6bf5833af2b..8930ffe9d58c40e83e169d0ceb3b964ed458cb10 100644 (file)
@@ -24,6 +24,7 @@ class DHT:
         self.bootstrap_node = False
         self.joining = None
         self.joined = False
+        self.outstandingJoins = 0
         self.foundAddrs = []
         self.storing = {}
         self.retrieving = {}
@@ -71,23 +72,35 @@ class DHT:
 
     def _join_gotIP(self, ip, port):
         """Called after an IP address has been found for a single bootstrap node."""
-        self.khashmir.addContact(ip, port, self._join_single)
+        self.outstandingJoins += 1
+        self.khashmir.addContact(ip, port, self._join_single, self._join_error)
     
     def _join_single(self, addr):
         """Called when a single bootstrap node has been added."""
+        self.outstandingJoins -= 1
         if addr:
             self.foundAddrs.append(addr)
+        if addr or self.outstandingJoins <= 0:
+            self.khashmir.findCloseNodes(self._join_complete, self._join_complete)
         log.msg('Got back from bootstrap node: %r' % (addr,))
-        self.khashmir.findCloseNodes(self._join_complete)
     
+    def _join_error(self, failure = None):
+        """Called when a single bootstrap node has failed."""
+        self.outstandingJoins -= 1
+        log.msg("bootstrap node could not be reached")
+        if self.outstandingJoins <= 0:
+            self.khashmir.findCloseNodes(self._join_complete, self._join_complete)
+
     def _join_complete(self, result):
         """Called when the tables have been initialized with nodes."""
-        if not self.joined:
+        if not self.joined and len(result) > 0:
             self.joined = True
+        if self.joining and self.outstandingJoins <= 0:
             df = self.joining
             self.joining = None
-            if len(result) > 0 or self.bootstrap_node:
-                df.callback(result)
+            if self.joined or self.bootstrap_node:
+                self.joined = True
+                df.callback(self.foundAddrs)
             else:
                 df.errback(DHTError('could not find any nodes to bootstrap to'))
         
index 80d811878260d54ac3f76bc69ce2150fb0063bf8..8c75c48df2727c4d98a60db575bfe24367eb3df3 100644 (file)
@@ -80,12 +80,12 @@ class KhashmirBase(protocol.Factory):
 
     #######
     #######  LOCAL INTERFACE    - use these methods!
-    def addContact(self, host, port, callback=None):
+    def addContact(self, host, port, callback=None, errback=None):
         """
             ping this node and add the contact info to the table on pong!
         """
         n = self.Node(NULL_ID, host, port)
-        self.sendJoin(n, callback=callback)
+        self.sendJoin(n, callback=callback, errback=errback)
 
     ## this call is async!
     def findNode(self, id, callback, errback=None):
@@ -133,7 +133,7 @@ class KhashmirBase(protocol.Factory):
             df = old.ping(self.node.id)
             df.addCallbacks(_notStaleNodeHandler, _staleNodeHandler)
 
-    def sendJoin(self, node, callback=None):
+    def sendJoin(self, node, callback=None, errback=None):
         """
             ping a node
         """
@@ -144,21 +144,23 @@ class KhashmirBase(protocol.Factory):
             self.insertNode(n)
             if callback:
                 callback((dict['rsp']['ip_addr'], dict['rsp']['port']))
-        def _defaultPong(err, node=node, table=self.table, callback=callback):
+        def _defaultPong(err, node=node, table=self.table, callback=callback, errback=errback):
             table.nodeFailed(node)
-            if callback:
+            if errback:
+                errback()
+            else:
                 callback(None)
         
         df.addCallbacks(_pongHandler,_defaultPong)
 
-    def findCloseNodes(self, callback=lambda a: None):
+    def findCloseNodes(self, callback=lambda a: None, errback = None):
         """
             This does a findNode on the ID one away from our own.  
             This will allow us to populate our table with nodes on our network closest to our own.
             This is called as soon as we start up with an empty table
         """
         id = self.node.id[:-1] + chr((ord(self.node.id[-1]) + 1) % 256)
-        self.findNode(id, callback)
+        self.findNode(id, callback, errback)
 
     def refreshTable(self, force=0):
         """