From 842adfabdd1ea04a07998c0bfbbb21ffa6ecbfad Mon Sep 17 00:00:00 2001 From: Cameron Dale Date: Thu, 10 Jan 2008 19:31:06 -0800 Subject: [PATCH] Added complicated testing to find our IP address. --- apt-dht.py | 5 +- apt_dht/MirrorManager.py | 7 ++- apt_dht/apt_dht.py | 104 ++++++++++++++++++++++++++++++++++- apt_dht_Khashmir/DHT.py | 23 ++++++-- apt_dht_Khashmir/khashmir.py | 16 +++--- 5 files changed, 136 insertions(+), 19 deletions(-) diff --git a/apt-dht.py b/apt-dht.py index 3cc5d94..70a1264 100644 --- a/apt-dht.py +++ b/apt-dht.py @@ -59,8 +59,6 @@ log.msg('Starting DHT') DHT = __import__(config.get('DEFAULT', 'DHT')+'.DHT', globals(), locals(), ['DHT']) assert IDHT.implementedBy(DHT.DHT), "You must provide a DHT implementation that implements the IDHT interface." myDHT = DHT.DHT() -myDHT.loadConfig(config, config.get('DEFAULT', 'DHT')) -myDHT.join() if not config.getboolean('DEFAULT', 'DHT-only'): log.msg('Starting main application server') @@ -69,6 +67,9 @@ if not config.getboolean('DEFAULT', 'DHT-only'): site = myapp.getSite() s = strports.service('tcp:'+config.get('DEFAULT', 'port'), channel.HTTPFactory(site)) s.setServiceParent(application) +else: + myDHT.loadConfig(config, config.get('DEFAULT', 'DHT')) + myDHT.join() if __name__ == '__main__': # Run on command line diff --git a/apt_dht/MirrorManager.py b/apt_dht/MirrorManager.py index dd1b163..70138e6 100644 --- a/apt_dht/MirrorManager.py +++ b/apt_dht/MirrorManager.py @@ -149,7 +149,7 @@ class ProxyFileStream(stream.SimpleStream): class MirrorManager: """Manages all requests for mirror objects.""" - def __init__(self, manager, cache_dir): + def __init__(self, cache_dir, manager = None): self.manager = manager self.cache_dir = cache_dir self.cache = filepath.FilePath(self.cache_dir) @@ -257,8 +257,9 @@ class MirrorManager: self.updatedFile(url, destFile.path) if ext: self.updatedFile(url[:-len(ext)], decFile.path) - - self.manager.download_complete(hash, url, destFile.path) + + if self.manager: + self.manager.download_complete(hash, url, destFile.path) else: log.msg("Hashes don't match %s != %s: %s" % (hash.hexexpected(), hash.hexdigest(), url)) diff --git a/apt_dht/apt_dht.py b/apt_dht/apt_dht.py index c3f1a76..9fc72ff 100644 --- a/apt_dht/apt_dht.py +++ b/apt_dht/apt_dht.py @@ -1,6 +1,6 @@ from binascii import b2a_hex -import os.path +import os, re from twisted.internet import defer from twisted.web2 import server, http, http_headers @@ -16,13 +16,113 @@ class AptDHT: def __init__(self, dht): log.msg('Initializing the main apt_dht application') self.dht = dht + self.dht.loadConfig(config, config.get('DEFAULT', 'DHT')) + self.dht.join().addCallbacks(self.joinComplete, self.joinError) self.http_server = TopLevel(config.get('DEFAULT', 'cache_dir'), self) self.http_site = server.Site(self.http_server) self.peers = PeerManager() - self.mirrors = MirrorManager(self, config.get('DEFAULT', 'cache_dir')) + self.mirrors = MirrorManager(config.get('DEFAULT', 'cache_dir'), self) + self.my_addr = None + self.isLocal = re.compile('^(192\.168\.[0-9]{1,3}\.[0-9]{1,3})|'+ + '(10\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})|'+ + '(172\.0?([1][6-9])|([2][0-9])|([3][0-1])\.[0-9]{1,3}\.[0-9]{1,3})|'+ + '(127\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})$') def getSite(self): return self.http_site + + def joinComplete(self, addrs): + log.msg("got addrs: %r" % (addrs,)) + + try: + ifconfig = os.popen("/sbin/ifconfig |/bin/grep inet|"+ + "/usr/bin/awk '{print $2}' | "+ + "sed -e s/.*://", "r").read().strip().split('\n') + except: + ifconfig = [] + + # Get counts for all the non-local addresses returned + addr_count = {} + for addr in ifconfig: + if not self.isLocal.match(addr): + addr_count.setdefault(addr, 0) + addr_count[addr] += 1 + + local_addrs = addr_count.keys() + if len(local_addrs) == 1: + self.my_addr = local_addrs[0] + log.msg('Found remote address from ifconfig: %r' % (self.my_addr,)) + + # Get counts for all the non-local addresses returned + addr_count = {} + port_count = {} + for addr in addrs: + if not self.isLocal.match(addr[0]): + addr_count.setdefault(addr[0], 0) + addr_count[addr[0]] += 1 + port_count.setdefault(addr[1], 0) + port_count[addr[1]] += 1 + + # Find the most popular address + popular_addr = [] + popular_count = 0 + for addr in addr_count: + if addr_count[addr] > popular_count: + popular_addr = [addr] + popular_count = addr_count[addr] + elif addr_count[addr] == popular_count: + popular_addr.append(addr) + + # Find the most popular port + popular_port = [] + popular_count = 0 + for port in port_count: + if port_count[port] > popular_count: + popular_port = [port] + popular_count = port_count[port] + elif port_count[port] == popular_count: + popular_port.append(port) + + port = config.getint(config.get('DEFAULT', 'DHT'), 'PORT') + if len(port_count.keys()) > 1: + log.msg('Problem, multiple ports have been found: %r' % (port_count,)) + if port not in port_count.keys(): + log.msg('And none of the ports found match the intended one') + elif len(port_count.keys()) == 1: + port = port_count.keys()[0] + else: + log.msg('Port was not found') + + if len(popular_addr) == 1: + log.msg('Found popular address: %r' % (popular_addr[0],)) + if self.my_addr and self.my_addr != popular_addr[0]: + log.msg('But the popular address does not match: %s != %s' % (popular_addr[0], self.my_addr)) + self.my_addr = popular_addr[0] + elif len(popular_addr) > 1: + log.msg('Found multiple popular addresses: %r' % (popular_addr,)) + if self.my_addr and self.my_addr not in popular_addr: + log.msg('And none of the addresses found match the ifconfig one') + else: + log.msg('No non-local addresses found: %r' % (popular_addr,)) + + if not self.my_addr: + log.err(RuntimeError("Remote IP Address could not be found for this machine")) + + def ipAddrFromChicken(self): + import urllib + ip_search = re.compile('\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}') + try: + f = urllib.urlopen("http://www.ipchicken.com") + data = f.read() + f.close() + current_ip = ip_search.findall(data) + return current_ip + except Exception: + return [] + + def joinError(self, failure): + log.msg("joining DHT failed miserably") + log.err(failure) def check_freshness(self, path, modtime, resp): log.msg('Checking if %s is still fresh' % path) diff --git a/apt_dht_Khashmir/DHT.py b/apt_dht_Khashmir/DHT.py index 933c353..8930ffe 100644 --- a/apt_dht_Khashmir/DHT.py +++ b/apt_dht_Khashmir/DHT.py @@ -24,6 +24,7 @@ class DHT: self.bootstrap_node = False self.joining = None self.joined = False + self.outstandingJoins = 0 self.foundAddrs = [] self.storing = {} self.retrieving = {} @@ -71,23 +72,35 @@ class DHT: def _join_gotIP(self, ip, port): """Called after an IP address has been found for a single bootstrap node.""" - self.khashmir.addContact(ip, port, self._join_single) + self.outstandingJoins += 1 + self.khashmir.addContact(ip, port, self._join_single, self._join_error) def _join_single(self, addr): """Called when a single bootstrap node has been added.""" + self.outstandingJoins -= 1 if addr: self.foundAddrs.append(addr) + if addr or self.outstandingJoins <= 0: + self.khashmir.findCloseNodes(self._join_complete, self._join_complete) log.msg('Got back from bootstrap node: %r' % (addr,)) - self.khashmir.findCloseNodes(self._join_complete) + def _join_error(self, failure = None): + """Called when a single bootstrap node has failed.""" + self.outstandingJoins -= 1 + log.msg("bootstrap node could not be reached") + if self.outstandingJoins <= 0: + self.khashmir.findCloseNodes(self._join_complete, self._join_complete) + def _join_complete(self, result): """Called when the tables have been initialized with nodes.""" - if not self.joined: + if not self.joined and len(result) > 0: self.joined = True + if self.joining and self.outstandingJoins <= 0: df = self.joining self.joining = None - if len(result) > 0 or self.bootstrap_node: - df.callback(result) + if self.joined or self.bootstrap_node: + self.joined = True + df.callback(self.foundAddrs) else: df.errback(DHTError('could not find any nodes to bootstrap to')) diff --git a/apt_dht_Khashmir/khashmir.py b/apt_dht_Khashmir/khashmir.py index 80d8118..8c75c48 100644 --- a/apt_dht_Khashmir/khashmir.py +++ b/apt_dht_Khashmir/khashmir.py @@ -80,12 +80,12 @@ class KhashmirBase(protocol.Factory): ####### ####### LOCAL INTERFACE - use these methods! - def addContact(self, host, port, callback=None): + def addContact(self, host, port, callback=None, errback=None): """ ping this node and add the contact info to the table on pong! """ n = self.Node(NULL_ID, host, port) - self.sendJoin(n, callback=callback) + self.sendJoin(n, callback=callback, errback=errback) ## this call is async! def findNode(self, id, callback, errback=None): @@ -133,7 +133,7 @@ class KhashmirBase(protocol.Factory): df = old.ping(self.node.id) df.addCallbacks(_notStaleNodeHandler, _staleNodeHandler) - def sendJoin(self, node, callback=None): + def sendJoin(self, node, callback=None, errback=None): """ ping a node """ @@ -144,21 +144,23 @@ class KhashmirBase(protocol.Factory): self.insertNode(n) if callback: callback((dict['rsp']['ip_addr'], dict['rsp']['port'])) - def _defaultPong(err, node=node, table=self.table, callback=callback): + def _defaultPong(err, node=node, table=self.table, callback=callback, errback=errback): table.nodeFailed(node) - if callback: + if errback: + errback() + else: callback(None) df.addCallbacks(_pongHandler,_defaultPong) - def findCloseNodes(self, callback=lambda a: None): + def findCloseNodes(self, callback=lambda a: None, errback = None): """ This does a findNode on the ID one away from our own. This will allow us to populate our table with nodes on our network closest to our own. This is called as soon as we start up with an empty table """ id = self.node.id[:-1] + chr((ord(self.node.id[-1]) + 1) % 256) - self.findNode(id, callback) + self.findNode(id, callback, errback) def refreshTable(self, force=0): """ -- 2.39.2