From 146f1044e3661716ec4689d3ea7e6ab4ab38ae77 Mon Sep 17 00:00:00 2001 From: Cameron Dale Date: Fri, 14 Mar 2008 16:40:05 -0700 Subject: [PATCH] Display DHT statistics to the HTTP user. Adds 2 new interfaces to the DHT implementation: * DHTStats: DHT that supports gathering statistics * DHTStatsFactory: DHT that supports creating it's own HTTP server apt_dht_Khashmir supports IDHTStats and formats stats for HTML. apt_dht_Khashmir also supports IDHTStatsFactory if twisted.web2 is found. Main script passes the DHT class to the main program. Main script (DHT only) starts the stats factory if the DHT supports it. Main program retrieves statistics from the DHT when asked by the HTTPServer. --- apt-p2p.py | 13 ++++--- apt_p2p/HTTPServer.py | 7 ++-- apt_p2p/apt_p2p.py | 24 ++++++++++--- apt_p2p/interfaces.py | 23 ++++++++++++ apt_p2p_Khashmir/DHT.py | 70 ++++++++++++++++++++++++++++++++++-- apt_p2p_Khashmir/khashmir.py | 6 ++++ apt_p2p_Khashmir/stats.py | 4 +-- 7 files changed, 129 insertions(+), 18 deletions(-) diff --git a/apt-p2p.py b/apt-p2p.py index 6873204..c3d48ef 100644 --- a/apt-p2p.py +++ b/apt-p2p.py @@ -14,10 +14,9 @@ import pwd,sys from twisted.application import service, internet, app, strports from twisted.internet import reactor from twisted.python import usage, log -from twisted.web2 import channel from apt_p2p.apt_p2p_conf import config, version, DEFAULT_CONFIG_FILES -from apt_p2p.interfaces import IDHT +from apt_p2p.interfaces import IDHT, IDHTStatsFactory config_file = '' @@ -67,16 +66,22 @@ application = service.Application("apt-p2p", uid, gid) log.msg('Starting DHT') DHT = __import__(config.get('DEFAULT', 'DHT')+'.DHT', globals(), locals(), ['DHT']) assert IDHT.implementedBy(DHT.DHT), "You must provide a DHT implementation that implements the IDHT interface." -myDHT = DHT.DHT() if not config.getboolean('DEFAULT', 'DHT-only'): log.msg('Starting main application server') from apt_p2p.apt_p2p import AptP2P - myapp = AptP2P(myDHT) + myapp = AptP2P(DHT.DHT) factory = myapp.getHTTPFactory() s = strports.service('tcp:'+config.get('DEFAULT', 'port'), factory) s.setServiceParent(application) else: + myDHT = DHT.DHT() + if IDHTStatsFactory.implementedBy(DHT.DHT): + log.msg("Starting the DHT's HTTP stats displayer") + factory = myDHT.getStatsFactory() + s = strports.service('tcp:'+config.get('DEFAULT', 'port'), factory) + s.setServiceParent(application) + myDHT.loadConfig(config, config.get('DEFAULT', 'DHT')) myDHT.join() diff --git a/apt_p2p/HTTPServer.py b/apt_p2p/HTTPServer.py index d252a63..03eea99 100644 --- a/apt_p2p/HTTPServer.py +++ b/apt_p2p/HTTPServer.py @@ -137,8 +137,7 @@ class TopLevel(resource.Resource): @type manager: L{apt_p2p.AptP2P} @ivar manager: the main program object to send requests to @type factory: L{twisted.web2.channel.HTTPFactory} or L{policies.ThrottlingFactory} - @ivar factory: the factory to use to server HTTP requests - + @ivar factory: the factory to use to serve HTTP requests """ addSlash = True @@ -172,9 +171,7 @@ class TopLevel(resource.Resource): return http.Response( 200, {'content-type': http_headers.MimeType('text', 'html')}, - """ -

Statistics

TODO: eventually some stats will be shown here.""") + self.manager.getStats()) def locateChild(self, request, segments): """Process the incoming request.""" diff --git a/apt_p2p/apt_p2p.py b/apt_p2p/apt_p2p.py index 83f7c2f..4a97d4c 100644 --- a/apt_p2p/apt_p2p.py +++ b/apt_p2p/apt_p2p.py @@ -18,6 +18,7 @@ from twisted.web2 import server, http, http_headers, static from twisted.python import log, failure from twisted.python.filepath import FilePath +from interfaces import IDHT, IDHTStats from apt_p2p_conf import config from PeerManager import PeerManager from HTTPServer import TopLevel @@ -38,12 +39,14 @@ class AptP2P: Contains all of the sub-components that do all the low-level work, and coordinates communication between them. + @type dhtClass: L{interfaces.IDHT} + @ivar dhtClass: the DHT class to use @type cache_dir: L{twisted.python.filepath.FilePath} @ivar cache_dir: the directory to use for storing all files @type db: L{db.DB} @ivar db: the database to use for tracking files and hashes @type dht: L{interfaces.IDHT} - @ivar dht: the DHT instance to use + @ivar dht: the DHT instance @type http_server: L{HTTPServer.TopLevel} @ivar http_server: the web server that will handle all requests from apt and from other peers @@ -59,18 +62,19 @@ class AptP2P: download information (IP address and port) """ - def __init__(self, dht): + def __init__(self, dhtClass): """Initialize all the sub-components. @type dht: L{interfaces.IDHT} - @param dht: the DHT instance to use + @param dht: the DHT class to use """ log.msg('Initializing the main apt_p2p application') + self.dhtClass = dhtClass self.cache_dir = FilePath(config.get('DEFAULT', 'cache_dir')) if not self.cache_dir.child(download_dir).exists(): self.cache_dir.child(download_dir).makedirs() self.db = DB(self.cache_dir.child('apt-p2p.db')) - self.dht = dht + self.dht = dhtClass() self.dht.loadConfig(config, config.get('DEFAULT', 'DHT')) self.dht.join().addCallbacks(self.joinComplete, self.joinError) self.http_server = TopLevel(self.cache_dir.child(download_dir), self.db, self) @@ -124,6 +128,18 @@ class AptP2P: storeDefer.addBoth(self._refreshFiles, hashes) else: reactor.callLater(60, self.refreshFiles) + + def getStats(self): + """Retrieve and format the statistics for the program. + + @rtype: C{string} + @return: the formatted HTML page containing the statistics + """ + out = '\n\n' + if IDHTStats.implementedBy(self.dhtClass): + out += self.dht.getStats() + out += '\n\n' + return out #{ Main workflow def check_freshness(self, req, url, modtime, resp): diff --git a/apt_p2p/interfaces.py b/apt_p2p/interfaces.py index b38de39..f72cd8e 100644 --- a/apt_p2p/interfaces.py +++ b/apt_p2p/interfaces.py @@ -41,3 +41,26 @@ class IDHT(Interface): The length of the key may be adjusted for use with the DHT. """ + +class IDHTStats(Interface): + """An abstract interface for DHTs that support statistics gathering.""" + + def getStats(self): + """Gather and format all the statistics for the DHT. + + The statistics will be formatted for inclusion in the body + of an HTML page. + + @rtype: C{string} + @return: the formatted statistics, suitable for displaying to the user + """ + +class IDHTStatsFactory(Interface): + """An abstract interface for DHTs that support statistics displaying.""" + + def getStatsFactory(self): + """Create and return an HTTP factory for displaying statistics. + + @rtype: + """ + \ No newline at end of file diff --git a/apt_p2p_Khashmir/DHT.py b/apt_p2p_Khashmir/DHT.py index 399babf..d0a40ee 100644 --- a/apt_p2p_Khashmir/DHT.py +++ b/apt_p2p_Khashmir/DHT.py @@ -5,6 +5,7 @@ """ from datetime import datetime +from StringIO import StringIO import os, sha, random from twisted.internet import defer, reactor @@ -13,10 +14,16 @@ from twisted.python import log from twisted.trial import unittest from zope.interface import implements -from apt_p2p.interfaces import IDHT +from apt_p2p.interfaces import IDHT, IDHTStats, IDHTStatsFactory from khashmir import Khashmir from bencode import bencode, bdecode +try: + from twisted.web2 import channel, server, resource, http, http_headers + _web2 = True +except ImportError: + _web2 = False + khashmir_dir = 'apt-p2p-Khashmir' class DHTError(Exception): @@ -51,6 +58,8 @@ class DHT: @type retrieved: C{dictionary} @ivar retrieved: keys are the keys for which getValue requests are active, values are list of the values returned so far + @type factory: L{twisted.web2.channel.HTTPFactory} + @ivar factory: the factory to use to serve HTTP requests for statistics @type config_parser: L{apt_p2p.apt_p2p_conf.AptP2PConfigParser} @ivar config_parser: the configuration info for the main program @type section: C{string} @@ -59,8 +68,11 @@ class DHT: @ivar khashmir: the khashmir DHT instance to use """ - implements(IDHT) - + if _web2: + implements(IDHT, IDHTStats, IDHTStatsFactory) + else: + implements(IDHT, IDHTStats) + def __init__(self): """Initialize the DHT.""" self.config = None @@ -74,6 +86,7 @@ class DHT: self.storing = {} self.retrieving = {} self.retrieved = {} + self.factory = None def loadConfig(self, config, section): """See L{apt_p2p.interfaces.IDHT}.""" @@ -260,6 +273,57 @@ class DHT: del self.storing[key][bvalue] if len(self.storing[key].keys()) == 0: del self.storing[key] + + def getStats(self): + """See L{apt_p2p.interfaces.IDHTStats}.""" + stats = self.khashmir.getStats() + out = StringIO() + out.write('

DHT Statistics

\n') + old_group = None + for stat in stats: + if stat['group'] != old_group: + if old_group is not None: + out.write('\n') + out.write('\n

' + stat['group'] + '

\n') + out.write("\n") + if stat['group'] != 'Actions': + out.write("\n") + else: + out.write("\n") + old_group = stat['group'] + if stat['group'] != 'Actions': + out.write("\n') + else: + actions = stat['value'].keys() + actions.sort() + for action in actions: + out.write("") + for i in xrange(5): + out.write("") + out.write('\n') + + return out.getvalue() + + def getStatsFactory(self): + """See L{apt_p2p.interfaces.IDHTStatsFactory}.""" + assert _web2, "NOT IMPLEMENTED: twisted.web2 must be installed to use the stats factory." + if self.factory is None: + # Create a simple HTTP factory for stats + class StatsResource(resource.Resource): + def __init__(self, manager): + self.manager = manager + def render(self, ctx): + return http.Response( + 200, + {'content-type': http_headers.MimeType('text', 'html')}, + '\n\n' + self.manager.getStats() + '\n\n') + def locateChild(self, request, segments): + log.msg('Got HTTP stats request from %s' % (request.remoteAddr, )) + return self, () + + self.factory = channel.HTTPFactory(server.Site(StatsResource(self))) + return self.factory + class TestSimpleDHT(unittest.TestCase): """Simple 2-node unit tests for the DHT.""" diff --git a/apt_p2p_Khashmir/khashmir.py b/apt_p2p_Khashmir/khashmir.py index 9bd3140..6734ff7 100644 --- a/apt_p2p_Khashmir/khashmir.py +++ b/apt_p2p_Khashmir/khashmir.py @@ -40,6 +40,8 @@ class KhashmirBase(protocol.Factory): @ivar table: the routing table @type token_secrets: C{list} of C{string} @ivar token_secrets: the current secrets to use to create tokens + @type stats: L{stats.StatsLogger} + @ivar stats: the statistics gatherer @type udp: L{krpc.hostbroker} @ivar udp: the factory for the KRPC protocol @type listenport: L{twisted.internet.interfaces.IListeningPort} @@ -294,6 +296,10 @@ class KhashmirBase(protocol.Factory): except: pass self.store.close() + + def getStats(self): + """Gather the statistics for the DHT.""" + return self.stats.gather() #{ Remote interface def krpc_ping(self, id, _krpc_sender): diff --git a/apt_p2p_Khashmir/stats.py b/apt_p2p_Khashmir/stats.py index 7a40adb..aeb979e 100644 --- a/apt_p2p_Khashmir/stats.py +++ b/apt_p2p_Khashmir/stats.py @@ -171,9 +171,9 @@ class StatsLogger: for stat in stats: val = getattr(self, stat['name'], None) if stat['name'] == 'uptime': - stat['value'] = dattime.now() - self.startTime + stat['value'] = datetime.now() - self.startTime elif stat['name'] == 'actions': - stat['value'] = deepcopy(actions) + stat['value'] = deepcopy(self.actions) elif val is not None: stat['value'] = val -- 2.30.2

Statistic	Value
Action	Sent	OK	Failed	Received	Error
" + stat['desc'] + '	' + str(stat['value']) + '
" + action + "	" + str(stat['value'][action][i]) + "