X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;f=apt_dht%2FHTTPServer.py;h=82a9babb2ca551f95d71056616661f38b9c4e186;hb=671642b5862a8309c342848843c1ab41d2e515a7;hp=0e53c237ef175bdca4934bf335b05300ca8e337b;hpb=8297bc9a2aa8132ea1a7363761d9d5c73a1efca2;p=quix0rs-apt-p2p.git diff --git a/apt_dht/HTTPServer.py b/apt_dht/HTTPServer.py index 0e53c23..82a9bab 100644 --- a/apt_dht/HTTPServer.py +++ b/apt_dht/HTTPServer.py @@ -1,10 +1,27 @@ +"""Serve local requests from apt and remote requests from peers.""" + +from urllib import unquote_plus +from binascii import b2a_hex + from twisted.python import log from twisted.internet import defer -from twisted.web2 import server, http, resource, channel +from twisted.web2 import server, http, resource, channel, stream from twisted.web2 import static, http_headers, responsecode +from policies import ThrottlingFactory +from apt_dht_Khashmir.bencode import bencode + class FileDownloader(static.File): + """Modified to make it suitable for apt requests. + + Tries to find requests in the cache. Found files are first checked for + freshness before being sent. Requests for unfound and stale files are + forwarded to the main program for downloading. + + @type manager: L{apt_dht.AptDHT} + @ivar manager: the main program to query + """ def __init__(self, path, manager, defaultType="text/plain", ignoredExts=(), processors=None, indexNames=None): self.manager = manager @@ -25,10 +42,10 @@ class FileDownloader(static.File): if self.manager: path = 'http:/' + req.uri if resp.code >= 200 and resp.code < 400: - return self.manager.check_freshness(path, resp.headers.getHeader('Last-Modified'), resp) + return self.manager.check_freshness(req, path, resp.headers.getHeader('Last-Modified'), resp) log.msg('Not found, trying other methods for %s' % req.uri) - return self.manager.get_resp(path) + return self.manager.get_resp(req, path) return resp @@ -36,23 +53,122 @@ class FileDownloader(static.File): return self.__class__(path, self.manager, self.defaultType, self.ignoredExts, self.processors, self.indexNames[:]) +class FileUploaderStream(stream.FileStream): + """Modified to make it suitable for streaming to peers. + + Streams the file is small chunks to make it easier to throttle the + streaming to peers. + + @ivar CHUNK_SIZE: the size of chunks of data to send at a time + """ + + CHUNK_SIZE = 4*1024 + + def read(self, sendfile=False): + if self.f is None: + return None + + length = self.length + if length == 0: + self.f = None + return None + # Remove the SendFileBuffer and mmap use, just use string reads and writes + + readSize = min(length, self.CHUNK_SIZE) + + self.f.seek(self.start) + b = self.f.read(readSize) + bytesRead = len(b) + if not bytesRead: + raise RuntimeError("Ran out of data reading file %r, expected %d more bytes" % (self.f, length)) + else: + self.length -= bytesRead + self.start += bytesRead + return b + + +class FileUploader(static.File): + """Modified to make it suitable for peer requests. + + Uses the modified L{FileUploaderStream} to stream the file for throttling, + and doesn't do any listing of directory contents. + """ + + def render(self, req): + if not self.fp.exists(): + return responsecode.NOT_FOUND + + if self.fp.isdir(): + # Don't try to render a directory listing + return responsecode.NOT_FOUND + + try: + f = self.fp.open() + except IOError, e: + import errno + if e[0] == errno.EACCES: + return responsecode.FORBIDDEN + elif e[0] == errno.ENOENT: + return responsecode.NOT_FOUND + else: + raise + + response = http.Response() + # Use the modified FileStream + response.stream = FileUploaderStream(f, 0, self.fp.getsize()) + + for (header, value) in ( + ("content-type", self.contentType()), + ("content-encoding", self.contentEncoding()), + ): + if value is not None: + response.headers.setHeader(header, value) + + return response + class TopLevel(resource.Resource): + """The HTTP server for all requests, both from peers and apt. + + @type directory: L{twisted.python.filepath.FilePath} + @ivar directory: the directory to check for cached files + @type db: L{db.DB} + @ivar db: the database to use for looking up files and hashes + @type manager: L{apt_dht.AptDHT} + @ivar manager: the main program object to send requests to + @type factory: L{twisted.web2.channel.HTTPFactory} or L{policies.ThrottlingFactory} + @ivar factory: the factory to use to server HTTP requests + + """ + addSlash = True - def __init__(self, directory, manager): + def __init__(self, directory, db, manager): + """Initialize the instance. + + @type directory: L{twisted.python.filepath.FilePath} + @param directory: the directory to check for cached files + @type db: L{db.DB} + @param db: the database to use for looking up files and hashes + @type manager: L{apt_dht.AptDHT} + @param manager: the main program object to send requests to + """ self.directory = directory + self.db = db self.manager = manager - self.subdirs = {} - - def setDirectories(self, dirs): - self.subdirs = {} - for k in dirs: - # Don't allow empty subdirectory - if k: - self.subdirs[k] = dirs[k] - + self.factory = None + + def getHTTPFactory(self): + """Initialize and get the factory for this HTTP server.""" + if self.factory is None: + self.factory = channel.HTTPFactory(server.Site(self), + **{'maxPipeline': 10, + 'betweenRequestsTimeOut': 60}) + self.factory = ThrottlingFactory(self.factory, writeLimit = 30*1024) + return self.factory + def render(self, ctx): + """Render a web page with descriptive statistics.""" return http.Response( 200, {'content-type': http_headers.MimeType('text', 'html')}, @@ -61,29 +177,66 @@ class TopLevel(resource.Resource):
TODO: eventually some stats will be shown here.