2 """Serve local requests from apt and remote requests from peers."""
4 from urllib import unquote_plus
5 from binascii import b2a_hex
7 from twisted.python import log
8 from twisted.internet import defer
9 from twisted.web2 import server, http, resource, channel, stream
10 from twisted.web2 import static, http_headers, responsecode
12 from policies import ThrottlingFactory
13 from apt_p2p_Khashmir.bencode import bencode
15 class FileDownloader(static.File):
16 """Modified to make it suitable for apt requests.
18 Tries to find requests in the cache. Found files are first checked for
19 freshness before being sent. Requests for unfound and stale files are
20 forwarded to the main program for downloading.
22 @type manager: L{apt_p2p.AptP2P}
23 @ivar manager: the main program to query
26 def __init__(self, path, manager, defaultType="text/plain", ignoredExts=(), processors=None, indexNames=None):
27 self.manager = manager
28 super(FileDownloader, self).__init__(path, defaultType, ignoredExts, processors, indexNames)
30 def renderHTTP(self, req):
31 log.msg('Got request for %s from %s' % (req.uri, req.remoteAddr))
32 resp = super(FileDownloader, self).renderHTTP(req)
33 if isinstance(resp, defer.Deferred):
34 resp.addCallback(self._renderHTTP_done, req)
36 resp = self._renderHTTP_done(resp, req)
39 def _renderHTTP_done(self, resp, req):
40 log.msg('Initial response to %s: %r' % (req.uri, resp))
43 path = 'http:/' + req.uri
44 if resp.code >= 200 and resp.code < 400:
45 return self.manager.check_freshness(req, path, resp.headers.getHeader('Last-Modified'), resp)
47 log.msg('Not found, trying other methods for %s' % req.uri)
48 return self.manager.get_resp(req, path)
52 def createSimilarFile(self, path):
53 return self.__class__(path, self.manager, self.defaultType, self.ignoredExts,
54 self.processors, self.indexNames[:])
56 class FileUploaderStream(stream.FileStream):
57 """Modified to make it suitable for streaming to peers.
59 Streams the file is small chunks to make it easier to throttle the
62 @ivar CHUNK_SIZE: the size of chunks of data to send at a time
67 def read(self, sendfile=False):
76 # Remove the SendFileBuffer and mmap use, just use string reads and writes
78 readSize = min(length, self.CHUNK_SIZE)
80 self.f.seek(self.start)
81 b = self.f.read(readSize)
84 raise RuntimeError("Ran out of data reading file %r, expected %d more bytes" % (self.f, length))
86 self.length -= bytesRead
87 self.start += bytesRead
91 class FileUploader(static.File):
92 """Modified to make it suitable for peer requests.
94 Uses the modified L{FileUploaderStream} to stream the file for throttling,
95 and doesn't do any listing of directory contents.
98 def render(self, req):
99 if not self.fp.exists():
100 return responsecode.NOT_FOUND
103 # Don't try to render a directory listing
104 return responsecode.NOT_FOUND
110 if e[0] == errno.EACCES:
111 return responsecode.FORBIDDEN
112 elif e[0] == errno.ENOENT:
113 return responsecode.NOT_FOUND
117 response = http.Response()
118 # Use the modified FileStream
119 response.stream = FileUploaderStream(f, 0, self.fp.getsize())
121 for (header, value) in (
122 ("content-type", self.contentType()),
123 ("content-encoding", self.contentEncoding()),
125 if value is not None:
126 response.headers.setHeader(header, value)
130 class TopLevel(resource.Resource):
131 """The HTTP server for all requests, both from peers and apt.
133 @type directory: L{twisted.python.filepath.FilePath}
134 @ivar directory: the directory to check for cached files
136 @ivar db: the database to use for looking up files and hashes
137 @type manager: L{apt_p2p.AptP2P}
138 @ivar manager: the main program object to send requests to
139 @type factory: L{twisted.web2.channel.HTTPFactory} or L{policies.ThrottlingFactory}
140 @ivar factory: the factory to use to server HTTP requests
146 def __init__(self, directory, db, manager):
147 """Initialize the instance.
149 @type directory: L{twisted.python.filepath.FilePath}
150 @param directory: the directory to check for cached files
152 @param db: the database to use for looking up files and hashes
153 @type manager: L{apt_p2p.AptP2P}
154 @param manager: the main program object to send requests to
156 self.directory = directory
158 self.manager = manager
161 def getHTTPFactory(self):
162 """Initialize and get the factory for this HTTP server."""
163 if self.factory is None:
164 self.factory = channel.HTTPFactory(server.Site(self),
165 **{'maxPipeline': 10,
166 'betweenRequestsTimeOut': 60})
167 self.factory = ThrottlingFactory(self.factory, writeLimit = 30*1024)
170 def render(self, ctx):
171 """Render a web page with descriptive statistics."""
172 return http.Response(
174 {'content-type': http_headers.MimeType('text', 'html')},
177 <p>TODO: eventually some stats will be shown here.</body></html>""")
179 def locateChild(self, request, segments):
180 """Process the incoming request."""
181 log.msg('Got HTTP request for %s from %s' % (request.uri, request.remoteAddr))
184 # If the request is for a shared file (from a peer)
186 if len(segments) != 2:
187 log.msg('Got a malformed request from %s' % request.remoteAddr)
190 # Find the file in the database
191 hash = unquote_plus(segments[1])
192 files = self.db.lookupHash(hash)
194 # If it is a file, return it
195 if 'path' in files[0]:
196 log.msg('Sharing %s with %s' % (files[0]['path'].path, request.remoteAddr))
197 return FileUploader(files[0]['path'].path), ()
199 # It's not for a file, but for a piece string, so return that
200 log.msg('Sending torrent string %s to %s' % (b2a_hex(hash), request.remoteAddr))
201 return static.Data(bencode({'t': files[0]['pieces']}), 'application/x-bencoded'), ()
203 log.msg('Hash could not be found in database: %s' % hash)
205 # Only local requests (apt) get past this point
206 if request.remoteAddr.host != "127.0.0.1":
207 log.msg('Blocked illegal access to %s from %s' % (request.uri, request.remoteAddr))
211 # It's a request from apt
212 return FileDownloader(self.directory.path, self.manager), segments[0:]
214 # Will render the statistics page
217 log.msg('Got a malformed request for "%s" from %s' % (request.uri, request.remoteAddr))
220 if __name__ == '__builtin__':
221 # Running from twistd -ny HTTPServer.py
223 # wget -S 'http://localhost:18080/~/whatever'
224 # wget -S 'http://localhost:18080/~/pieces'
227 from twisted.python.filepath import FilePath
230 def lookupHash(self, hash):
232 return [{'pieces': 'abcdefghij0123456789\xca\xec\xb8\x0c\x00\xe7\x07\xf8~])\x8f\x9d\xe5_B\xff\x1a\xc4!'}]
233 return [{'path': FilePath(os.path.expanduser('~/school/optout'))}]
235 t = TopLevel(FilePath(os.path.expanduser('~')), DB(), None)
236 factory = t.getHTTPFactory()
238 # Standard twisted application Boilerplate
239 from twisted.application import service, strports
240 application = service.Application("demoserver")
241 s = strports.service('tcp:18080', factory)
242 s.setServiceParent(application)