Added modification time tracking to the MirrorManager cache.

Had to switch the HTTPServer to use renderHTTP() instead of
render() so that the last-modified HTTP header would be set.

diff --git a/apt_dht/HTTPDownloader.py b/apt_dht/HTTPDownloader.py
index f43b6e1c6f8f041b1d405acd3b5102f8e7a03460..91ce7c14c598cb70af1cb545547225674754ea32 100644 (file)
--- a/apt_dht/HTTPDownloader.py
+++ b/apt_dht/HTTPDownloader.py
@@ -76,12 +76,11 @@ class HTTPClientManager(ClientFactory):
         req = self.request_queue.pop(0)
         self.response_queue.append(req)
         req.deferResponse = self.proto.submitRequest(req, False)
-        req.deferResponse.addCallback(self.requestComplete)
-        req.deferResponse.addErrback(self.requestError)
+        req.deferResponse.addCallbacks(self.requestComplete, self.requestError)
 
     def requestComplete(self, resp):
         req = self.response_queue.pop(0)
-        log.msg('Download of %s completed with code %d' % (req.uri, resp.code))
+        log.msg('%s of %s completed with code %d' % (req.method, req.uri, resp.code))
         req.deferRequest.callback(resp)
 
     def requestError(self, error):

diff --git a/apt_dht/HTTPServer.py b/apt_dht/HTTPServer.py
index 1e049c392a789a0f4250bf6a05ef8664e8f00709..181da4e4575f20a1ab66a6fba71f70dd62b71db2 100644 (file)
--- a/apt_dht/HTTPServer.py
+++ b/apt_dht/HTTPServer.py
@@ -1,6 +1,7 @@
 import os.path, time
 
 from twisted.python import log
+from twisted.internet import defer
 from twisted.web2 import server, http, resource, channel
 from twisted.web2 import static, http_headers, responsecode
 
@@ -10,15 +11,21 @@ class FileDownloader(static.File):
         self.manager = manager
         super(FileDownloader, self).__init__(path, defaultType, ignoredExts, processors, indexNames)
         
-    def render(self, req):
+    def renderHTTP(self, req):
         log.msg('Got request for %s from %s' % (req.uri, req.remoteAddr))
-        resp = super(FileDownloader, self).render(req)
+        resp = super(FileDownloader, self).renderHTTP(req)
+        if isinstance(resp, defer.Deferred):
+            resp.addCallback(self._renderHTTP_done, req)
+        else:
+            resp = self._renderHTTP_done(resp, req)
+        return resp
+        
+    def _renderHTTP_done(self, resp, req):
         log.msg('Initial response to %s: %r' % (req.uri, resp))
         
         if self.manager:
             path = 'http:/' + req.uri
-            if resp != responsecode.NOT_FOUND:
-                log.msg('Checking freshness of %s' % req.uri)
+            if resp.code >= 200 and resp.code < 400:
                 return self.manager.check_freshness(path, resp.headers.getHeader('Last-Modified'), resp)
             
             log.msg('Not found, trying other methods for %s' % req.uri)

diff --git a/apt_dht/MirrorManager.py b/apt_dht/MirrorManager.py
index e39e99b3f408bdde057d70eeeafc2593798658b8..b4995e9445a1a9ca175399430b4196b5582a1e1e 100644 (file)
--- a/apt_dht/MirrorManager.py
+++ b/apt_dht/MirrorManager.py
@@ -18,26 +18,33 @@ class MirrorError(Exception):
 class ProxyFileStream(stream.SimpleStream):
     """Saves a stream to a file while providing a new stream."""
     
-    def __init__(self, stream, outFile):
+    def __init__(self, stream, outFile, modtime = None):
         """Initializes the proxy.
         
         @type stream: C{twisted.web2.stream.IByteStream}
         @param stream: the input stream to read from
         @type outFile: C{twisted.python.filepath.FilePath}
         @param outFile: the file to write to
+        @type modtime: C{int}
+        @param modtime: the modification time to set for the file
         """
         self.stream = stream
-        self.outFile = outFile.open('w')
+        self.outFile = outFile
+        self.openFile = outFile.open('w')
+        self.modtime = modtime
         self.length = self.stream.length
         self.start = 0
 
     def _done(self):
         """Close the output file."""
-        self.outFile.close()
+        if not self.openFile.closed:
+            self.openFile.close()
+            if self.modtime:
+                os.utime(self.outFile.path, (self.modtime, self.modtime))
     
     def read(self):
         """Read some data from the stream."""
-        if self.outFile.closed:
+        if self.openFile.closed:
             return None
         
         data = self.stream.read()
@@ -54,7 +61,7 @@ class ProxyFileStream(stream.SimpleStream):
             self._done()
             return data
         
-        self.outFile.write(data)
+        self.openFile.write(data)
         return data
     
     def close(self):
@@ -99,7 +106,6 @@ class MirrorManager:
                         baseDir = base_match
             log.msg("Settled on baseDir: %s" % baseDir)
         
-        log.msg("Parsing '%s' gave '%s', '%s', '%s'" % (url, site, baseDir, path))
         return site, baseDir, path
         
     def init(self, site, baseDir):
@@ -116,7 +122,6 @@ class MirrorManager:
         self.apt_caches[site][baseDir].file_updated(path, file_path)
     
     def findHash(self, url):
-        log.msg('Trying to find hash for %s' % url)
         site, baseDir, path = self.extractPath(url)
         if site in self.apt_caches and baseDir in self.apt_caches[site]:
             return self.apt_caches[site][baseDir].findHash(path)
@@ -130,18 +135,16 @@ class MirrorManager:
         
         parsed = urlparse(url)
         destFile = self.cache.preauthChild(parsed[1] + parsed[2])
-        log.msg('Cache file: %s' % destFile.path)
+        log.msg('Saving returned %r byte file to cache: %s' % (response.stream.length, destFile.path))
         
         if destFile.exists():
-            log.err('File already exists: %s', destFile.path)
-            d.callback(response)
-            return
-        
-        destFile.parent().makedirs()
-        log.msg('Saving returned %i byte file to: %s' % (response.stream.length, destFile.path))
+            log.msg('File already exists, removing: %s' % destFile.path)
+            destFile.remove()
+        else:
+            destFile.parent().makedirs()
         
         orig_stream = response.stream
-        response.stream = ProxyFileStream(orig_stream, destFile)
+        response.stream = ProxyFileStream(orig_stream, destFile, response.headers.getHeader('Last-Modified'))
         return response
 
     def save_error(self, failure, url):

diff --git a/apt_dht/apt_dht.py b/apt_dht/apt_dht.py
index 2e55d8e05a0c29bc554e2e5390045a189a157e9e..e9b5df30a8928e45cc60ea257676a43ed540c49b 100644 (file)
--- a/apt_dht/apt_dht.py
+++ b/apt_dht/apt_dht.py
@@ -21,14 +21,14 @@ class AptDHT:
         return self.http_site
     
     def check_freshness(self, path, modtime, resp):
-        log.msg('Checking if %s is still fresh: %r' % (path, modtime))
+        log.msg('Checking if %s is still fresh' % path)
         d = self.peers.get([path], "HEAD", modtime)
         d.addCallback(self.check_freshness_done, path, resp)
         return d
     
     def check_freshness_done(self, resp, path, orig_resp):
-        if resp.code == "304":
-            log.msg('Still fresh: %s' % path)
+        if resp.code == 304:
+            log.msg('Still fresh, returning: %s' % path)
             return orig_resp
         else:
             log.msg('Stale, need to redownload: %s' % path)