Non-DHT files are marked as such.
HTTP server looks up the cache path to decide whether to return a file.
-Add all cache files to the database.
-
-All files in the cache should be added to the database, so that they can
-be checked to make sure nothing has happened to them. The database would
-then need a flag to indicate files that are hashed and available, but
-that shouldn't be added to the DHT.
-
-
Packages.diff files need to be considered.
The Packages.diff/Index files contain hashes of Packages.diff/rred.gz
url = 'http:/' + file.path[len(self.cache_dir.path):]
# Store the hashed file in the database
- new_hash = self.db.storeFile(file, result.digest(),
+ new_hash = self.db.storeFile(file, result.digest(), True,
''.join(result.pieceDigests()))
# Tell the main program to handle the new cache file
@param decFile: the file where the decompressed download was written to
(optional, defaults to the file not having been compressed)
"""
- if modtime:
- os.utime(destFile.path, (modtime, modtime))
- if decFile:
- os.utime(decFile.path, (modtime, modtime))
-
result = hash.verify()
if result or result is None:
+ if modtime:
+ os.utime(destFile.path, (modtime, modtime))
+
if result:
log.msg('Hashes match: %s' % url)
+ dht = True
else:
log.msg('Hashed file to %s: %s' % (hash.hexdigest(), url))
+ dht = False
- new_hash = self.db.storeFile(destFile, hash.digest(),
+ new_hash = self.db.storeFile(destFile, hash.digest(), dht,
''.join(hash.pieceDigests()))
- log.msg('now avaliable: %s' % (url))
if self.manager:
self.manager.new_cached_file(destFile, hash, new_hash, url)
- if decFile:
- ext_len = len(destFile.path) - len(decFile.path)
- self.manager.new_cached_file(decFile, None, False, url[:-ext_len])
+
+ if decFile:
+ # Hash the decompressed file and add it to the DB
+ decHash = HashObject()
+ ext_len = len(destFile.path) - len(decFile.path)
+ df = decHash.hashInThread(decFile)
+ df.addCallback(self._save_complete, url[:-ext_len], decFile, modtime)
+ df.addErrback(self._save_error, url[:-ext_len], decFile)
else:
log.msg("Hashes don't match %s != %s: %s" % (hash.hexexpected(), hash.hexdigest(), url))
destFile.remove()
def __init__(self, path, manager, defaultType="text/plain", ignoredExts=(), processors=None, indexNames=None):
self.manager = manager
super(FileDownloader, self).__init__(path, defaultType, ignoredExts, processors, indexNames)
-
+
+ def locateChild(self, req, segments):
+ child, segments = super(FileDownloader, self).locateChild(req, segments)
+ # Make sure we always call renderHTTP()
+ if isinstance(child, FileDownloader):
+ return child, segments
+ else:
+ return self, server.StopTraversal
+
def renderHTTP(self, req):
log.msg('Got request for %s from %s' % (req.uri, req.remoteAddr))
+
+ # Make sure the file is in the DB and unchanged
+ if self.manager and not self.manager.db.isUnchanged(self.fp):
+ if self.fp.exists() and self.fp.isfile():
+ self.fp.remove()
+ return self._renderHTTP_done(http.Response(404,
+ {'content-type': http_headers.MimeType('text', 'html')},
+ '<html><body><p>File found but it has changed.</body></html>'),
+ req)
+
resp = super(FileDownloader, self).renderHTTP(req)
if isinstance(resp, defer.Deferred):
resp.addCallbacks(self._renderHTTP_done, self._renderHTTP_error,
self.conn = sqlite.connect(database=self.db.path, detect_types=sqlite.PARSE_DECLTYPES)
c = self.conn.cursor()
c.execute("CREATE TABLE files (path TEXT PRIMARY KEY UNIQUE, hashID INTEGER, " +
- "size NUMBER, mtime NUMBER)")
+ "dht BOOL, size NUMBER, mtime NUMBER)")
c.execute("CREATE TABLE hashes (hashID INTEGER PRIMARY KEY AUTOINCREMENT, " +
"hash KHASH UNIQUE, pieces KHASH, " +
"piecehash KHASH, refreshed TIMESTAMP)")
c.close()
return res
- def storeFile(self, file, hash, pieces = ''):
+ def storeFile(self, file, hash, dht = True, pieces = ''):
"""Store or update a file in the database.
@type file: L{twisted.python.filepath.FilePath}
@param file: the file to check
@type hash: C{string}
@param hash: the hash of the file
+ @param dht: whether the file is added to the DHT
+ (optional, defaults to true)
@type pieces: C{string}
@param pieces: the concatenated list of the hashes of the pieces of
the file (optional, defaults to the empty string)
# Add the file to the database
file.restat()
- c.execute("INSERT OR REPLACE INTO files (path, hashID, size, mtime) VALUES (?, ?, ?, ?)",
- (file.path, hashID, file.getsize(), file.getmtime()))
+ c.execute("INSERT OR REPLACE INTO files (path, hashID, dht, size, mtime) VALUES (?, ?, ?, ?, ?)",
+ (file.path, hashID, dht, file.getsize(), file.getmtime()))
self.conn.commit()
c.close()
res['pieces'] = row['pieces']
row = c.fetchone()
- # Make sure there are still valid files for each hash
+ # Make sure there are still valid DHT files for each hash
for hash in expired.values():
- valid = False
- c.execute("SELECT path, size, mtime FROM files WHERE hashID = ?", (hash['hashID'], ))
+ dht = False
+ non_dht = False
+ c.execute("SELECT path, dht, size, mtime FROM files WHERE hashID = ?", (hash['hashID'], ))
row = c.fetchone()
while row:
res = self._removeChanged(FilePath(row['path']), row)
if res:
- valid = True
+ if row['dht']:
+ dht = True
+ else:
+ non_dht = True
row = c.fetchone()
- if not valid:
- # Remove hashes for which no files are still available
+ if not dht:
+ # Remove hashes for which no DHT files are still available
del expired[hash['hash']]
- c.execute("DELETE FROM hashes WHERE hashID = ?", (hash['hashID'], ))
+ if not non_dht:
+ # Remove hashes for which no files are still available
+ c.execute("DELETE FROM hashes WHERE hashID = ?", (hash['hashID'], ))
+ else:
+ # There are still some non-DHT files available, so refresh them
+ c.execute("UPDATE hashes SET refreshed = ? WHERE hashID = ?",
+ (datetime.now(), hash['hashID']))
self.conn.commit()
c.close()
projects / quix0rs-apt-p2p.git / commitdiff