From: Cameron Dale Date: Wed, 23 Apr 2008 21:31:03 +0000 (-0700) Subject: Don't check file when listing expired hashes, instead check files at startup. X-Git-Url: https://git.mxchange.org/?a=commitdiff_plain;h=253dd262912897565da6b65708905586efc4b542;p=quix0rs-apt-p2p.git Don't check file when listing expired hashes, instead check files at startup. --- diff --git a/TODO b/TODO index c6c74bd..d48ea93 100644 --- a/TODO +++ b/TODO @@ -1,9 +1,6 @@ Some last few things to do before release. - Handle/investigate the HTTP client pipeline errors -- DB should not always restat files (especially for expired hashes) -- remove missing files at startup (in DB's removeUntracked) -- when files modtime but not size changes, rehash them to be sure - remove files from the peer's download cache - update the modtime of files downloaded from peers - also set the Last-Modified header for the return to Apt @@ -41,6 +38,14 @@ and not the headers, and also misses the requests for downloads sent to other peers. +Rehash changed files instead of removing them. + +When the modification time of a file changes but the size does not, +the file could be rehased to verify it is the same instead of +automatically removing it. The DB would have to be modified to return +deferred's for a lot of its functions. + + Consider storing deltas of packages. Instead of downloading full package files when a previous version of diff --git a/apt_p2p/db.py b/apt_p2p/db.py index f72a326..2640987 100644 --- a/apt_p2p/db.py +++ b/apt_p2p/db.py @@ -263,12 +263,10 @@ class DB: c.execute("SELECT path, dht, size, mtime FROM files WHERE hashID = ?", (hash['hashID'], )) row = c.fetchone() while row: - res = self._removeChanged(FilePath(row['path']), row) - if res: - if row['dht']: - dht = True - else: - non_dht = True + if row['dht']: + dht = True + else: + non_dht = True row = c.fetchone() if not dht: # Remove hashes for which no DHT files are still available @@ -315,6 +313,15 @@ class DB: # Delete all the removed files from the database if removed: c.execute("DELETE FROM files " + sql, newdirs) + self.conn.commit() + + c.execute("SELECT path FROM files") + rows = c.fetchall() + for row in rows: + if not os.path.exists(row['path']): + # Leave hashes, they will be removed on next refresh + c.execute("DELETE FROM files WHERE path = ?", (row['path'], )) + removed.append(FilePath(row['path'])) self.conn.commit() return removed @@ -460,11 +467,19 @@ class TestDB(unittest.TestCase): def test_removeUntracked(self): """Tests removing untracked files from the database.""" self.build_dirs() + file = self.dirs[0].child('test.khashmir') + file.setContent(file.path) + file.touch() + self.store.storeFile(file, self.hash) res = self.store.removeUntrackedFiles(self.dirs) self.failUnlessEqual(len(res), 1, 'Got removed paths: %r' % res) self.failUnlessEqual(res[0], self.file, 'Got removed paths: %r' % res) res = self.store.removeUntrackedFiles(self.dirs) self.failUnlessEqual(len(res), 0, 'Got removed paths: %r' % res) + file.remove() + res = self.store.removeUntrackedFiles(self.dirs) + self.failUnlessEqual(len(res), 1, 'Got removed paths: %r' % res) + self.failUnlessEqual(res[0], self.dirs[0].child('test.khashmir'), 'Got removed paths: %r' % res) res = self.store.removeUntrackedFiles(self.dirs[1:]) self.failUnlessEqual(len(res), 1, 'Got removed paths: %r' % res) self.failUnlessEqual(res[0], self.dirs[0].preauthChild(self.testfile), 'Got removed paths: %r' % res)