Don't check file when listing expired hashes, instead check files at startup.

diff --git a/TODO b/TODO
index c6c74bd8de2fe762962f4d701af0a1ad073bda4b..d48ea9373b8cf927f3eb4519bb8bb51f10590d0e 100644 (file)
--- a/TODO
+++ b/TODO
@@ -1,9 +1,6 @@
 Some last few things to do before release.
 
 - Handle/investigate the HTTP client pipeline errors
 Some last few things to do before release.
 
 - Handle/investigate the HTTP client pipeline errors

-- DB should not always restat files (especially for expired hashes)
-- remove missing files at startup (in DB's removeUntracked)
-- when files modtime but not size changes, rehash them to be sure

 - remove files from the peer's download cache
 - update the modtime of files downloaded from peers
   - also set the Last-Modified header for the return to Apt
 - remove files from the peer's download cache
 - update the modtime of files downloaded from peers
   - also set the Last-Modified header for the return to Apt


@@ -41,6 +38,14 @@ and not the headers, and also misses the requests for downloads sent to
 other peers.
 
 
 other peers.
 
 

+Rehash changed files instead of removing them.
+
+When the modification time of a file changes but the size does not,
+the file could be rehased to verify it is the same instead of
+automatically removing it. The DB would have to be modified to return
+deferred's for a lot of its functions.
+
+

 Consider storing deltas of packages.
 
 Instead of downloading full package files when a previous version of
 Consider storing deltas of packages.
 
 Instead of downloading full package files when a previous version of

diff --git a/apt_p2p/db.py b/apt_p2p/db.py
index f72a3268603d2e020e73ecbe8efea75eb33fe596..26409875f3339c37bb74fb086d7cc07fc8ae972f 100644 (file)
--- a/apt_p2p/db.py
+++ b/apt_p2p/db.py
@@ -263,12 +263,10 @@ class DB:
             c.execute("SELECT path, dht, size, mtime FROM files WHERE hashID = ?", (hash['hashID'], ))
             row = c.fetchone()
             while row:
             c.execute("SELECT path, dht, size, mtime FROM files WHERE hashID = ?", (hash['hashID'], ))
             row = c.fetchone()
             while row:

-                res = self._removeChanged(FilePath(row['path']), row)
-                if res:
-                    if row['dht']:
-                        dht = True
-                    else:
-                        non_dht = True
+                if row['dht']:
+                    dht = True
+                else:
+                    non_dht = True

                 row = c.fetchone()
             if not dht:
                 # Remove hashes for which no DHT files are still available
                 row = c.fetchone()
             if not dht:
                 # Remove hashes for which no DHT files are still available


@@ -315,6 +313,15 @@ class DB:
         # Delete all the removed files from the database
         if removed:
             c.execute("DELETE FROM files " + sql, newdirs)
         # Delete all the removed files from the database
         if removed:
             c.execute("DELETE FROM files " + sql, newdirs)

+            self.conn.commit()
+        
+        c.execute("SELECT path FROM files")
+        rows = c.fetchall()
+        for row in rows:
+            if not os.path.exists(row['path']):
+                # Leave hashes, they will be removed on next refresh
+                c.execute("DELETE FROM files WHERE path = ?", (row['path'], ))
+                removed.append(FilePath(row['path']))

         self.conn.commit()
 
         return removed
         self.conn.commit()
 
         return removed


@@ -460,11 +467,19 @@ class TestDB(unittest.TestCase):
     def test_removeUntracked(self):
         """Tests removing untracked files from the database."""
         self.build_dirs()
     def test_removeUntracked(self):
         """Tests removing untracked files from the database."""
         self.build_dirs()

+        file = self.dirs[0].child('test.khashmir')
+        file.setContent(file.path)
+        file.touch()
+        self.store.storeFile(file, self.hash)

         res = self.store.removeUntrackedFiles(self.dirs)
         self.failUnlessEqual(len(res), 1, 'Got removed paths: %r' % res)
         self.failUnlessEqual(res[0], self.file, 'Got removed paths: %r' % res)
         res = self.store.removeUntrackedFiles(self.dirs)
         self.failUnlessEqual(len(res), 0, 'Got removed paths: %r' % res)
         res = self.store.removeUntrackedFiles(self.dirs)
         self.failUnlessEqual(len(res), 1, 'Got removed paths: %r' % res)
         self.failUnlessEqual(res[0], self.file, 'Got removed paths: %r' % res)
         res = self.store.removeUntrackedFiles(self.dirs)
         self.failUnlessEqual(len(res), 0, 'Got removed paths: %r' % res)

+        file.remove()
+        res = self.store.removeUntrackedFiles(self.dirs)
+        self.failUnlessEqual(len(res), 1, 'Got removed paths: %r' % res)
+        self.failUnlessEqual(res[0], self.dirs[0].child('test.khashmir'), 'Got removed paths: %r' % res)

         res = self.store.removeUntrackedFiles(self.dirs[1:])
         self.failUnlessEqual(len(res), 1, 'Got removed paths: %r' % res)
         self.failUnlessEqual(res[0], self.dirs[0].preauthChild(self.testfile), 'Got removed paths: %r' % res)
         res = self.store.removeUntrackedFiles(self.dirs[1:])
         self.failUnlessEqual(len(res), 1, 'Got removed paths: %r' % res)
         self.failUnlessEqual(res[0], self.dirs[0].preauthChild(self.testfile), 'Got removed paths: %r' % res)