# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ """Manage a mirror's index files.
+
+ @type TRACKED_FILES: C{list} of C{string}
+ @var TRACKED_FILES: the file names of files that contain index information
+ """
+
# Disable the FutureWarning from the apt module
import warnings
warnings.simplefilter("ignore", FutureWarning)
TRACKED_FILES = ['release', 'sources', 'packages']
class PackageFileList(DictMixin):
- """Manages a list of package files belonging to a backend.
+ """Manages a list of index files belonging to a mirror.
+ @type cache_dir: L{twisted.python.filepath.FilePath}
+ @ivar cache_dir: the directory to use for storing all files
@type packages: C{shelve dictionary}
- @ivar packages: the files stored for this backend
+ @ivar packages: the files tracked for this mirror
"""
def __init__(self, cache_dir):
+ """Initialize the list by opening the dictionary."""
self.cache_dir = cache_dir
self.cache_dir.restat(False)
if not self.cache_dir.exists():
self.open()
def open(self):
- """Open the persistent dictionary of files in this backend."""
+ """Open the persistent dictionary of files for this mirror."""
if self.packages is None:
self.packages = shelve.open(self.cache_dir.child('packages.db').path)
Called from the mirror manager when files get updated so we can update our
fake lists and sources.list.
+
+ @type cache_path: C{string}
+ @param cache_path: the location of the file within the mirror
+ @type file_path: L{twisted.python.filepath.FilePath}
+ @param file_path: The location of the file in the file system
+ @rtype: C{boolean}
+ @return: whether the file is an index file
"""
filename = cache_path.split('/')[-1]
if filename.lower() in TRACKED_FILES:
return False
def check_files(self):
- """Check all files in the database to make sure they exist."""
+ """Check all files in the database to remove any that don't exist."""
files = self.packages.keys()
for f in files:
self.packages[f].restat(False)
log.msg("File in packages database has been deleted: "+f)
del self.packages[f]
- # Standard dictionary implementation so this class can be used like a dictionary.
+ #{ Dictionary interface details
def __getitem__(self, key): return self.packages[key]
def __setitem__(self, key, item): self.packages[key] = item
def __delitem__(self, key): del self.packages[key]
def keys(self): return self.packages.keys()
class AptPackages:
- """Uses python-apt to answer queries about packages.
-
- Makes a fake configuration for python-apt for each backend.
+ """Answers queries about packages available from a mirror.
+
+ Uses the python-apt tools to parse and provide information about the
+ files that are available on a single mirror.
+
+ @ivar DEFAULT_APT_CONFIG: the default configuration parameters to use for apt
+ @ivar essential_dirs: directories that must be created for apt to work
+ @ivar essential_files: files that must be created for apt to work
+ @type cache_dir: L{twisted.python.filepath.FilePath}
+ @ivar cache_dir: the directory to use for storing all files
+ @type unload_delay: C{int}
+ @ivar unload_delay: the time to wait before unloading the apt cache
+ @ivar apt_config: the configuration parameters to use for apt
+ @type packages: L{PackageFileList}
+ @ivar packages: the persistent storage of tracked apt index files
+ @type loaded: C{boolean}
+ @ivar loaded: whether the apt cache is currently loaded
+ @type loading: L{twisted.internet.defer.Deferred}
+ @ivar loading: if the cache is currently being loaded, this will be
+ called when it is loaded, otherwise it is None
+ @type unload_later: L{twisted.internet.interfaces.IDelayedCall}
+ @ivar unload_later: the delayed call to unload the apt cache
+ @type indexrecords: C{dictionary}
+ @ivar indexrecords: the hashes of index files for the mirror, keys are
+ mirror directories, values are dictionaries with keys the path to the
+ index file in the mirror directory and values are dictionaries with
+ keys the hash type and values the hash
+ @type cache: C{apt_pkg.GetCache()}
+ @ivar cache: the apt cache of the mirror
+ @type records: C{apt_pkg.GetPkgRecords()}
+ @ivar records: the apt package records for all binary packages in a mirror
+ @type srcrecords: C{apt_pkg.GetPkgSrcRecords}
+ @ivar srcrecords: the apt package records for all source packages in a mirror
"""
DEFAULT_APT_CONFIG = {
def __init__(self, cache_dir, unload_delay):
"""Construct a new packages manager.
- @param cache_dir: cache directory from config file
+ @param cache_dir: directory to use to store files for this mirror
"""
self.cache_dir = cache_dir
self.unload_delay = unload_delay
self.apt_config = deepcopy(self.DEFAULT_APT_CONFIG)
+ # Create the necessary files and directories for apt
for dir in self.essential_dirs:
path = self.cache_dir.preauthChild(dir)
if not path.exists():
self.apt_config['Dir'] = self.cache_dir.path
self.apt_config['Dir::State::status'] = self.cache_dir.preauthChild(self.apt_config['Dir::State']).preauthChild(self.apt_config['Dir::State::status']).path
self.packages = PackageFileList(cache_dir)
- self.loaded = 0
+ self.loaded = False
self.loading = None
self.unload_later = None
def __del__(self):
self.cleanup()
- self.packages.close()
def addRelease(self, cache_path, file_path):
- """Dirty hack until python-apt supports apt-pkg/indexrecords.h
+ """Add a Release file's info to the list of index files.
+
+ Dirty hack until python-apt supports apt-pkg/indexrecords.h
(see Bug #456141)
"""
self.indexrecords[cache_path] = {}
read_packages = False
f = file_path.open('r')
+ # Use python-debian routines to parse the file for hashes
rel = deb822.Release(f, fields = ['MD5Sum', 'SHA1', 'SHA256'])
for hash_type in rel:
for file in rel[hash_type]:
f.close()
def file_updated(self, cache_path, file_path):
- """A file in the backend has changed, manage it.
+ """A file in the mirror has changed or been added.
- If this affects us, unload our apt database
+ If this affects us, unload our apt database.
+ @see: L{PackageFileList.update_file}
"""
if self.packages.update_file(cache_path, file_path):
self.unload()
def load(self):
- """Make sure the package is initialized and loaded."""
+ """Make sure the package cache is initialized and loaded."""
+ # Reset the pending unload call
if self.unload_later and self.unload_later.active():
self.unload_later.reset(self.unload_delay)
else:
self.unload_later = reactor.callLater(self.unload_delay, self.unload)
+
+ # Make sure it's not already being loaded
if self.loading is None:
log.msg('Loading the packages cache')
self.loading = threads.deferToThread(self._load)
return loadResult
def _load(self):
- """Regenerates the fake configuration and load the packages cache."""
+ """Regenerates the fake configuration and loads the packages caches."""
if self.loaded: return True
+
+ # Modify the default configuration to create the fake one.
apt_pkg.InitSystem()
self.cache_dir.preauthChild(self.apt_config['Dir::State']
).preauthChild(self.apt_config['Dir::State::Lists']).remove()
deb_src_added = False
self.packages.check_files()
self.indexrecords = {}
+
+ # Create an entry in sources.list for each needed index file
for f in self.packages:
# we should probably clear old entries from self.packages and
# take into account the recorded mtime as optimization
else:
self.srcrecords = None
- self.loaded = 1
+ self.loaded = True
return True
def unload(self):
self.unload_later = None
if self.loaded:
log.msg('Unloading the packages cache')
+ # This should save memory
del self.cache
del self.records
del self.srcrecords
del self.indexrecords
- self.loaded = 0
+ self.loaded = False
def cleanup(self):
"""Cleanup and close any loaded caches."""
self.unload()
+ if self.unload_later and self.unload_later.active():
+ self.unload_later.cancel()
self.packages.close()
def findHash(self, path):
"""Find the hash for a given path in this mirror.
- Returns a deferred so it can make sure the cache is loaded first.
+ @type path: C{string}
+ @param path: the path within the mirror of the file to lookup
+ @rtype: L{twisted.internet.defer.Deferred}
+ @return: a deferred so it can make sure the cache is loaded first
"""
d = defer.Deferred()
return d
def _findHash_error(self, failure, path, d):
- """An error occurred while trying to find a hash."""
+ """An error occurred, return an empty hash."""
log.msg('An error occurred while looking up a hash for: %s' % path)
log.err(failure)
d.callback(HashObject())
+ return failure
def _findHash(self, loadResult, path, d):
- """Really find the hash for a path.
+ """Search the records for the hash of a path.
- Have to pass the returned loadResult on in case other calls to this
- function are pending.
+ @type loadResult: C{boolean}
+ @param loadResult: whether apt's cache was successfully loaded
+ @type path: C{string}
+ @param path: the path within the mirror of the file to lookup
+ @type d: L{twisted.internet.defer.Deferred}
+ @param d: the deferred to callback with the result
"""
if not loadResult:
d.callback(HashObject())
return loadResult
+ h = HashObject()
+
# First look for the path in the cache of index files
for release in self.indexrecords:
if path.startswith(release[:-7]):
for indexFile in self.indexrecords[release]:
if release[:-7] + indexFile == path:
- h = HashObject()
h.setFromIndexRecord(self.indexrecords[release][indexFile])
d.callback(h)
return loadResult
for verFile in version.FileList:
if self.records.Lookup(verFile):
if '/' + self.records.FileName == path:
- h = HashObject()
h.setFromPkgRecord(self.records, size)
d.callback(h)
return loadResult
if self.srcrecords.Lookup(package):
for f in self.srcrecords.Files:
if path == '/' + f[2]:
- h = HashObject()
h.setFromSrcRecord(f)
d.callback(h)
return loadResult
- d.callback(HashObject())
+ d.callback(h)
+
+ # Have to pass the returned loadResult on in case other calls to this function are pending.
return loadResult
class TestAptPackages(unittest.TestCase):
releaseFile = ''
def setUp(self):
+ """Initializes the cache with files found in the traditional apt location."""
self.client = AptPackages(FilePath('/tmp/.apt-dht'), 300)
+ # Find the largest index files that are for 'main'
self.packagesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "_main_.*Packages$" | tail -n 1').read().rstrip('\n')
self.sourcesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "_main_.*Sources$" | tail -n 1').read().rstrip('\n')
+
+ # Find the Release file corresponding to the found Packages file
for f in os.walk('/var/lib/apt/lists').next()[2]:
if f[-7:] == "Release" and self.packagesFile.startswith(f[:-7]):
self.releaseFile = f
break
-
+
+ # Add all the found files to the PackageFileList
self.client.file_updated(self.releaseFile[self.releaseFile.find('_dists_'):].replace('_','/'),
FilePath('/var/lib/apt/lists/' + self.releaseFile))
self.client.file_updated(self.packagesFile[self.packagesFile.find('_dists_'):].replace('_','/'),
FilePath('/var/lib/apt/lists/' + self.sourcesFile))
def test_pkg_hash(self):
+ """Tests loading the binary package records cache."""
self.client._load()
self.client.records.Lookup(self.client.cache['dpkg'].VersionList[0].FileList[0])
"Hashes don't match: %s != %s" % (self.client.records.SHA1Hash, pkg_hash))
def test_src_hash(self):
+ """Tests loading the source package records cache."""
self.client._load()
self.client.srcrecords.Lookup('dpkg')
self.failUnless(f[0] in src_hashes, "Couldn't find %s in: %r" % (f[0], src_hashes))
def test_index_hash(self):
+ """Tests loading the cache of index file information."""
self.client._load()
indexhash = self.client.indexrecords[self.releaseFile[self.releaseFile.find('_dists_'):].replace('_','/')]['main/binary-i386/Packages.bz2']['SHA1'][0]
"%s hashes don't match: %s != %s" % (path, found_hash.hexexpected(), true_hash))
def test_findIndexHash(self):
+ """Tests finding the hash of a single index file."""
lastDefer = defer.Deferred()
idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
return lastDefer
def test_findPkgHash(self):
+ """Tests finding the hash of a single binary package."""
lastDefer = defer.Deferred()
pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
return lastDefer
def test_findSrcHash(self):
+ """Tests finding the hash of a single source package."""
lastDefer = defer.Deferred()
src_dir = '/' + os.popen('grep -A 30 -E "^Package: dpkg$" ' +
return lastDefer
def test_multipleFindHash(self):
+ """Tests finding the hash of an index file, binary package, source package, and another index file."""
lastDefer = defer.Deferred()
+ # Lookup a Packages.bz2 file
idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
'/var/lib/apt/lists/' + self.releaseFile +
' | grep -E " main/binary-i386/Packages.bz2$"'
d = self.client.findHash(idx_path)
d.addCallback(self.verifyHash, idx_path, idx_hash)
+ # Lookup the binary 'dpkg' package
pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
'/var/lib/apt/lists/' + self.packagesFile +
' | grep -E "^SHA1:" | head -n 1' +
d = self.client.findHash(pkg_path)
d.addCallback(self.verifyHash, pkg_path, pkg_hash)
+ # Lookup the source 'dpkg' package
src_dir = '/' + os.popen('grep -A 30 -E "^Package: dpkg$" ' +
'/var/lib/apt/lists/' + self.sourcesFile +
' | grep -E "^Directory:" | head -n 1' +
d = self.client.findHash(src_dir + '/' + src_paths[i])
d.addCallback(self.verifyHash, src_dir + '/' + src_paths[i], src_hashes[i])
+ # Lookup a Sources.bz2 file
idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
'/var/lib/apt/lists/' + self.releaseFile +
' | grep -E " main/source/Sources.bz2$"'
+ """Manage the multiple mirrors that may be requested.
+
+ @var aptpkg_dir: the name of the directory to use for mirror files
+ """
+
from urlparse import urlparse
import os
"""Exception raised when there's a problem with the mirror."""
class MirrorManager:
- """Manages all requests for mirror objects."""
+ """Manages all requests for mirror information.
+
+ @type cache_dir: L{twisted.python.filepath.FilePath}
+ @ivar cache_dir: the directory to use for storing all files
+ @type unload_delay: C{int}
+ @ivar unload_delay: the time to wait before unloading the apt cache
+ @type apt_caches: C{dictionary}
+ @ivar apt_caches: the avaliable mirrors
+ """
def __init__(self, cache_dir, unload_delay):
self.cache_dir = cache_dir
self.apt_caches = {}
def extractPath(self, url):
+ """Break the full URI down into the site, base directory and path.
+
+ Site is the host and port of the mirror. Base directory is the
+ directory to the mirror location (usually just '/debian'). Path is
+ the remaining path to get to the file.
+
+ E.g. http://ftp.debian.org/debian/dists/sid/binary-i386/Packages.bz2
+ would return ('ftp.debian.org:80', '/debian',
+ '/dists/sid/binary-i386/Packages.bz2').
+
+ @param url: the URI of the file's location on the mirror
+ @rtype: (C{string}, C{string}, C{string})
+ @return: the site, base directory and path to the file
+ """
+ # Extract the host and port
parsed = urlparse(url)
host, port = splitHostPort(parsed[0], parsed[1])
site = host + ":" + str(port)
path = parsed[2]
-
+
+ # Try to find the base directory (most can be found this way)
i = max(path.rfind('/dists/'), path.rfind('/pool/'))
if i >= 0:
baseDir = path[:i]
else:
# Uh oh, this is not good
log.msg("Couldn't find a good base directory for path: %s" % (site + path))
+
+ # Try to find an existing cache that starts with this one
+ # (fallback to using an empty base directory)
baseDir = ''
if site in self.apt_caches:
longest_match = 0
return site, baseDir, path
def init(self, site, baseDir):
+ """Make sure an L{AptPackages} exists for this mirror."""
if site not in self.apt_caches:
self.apt_caches[site] = {}
self.apt_caches[site][baseDir] = AptPackages(site_cache, self.unload_delay)
def updatedFile(self, url, file_path):
+ """A file in the mirror has changed or been added.
+
+ @see: L{AptPackages.PackageFileList.update_file}
+ """
site, baseDir, path = self.extractPath(url)
self.init(site, baseDir)
self.apt_caches[site][baseDir].file_updated(path, file_path)
def findHash(self, url):
+ """Find the hash for a given url.
+
+ @param url: the URI of the file's location on the mirror
+ @rtype: L{twisted.internet.defer.Deferred}
+ @return: a deferred that will fire with the returned L{Hash.HashObject}
+ """
site, baseDir, path = self.extractPath(url)
if site in self.apt_caches and baseDir in self.apt_caches[site]:
return self.apt_caches[site][baseDir].findHash(path)
d.errback(MirrorError("Site Not Found"))
return d
+ def cleanup(self):
+ for site in self.apt_caches.keys():
+ for baseDir in self.apt_caches[site].keys():
+ self.apt_caches[site][baseDir].cleanup()
+ del self.apt_caches[site][baseDir]
+ del self.apt_caches[site]
+
class TestMirrorManager(unittest.TestCase):
"""Unit tests for the mirror manager."""
self.client = MirrorManager(FilePath('/tmp/.apt-dht'), 300)
def test_extractPath(self):
+ """Test extracting the site and base directory from various mirrors."""
site, baseDir, path = self.client.extractPath('http://ftp.us.debian.org/debian/dists/unstable/Release')
self.failUnless(site == "ftp.us.debian.org:80", "no match: %s" % site)
self.failUnless(baseDir == "/debian", "no match: %s" % baseDir)
"%s hashes don't match: %s != %s" % (path, found_hash.hexexpected(), true_hash))
def test_findHash(self):
+ """Tests finding the hash of an index file, binary package, source package, and another index file."""
+ # Find the largest index files that are for 'main'
self.packagesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "_main_.*Packages$" | tail -n 1').read().rstrip('\n')
self.sourcesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "_main_.*Sources$" | tail -n 1').read().rstrip('\n')
+
+ # Find the Release file corresponding to the found Packages file
for f in os.walk('/var/lib/apt/lists').next()[2]:
if f[-7:] == "Release" and self.packagesFile.startswith(f[:-7]):
self.releaseFile = f
break
+ # Add all the found files to the mirror
self.client.updatedFile('http://' + self.releaseFile.replace('_','/'),
FilePath('/var/lib/apt/lists/' + self.releaseFile))
self.client.updatedFile('http://' + self.releaseFile[:self.releaseFile.find('_dists_')+1].replace('_','/') +
lastDefer = defer.Deferred()
+ # Lookup a Packages.bz2 file
idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
'/var/lib/apt/lists/' + self.releaseFile +
' | grep -E " main/binary-i386/Packages.bz2$"'
d = self.client.findHash(idx_path)
d.addCallback(self.verifyHash, idx_path, idx_hash)
+ # Lookup the binary 'dpkg' package
pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
'/var/lib/apt/lists/' + self.packagesFile +
' | grep -E "^SHA1:" | head -n 1' +
d = self.client.findHash(pkg_path)
d.addCallback(self.verifyHash, pkg_path, pkg_hash)
+ # Lookup the source 'dpkg' package
src_dir = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
'/var/lib/apt/lists/' + self.sourcesFile +
' | grep -E "^Directory:" | head -n 1' +
d = self.client.findHash(src_path)
d.addCallback(self.verifyHash, src_path, src_hashes[i])
+ # Lookup a Sources.bz2 file
idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
'/var/lib/apt/lists/' + self.releaseFile +
' | grep -E " main/source/Sources.bz2$"'
for p in self.pending_calls:
if p.active():
p.cancel()
+ self.client.cleanup()
self.client = None
+ """An sqlite database for storing persistent files and hashes."""
+
from datetime import datetime, timedelta
from pysqlite2 import dbapi2 as sqlite
from binascii import a2b_base64, b2a_base64
assert sqlite.version_info >= (2, 1)
class DBExcept(Exception):
+ """An error occurred in accessing the database."""
pass
class khash(str):
"""Dummy class to convert all hashes to base64 for storing in the DB."""
-
+
+ # Initialize the database to work with 'khash' objects (binary strings)
sqlite.register_adapter(khash, b2a_base64)
sqlite.register_converter("KHASH", a2b_base64)
sqlite.register_converter("khash", a2b_base64)
sqlite.enable_callback_tracebacks(True)
class DB:
- """Database access for storing persistent data."""
+ """An sqlite database for storing persistent files and hashes.
+
+ @type db: L{twisted.python.filepath.FilePath}
+ @ivar db: the database file to use
+ @type conn: L{pysqlite2.dbapi2.Connection}
+ @ivar conn: an open connection to the sqlite database
+ """
def __init__(self, db):
+ """Load or create the database file.
+
+ @type db: L{twisted.python.filepath.FilePath}
+ @param db: the database file to use
+ """
self.db = db
self.db.restat(False)
if self.db.exists():
self.conn.row_factory = sqlite.Row
def _loadDB(self):
+ """Open a new connection to the existing database file"""
try:
self.conn = sqlite.connect(database=self.db.path, detect_types=sqlite.PARSE_DECLTYPES)
except:
raise DBExcept, "Couldn't open DB", traceback.format_exc()
def _createNewDB(self):
+ """Open a connection to a new database and create the necessary tables."""
if not self.db.parent().exists():
self.db.parent().makedirs()
self.conn = sqlite.connect(database=self.db.path, detect_types=sqlite.PARSE_DECLTYPES)
self.conn.commit()
def _removeChanged(self, file, row):
+ """If the file has changed or is missing, remove it from the DB.
+
+ @type file: L{twisted.python.filepath.FilePath}
+ @param file: the file to check
+ @type row: C{dictionary}-like object
+ @param row: contains the expected 'size' and 'mtime' of the file
+ @rtype: C{boolean}
+ @return: True if the file is unchanged, False if it is changed,
+ and None if it is missing
+ """
res = None
if row:
file.restat(False)
if file.exists():
+ # Compare the current with the expected file properties
res = (row['size'] == file.getsize() and row['mtime'] == file.getmtime())
if not res:
+ # Remove the file from the database
c = self.conn.cursor()
c.execute("DELETE FROM files WHERE path = ?", (file.path, ))
self.conn.commit()
def storeFile(self, file, hash, pieces = ''):
"""Store or update a file in the database.
+ @type file: L{twisted.python.filepath.FilePath}
+ @param file: the file to check
+ @type hash: C{string}
+ @param hash: the hash of the file
+ @type pieces: C{string}
+ @param pieces: the concatenated list of the hashes of the pieces of
+ the file (optional, defaults to the empty string)
@return: True if the hash was not in the database before
(so it needs to be added to the DHT)
"""
+ # Hash the pieces to get the piecehash
piecehash = ''
if pieces:
s = sha.new().update(pieces)
piecehash = sha.digest()
+
+ # Check the database for the hash
c = self.conn.cursor()
c.execute("SELECT hashID, piecehash FROM hashes WHERE hash = ?", (khash(hash), ))
row = c.fetchone()
new_hash = False
hashID = row['hashID']
else:
+ # Add the new hash to the database
c = self.conn.cursor()
- c.execute("INSERT OR REPLACE INTO hashes (hash, pieces, piecehash, refreshed) VALUES (?, ?, ?)",
+ c.execute("INSERT OR REPLACE INTO hashes (hash, pieces, piecehash, refreshed) VALUES (?, ?, ?, ?)",
(khash(hash), khash(pieces), khash(piecehash), datetime.now()))
self.conn.commit()
new_hash = True
hashID = c.lastrowid
-
+
+ # Add the file to the database
file.restat()
c.execute("INSERT OR REPLACE INTO files (path, hashID, size, mtime) VALUES (?, ?, ?, ?)",
(file.path, hashID, file.getsize(), file.getmtime()))
If it has changed or is missing, it is removed from the database.
+ @type file: L{twisted.python.filepath.FilePath}
+ @param file: the file to check
@return: dictionary of info for the file, False if changed, or
None if not in database or missing
"""
@return: list of dictionaries of info for the found files
"""
+ # Try to find the hash in the files table
c = self.conn.cursor()
c.execute("SELECT path, size, mtime, refreshed, pieces FROM files JOIN hashes USING (hashID) WHERE hash = ?", (khash(hash), ))
row = c.fetchone()
files = []
while row:
+ # Save the file to the list of found files
file = FilePath(row['path'])
res = self._removeChanged(file, row)
if res:
row = c.fetchone()
if not filesOnly and not files:
+ # No files were found, so check the piecehashes as well
c.execute("SELECT refreshed, pieces, piecehash FROM hashes WHERE piecehash = ?", (khash(hash), ))
row = c.fetchone()
if row:
def isUnchanged(self, file):
"""Check if a file in the file system has changed.
- If it has changed, it is removed from the table.
+ If it has changed, it is removed from the database.
@return: True if unchanged, False if changed, None if not in database
"""
return self._removeChanged(file, row)
def refreshHash(self, hash):
- """Refresh the publishing time all files with a hash."""
+ """Refresh the publishing time of a hash."""
c = self.conn.cursor()
c.execute("UPDATE hashes SET refreshed = ? WHERE hash = ?", (datetime.now(), khash(hash)))
c.close()
"""
t = datetime.now() - timedelta(seconds=expireAfter)
- # First find the hashes that need refreshing
+ # Find all the hashes that need refreshing
c = self.conn.cursor()
c.execute("SELECT hashID, hash, pieces FROM hashes WHERE refreshed < ?", (t, ))
row = c.fetchone()
valid = True
row = c.fetchone()
if not valid:
+ # Remove hashes for which no files are still available
del expired[hash['hash']]
c.execute("DELETE FROM hashes WHERE hashID = ?", (hash['hashID'], ))
return expired
def removeUntrackedFiles(self, dirs):
- """Find files that are no longer tracked and so should be removed.
-
- Also removes the entries from the table.
+ """Remove files that are no longer tracked by the program.
+ @type dirs: C{list} of L{twisted.python.filepath.FilePath}
+ @param dirs: a list of the directories that we are tracking
@return: list of files that were removed
"""
assert len(dirs) >= 1
+
+ # Create a list of globs and an SQL statement for the directories
newdirs = []
sql = "WHERE"
for dir in dirs:
sql += " path NOT GLOB ? AND"
sql = sql[:-4]
+ # Get a listing of all the files that will be removed
c = self.conn.cursor()
c.execute("SELECT path FROM files " + sql, newdirs)
row = c.fetchone()
removed.append(FilePath(row['path']))
row = c.fetchone()
+ # Delete all the removed files from the database
if removed:
c.execute("DELETE FROM files " + sql, newdirs)
self.conn.commit()
+
return removed
def close(self):
+ """Close the database connection."""
self.conn.close()
class TestDB(unittest.TestCase):
self.store.storeFile(self.file, self.hash)
def test_openExistingDB(self):
+ """Tests opening an existing database."""
self.store.close()
self.store = None
sleep(1)
self.failUnless(res)
def test_getFile(self):
+ """Tests retrieving a file from the database."""
res = self.store.getFile(self.file)
self.failUnless(res)
self.failUnlessEqual(res['hash'], self.hash)
def test_lookupHash(self):
+ """Tests looking up a hash in the database."""
res = self.store.lookupHash(self.hash)
self.failUnless(res)
self.failUnlessEqual(len(res), 1)
self.failUnlessEqual(res[0]['path'].path, self.file.path)
def test_isUnchanged(self):
+ """Tests checking if a file in the database is unchanged."""
res = self.store.isUnchanged(self.file)
self.failUnless(res)
sleep(2)
self.failUnless(res is None)
def test_expiry(self):
+ """Tests retrieving the files from the database that have expired."""
res = self.store.expiredHashes(1)
self.failUnlessEqual(len(res.keys()), 0)
sleep(2)
self.store.storeFile(file, self.hash)
def test_multipleHashes(self):
+ """Tests looking up a hash with multiple files in the database."""
self.build_dirs()
res = self.store.expiredHashes(1)
self.failUnlessEqual(len(res.keys()), 0)
self.failUnlessEqual(len(res.keys()), 0)
def test_removeUntracked(self):
+ """Tests removing untracked files from the database."""
self.build_dirs()
res = self.store.removeUntrackedFiles(self.dirs)
self.failUnlessEqual(len(res), 1, 'Got removed paths: %r' % res)
projects / quix0rs-apt-p2p.git / commitdiff