From be126fc5cd40e26479c50fee4f64cc478f1eb55a Mon Sep 17 00:00:00 2001 From: Cameron Dale Date: Fri, 22 Feb 2008 11:22:28 -0800 Subject: [PATCH] Unload the AptPackages caches after a period of inactivity. --- TODO | 8 -------- apt-dht.conf | 5 +++++ apt_dht/AptPackages.py | 17 ++++++++++++++--- apt_dht/MirrorManager.py | 7 ++++--- apt_dht/apt_dht.py | 2 +- apt_dht/apt_dht_conf.py | 5 +++++ debian/apt-dht.conf.sgml | 8 ++++++++ test.py | 5 +++++ 8 files changed, 42 insertions(+), 15 deletions(-) diff --git a/TODO b/TODO index fac9201..b204ce7 100644 --- a/TODO +++ b/TODO @@ -1,11 +1,3 @@ -Reduce the memory footprint by clearing the AptPackages caches. - -The memory usage is a little bit high due to keeping the AptPackages -caches always. Instead, they should timeout after a period of inactivity -(say 15 minutes), and unload themselves from meory. It only takes a few -seconds to reload, so this should not be an issue. - - Packages.diff files need to be considered. The Packages.diff/Index files contain hashes of Packages.diff/rred.gz diff --git a/apt-dht.conf b/apt-dht.conf index 0b06e20..7f42e4d 100644 --- a/apt-dht.conf +++ b/apt-dht.conf @@ -29,6 +29,11 @@ CACHE_DIR = /var/cache/apt-dht # Whether it's OK to use an IP addres from a known local/private range LOCAL_OK = no +# Unload the packages cache after an interval of inactivity this long. +# The packages cache uses a lot of memory, and only takes a few seconds +# to reload when a new request arrives. +UNLOAD_PACKAGES_CACHE = 5m + # Which DHT implementation to use. # It must be possile to do "from .DHT import DHT" to get a class that # implements the IDHT interface. There should also be a similarly named diff --git a/apt_dht/AptPackages.py b/apt_dht/AptPackages.py index 0711802..48dd448 100644 --- a/apt_dht/AptPackages.py +++ b/apt_dht/AptPackages.py @@ -25,7 +25,7 @@ from shutil import rmtree from copy import deepcopy from UserDict import DictMixin -from twisted.internet import threads, defer +from twisted.internet import threads, defer, reactor from twisted.python import log from twisted.python.filepath import FilePath from twisted.trial import unittest @@ -135,12 +135,13 @@ class AptPackages: 'apt/lists/partial') essential_files = ('apt/dpkg/status', 'apt/etc/sources.list',) - def __init__(self, cache_dir): + def __init__(self, cache_dir, unload_delay): """Construct a new packages manager. @param cache_dir: cache directory from config file """ self.cache_dir = cache_dir + self.unload_delay = unload_delay self.apt_config = deepcopy(self.DEFAULT_APT_CONFIG) for dir in self.essential_dirs: @@ -157,6 +158,7 @@ class AptPackages: self.packages = PackageFileList(cache_dir) self.loaded = 0 self.loading = None + self.unload_later = None def __del__(self): self.cleanup() @@ -188,7 +190,12 @@ class AptPackages: def load(self): """Make sure the package is initialized and loaded.""" + if self.unload_later and self.unload_later.active(): + self.unload_later.reset(self.unload_delay) + else: + self.unload_later = reactor.callLater(self.unload_delay, self.unload) if self.loading is None: + log.msg('Loading the packages cache') self.loading = threads.deferToThread(self._load) self.loading.addCallback(self.doneLoading) return self.loading @@ -261,7 +268,11 @@ class AptPackages: def unload(self): """Tries to make the packages server quit.""" + if self.unload_later and self.unload_later.active(): + self.unload_later.cancel() + self.unload_later = None if self.loaded: + log.msg('Unloading the packages cache') del self.cache del self.records del self.srcrecords @@ -353,7 +364,7 @@ class TestAptPackages(unittest.TestCase): releaseFile = '' def setUp(self): - self.client = AptPackages(FilePath('/tmp/.apt-dht')) + self.client = AptPackages(FilePath('/tmp/.apt-dht'), 300) self.packagesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "_main_.*Packages$" | tail -n 1').read().rstrip('\n') self.sourcesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "_main_.*Sources$" | tail -n 1').read().rstrip('\n') diff --git a/apt_dht/MirrorManager.py b/apt_dht/MirrorManager.py index 84065c9..738fdeb 100644 --- a/apt_dht/MirrorManager.py +++ b/apt_dht/MirrorManager.py @@ -18,8 +18,9 @@ class MirrorError(Exception): class MirrorManager: """Manages all requests for mirror objects.""" - def __init__(self, cache_dir): + def __init__(self, cache_dir, unload_delay): self.cache_dir = cache_dir + self.unload_delay = unload_delay self.apt_caches = {} def extractPath(self, url): @@ -59,7 +60,7 @@ class MirrorManager: if baseDir not in self.apt_caches[site]: site_cache = self.cache_dir.child(aptpkg_dir).child('mirrors').child(site + baseDir.replace('/', '_')) site_cache.makedirs - self.apt_caches[site][baseDir] = AptPackages(site_cache) + self.apt_caches[site][baseDir] = AptPackages(site_cache, self.unload_delay) def updatedFile(self, url, file_path): site, baseDir, path = self.extractPath(url) @@ -82,7 +83,7 @@ class TestMirrorManager(unittest.TestCase): client = None def setUp(self): - self.client = MirrorManager(FilePath('/tmp/.apt-dht')) + self.client = MirrorManager(FilePath('/tmp/.apt-dht'), 300) def test_extractPath(self): site, baseDir, path = self.client.extractPath('http://ftp.us.debian.org/debian/dists/unstable/Release') diff --git a/apt_dht/apt_dht.py b/apt_dht/apt_dht.py index 25db818..eb103bd 100644 --- a/apt_dht/apt_dht.py +++ b/apt_dht/apt_dht.py @@ -32,7 +32,7 @@ class AptDHT: self.http_server = TopLevel(self.cache_dir.child(download_dir), self.db, self) self.getHTTPFactory = self.http_server.getHTTPFactory self.peers = PeerManager() - self.mirrors = MirrorManager(self.cache_dir) + self.mirrors = MirrorManager(self.cache_dir, config.gettime('DEFAULT', 'UNLOAD_PACKAGES_CACHE')) other_dirs = [FilePath(f) for f in config.getstringlist('DEFAULT', 'OTHER_DIRS')] self.cache = CacheManager(self.cache_dir.child(download_dir), self.db, other_dirs, self) self.my_addr = None diff --git a/apt_dht/apt_dht_conf.py b/apt_dht/apt_dht_conf.py index 42a3d38..16f10c1 100644 --- a/apt_dht/apt_dht_conf.py +++ b/apt_dht/apt_dht_conf.py @@ -38,6 +38,11 @@ DEFAULTS = { # Whether it's OK to use an IP addres from a known local/private range 'LOCAL_OK': 'no', + # Unload the packages cache after an interval of inactivity this long. + # The packages cache uses a lot of memory, and only takes a few seconds + # to reload when a new request arrives. + 'UNLOAD_PACKAGES_CACHE': '5m', + # Which DHT implementation to use. # It must be possile to do "from .DHT import DHT" to get a class that # implements the IDHT interface. diff --git a/debian/apt-dht.conf.sgml b/debian/apt-dht.conf.sgml index 6911eb8..6b7762d 100644 --- a/debian/apt-dht.conf.sgml +++ b/debian/apt-dht.conf.sgml @@ -115,6 +115,14 @@ (Default is false) + + + + The time of inactivity to wait for before unloading the + packages cache. The packages cache uses a lot of memory, and only takes a few seconds + to reload when a new request arrives. (Default is 5 minutes.) + + diff --git a/test.py b/test.py index f3a4dbc..cfac85b 100755 --- a/test.py +++ b/test.py @@ -332,6 +332,11 @@ CACHE_DIR = %(CACHE_DIR)s # Whether it's OK to use an IP addres from a known local/private range LOCAL_OK = yes +# Unload the packages cache after an interval of inactivity this long. +# The packages cache uses a lot of memory, and only takes a few seconds +# to reload when a new request arrives. +UNLOAD_PACKAGES_CACHE = 5m + # Which DHT implementation to use. # It must be possile to do "from .DHT import DHT" to get a class that # implements the IDHT interface. -- 2.39.2