Unload the AptPackages caches after a period of inactivity.

diff --git a/TODO b/TODO
index fac920195d4af7f15becfd6dd8ee53e03e08a170..b204ce719843ea8c1f22c6d1381987c38e9b7b63 100644 (file)
--- a/TODO
+++ b/TODO
@@ -1,11 +1,3 @@
-Reduce the memory footprint by clearing the AptPackages caches.
-
-The memory usage is a little bit high due to keeping the AptPackages
-caches always. Instead, they should timeout after a period of inactivity
-(say 15 minutes), and unload themselves from meory. It only takes a few
-seconds to reload, so this should not be an issue.
-
-
 Packages.diff files need to be considered.
 
 The Packages.diff/Index files contain hashes of Packages.diff/rred.gz 

diff --git a/apt-dht.conf b/apt-dht.conf
index 0b06e20eb50df6dc4d709568db14ebf6bd06b0bb..7f42e4d31822fe30147cbb5a1c7499b52495a2cd 100644 (file)
--- a/apt-dht.conf
+++ b/apt-dht.conf
@@ -29,6 +29,11 @@ CACHE_DIR = /var/cache/apt-dht
 # Whether it's OK to use an IP addres from a known local/private range
 LOCAL_OK = no
 
+# Unload the packages cache after an interval of inactivity this long.
+# The packages cache uses a lot of memory, and only takes a few seconds
+# to reload when a new request arrives.
+UNLOAD_PACKAGES_CACHE = 5m
+
 # Which DHT implementation to use.
 # It must be possile to do "from <DHT>.DHT import DHT" to get a class that
 # implements the IDHT interface. There should also be a similarly named

diff --git a/apt_dht/AptPackages.py b/apt_dht/AptPackages.py
index 0711802cf94debd10c96869de0cb67f641fae3c6..48dd4480fd729dc3b31aeb5f7a213f0bffff712f 100644 (file)
--- a/apt_dht/AptPackages.py
+++ b/apt_dht/AptPackages.py
@@ -25,7 +25,7 @@ from shutil import rmtree
 from copy import deepcopy
 from UserDict import DictMixin
 
-from twisted.internet import threads, defer
+from twisted.internet import threads, defer, reactor
 from twisted.python import log
 from twisted.python.filepath import FilePath
 from twisted.trial import unittest
@@ -135,12 +135,13 @@ class AptPackages:
                       'apt/lists/partial')
     essential_files = ('apt/dpkg/status', 'apt/etc/sources.list',)
         
-    def __init__(self, cache_dir):
+    def __init__(self, cache_dir, unload_delay):
         """Construct a new packages manager.
 
         @param cache_dir: cache directory from config file
         """
         self.cache_dir = cache_dir
+        self.unload_delay = unload_delay
         self.apt_config = deepcopy(self.DEFAULT_APT_CONFIG)
 
         for dir in self.essential_dirs:
@@ -157,6 +158,7 @@ class AptPackages:
         self.packages = PackageFileList(cache_dir)
         self.loaded = 0
         self.loading = None
+        self.unload_later = None
         
     def __del__(self):
         self.cleanup()
@@ -188,7 +190,12 @@ class AptPackages:
 
     def load(self):
         """Make sure the package is initialized and loaded."""
+        if self.unload_later and self.unload_later.active():
+            self.unload_later.reset(self.unload_delay)
+        else:
+            self.unload_later = reactor.callLater(self.unload_delay, self.unload)
         if self.loading is None:
+            log.msg('Loading the packages cache')
             self.loading = threads.deferToThread(self._load)
             self.loading.addCallback(self.doneLoading)
         return self.loading
@@ -261,7 +268,11 @@ class AptPackages:
 
     def unload(self):
         """Tries to make the packages server quit."""
+        if self.unload_later and self.unload_later.active():
+            self.unload_later.cancel()
+        self.unload_later = None
         if self.loaded:
+            log.msg('Unloading the packages cache')
             del self.cache
             del self.records
             del self.srcrecords
@@ -353,7 +364,7 @@ class TestAptPackages(unittest.TestCase):
     releaseFile = ''
     
     def setUp(self):
-        self.client = AptPackages(FilePath('/tmp/.apt-dht'))
+        self.client = AptPackages(FilePath('/tmp/.apt-dht'), 300)
     
         self.packagesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "_main_.*Packages$" | tail -n 1').read().rstrip('\n')
         self.sourcesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "_main_.*Sources$" | tail -n 1').read().rstrip('\n')

diff --git a/apt_dht/MirrorManager.py b/apt_dht/MirrorManager.py
index 84065c9477c7c7afba5447ffde7262b94b7715b7..738fdebc4d7e668d6dee6e42ee7ce2860c68ceb7 100644 (file)
--- a/apt_dht/MirrorManager.py
+++ b/apt_dht/MirrorManager.py
@@ -18,8 +18,9 @@ class MirrorError(Exception):
 class MirrorManager:
     """Manages all requests for mirror objects."""
     
-    def __init__(self, cache_dir):
+    def __init__(self, cache_dir, unload_delay):
         self.cache_dir = cache_dir
+        self.unload_delay = unload_delay
         self.apt_caches = {}
     
     def extractPath(self, url):
@@ -59,7 +60,7 @@ class MirrorManager:
         if baseDir not in self.apt_caches[site]:
             site_cache = self.cache_dir.child(aptpkg_dir).child('mirrors').child(site + baseDir.replace('/', '_'))
             site_cache.makedirs
-            self.apt_caches[site][baseDir] = AptPackages(site_cache)
+            self.apt_caches[site][baseDir] = AptPackages(site_cache, self.unload_delay)
     
     def updatedFile(self, url, file_path):
         site, baseDir, path = self.extractPath(url)
@@ -82,7 +83,7 @@ class TestMirrorManager(unittest.TestCase):
     client = None
     
     def setUp(self):
-        self.client = MirrorManager(FilePath('/tmp/.apt-dht'))
+        self.client = MirrorManager(FilePath('/tmp/.apt-dht'), 300)
         
     def test_extractPath(self):
         site, baseDir, path = self.client.extractPath('http://ftp.us.debian.org/debian/dists/unstable/Release')

diff --git a/apt_dht/apt_dht.py b/apt_dht/apt_dht.py
index 25db818afa1dd994eea66e6e043aabe94f3e91b9..eb103bd34cd2f4eb285fa2c9d07437f50f8e9073 100644 (file)
--- a/apt_dht/apt_dht.py
+++ b/apt_dht/apt_dht.py
@@ -32,7 +32,7 @@ class AptDHT:
         self.http_server = TopLevel(self.cache_dir.child(download_dir), self.db, self)
         self.getHTTPFactory = self.http_server.getHTTPFactory
         self.peers = PeerManager()
-        self.mirrors = MirrorManager(self.cache_dir)
+        self.mirrors = MirrorManager(self.cache_dir, config.gettime('DEFAULT', 'UNLOAD_PACKAGES_CACHE'))
         other_dirs = [FilePath(f) for f in config.getstringlist('DEFAULT', 'OTHER_DIRS')]
         self.cache = CacheManager(self.cache_dir.child(download_dir), self.db, other_dirs, self)
         self.my_addr = None

diff --git a/apt_dht/apt_dht_conf.py b/apt_dht/apt_dht_conf.py
index 42a3d38d3bdd002b0e5fc852ab807b7e4123d3a7..16f10c1c3914731a0ec2a6fece3e1c98e331d823 100644 (file)
--- a/apt_dht/apt_dht_conf.py
+++ b/apt_dht/apt_dht_conf.py
@@ -38,6 +38,11 @@ DEFAULTS = {
     # Whether it's OK to use an IP addres from a known local/private range
     'LOCAL_OK': 'no',
 
+    # Unload the packages cache after an interval of inactivity this long.
+    # The packages cache uses a lot of memory, and only takes a few seconds
+    # to reload when a new request arrives.
+    'UNLOAD_PACKAGES_CACHE': '5m',
+
     # Which DHT implementation to use.
     # It must be possile to do "from <DHT>.DHT import DHT" to get a class that
     # implements the IDHT interface.

diff --git a/debian/apt-dht.conf.sgml b/debian/apt-dht.conf.sgml
index 6911eb8226f8bf4f6d08d1eb98af09450a7b9a46..6b7762ddd4c8f27a565f50bc3576425dd7c41f3c 100644 (file)
--- a/debian/apt-dht.conf.sgml
+++ b/debian/apt-dht.conf.sgml
@@ -115,6 +115,14 @@
                (Default is false)</para>
            </listitem>
          </varlistentry>
+         <varlistentry>
+           <term><option>UNLOAD_PACKAGES_CACHE = <replaceable>time</replaceable></option></term>
+            <listitem>
+             <para>The <replaceable>time</replaceable> of inactivity to wait for before unloading the
+                 packages cache. The packages cache uses a lot of memory, and only takes a few seconds
+                 to reload when a new request arrives. (Default is 5 minutes.)</para>
+           </listitem>
+         </varlistentry>
          <varlistentry>
            <term><option>DHT = <replaceable>string</replaceable></option></term>
             <listitem>

diff --git a/test.py b/test.py
index f3a4dbcfd3fd2a3bb4f42fbfb5ce8a7deb32dd63..cfac85b572851815e030689792ed4611fbaabd10 100755 (executable)
--- a/test.py
+++ b/test.py
@@ -332,6 +332,11 @@ CACHE_DIR = %(CACHE_DIR)s
 # Whether it's OK to use an IP addres from a known local/private range
 LOCAL_OK = yes
 
+# Unload the packages cache after an interval of inactivity this long.
+# The packages cache uses a lot of memory, and only takes a few seconds
+# to reload when a new request arrives.
+UNLOAD_PACKAGES_CACHE = 5m
+
 # Which DHT implementation to use.
 # It must be possile to do "from <DHT>.DHT import DHT" to get a class that
 # implements the IDHT interface.