Continued:

- reduce more `conig.get()` invocations by "caching" (locally referencing) the
  value

diff --git a/fba/commands.py b/fba/commands.py
index 6a3a3a584a043878fe25f76d52f1b9160b156e97..bdfb163dec87b94a76a4c173763e4a77b865aeef 100644 (file)
--- a/fba/commands.py
+++ b/fba/commands.py
@@ -675,6 +675,8 @@ def fetch_todon_wiki(args: argparse.Namespace) -> int:
     instances.set_total_blocks(blocker, blocking)
 
     blockdict = []
+
+    logger.info("Checking %d block lists ...", len(blocklist))
     for block_level in blocklist:
         logger.debug("Checking %d blocker entries for block_level='%s' ...", len(blocklist[block_level]), block_level)
         for blocked in blocklist[block_level]:

diff --git a/fba/helpers/domain.py b/fba/helpers/domain.py
index 965796a08a166aadd8ecc833cd0498f8b17d83dd..6948fb81bf077a97ae776e327e95d1c75789a559 100644 (file)
--- a/fba/helpers/domain.py
+++ b/fba/helpers/domain.py
@@ -30,6 +30,9 @@ logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 #logger.setLevel(logging.DEBUG)
 
+# "Cache" configuration get() invocations
+_allow_i2p_domain = config.get("allow_i2p_domain")
+
 def raise_on(domain: str) -> None:
     logger.debug("domain='%s' - CALLED!", domain)
 
@@ -45,7 +48,7 @@ def raise_on(domain: str) -> None:
         raise ValueError(f"domain='{domain}' is not a valid domain")
     elif domain.endswith(".onion"):
         raise ValueError(f"domain='{domain}' is a TOR, please don't crawl them!")
-    elif domain.endswith(".i2p") and not config.get("allow_i2p_domain"):
+    elif domain.endswith(".i2p") and not _allow_i2p_domain:
         raise ValueError(f"domain='{domain}' is an I2P, please don't crawl them!")
     elif domain.endswith(".arpa"):
         raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!")
@@ -93,7 +96,7 @@ def is_tld_wanted(domain: str) -> bool:
     if domain.endswith(".onion"):
         logger.debug("domain='%s' is a TOR .onion domain - setting wanted=False ...", domain)
         wanted = False
-    elif domain.endswith(".i2p") and not config.get("allow_i2p_domain"):
+    elif domain.endswith(".i2p") and not _allow_i2p_domain:
         logger.debug("domain='%s' is an I2P .onion domain - setting wanted=False ...", domain)
         wanted = False
     elif domain.endswith(".arpa"):

diff --git a/fba/helpers/processing.py b/fba/helpers/processing.py
index 6472afdda2bc603af070d49d57c04a3338a8c187..c4d1f341913958130f7212b3940a7a533c3be01d 100644 (file)
--- a/fba/helpers/processing.py
+++ b/fba/helpers/processing.py
@@ -37,6 +37,9 @@ logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 #logger.setLevel(logging.DEBUG)
 
+# "Cache" configuration get() invocations
+_bot_enabled = config.get("bot_enabled")
+
 def instance(blocked: str, blocker: str, command: str, force: bool = False) -> bool:
     logger.debug("blocked='%s',blocker='%s',command='%s',force='%s' - CALLED!", blocked, blocker, command, force)
     domain_helper.raise_on(blocked)
@@ -226,7 +229,7 @@ def csv_block(blocker: str, url: str, command: str) -> None:
         processed = instance(domain, blocker, command)
         logger.debug("processed='%s'", processed)
 
-        if block(blocker, domain, reason, severity) and config.get("bot_enabled"):
+        if block(blocker, domain, reason, severity) and _bot_enabled:
             logger.debug("Appending blocked='%s',reason='%s' for blocker='%s' ...", domain, reason, blocker)
             blockdict.append({
                 "blocked": domain,
@@ -254,8 +257,8 @@ def csv_block(blocker: str, url: str, command: str) -> None:
         logger.debug("Flushing updates for blocker='%s' ...", blocker)
         instances.update(blocker)
 
-    logger.debug("config.get(bot_enabled)='%s',blockdict()=%d", config.get("bot_enabled"), len(blockdict))
-    if config.get("bot_enabled") and len(blockdict) > 0:
+    logger.debug("config.get(bot_enabled)='%s',blockdict()=%d", _bot_enabled, len(blockdict))
+    if _bot_enabled and len(blockdict) > 0:
         logger.info("Sending bot POST for blocker='%s',blockdict()=%d ...", blocker, len(blockdict))
         network.send_bot_post(blocker, blockdict)
 

diff --git a/fba/models/error_log.py b/fba/models/error_log.py
index 718540d30e17b941d7a639e1e02a18bddbd9e819..5ece4b36470c692832c39ddcb4f8a1cd12f88da0 100644 (file)
--- a/fba/models/error_log.py
+++ b/fba/models/error_log.py
@@ -28,7 +28,8 @@ logger = logging.getLogger(__name__)
 #logger.setLevel(logging.DEBUG)
 
 # "Cached" configuration values
-_write_error_log = config.get("write_error_log")
+_write_error_log   = config.get("write_error_log")
+_error_log_cleanup = config.get("error_log_cleanup")
 
 def add(domain: str, error: dict) -> None:
     logger.debug("domain='%s',error[]='%s' - CALLED!", domain, type(error))
@@ -59,7 +60,7 @@ def add(domain: str, error: dict) -> None:
         ])
 
     # Cleanup old entries
-    logger.debug("Purging old records (distance: %d)", config.get('error_log_cleanup'))
-    database.cursor.execute("DELETE FROM error_log WHERE created < ?", [time.time() - config.get("error_log_cleanup")])
+    logger.debug("Purging old records (distance: %d)", _error_log_cleanup)
+    database.cursor.execute("DELETE FROM error_log WHERE created < ?", [time.time() - _error_log_cleanup])
 
     logger.debug("EXIT!")