From 4945f9f0026cd0861aa8d8d2c55f7afd12a8849f Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Tue, 28 Nov 2023 15:50:35 +0100 Subject: [PATCH] Continued: - first check parameter (better performance) - cache is_wanted() and is_in_url() invocations --- fba/commands.py | 2 +- fba/helpers/domain.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/fba/commands.py b/fba/commands.py index faabb9e..c85f5fe 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -311,7 +311,7 @@ def fetch_blocks(args: argparse.Namespace) -> int: if not domain_helper.is_wanted(blocker): logger.warning("blocker='%s' is not wanted - SKIPPED!", blocker) continue - elif instances.is_recent(blocker) and not args.force: + elif not args.force and instances.is_recent(blocker, "last_blocked"): logger.debug("blocker='%s' has been recently accessed - SKIPPED!", blocker) continue diff --git a/fba/helpers/domain.py b/fba/helpers/domain.py index 00bf938..d79fb20 100644 --- a/fba/helpers/domain.py +++ b/fba/helpers/domain.py @@ -28,6 +28,9 @@ from fba.models import instances logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) +# In-function cache +_cache = {} + def raise_on(domain: str): logger.debug("domain='%s' - CALLED!", domain) @@ -58,6 +61,12 @@ def is_in_url(domain: str, url: str) -> bool: raise ValueError(f"Parameter url[]='{type(url)}' is not of type 'str'") elif url == "": raise ValueError("Parameter 'url' is empty") + elif "is_in_url" in _cache and domain + url in _cache["is_in_url"]: + logger.debug("Returning cached is_found='%s' - EXIT!", _cache["is_in_url"][domain + url]) + return _cache["is_in_url"][domain + url] + elif "is_in_url" not in _cache: + logger.debug("Initializing cache for function 'is_in_url' ...") + _cache["is_in_url"] = {} punycode = domain.encode("idna").decode("utf-8") @@ -65,6 +74,7 @@ def is_in_url(domain: str, url: str) -> bool: logger.debug("components[]='%s',punycode='%s'", type(components), punycode) is_found = (punycode in [components.netloc, components.hostname]) + _cache["is_in_url"][domain + url] = is_found logger.debug("is_found='%s' - EXIT!", is_found) return is_found @@ -76,6 +86,12 @@ def is_wanted(domain: str) -> bool: raise ValueError(f"Parameter domain[]='{type(domain)}' is not of type 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") + elif "is_wanted" in _cache and domain in _cache["is_wanted"]: + logger.debug("Returning cached is_found='%s' - EXIT!", _cache["is_wanted"][domain]) + return _cache["is_wanted"][domain] + elif "is_wanted" not in _cache: + logger.debug("Initializing cache for function 'is_wanted' ...") + _cache["is_wanted"] = {} wanted = True if domain.lower() != domain: @@ -106,5 +122,6 @@ def is_wanted(domain: str) -> bool: logger.debug("domain='%s' is a tag", domain) wanted = False + _cache["is_wanted"][domain] = wanted logger.debug("wanted='%s' - EXIT!", wanted) return wanted -- 2.39.2