From b89243605d1b0462187701c037143d5f302d45f7 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Sat, 27 Jul 2024 23:36:07 +0200 Subject: [PATCH] Continued: - reworked parts of finding out if blocker has blocked domains with placeholders like '*' or '?' in it - improved some logger messages --- fba/commands.py | 22 ++++++++++++---------- fba/networks/pleroma.py | 2 +- fba/utils.py | 3 --- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/fba/commands.py b/fba/commands.py index b845499..45bfbd7 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -1341,7 +1341,7 @@ def recheck_obfuscation(args: argparse.Namespace) -> int: if isinstance(args.domain, str) and args.domain != "" and domain_helper.is_wanted(args.domain): logger.debug("Fetching record for args.domain='%s' ...", args.domain) - database.cursor.execute("SELECT domain, software, nodeinfo_url FROM instances WHERE (has_obfuscation = 1 OR has_obfuscation IS NULL) AND domain = ? ORDER BY last_blocked ASC", [args.domain]) + database.cursor.execute("SELECT domain, software, nodeinfo_url FROM instances WHERE domain = ? LIMIT 1", [args.domain]) elif isinstance(args.software, str) and args.software != "" and validators.domain(args.software, rfc_2782=True) == args.software: logger.debug("Fetching records for args.software='%s' ...", args.software) database.cursor.execute("SELECT domain, software, nodeinfo_url FROM instances WHERE (has_obfuscation = 1 OR has_obfuscation IS NULL) AND software = ? ORDER BY last_blocked ASC", [args.software]) @@ -1373,6 +1373,9 @@ def recheck_obfuscation(args: argparse.Namespace) -> int: logger.debug("Invoking federation.fetch_blocks(%s) ...", row["domain"]) blocking = federation.fetch_blocks(row["domain"]) + logger.debug("Setting has_obfuscation=False for row[domain]='%s' ...", row["domain"]) + instances.set_has_obfuscation(row["domain"], False) + logger.debug("blocking()=%d", len(blocking)) if len(blocking) == 0: logger.debug("Empty blocking list, trying individual fetch_blocks() for row[software]='%s' ...", row["software"]) @@ -1404,7 +1407,7 @@ def recheck_obfuscation(args: argparse.Namespace) -> int: obfuscated = 0 blockdict = list() - logger.info("Checking %d block(s) from domain='%s' ...", len(blocking), row["domain"]) + logger.info("Checking %d block(s) from row[domain]='%s' ...", len(blocking), row["domain"]) for block in blocking: logger.debug("block[blocked]='%s'", block["blocked"]) blocked = None @@ -1413,26 +1416,26 @@ def recheck_obfuscation(args: argparse.Namespace) -> int: logger.debug("block[blocked] is empty - SKIPPED!") continue elif block["blocked"].endswith(".onion"): - logger.debug("blocked='%s' is a TOR onion domain name - SKIPPED!", block["blocked"]) + logger.debug("block[blocked]='%s' is a TOR onion domain name - SKIPPED!", block["blocked"]) continue elif block["blocked"].endswith(".i2p") and not config.get("allow_i2p_domain"): - logger.debug("blocked='%s' is an I2P onion domain name - SKIPPED!", block["blocked"]) + logger.debug("block[blocked]='%s' is an I2P onion domain name - SKIPPED!", block["blocked"]) continue elif block["blocked"].endswith(".arpa"): - logger.debug("blocked='%s' is a reversed IP address - SKIPPED!", block["blocked"]) + logger.debug("block[blocked]='%s' is a reversed IP address - SKIPPED!", block["blocked"]) continue elif block["blocked"].endswith(".tld"): - logger.debug("blocked='%s' is a fake domain name - SKIPPED!", block["blocked"]) + logger.debug("block[blocked]='%s' is a fake domain name - SKIPPED!", block["blocked"]) continue elif block["blocked"].find("*") >= 0 or block["blocked"].find("?") >= 0: - logger.debug("block='%s' is obfuscated.", block["blocked"]) + logger.debug("block[blocked]='%s' is obfuscated.", block["blocked"]) obfuscated = obfuscated + 1 blocked = utils.deobfuscate(block["blocked"], row["domain"], block["digest"] if "digest" in block else None) elif not domain_helper.is_wanted(block["blocked"]): - logger.debug("blocked='%s' is not wanted - SKIPPED!", block["blocked"]) + logger.debug("block[blocked]='%s' is not wanted - SKIPPED!", block["blocked"]) continue elif blocks.is_instance_blocked(row["domain"], block["blocked"]): - logger.debug("blocked='%s' is already blocked - SKIPPED!", block["blocked"]) + logger.debug("block[blocked]='%s' is already blocked - SKIPPED!", block["blocked"]) continue logger.debug("blocked[%s]='%s',block[blocked]='%s'", type(blocked), blocked, block["blocked"]) @@ -1460,7 +1463,6 @@ def recheck_obfuscation(args: argparse.Namespace) -> int: }) logger.debug("Setting obfuscated=%d for row[domain]='%s' ...", obfuscated, row["domain"]) - instances.set_has_obfuscation(row["domain"], (obfuscated > 0)) instances.set_obfuscated_blocks(row["domain"], obfuscated) logger.info("domain='%s' has %d obfuscated domain(s)", row["domain"], obfuscated) diff --git a/fba/networks/pleroma.py b/fba/networks/pleroma.py index 07d2a2e..9c44e88 100644 --- a/fba/networks/pleroma.py +++ b/fba/networks/pleroma.py @@ -139,7 +139,7 @@ def fetch_blocks(domain: str) -> list: if blocked in [None, ""]: logger.warning("instance[host]='%s' is None or empty after tidyup.domain() - SKIPPED!", instance["host"]) continue - elif not domain_helper.is_wanted(blocked): + elif blocked.find("*") == -1 and blocked.find("?") == -1 and not domain_helper.is_wanted(blocked): logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked) continue diff --git a/fba/utils.py b/fba/utils.py index d56110d..92d2b1b 100644 --- a/fba/utils.py +++ b/fba/utils.py @@ -135,9 +135,6 @@ def deobfuscate(domain: str, blocker: str, domain_hash: str = None) -> str: elif not isinstance(domain_hash, str) and domain_hash is not None: raise ValueError(f"Parameter domain_hash[]='{type(domain_hash)}' is not of type 'str'") - logger.debug("Setting has_obfuscation=False for blocker='%s' ...", blocker) - instances.set_has_obfuscation(blocker, False) - logger.debug("Checking domain='%s' ...", domain) if domain.find("*") >= 0: logger.debug("blocker='%s' uses obfuscated domains", blocker) -- 2.39.5