From 449ef58c0aca6a0e709b967764bb8c47f246a922 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Mon, 10 Jul 2023 21:12:11 +0200 Subject: [PATCH] Continued: - renamed utils.deobfuscate_domain() to deobfuscate() - oliphant blocklists may contain obfuscated domains, need to deobfuscate them first to get actual domain names --- fba/commands.py | 23 ++++++++++++++++++++--- fba/networks/friendica.py | 2 +- fba/networks/pleroma.py | 20 ++++++++++---------- fba/utils.py | 4 ++-- 4 files changed, 33 insertions(+), 16 deletions(-) diff --git a/fba/commands.py b/fba/commands.py index 6b63c8d..e4ae221 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -1023,8 +1023,25 @@ def fetch_oliphant(args: argparse.Namespace) -> int: if domain == "": logger.debug("domain is empty - SKIPPED!") continue - elif not utils.is_domain_wanted(domain): - logger.warning("domain='%s' is not wanted - SKIPPED!", domain) + elif domain.endswith(".onion"): + logger.debug("domain='%s' is a TOR .onion domain - SKIPPED", domain) + continue + elif domain.endswith(".arpa"): + logger.debug("domain='%s' is a reverse IP address - SKIPPED", domain) + continue + elif domain.endswith(".tld"): + logger.debug("domain='%s' is a fake domain - SKIPPED", domain) + continue + elif domain.find("*") >= 0 or domain.find("?") >= 0: + logger.debug("domain='%s' is obfuscated - Invoking utils.deobfuscate(%s, %s) ...", domain, domain, block["blocker"]) + domain = utils.deobfuscate(domain, block["blocker"]) + logger.debug("domain='%s' - AFTER!", domain) + + if not validators.domain(domain): + logger.debug("domain='%s' is not a valid domain - SKIPPED!") + continue + elif blacklist.is_blacklisted(domain): + logger.warning("domain='%s' is blacklisted - SKIPPED!", domain) continue logger.debug("Marking domain='%s' as handled", domain) @@ -1404,7 +1421,7 @@ def recheck_obfuscation(args: argparse.Namespace) -> int: elif block["blocked"].find("*") >= 0 or block["blocked"].find("?") >= 0: logger.debug("block='%s' is obfuscated.", block["blocked"]) obfuscated = obfuscated + 1 - blocked = utils.deobfuscate_domain(block["blocked"], row["domain"], block["hash"] if "hash" in block else None) + blocked = utils.deobfuscate(block["blocked"], row["domain"], block["hash"] if "hash" in block else None) elif not utils.is_domain_wanted(block["blocked"]): logger.warning("blocked='%s' is not wanted - SKIPPED!", block["blocked"]) continue diff --git a/fba/networks/friendica.py b/fba/networks/friendica.py index ab985c7..e3fdb71 100644 --- a/fba/networks/friendica.py +++ b/fba/networks/friendica.py @@ -86,7 +86,7 @@ def fetch_blocks(domain: str) -> list: continue logger.debug("blocked='%s',domain='%s' - BEFORE!", blocked, domain) - blocked = utils.deobfuscate_domain(blocked, domain) + blocked = utils.deobfuscate(blocked, domain) logger.debug("blocked[%s]='%s' - DEOBFUSCATED!", type(blocked), blocked) if not utils.is_domain_wanted(blocked): diff --git a/fba/networks/pleroma.py b/fba/networks/pleroma.py index abbf88d..f7964be 100644 --- a/fba/networks/pleroma.py +++ b/fba/networks/pleroma.py @@ -130,8 +130,8 @@ def fetch_blocks(domain: str, nodeinfo_url: str) -> list: logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked) continue - logger.debug("Invoking utils.deobfuscate_domain(%s, %s) ...", blocked, domain) - blocked = utils.deobfuscate_domain(blocked, domain) + logger.debug("Invoking utils.deobfuscate(%s, %s) ...", blocked, domain) + blocked = utils.deobfuscate(blocked, domain) logger.debug("blocked='%s' - DEOBFUSCATED!", blocked) if not utils.is_domain_wanted(blocked): @@ -163,8 +163,8 @@ def fetch_blocks(domain: str, nodeinfo_url: str) -> list: logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked) continue - logger.debug("Invoking utils.deobfuscate_domain(%s, %s) ...", blocked, domain) - blocked = utils.deobfuscate_domain(blocked, domain) + logger.debug("Invoking utils.deobfuscate(%s, %s) ...", blocked, domain) + blocked = utils.deobfuscate(blocked, domain) logger.debug("blocked='%s' - DEOBFUSCATED!", blocked) if not utils.is_domain_wanted(blocked): @@ -232,8 +232,8 @@ def fetch_blocks(domain: str, nodeinfo_url: str) -> list: logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked) continue - logger.debug("Invoking utils.deobfuscate_domain(%s, %s) ...", blocked, domain) - blocked = utils.deobfuscate_domain(blocked, domain) + logger.debug("Invoking utils.deobfuscate(%s, %s) ...", blocked, domain) + blocked = utils.deobfuscate(blocked, domain) logger.debug("blocked='%s' - DEOBFUSCATED!", blocked) logger.debug("Checking %d blockdict records ...", len(blockdict)) @@ -266,8 +266,8 @@ def fetch_blocks(domain: str, nodeinfo_url: str) -> list: logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked) continue - logger.debug("Invoking utils.deobfuscate_domain(%s, %s) ...", blocked, domain) - blocked = utils.deobfuscate_domain(blocked, domain) + logger.debug("Invoking utils.deobfuscate(%s, %s) ...", blocked, domain) + blocked = utils.deobfuscate(blocked, domain) logger.debug("blocked='%s' - DEOBFUSCATED!", blocked) if not utils.is_domain_wanted(blocked): @@ -296,8 +296,8 @@ def fetch_blocks(domain: str, nodeinfo_url: str) -> list: logger.debug("rows[%s]()=%d'", type(rows), len(rows)) for block in rows: - logger.debug("Invoking utils.deobfuscate_domain(%s, %s) ...", block["blocked"], domain) - block["blocked"] = utils.deobfuscate_domain(block["blocked"], domain) + logger.debug("Invoking utils.deobfuscate(%s, %s) ...", block["blocked"], domain) + block["blocked"] = utils.deobfuscate(block["blocked"], domain) logger.debug("block[blocked]='%s' - DEOBFUSCATED!", block["blocked"]) if not utils.is_domain_wanted(block["blocked"]): diff --git a/fba/utils.py b/fba/utils.py index 30f3a1c..727af2b 100644 --- a/fba/utils.py +++ b/fba/utils.py @@ -82,7 +82,7 @@ def process_domain(domain: str, blocker: str, command: str) -> bool: raise ValueError("Parameter 'command' is empty") logger.debug("domain='%s' - BEFORE!", domain) - domain = deobfuscate_domain(domain, blocker) + domain = deobfuscate(domain, blocker) logger.debug("domain='%s' - DEOBFUSCATED!", domain) if instances.has_pending(blocker): @@ -178,7 +178,7 @@ def is_domain_wanted(domain: str) -> bool: logger.debug("wanted='%s' - EXIT!", wanted) return wanted -def deobfuscate_domain(domain: str, blocker: str, domain_hash: str = None) -> str: +def deobfuscate(domain: str, blocker: str, domain_hash: str = None) -> str: logger.debug("domain='%s',blocker='%s',domain_hash='%s' - CALLED!", domain, blocker, domain_hash) domain_helper.raise_on(blocker) -- 2.39.5