From a018004e255b9aa245a1cdb52c083c4c580ee98f Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Fri, 23 Jun 2023 19:24:41 +0200 Subject: [PATCH] Continued: - introduced utils.deobfuscate_domain() which encapsulates deobfuscating domain names --- fba/networks/friendica.py | 30 +------- fba/networks/pleroma.py | 153 ++++---------------------------------- fba/utils.py | 33 ++++++++ 3 files changed, 49 insertions(+), 167 deletions(-) diff --git a/fba/networks/friendica.py b/fba/networks/friendica.py index b3c4077..e8c5b4e 100644 --- a/fba/networks/friendica.py +++ b/fba/networks/friendica.py @@ -81,34 +81,8 @@ def fetch_blocks(domain: str) -> list: if blocked == "": logger.debug("line[]='%s' returned empty blocked domain - SKIPPED!") continue - elif blocked.count("*") > 0: - logger.debug("domain='%s' uses obfuscated domains, marking ...", domain) - instances.set_has_obfuscation(domain, True) - - # Obscured domain name with no hash - row = instances.deobfuscate("*", blocked) - - logger.debug("row[]='%s'", type(row)) - if row is None: - logger.warning("Cannot deobfuscate blocked='%s',domain='%s' - SKIPPED!", blocked, domain) - continue - - logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0]) - blocked = row[0] - elif blocked.count("?") > 0: - logger.debug("domain='%s' uses obfuscated domains, marking ...", domain) - instances.set_has_obfuscation(domain, True) - - # Obscured domain name with no hash - row = instances.deobfuscate("?", blocked) - - logger.debug("row[]='%s'", type(row)) - if row is None: - logger.warning("Cannot deobfuscate blocked='%s',domain='%s' - SKIPPED!", blocked, domain) - continue - - logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0]) - blocked = row[0] + + blocked = utils.deobfuscate_domain(blocked, domain) logger.debug("blocked[%s]='%s'", type(blocked), blocked) if not utils.is_domain_wanted(blocked): diff --git a/fba/networks/pleroma.py b/fba/networks/pleroma.py index 0680fa4..84a535f 100644 --- a/fba/networks/pleroma.py +++ b/fba/networks/pleroma.py @@ -115,36 +115,11 @@ def fetch_blocks(domain: str, nodeinfo_url: str) -> list: elif blocked.endswith(".tld"): logger.debug("blocked='%s' is a fake domain - SKIPPED!", blocked) continue - elif blocked.count("*") > 0: - logger.debug("domain='%s' uses obfuscated domains, marking ...", domain) - instances.set_has_obfuscation(domain, True) - # Obscured domain name with no hash - row = instances.deobfuscate("*", blocked) + logger.debug("Invoking utils.deobfuscate_domain(%s, %s) ...", blocked, domain) + blocked = utils.deobfuscate_domain(blocked, domain) - logger.debug("row[]='%s'", type(row)) - if row is None: - logger.warning("Cannot deobfuscate blocked='%s',domain='%s' - SKIPPED!", blocked, domain) - continue - - logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0]) - blocked = row[0] - elif blocked.count("?") > 0: - logger.debug("domain='%s' uses obfuscated domains, marking ...", domain) - instances.set_has_obfuscation(domain, True) - - # Obscured domain name with no hash - row = instances.deobfuscate("?", blocked) - - logger.debug("row[]='%s'", type(row)) - if row is None: - logger.warning("Cannot deobfuscate blocked='%s',domain='%s' - SKIPPED!", blocked, domain) - continue - - logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0]) - blocked = row[0] - - logger.debug("blocked='%s'", blocked) + logger.debug("blocked='%s' - DEOBFUSCATED!", blocked) if not utils.is_domain_wanted(blocked): logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked) continue @@ -176,34 +151,9 @@ def fetch_blocks(domain: str, nodeinfo_url: str) -> list: elif blocked.endswith(".tld"): logger.debug("blocked='%s' is a fake domain - SKIPPED!", blocked) continue - elif blocked.count("*") > 0: - logger.debug("domain='%s' uses obfuscated domains, marking ...", domain) - instances.set_has_obfuscation(domain, True) - - # Obscured domain name with no hash - row = instances.deobfuscate("*", blocked) - - logger.debug("row[]='%s'", type(row)) - if row is None: - logger.warning("Cannot deobfuscate blocked='%s',domain='%s' - SKIPPED!", blocked, domain) - continue - - logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0]) - blocked = row[0] - elif blocked.count("?") > 0: - logger.debug("domain='%s' uses obfuscated domains, marking ...", domain) - instances.set_has_obfuscation(domain, True) - - # Obscured domain name with no hash - row = instances.deobfuscate("?", blocked) - - logger.debug("row[]='%s'", type(row)) - if row is None: - logger.warning("Cannot deobfuscate blocked='%s',domain='%s' - SKIPPED!", blocked, domain) - continue - logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0]) - blocked = row[0] + logger.debug("Invoking utils.deobfuscate_domain(%s, %s) ...", blocked, domain) + blocked = utils.deobfuscate_domain(blocked, domain) logger.debug("blocked='%s' - DEOBFUSCATED!", blocked) if not utils.is_domain_wanted(blocked): @@ -271,34 +221,9 @@ def fetch_blocks(domain: str, nodeinfo_url: str) -> list: if blocked == "": logger.warning("blocked is empty after tidyup.domain(): domain='%s',block_level='%s'", domain, block_level) continue - elif blocked.count("*") > 0: - logger.debug("domain='%s' uses obfuscated domains, marking ...", domain) - instances.set_has_obfuscation(domain, True) - - # Obscured domain name with no hash - row = instances.deobfuscate("*", blocked) - - logger.debug("row[]='%s'", type(row)) - if row is None: - logger.warning("Cannot deobfuscate blocked='%s',domain='%s' - SKIPPED!", blocked, domain) - continue - logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0]) - blocked = row[0] - elif blocked.count("?") > 0: - logger.debug("domain='%s' uses obfuscated domains, marking ...", domain) - instances.set_has_obfuscation(domain, True) - - # Obscured domain name with no hash - row = instances.deobfuscate("?", blocked) - - logger.debug("row[]='%s'", type(row)) - if row is None: - logger.warning("Cannot deobfuscate blocked='%s',domain='%s' - SKIPPED!", blocked, domain) - continue - - logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0]) - blocked = row[0] + logger.debug("Invoking utils.deobfuscate_domain(%s, %s) ...", blocked, domain) + blocked = utils.deobfuscate_domain(blocked, domain) logger.debug("blocked='%s' - DEOBFUSCATED!", blocked) if not utils.is_domain_wanted(blocked): @@ -334,36 +259,11 @@ def fetch_blocks(domain: str, nodeinfo_url: str) -> list: if blocked == "": logger.warning("blocked is empty after tidyup.domain(): domain='%s',block_level='%s'", domain, block_level) continue - elif blocked.count("*") > 0: - logger.debug("domain='%s' uses obfuscated domains, marking ...", domain) - instances.set_has_obfuscation(domain, True) - - # Obscured domain name with no hash - row = instances.deobfuscate("*", blocked) - - logger.debug("row[]='%s'", type(row)) - if row is None: - logger.warning("Cannot deobfuscate blocked='%s',domain='%s' - SKIPPED!", blocked, domain) - continue - - logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0]) - blocked = row[0] - elif blocked.count("?") > 0: - logger.debug("domain='%s' uses obfuscated domains, marking ...", domain) - instances.set_has_obfuscation(domain, True) - # Obscured domain name with no hash - row = instances.deobfuscate("?", blocked) + logger.debug("Invoking utils.deobfuscate_domain(%s, %s) ...", blocked, domain) + blocked = utils.deobfuscate_domain(blocked, domain) - logger.debug("row[]='%s'", type(row)) - if row is None: - logger.warning("Cannot deobfuscate blocked='%s',domain='%s' - SKIPPED!", blocked, domain) - continue - - logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0]) - blocked = row[0] - - logger.debug("blocked='%s'", blocked) + logger.debug("blocked='%s' - DEOBFUSCATED!", blocked) if not utils.is_domain_wanted(blocked): logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked) continue @@ -386,8 +286,8 @@ def fetch_blocks(domain: str, nodeinfo_url: str) -> list: logger.info("Checking %d record(s) ...", len(blocklist)) for block_level in blocklist: logger.debug("block_level='%s'", block_level) - rows = blocklist[block_level] + logger.debug("rows[%s]()=%d'", type(rows), len(rows)) for record in rows: logger.debug("record[]='%s'", type(record)) @@ -398,34 +298,9 @@ def fetch_blocks(domain: str, nodeinfo_url: str) -> list: if blocked == "": logger.warning("blocked is empty after tidyup.domain(): domain='%s',block_level='%s'", domain, block_level) continue - elif blocked.count("*") > 0: - logger.debug("domain='%s' uses obfuscated domains, marking ...", domain) - instances.set_has_obfuscation(domain, True) - - # Obscured domain name with no hash - row = instances.deobfuscate("*", blocked) - - logger.debug("row[]='%s'", type(row)) - if row is None: - logger.warning("Cannot deobfuscate blocked='%s',domain='%s' - SKIPPED!", blocked, domain) - continue - - logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0]) - blocked = row[0] - elif blocked.count("?") > 0: - logger.debug("domain='%s' uses obfuscated domains, marking ...", domain) - instances.set_has_obfuscation(domain, True) - - # Obscured domain name with no hash - row = instances.deobfuscate("?", blocked) - - logger.debug("row[]='%s'", type(row)) - if row is None: - logger.warning("Cannot deobfuscate blocked='%s',domain='%s' - SKIPPED!", blocked, domain) - continue - - logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0]) - blocked = row[0] + + logger.debug("Invoking utils.deobfuscate_domain(%s, %s) ...", blocked, domain) + blocked = utils.deobfuscate_domain(blocked, domain) logger.debug("blocked='%s' - DEOBFUSCATED!", blocked) if not utils.is_domain_wanted(blocked): diff --git a/fba/utils.py b/fba/utils.py index 577e9ab..bdda207 100644 --- a/fba/utils.py +++ b/fba/utils.py @@ -196,3 +196,36 @@ def is_domain_wanted(domain: str) -> bool: logger.debug("wanted='%s' - EXIT!", wanted) return wanted + +def deobfuscate_domain(domain: str, blocker: str) -> str: + logger.debug("domain='%s',blocker='%s' - CALLED!", domain, blocker) + domain_helper.raise_on(domain) + domain_helper.raise_on(blocker) + + if domain.count("*") > 0: + logger.debug("blocker='%s' uses obfuscated domains, marking ...", blocker) + instances.set_has_obfuscation(blocker, True) + + # Obscured domain name with no hash + row = instances.deobfuscate("*", domain) + + logger.debug("row[]='%s'", type(row)) + if row is not None: + logger.debug("domain='%s' de-obscured to '%s'", domain, row[0]) + domain = row[0] + elif domain.count("?") > 0: + logger.debug("blocker='%s' uses obfuscated domains, marking ...", blocker) + instances.set_has_obfuscation(blocker, True) + + # Obscured domain name with no hash + row = instances.deobfuscate("?", domain) + + logger.debug("row[]='%s'", type(row)) + if row is not None: + logger.debug("domain='%s' de-obscured to '%s'", domain, row[0]) + domain = row[0] + else: + logger.debug("domain='%s' is not obfuscated", domain) + + logger.debug("domain='%s' - EXIT!", domain) + return domain -- 2.39.5