Continued:

- renamed utils.deobfuscate_domain() to deobfuscate()
- oliphant blocklists may contain obfuscated domains, need to deobfuscate them
  first to get actual domain names

diff --git a/fba/commands.py b/fba/commands.py
index 6b63c8debf8527e3a5838aa03d4d77b519b03ad1..e4ae2212c82bd3f5a6f14fa7f53af1a78f33a882 100644 (file)
--- a/fba/commands.py
+++ b/fba/commands.py
@@ -1023,8 +1023,25 @@ def fetch_oliphant(args: argparse.Namespace) -> int:
             if domain == "":
                 logger.debug("domain is empty - SKIPPED!")
                 continue
-            elif not utils.is_domain_wanted(domain):
-                logger.warning("domain='%s' is not wanted - SKIPPED!", domain)
+            elif domain.endswith(".onion"):
+                logger.debug("domain='%s' is a TOR .onion domain - SKIPPED", domain)
+                continue
+            elif domain.endswith(".arpa"):
+                logger.debug("domain='%s' is a reverse IP address - SKIPPED", domain)
+                continue
+            elif domain.endswith(".tld"):
+                logger.debug("domain='%s' is a fake domain - SKIPPED", domain)
+                continue
+            elif domain.find("*") >= 0 or domain.find("?") >= 0:
+                logger.debug("domain='%s' is obfuscated - Invoking utils.deobfuscate(%s, %s) ...", domain, domain, block["blocker"])
+                domain = utils.deobfuscate(domain, block["blocker"])
+                logger.debug("domain='%s' - AFTER!", domain)
+
+            if not validators.domain(domain):
+                logger.debug("domain='%s' is not a valid domain - SKIPPED!")
+                continue
+            elif blacklist.is_blacklisted(domain):
+                logger.warning("domain='%s' is blacklisted - SKIPPED!", domain)
                 continue
 
             logger.debug("Marking domain='%s' as handled", domain)
@@ -1404,7 +1421,7 @@ def recheck_obfuscation(args: argparse.Namespace) -> int:
             elif block["blocked"].find("*") >= 0 or block["blocked"].find("?") >= 0:
                 logger.debug("block='%s' is obfuscated.", block["blocked"])
                 obfuscated = obfuscated + 1
-                blocked = utils.deobfuscate_domain(block["blocked"], row["domain"], block["hash"] if "hash" in block else None)
+                blocked = utils.deobfuscate(block["blocked"], row["domain"], block["hash"] if "hash" in block else None)
             elif not utils.is_domain_wanted(block["blocked"]):
                 logger.warning("blocked='%s' is not wanted - SKIPPED!", block["blocked"])
                 continue

diff --git a/fba/networks/friendica.py b/fba/networks/friendica.py
index ab985c7f3f69413e4f56a4d53939f187a2b35dbc..e3fdb7111fb743a15bdb3b7e8310e6b3824607e9 100644 (file)
--- a/fba/networks/friendica.py
+++ b/fba/networks/friendica.py
@@ -86,7 +86,7 @@ def fetch_blocks(domain: str) -> list:
             continue
 
         logger.debug("blocked='%s',domain='%s' - BEFORE!", blocked, domain)
-        blocked = utils.deobfuscate_domain(blocked, domain)
+        blocked = utils.deobfuscate(blocked, domain)
 
         logger.debug("blocked[%s]='%s' - DEOBFUSCATED!", type(blocked), blocked)
         if not utils.is_domain_wanted(blocked):

diff --git a/fba/networks/pleroma.py b/fba/networks/pleroma.py
index abbf88d4caf8b2d43a611f3534eb9d05b76262c7..f7964be4e486d835d89c0bb3455a34c93492d572 100644 (file)
--- a/fba/networks/pleroma.py
+++ b/fba/networks/pleroma.py
@@ -130,8 +130,8 @@ def fetch_blocks(domain: str, nodeinfo_url: str) -> list:
                         logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
                         continue
 
-                    logger.debug("Invoking utils.deobfuscate_domain(%s, %s) ...", blocked, domain)
-                    blocked = utils.deobfuscate_domain(blocked, domain)
+                    logger.debug("Invoking utils.deobfuscate(%s, %s) ...", blocked, domain)
+                    blocked = utils.deobfuscate(blocked, domain)
 
                     logger.debug("blocked='%s' - DEOBFUSCATED!", blocked)
                     if not utils.is_domain_wanted(blocked):
@@ -163,8 +163,8 @@ def fetch_blocks(domain: str, nodeinfo_url: str) -> list:
                 logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
                 continue
 
-            logger.debug("Invoking utils.deobfuscate_domain(%s, %s) ...", blocked, domain)
-            blocked = utils.deobfuscate_domain(blocked, domain)
+            logger.debug("Invoking utils.deobfuscate(%s, %s) ...", blocked, domain)
+            blocked = utils.deobfuscate(blocked, domain)
 
             logger.debug("blocked='%s' - DEOBFUSCATED!", blocked)
             if not utils.is_domain_wanted(blocked):
@@ -232,8 +232,8 @@ def fetch_blocks(domain: str, nodeinfo_url: str) -> list:
                     logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
                     continue
 
-                logger.debug("Invoking utils.deobfuscate_domain(%s, %s) ...", blocked, domain)
-                blocked = utils.deobfuscate_domain(blocked, domain)
+                logger.debug("Invoking utils.deobfuscate(%s, %s) ...", blocked, domain)
+                blocked = utils.deobfuscate(blocked, domain)
                 logger.debug("blocked='%s' - DEOBFUSCATED!", blocked)
 
                 logger.debug("Checking %d blockdict records ...", len(blockdict))
@@ -266,8 +266,8 @@ def fetch_blocks(domain: str, nodeinfo_url: str) -> list:
                 logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
                 continue
 
-            logger.debug("Invoking utils.deobfuscate_domain(%s, %s) ...", blocked, domain)
-            blocked = utils.deobfuscate_domain(blocked, domain)
+            logger.debug("Invoking utils.deobfuscate(%s, %s) ...", blocked, domain)
+            blocked = utils.deobfuscate(blocked, domain)
 
             logger.debug("blocked='%s' - DEOBFUSCATED!", blocked)
             if not utils.is_domain_wanted(blocked):
@@ -296,8 +296,8 @@ def fetch_blocks(domain: str, nodeinfo_url: str) -> list:
 
                 logger.debug("rows[%s]()=%d'", type(rows), len(rows))
                 for block in rows:
-                    logger.debug("Invoking utils.deobfuscate_domain(%s, %s) ...", block["blocked"], domain)
-                    block["blocked"] = utils.deobfuscate_domain(block["blocked"], domain)
+                    logger.debug("Invoking utils.deobfuscate(%s, %s) ...", block["blocked"], domain)
+                    block["blocked"] = utils.deobfuscate(block["blocked"], domain)
 
                     logger.debug("block[blocked]='%s' - DEOBFUSCATED!", block["blocked"])
                     if not utils.is_domain_wanted(block["blocked"]):

diff --git a/fba/utils.py b/fba/utils.py
index 30f3a1c0b43d019173ec068cc58f158f5e085bb9..727af2bc81e4c576d661f9e40f875387ff8db4ab 100644 (file)
--- a/fba/utils.py
+++ b/fba/utils.py
@@ -82,7 +82,7 @@ def process_domain(domain: str, blocker: str, command: str) -> bool:
         raise ValueError("Parameter 'command' is empty")
 
     logger.debug("domain='%s' - BEFORE!", domain)
-    domain = deobfuscate_domain(domain, blocker)
+    domain = deobfuscate(domain, blocker)
 
     logger.debug("domain='%s' - DEOBFUSCATED!", domain)
     if instances.has_pending(blocker):
@@ -178,7 +178,7 @@ def is_domain_wanted(domain: str) -> bool:
     logger.debug("wanted='%s' - EXIT!", wanted)
     return wanted
 
-def deobfuscate_domain(domain: str, blocker: str, domain_hash: str = None) -> str:
+def deobfuscate(domain: str, blocker: str, domain_hash: str = None) -> str:
     logger.debug("domain='%s',blocker='%s',domain_hash='%s' - CALLED!", domain, blocker, domain_hash)
     domain_helper.raise_on(blocker)