Continued:

- introduced utils.deobfuscate_domain() which encapsulates deobfuscating domain
  names

diff --git a/fba/networks/friendica.py b/fba/networks/friendica.py
index b3c4077f92c0407d180cbde39ae09f5c6b95511a..e8c5b4ed44d3e60d9fce624c918353bd9cdf8dbf 100644 (file)
--- a/fba/networks/friendica.py
+++ b/fba/networks/friendica.py
@@ -81,34 +81,8 @@ def fetch_blocks(domain: str) -> list:
         if blocked == "":
             logger.debug("line[]='%s' returned empty blocked domain - SKIPPED!")
             continue
-        elif blocked.count("*") > 0:
-            logger.debug("domain='%s' uses obfuscated domains, marking ...", domain)
-            instances.set_has_obfuscation(domain, True)
-
-            # Obscured domain name with no hash
-            row = instances.deobfuscate("*", blocked)
-
-            logger.debug("row[]='%s'", type(row))
-            if row is None:
-                logger.warning("Cannot deobfuscate blocked='%s',domain='%s' - SKIPPED!", blocked, domain)
-                continue
-
-            logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0])
-            blocked = row[0]
-        elif blocked.count("?") > 0:
-            logger.debug("domain='%s' uses obfuscated domains, marking ...", domain)
-            instances.set_has_obfuscation(domain, True)
-
-            # Obscured domain name with no hash
-            row = instances.deobfuscate("?", blocked)
-
-            logger.debug("row[]='%s'", type(row))
-            if row is None:
-                logger.warning("Cannot deobfuscate blocked='%s',domain='%s' - SKIPPED!", blocked, domain)
-                continue
-
-            logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0])
-            blocked = row[0]
+
+        blocked = utils.deobfuscate_domain(blocked, domain)
 
         logger.debug("blocked[%s]='%s'", type(blocked), blocked)
         if not utils.is_domain_wanted(blocked):

diff --git a/fba/networks/pleroma.py b/fba/networks/pleroma.py
index 0680fa42b83626551afa3cda6440d38ad7dc47e4..84a535fee0adf4e0597327a09d39ca50a20c2266 100644 (file)
--- a/fba/networks/pleroma.py
+++ b/fba/networks/pleroma.py
@@ -115,36 +115,11 @@ def fetch_blocks(domain: str, nodeinfo_url: str) -> list:
                     elif blocked.endswith(".tld"):
                         logger.debug("blocked='%s' is a fake domain - SKIPPED!", blocked)
                         continue
-                    elif blocked.count("*") > 0:
-                        logger.debug("domain='%s' uses obfuscated domains, marking ...", domain)
-                        instances.set_has_obfuscation(domain, True)
 
-                        # Obscured domain name with no hash
-                        row = instances.deobfuscate("*", blocked)
+                    logger.debug("Invoking utils.deobfuscate_domain(%s, %s) ...", blocked, domain)
+                    blocked = utils.deobfuscate_domain(blocked, domain)
 
-                        logger.debug("row[]='%s'", type(row))
-                        if row is None:
-                            logger.warning("Cannot deobfuscate blocked='%s',domain='%s' - SKIPPED!", blocked, domain)
-                            continue
-
-                        logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0])
-                        blocked = row[0]
-                    elif blocked.count("?") > 0:
-                        logger.debug("domain='%s' uses obfuscated domains, marking ...", domain)
-                        instances.set_has_obfuscation(domain, True)
-
-                        # Obscured domain name with no hash
-                        row = instances.deobfuscate("?", blocked)
-
-                        logger.debug("row[]='%s'", type(row))
-                        if row is None:
-                            logger.warning("Cannot deobfuscate blocked='%s',domain='%s' - SKIPPED!", blocked, domain)
-                            continue
-
-                        logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0])
-                        blocked = row[0]
-
-                    logger.debug("blocked='%s'", blocked)
+                    logger.debug("blocked='%s' - DEOBFUSCATED!", blocked)
                     if not utils.is_domain_wanted(blocked):
                         logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
                         continue
@@ -176,34 +151,9 @@ def fetch_blocks(domain: str, nodeinfo_url: str) -> list:
             elif blocked.endswith(".tld"):
                 logger.debug("blocked='%s' is a fake domain - SKIPPED!", blocked)
                 continue
-            elif blocked.count("*") > 0:
-                logger.debug("domain='%s' uses obfuscated domains, marking ...", domain)
-                instances.set_has_obfuscation(domain, True)
-
-                # Obscured domain name with no hash
-                row = instances.deobfuscate("*", blocked)
-
-                logger.debug("row[]='%s'", type(row))
-                if row is None:
-                    logger.warning("Cannot deobfuscate blocked='%s',domain='%s' - SKIPPED!", blocked, domain)
-                    continue
-
-                logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0])
-                blocked = row[0]
-            elif blocked.count("?") > 0:
-                logger.debug("domain='%s' uses obfuscated domains, marking ...", domain)
-                instances.set_has_obfuscation(domain, True)
-
-                # Obscured domain name with no hash
-                row = instances.deobfuscate("?", blocked)
-
-                logger.debug("row[]='%s'", type(row))
-                if row is None:
-                    logger.warning("Cannot deobfuscate blocked='%s',domain='%s' - SKIPPED!", blocked, domain)
-                    continue
 
-                logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0])
-                blocked = row[0]
+            logger.debug("Invoking utils.deobfuscate_domain(%s, %s) ...", blocked, domain)
+            blocked = utils.deobfuscate_domain(blocked, domain)
 
             logger.debug("blocked='%s' - DEOBFUSCATED!", blocked)
             if not utils.is_domain_wanted(blocked):
@@ -271,34 +221,9 @@ def fetch_blocks(domain: str, nodeinfo_url: str) -> list:
                 if blocked == "":
                     logger.warning("blocked is empty after tidyup.domain(): domain='%s',block_level='%s'", domain, block_level)
                     continue
-                elif blocked.count("*") > 0:
-                    logger.debug("domain='%s' uses obfuscated domains, marking ...", domain)
-                    instances.set_has_obfuscation(domain, True)
-
-                    # Obscured domain name with no hash
-                    row = instances.deobfuscate("*", blocked)
-
-                    logger.debug("row[]='%s'", type(row))
-                    if row is None:
-                        logger.warning("Cannot deobfuscate blocked='%s',domain='%s' - SKIPPED!", blocked, domain)
-                        continue
 
-                    logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0])
-                    blocked = row[0]
-                elif blocked.count("?") > 0:
-                    logger.debug("domain='%s' uses obfuscated domains, marking ...", domain)
-                    instances.set_has_obfuscation(domain, True)
-
-                    # Obscured domain name with no hash
-                    row = instances.deobfuscate("?", blocked)
-
-                    logger.debug("row[]='%s'", type(row))
-                    if row is None:
-                        logger.warning("Cannot deobfuscate blocked='%s',domain='%s' - SKIPPED!", blocked, domain)
-                        continue
-
-                    logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0])
-                    blocked = row[0]
+                logger.debug("Invoking utils.deobfuscate_domain(%s, %s) ...", blocked, domain)
+                blocked = utils.deobfuscate_domain(blocked, domain)
 
                 logger.debug("blocked='%s' - DEOBFUSCATED!", blocked)
                 if not utils.is_domain_wanted(blocked):
@@ -334,36 +259,11 @@ def fetch_blocks(domain: str, nodeinfo_url: str) -> list:
             if blocked == "":
                 logger.warning("blocked is empty after tidyup.domain(): domain='%s',block_level='%s'", domain, block_level)
                 continue
-            elif blocked.count("*") > 0:
-                logger.debug("domain='%s' uses obfuscated domains, marking ...", domain)
-                instances.set_has_obfuscation(domain, True)
-
-                # Obscured domain name with no hash
-                row = instances.deobfuscate("*", blocked)
-
-                logger.debug("row[]='%s'", type(row))
-                if row is None:
-                    logger.warning("Cannot deobfuscate blocked='%s',domain='%s' - SKIPPED!", blocked, domain)
-                    continue
-
-                logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0])
-                blocked = row[0]
-            elif blocked.count("?") > 0:
-                logger.debug("domain='%s' uses obfuscated domains, marking ...", domain)
-                instances.set_has_obfuscation(domain, True)
 
-                # Obscured domain name with no hash
-                row = instances.deobfuscate("?", blocked)
+            logger.debug("Invoking utils.deobfuscate_domain(%s, %s) ...", blocked, domain)
+            blocked = utils.deobfuscate_domain(blocked, domain)
 
-                logger.debug("row[]='%s'", type(row))
-                if row is None:
-                    logger.warning("Cannot deobfuscate blocked='%s',domain='%s' - SKIPPED!", blocked, domain)
-                    continue
-
-                logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0])
-                blocked = row[0]
-
-            logger.debug("blocked='%s'", blocked)
+            logger.debug("blocked='%s' - DEOBFUSCATED!", blocked)
             if not utils.is_domain_wanted(blocked):
                 logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
                 continue
@@ -386,8 +286,8 @@ def fetch_blocks(domain: str, nodeinfo_url: str) -> list:
             logger.info("Checking %d record(s) ...", len(blocklist))
             for block_level in blocklist:
                 logger.debug("block_level='%s'", block_level)
-
                 rows = blocklist[block_level]
+
                 logger.debug("rows[%s]()=%d'", type(rows), len(rows))
                 for record in rows:
                     logger.debug("record[]='%s'", type(record))
@@ -398,34 +298,9 @@ def fetch_blocks(domain: str, nodeinfo_url: str) -> list:
                     if blocked == "":
                         logger.warning("blocked is empty after tidyup.domain(): domain='%s',block_level='%s'", domain, block_level)
                         continue
-                    elif blocked.count("*") > 0:
-                        logger.debug("domain='%s' uses obfuscated domains, marking ...", domain)
-                        instances.set_has_obfuscation(domain, True)
-
-                        # Obscured domain name with no hash
-                        row = instances.deobfuscate("*", blocked)
-
-                        logger.debug("row[]='%s'", type(row))
-                        if row is None:
-                            logger.warning("Cannot deobfuscate blocked='%s',domain='%s' - SKIPPED!", blocked, domain)
-                            continue
-
-                        logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0])
-                        blocked = row[0]
-                    elif blocked.count("?") > 0:
-                        logger.debug("domain='%s' uses obfuscated domains, marking ...", domain)
-                        instances.set_has_obfuscation(domain, True)
-
-                        # Obscured domain name with no hash
-                        row = instances.deobfuscate("?", blocked)
-
-                        logger.debug("row[]='%s'", type(row))
-                        if row is None:
-                            logger.warning("Cannot deobfuscate blocked='%s',domain='%s' - SKIPPED!", blocked, domain)
-                            continue
-
-                        logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0])
-                        blocked = row[0]
+
+                    logger.debug("Invoking utils.deobfuscate_domain(%s, %s) ...", blocked, domain)
+                    blocked = utils.deobfuscate_domain(blocked, domain)
 
                     logger.debug("blocked='%s' - DEOBFUSCATED!", blocked)
                     if not utils.is_domain_wanted(blocked):

diff --git a/fba/utils.py b/fba/utils.py
index 577e9ab4635c317cd06064985bc308570b73db9f..bdda2072c31c5587c4325dbdea7a3c2414813514 100644 (file)
--- a/fba/utils.py
+++ b/fba/utils.py
@@ -196,3 +196,36 @@ def is_domain_wanted(domain: str) -> bool:
 
     logger.debug("wanted='%s' - EXIT!", wanted)
     return wanted
+
+def deobfuscate_domain(domain: str, blocker: str) -> str:
+    logger.debug("domain='%s',blocker='%s' - CALLED!", domain, blocker)
+    domain_helper.raise_on(domain)
+    domain_helper.raise_on(blocker)
+
+    if domain.count("*") > 0:
+        logger.debug("blocker='%s' uses obfuscated domains, marking ...", blocker)
+        instances.set_has_obfuscation(blocker, True)
+
+        # Obscured domain name with no hash
+        row = instances.deobfuscate("*", domain)
+
+        logger.debug("row[]='%s'", type(row))
+        if row is not None:
+            logger.debug("domain='%s' de-obscured to '%s'", domain, row[0])
+            domain = row[0]
+    elif domain.count("?") > 0:
+        logger.debug("blocker='%s' uses obfuscated domains, marking ...", blocker)
+        instances.set_has_obfuscation(blocker, True)
+
+        # Obscured domain name with no hash
+        row = instances.deobfuscate("?", domain)
+
+        logger.debug("row[]='%s'", type(row))
+        if row is not None:
+            logger.debug("domain='%s' de-obscured to '%s'", domain, row[0])
+            domain = row[0]
+    else:
+        logger.debug("domain='%s' is not obfuscated", domain)
+
+    logger.debug("domain='%s' - EXIT!", domain)
+    return domain