Continued:

- okay, then don't check if they are punycode and then raise an exception ...

diff --git a/fba/commands.py b/fba/commands.py
index bd3d3938abaa381a12042645453c4d02a84797f6..6c0657993155212946b1c1e9c1f12a63c64c7ffe 100644 (file)
--- a/fba/commands.py
+++ b/fba/commands.py
@@ -363,8 +363,6 @@ def fetch_blocks(args: argparse.Namespace) -> int:
             elif block["blocked"].endswith(".tld"):
                 logger.debug("blocked='%s' is a fake domain - SKIPPED", block["blocked"])
                 continue
-            elif "xn--" in block["blocked"]:
-                raise ValueError(f"blocked='{block['blocked']}' is a punycode domain, please don't crawl them!")
             elif block["blocked"].find("*") >= 0:
                 logger.debug("blocker='%s' uses obfuscated domains", blocker)
 
@@ -1075,8 +1073,6 @@ def fetch_oliphant(args: argparse.Namespace) -> int:
             elif domain.endswith(".tld"):
                 logger.debug("domain='%s' is a fake domain - SKIPPED", domain)
                 continue
-            elif "xn--" in domain:
-                raise ValueError(f"domain='{domain}' is a punycode domain, please translate them back!")
             elif domain.find("*") >= 0 or domain.find("?") >= 0:
                 logger.debug("domain='%s' is obfuscated - Invoking utils.deobfuscate(%s, %s) ...", domain, domain, block["blocker"])
                 domain = utils.deobfuscate(domain, block["blocker"])
@@ -1467,8 +1463,6 @@ def recheck_obfuscation(args: argparse.Namespace) -> int:
             elif block["blocked"].endswith(".tld"):
                 logger.debug("blocked='%s' is a fake domain name - SKIPPED!", block["blocked"])
                 continue
-            elif "xn--" in block["blocked"]:
-                raise ValueError(f"blocked='{block['blocked']}' is a punycode domain, please translate them back!")
             elif block["blocked"].endswith(".onion"):
                 logger.debug("blocked='%s' is a TOR onion domain name - SKIPPED!", block["blocked"])
                 continue

diff --git a/fba/helpers/domain.py b/fba/helpers/domain.py
index 75f97a5d3020b7de9ea406ba7f184c328b84bb36..5328e3f37eeddc8a25043b390249704a7486de82 100644 (file)
--- a/fba/helpers/domain.py
+++ b/fba/helpers/domain.py
@@ -39,8 +39,6 @@ def raise_on(domain: str):
         raise ValueError(f"domain='{domain}' is a TOR, please don't crawl them!")
     elif domain.endswith(".tld"):
         raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
-    elif "xn--" in domain:
-        raise ValueError(f"domain='{domain}' is a punycode domain, please don't crawl them!")
 
     logger.debug("EXIT!")
 

diff --git a/fba/utils.py b/fba/utils.py
index 019938c4f6c89d14be0c3a91f29b33ec89b9a652..600763558b8b9943aa4397c32c3345ae2863a379 100644 (file)
--- a/fba/utils.py
+++ b/fba/utils.py
@@ -174,8 +174,6 @@ def is_domain_wanted(domain: str) -> bool:
     elif domain.endswith(".tld"):
         logger.debug("domain='%s' is a fake domain - settings False ...", domain)
         wanted = False
-    elif "xn--" in domain:
-        raise ValueError(f"domain='{domain}' is a punycode domain, please don't crawl them!")
     elif blacklist.is_blacklisted(domain):
         logger.debug("domain='%s' is blacklisted - settings False ...", domain)
         wanted = False