Continued:

- TOR .onion instances are not analyzed here, maybe later a switch --onion will
  add TOR-only support?

diff --git a/fba/helpers/domain.py b/fba/helpers/domain.py
index 2606f14b555e5a586b97c8db71713698115a37b4..76738acff05730619db1c434d31eb511c8deb81a 100644 (file)
--- a/fba/helpers/domain.py
+++ b/fba/helpers/domain.py
@@ -33,6 +33,8 @@ def raise_on(domain: str):
         raise ValueError(f"domain='{domain}' is not a valid domain")
     elif domain.endswith(".arpa"):
         raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!")
+    elif domain.endswith(".onion"):
+        raise ValueError(f"domain='{domain}' is a TOR, please don't crawl them!")
     elif domain.endswith(".tld"):
         raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
 

diff --git a/fba/models/instances.py b/fba/models/instances.py
index f113beef563d9cf5652f00619bac2e1b600c57ba..d2555063131e14cee95062574667fd8361a233ad 100644 (file)
--- a/fba/models/instances.py
+++ b/fba/models/instances.py
@@ -317,6 +317,8 @@ def deobfuscate(char: str, domain: str, blocked_hash: str = None) -> tuple:
         raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
     elif domain.endswith(".arpa"):
         raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!")
+    elif domain.endswith(".onion"):
+        raise ValueError(f"domain='{domain}' is a TOR domain, please don't crawl them!")
     elif domain.endswith(".tld"):
         raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
     elif not isinstance(char, str):

diff --git a/fba/utils.py b/fba/utils.py
index c7f35d5b481852c92d34a9b20e041f5679945a51..7ef3e0126bf470d62322d10e1670a79cfb56ea97 100644 (file)
--- a/fba/utils.py
+++ b/fba/utils.py
@@ -160,6 +160,9 @@ def is_domain_wanted(domain: str) -> bool:
     elif domain.endswith(".arpa"):
         logger.debug("domain='%s' is a domain for reversed IP addresses - settings False ...", domain)
         wanted = False
+    elif domain.endswith(".onion"):
+        logger.debug("domain='%s' is a TOR .onion domain - settings False ...", domain)
+        wanted = False
     elif domain.endswith(".tld"):
         logger.debug("domain='%s' is a fake domain - settings False ...", domain)
         wanted = False