From 7979c111794ae44fa5e8e79ea4577ac97c79cf23 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Sun, 25 Jun 2023 22:18:22 +0200 Subject: [PATCH] Continued: - TOR .onion instances are not analyzed here, maybe later a switch --onion will add TOR-only support? --- fba/helpers/domain.py | 2 ++ fba/models/instances.py | 2 ++ fba/utils.py | 3 +++ 3 files changed, 7 insertions(+) diff --git a/fba/helpers/domain.py b/fba/helpers/domain.py index 2606f14..76738ac 100644 --- a/fba/helpers/domain.py +++ b/fba/helpers/domain.py @@ -33,6 +33,8 @@ def raise_on(domain: str): raise ValueError(f"domain='{domain}' is not a valid domain") elif domain.endswith(".arpa"): raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!") + elif domain.endswith(".onion"): + raise ValueError(f"domain='{domain}' is a TOR, please don't crawl them!") elif domain.endswith(".tld"): raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!") diff --git a/fba/models/instances.py b/fba/models/instances.py index f113bee..d255506 100644 --- a/fba/models/instances.py +++ b/fba/models/instances.py @@ -317,6 +317,8 @@ def deobfuscate(char: str, domain: str, blocked_hash: str = None) -> tuple: raise ValueError(f"Parameter domain='{domain}' must be all lower-case") elif domain.endswith(".arpa"): raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!") + elif domain.endswith(".onion"): + raise ValueError(f"domain='{domain}' is a TOR domain, please don't crawl them!") elif domain.endswith(".tld"): raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!") elif not isinstance(char, str): diff --git a/fba/utils.py b/fba/utils.py index c7f35d5..7ef3e01 100644 --- a/fba/utils.py +++ b/fba/utils.py @@ -160,6 +160,9 @@ def is_domain_wanted(domain: str) -> bool: elif domain.endswith(".arpa"): logger.debug("domain='%s' is a domain for reversed IP addresses - settings False ...", domain) wanted = False + elif domain.endswith(".onion"): + logger.debug("domain='%s' is a TOR .onion domain - settings False ...", domain) + wanted = False elif domain.endswith(".tld"): logger.debug("domain='%s' is a fake domain - settings False ...", domain) wanted = False -- 2.39.5