From 0e46573ec6d9356f55076d525e99d4337392c232 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Fri, 19 Jul 2024 00:16:38 +0200 Subject: [PATCH] Continued: - split off question mark (mostly comes from WordPress instances) --- fba/helpers/domain.py | 2 +- fba/http/federation.py | 25 +++++++++++++------------ 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/fba/helpers/domain.py b/fba/helpers/domain.py index f056047..6b0fd10 100644 --- a/fba/helpers/domain.py +++ b/fba/helpers/domain.py @@ -122,7 +122,7 @@ def is_wanted(domain: str) -> bool: @lru_cache def encode_idna(domain: str) -> str: - logger.debug("domain='%s' - CALLED!") + logger.debug("domain='%s' - CALLED!", domain) raise_on(domain) punycode = domain.lstrip(".").split("?")[0] diff --git a/fba/http/federation.py b/fba/http/federation.py index d62aca4..ad10c0c 100644 --- a/fba/http/federation.py +++ b/fba/http/federation.py @@ -171,25 +171,26 @@ def fetch_instances(domain: str, origin: str, software: str, command: str, path: logger.warning("instance='%s' contains double-dot, removing ...", instance) instance = instance.replace("..", ".") - logger.debug("instance='%s'", instance) - if not validators.domain(instance, rfc_2782=True): - logger.warning("instance='%s' is not a valid domain - SKIPPED!", instance) + probe = instance.split("/")[0] + logger.debug("instance='%s',probe='%s'", instance, probe) + if not validators.domain(probe, rfc_2782=True): + logger.warning("probe='%s' is not a valid domain - SKIPPED!", probe) continue - elif instance.endswith(".onion"): - logger.debug("instance='%s' is a TOR .onion instance - SKIPPED!", instance) + elif probe.endswith(".onion"): + logger.debug("probe='%s' is a TOR .onion probe - SKIPPED!", probe) continue - elif instance.endswith(".i2p") and not config.get("allow_i2p_domain"): - logger.debug("instance='%s' is an I2P .onion instance - SKIPPED!", instance) + elif probe.endswith(".i2p") and not config.get("allow_i2p_domain"): + logger.debug("probe='%s' is an I2P .onion probe - SKIPPED!", probe) continue - elif instance.endswith(".arpa"): - logger.debug("instance='%s' is a reverse IP address - SKIPPED!", instance) + elif probe.endswith(".arpa"): + logger.debug("probe='%s' is a reverse IP address - SKIPPED!", probe) continue - elif instance.endswith(".tld"): - logger.debug("instance='%s' is a fake instance - SKIPPED!", instance) + elif probe.endswith(".tld"): + logger.debug("probe='%s' is a fake probe - SKIPPED!", probe) continue logger.debug("instance='%s' - BEFORE!", instance) - instance = domain_helper.encode_idna(instance) + instance = domain_helper.encode_idna(instance.split("?")[0]) logger.debug("instance='%s' - AFTER!", instance) if not domain_helper.is_wanted(instance): -- 2.39.5