From cb575ce5f23824862a90a09222e5276f7e246b08 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Sun, 18 Jun 2023 18:54:39 +0200 Subject: [PATCH] Continued: - skip fake domains (.tld) - don't invoke instances.add() with fake domains (.tld) --- fba/federation.py | 3 +++ fba/models/instances.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/fba/federation.py b/fba/federation.py index bdb6574..f5a5f1b 100644 --- a/fba/federation.py +++ b/fba/federation.py @@ -127,6 +127,9 @@ def fetch_instances(domain: str, origin: str, software: str, command: str, path: elif instance.find("/profile/") > 0 or instance.find("/users/") > 0: # DEBUG: print(f"DEBUG: instance='{instance}' is a link to a single user profile - SKIPPED!") continue + elif instance.endswith(".tld"): + # DEBUG: print(f"DEBUG: instance='{instance}' is a fake domain - SKIPPED!") + continue elif not instances.is_registered(instance): # DEBUG: print("DEBUG: Adding new instance:", instance, domain) instances.add(instance, domain, command) diff --git a/fba/models/instances.py b/fba/models/instances.py index 7cf70e6..e1a1143 100644 --- a/fba/models/instances.py +++ b/fba/models/instances.py @@ -178,6 +178,8 @@ def add(domain: str, origin: str, command: str, path: str = None, software: str raise Exception(f"domain='{domain}' is blacklisted, but method invoked") elif domain.find("/profile/") > 0 or domain.find("/users/") > 0 or (software == "lemmy" and domain.find("/c/") > 0): raise Exception(f"domain='{domain}' is a single user") + elif domain.endswith(".tld"): + raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!") if software is None: try: -- 2.39.5