Continued:

- skip fake domains (.tld)
- don't invoke instances.add() with fake domains (.tld)

diff --git a/fba/federation.py b/fba/federation.py
index bdb6574234922db04114aef49ba9402c0a0662ef..f5a5f1ba261f185011e5751d7002add83a27263f 100644 (file)
--- a/fba/federation.py
+++ b/fba/federation.py
@@ -127,6 +127,9 @@ def fetch_instances(domain: str, origin: str, software: str, command: str, path:
         elif instance.find("/profile/") > 0 or instance.find("/users/") > 0:
             # DEBUG: print(f"DEBUG: instance='{instance}' is a link to a single user profile - SKIPPED!")
             continue
+        elif instance.endswith(".tld"):
+            # DEBUG: print(f"DEBUG: instance='{instance}' is a fake domain - SKIPPED!")
+            continue
         elif not instances.is_registered(instance):
             # DEBUG: print("DEBUG: Adding new instance:", instance, domain)
             instances.add(instance, domain, command)

diff --git a/fba/models/instances.py b/fba/models/instances.py
index 7cf70e6524c290edd249a6908b29bb19be56d873..e1a11430645241f4090eb6c8c38f6ddaf22b39c3 100644 (file)
--- a/fba/models/instances.py
+++ b/fba/models/instances.py
@@ -178,6 +178,8 @@ def add(domain: str, origin: str, command: str, path: str = None, software: str
         raise Exception(f"domain='{domain}' is blacklisted, but method invoked")
     elif domain.find("/profile/") > 0 or domain.find("/users/") > 0 or (software == "lemmy" and domain.find("/c/") > 0):
         raise Exception(f"domain='{domain}' is a single user")
+    elif domain.endswith(".tld"):
+        raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
 
     if software is None:
         try: