Continued:

- checks against blacklist added

diff --git a/fba/networks/lemmy.py b/fba/networks/lemmy.py
index a7079d0680ee3ec44cbad99490183c5c55aa99ab..e05c2ccb0442a43d9d0ef9355dd4fb951601b524 100644 (file)
--- a/fba/networks/lemmy.py
+++ b/fba/networks/lemmy.py
@@ -183,10 +183,7 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str):
 
                 # DEBUG: print(f"DEBUG: blocked='{blocked}'")
                 if not validators.domain(blocked):
-                    # DEBUG: print(f"DEBUG: blocked='{blocked}' is not a valid domain - SKIPPED!")
-                    continue
-                elif blacklist.is_blacklisted(blocked):
-                    # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - SKIPPED!")
+                    print(f"WARNING: blocked='{blocked}' is not a valid domain - SKIPPED!")
                     continue
                 elif blocked.endswith(".arpa"):
                     print(f"WARNING: blocked='{blocked}' is a reversed .arpa domain and should not be used generally.")
@@ -194,6 +191,9 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str):
                 elif blocked.endswith(".tld"):
                     print(f"WARNING: blocked='{blocked}' is a fake domain, please don't crawl them!")
                     continue
+                elif blacklist.is_blacklisted(blocked):
+                    # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - SKIPPED!")
+                    continue
                 elif not instances.is_registered(blocked):
                     # DEBUG: print("DEBUG: Hash wasn't found, adding:", blocked, domain)
                     instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url)

diff --git a/fba/networks/mastodon.py b/fba/networks/mastodon.py
index dd215196851773a0bc8c1d4521dbe074ab8733d1..61971afbe3a13fcbe800fa654a52eb1fc7a13571 100644 (file)
--- a/fba/networks/mastodon.py
+++ b/fba/networks/mastodon.py
@@ -287,6 +287,9 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str):
                 elif blocked.endswith(".tld"):
                     print(f"WARNING: blocked='{blocked}' is a fake domain, please don't crawl them!")
                     continue
+                elif blacklist.is_blacklisted(blocked):
+                    # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - SKIPPED!")
+                    continue
                 elif not instances.is_registered(blocked):
                     # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'")
                     instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url)
@@ -301,6 +304,9 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str):
                 elif blocked.endswith(".tld"):
                     print(f"WARNING: blocked='{blocked}' is a fake domain, please don't crawl them!")
                     continue
+                elif blacklist.is_blacklisted(blocked):
+                    # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - SKIPPED!")
+                    continue
                 elif not instances.is_registered(blocked):
                     # DEBUG: print("DEBUG: Hash wasn't found, adding:", blocked, domain)
                     instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url)

diff --git a/fba/networks/pleroma.py b/fba/networks/pleroma.py
index f23393cabc70aaf60f5610d1075f1bab1865505f..546d7c59cb3db9445dbbf0e2e6ee4750984be1c8 100644 (file)
--- a/fba/networks/pleroma.py
+++ b/fba/networks/pleroma.py
@@ -146,6 +146,9 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str):
                     elif blocked.endswith(".tld"):
                         print(f"WARNING: blocked='{blocked}' is a fake domain, please don't crawl them!")
                         continue
+                    elif blacklist.is_blacklisted(blocked):
+                        # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - SKIPPED!")
+                        continue
                     elif not instances.is_registered(blocked):
                         # Commit changes
                         fba.connection.commit()
@@ -219,6 +222,9 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str):
             elif blocked.endswith(".tld"):
                 print(f"WARNING: blocked='{blocked}' is a fake domain, please don't crawl them!")
                 continue
+            elif blacklist.is_blacklisted(blocked):
+                # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - SKIPPED!")
+                continue
             elif not instances.is_registered(blocked):
                 # Commit changes
                 fba.connection.commit()
@@ -325,6 +331,9 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str):
                 elif blocked.endswith(".tld"):
                     print(f"WARNING: blocked='{blocked}' is a fake domain, please don't crawl them!")
                     continue
+                elif blacklist.is_blacklisted(blocked):
+                    # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - SKIPPED!")
+                    continue
                 elif not instances.is_registered(blocked):
                     # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'")
                     instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url)
@@ -400,6 +409,9 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str):
             elif blocked.endswith(".tld"):
                 print(f"WARNING: blocked='{blocked}' is a fake domain, please don't crawl them!")
                 continue
+            elif blacklist.is_blacklisted(blocked):
+                # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - SKIPPED!")
+                continue
             elif not instances.is_registered(blocked):
                 # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'")
                 instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url)