Continued:

- fixed bad handling of errors during nodeinfo discovery

diff --git a/fba.py b/fba.py
index 03800f2841a7725dcacdedadddc35b88c0e44c97..6d20d6e775aedf2b7c482da78f3ced1c1c9c70cf 100644 (file)
--- a/fba.py
+++ b/fba.py
@@ -28,23 +28,18 @@ connection = sqlite3.connect("blocks.db")
 cursor = connection.cursor()
 
 def is_blacklisted(domain: str) -> bool:
-    # NOISY-DEBUG: print("DEBUG: Checking blacklist for domain:", domain)
     blacklisted = False
     for peer in blacklist:
-        # NOISY-DEBUG: print("DEBUG: domain,peer:", domain, peer)
         if peer in domain:
             blacklisted = True
 
-    # NOISY-DEBUG: print("DEBUG: blacklisted:", blacklisted)
     return blacklisted
 
 def get_hash(domain: str) -> str:
-    # NOISY-DEBUG: print("DEBUG: Calculating hash for domain:", domain)
     return sha256(domain.encode("utf-8")).hexdigest()
 
 def update_last_blocked(domain: str):
     # NOISY-DEBUG: print("DEBUG: Updating last_blocked for domain", domain)
-
     try:
         cursor.execute("UPDATE instances SET last_blocked = ?, last_updated = ? WHERE domain = ?", [
             time.time(),
@@ -109,6 +104,7 @@ def get_peers(domain: str) -> list:
 
 def post_json_api(domain: str, path: str, data: str) -> list:
     # NOISY-DEBUG: print("DEBUG: Sending POST to domain,path,data:", domain, path, data)
+    json = {}
     try:
         res = reqto.post(f"https://{domain}{path}", data=data, headers=headers, timeout=(config["connection_timeout"], config["read_timeout"]))
 
@@ -137,60 +133,61 @@ def fetch_nodeinfo(domain: str) -> list:
        f"https://{domain}/api/v1/instance"
     ]
 
-    json = None
+    json = {}
     for request in requests:
-        # NOISY-DEBUG: print("DEBUG: Fetching request:", request)
-        res = reqto.get(request, headers=headers, timeout=(config["connection_timeout"], config["read_timeout"]))
-
-        # NOISY-DEBUG: print("DEBUG: res.ok,res.json[]:", res.ok, type(res.json()))
-        if res.ok and res.json() is not None:
-            # NOISY-DEBUG: print("DEBUG: Success:", request)
-            json = res.json()
-            break
-        elif not res.ok or res.status_code >= 400:
-            # NOISY-DEBUG: print("DEBUG: Failed fetching nodeinfo from domain:", domain)
-            update_last_error(domain, res)
-            continue
+        try:
+            # NOISY-DEBUG: print("DEBUG: Fetching request:", request)
+            res = reqto.get(request, headers=headers, timeout=(config["connection_timeout"], config["read_timeout"]))
+
+            # NOISY-DEBUG: print("DEBUG: res.ok,res.json[]:", res.ok, type(res.json()))
+            if res.ok and res.json() is not None:
+                # NOISY-DEBUG: print("DEBUG: Success:", request)
+                json = res.json()
+                break
+            elif not res.ok or res.status_code >= 400:
+                # NOISY-DEBUG: print("DEBUG: Failed fetching nodeinfo from domain:", domain)
+                update_last_error(domain, res)
+                continue
+
+        except:
+            # NOISY-DEBUG: print("DEBUG: Cannot fetch API request:", request)
+            pass
 
     if json is None:
         print("WARNING: Failed fetching nodeinfo from domain:", domain)
 
-    # NOISY-DEBUG: print("DEBUG: Updating last_nodeinfo for domain:", domain)
-    update_last_nodeinfo(domain)
-
-    # NOISY-DEBUG: print("DEBUG: Returning json():", len(json))
+    # NOISY-DEBUG: print("DEBUG: Returning json[]:", type(json))
     return json
 
 def determine_software(domain: str) -> str:
     # NOISY-DEBUG: print("DEBUG: Determining software for domain:", domain)
     software = None
 
-    try:
-        json = fetch_nodeinfo(domain)
-
-        # NOISY-DEBUG: print("DEBUG: json():", len(json))
-        software = tidyup(json["software"]["name"])
-
-        # NOISY-DEBUG: print("DEBUG: BEFORE software:", software)
-        if software in ["akkoma", "rebased"]:
-            # NOISY-DEBUG: print("DEBUG: Setting pleroma:", domain, software)
-            software = "pleroma"
-        elif software in ["hometown", "ecko"]:
-            # NOISY-DEBUG: print("DEBUG: Setting mastodon:", domain, software)
-            software = "mastodon"
-        elif software in ["calckey", "groundpolis", "foundkey", "cherrypick"]:
-            # NOISY-DEBUG: print("DEBUG: Setting misskey:", domain, software)
-            software = "misskey"
-        elif software.find("/") > 0:
-            print("WARNING: Spliting of path:", software)
-            software = software.split("/")[-1];
-
-        if software == "":
-            print("WARNING: tidyup() left no software name behind:", domain)
-            software = None
-        # NOISY-DEBUG: print("DEBUG: AFTER software:", software)
-    except:
-        print("WARNING: Could not determine software type:", domain)
+    json = fetch_nodeinfo(domain)
+    if len(json) == 0:
+        print("DEBUG: Could not determine software type:", domain)
+        return None
+
+    # NOISY-DEBUG: print("DEBUG: json():", len(json))
+    software = tidyup(json["software"]["name"])
+
+    # NOISY-DEBUG: print("DEBUG: tidyup software:", software)
+    if software in ["akkoma", "rebased"]:
+        # NOISY-DEBUG: print("DEBUG: Setting pleroma:", domain, software)
+        software = "pleroma"
+    elif software in ["hometown", "ecko"]:
+        # NOISY-DEBUG: print("DEBUG: Setting mastodon:", domain, software)
+        software = "mastodon"
+    elif software in ["calckey", "groundpolis", "foundkey", "cherrypick"]:
+        # NOISY-DEBUG: print("DEBUG: Setting misskey:", domain, software)
+        software = "misskey"
+    elif software.find("/") > 0:
+        print("WARNING: Spliting of path:", software)
+        software = software.split("/")[-1];
+
+    if software == "":
+        print("WARNING: tidyup() left no software name behind:", domain)
+        software = None
 
     # NOISY-DEBUG: print("DEBUG: Returning domain,software:", domain, software)
     return software
@@ -266,6 +263,9 @@ def add_instance(domain: str, origin: str, originator: str):
         print("WARNING: Bad origin name:", origin)
         raise
 
+    software = determine_software(domain)
+    # NOISY-DEBUG: print("DEBUG: Determined software:", software)
+
     print(f"--- Adding new instance {domain} (origin: {origin})")
     try:
         cursor.execute(
@@ -275,7 +275,7 @@ def add_instance(domain: str, origin: str, originator: str):
                origin,
                originator,
                get_hash(domain),
-               determine_software(domain),
+               software,
                time.time()
             ),
         )
@@ -283,6 +283,9 @@ def add_instance(domain: str, origin: str, originator: str):
     except:
         print("ERROR: failed SQL query:", domain)
         sys.exit(255)
+    else:
+        # NOISY-DEBUG: print("DEBUG: Updating nodeinfo for domain:", domain)
+        update_last_nodeinfo(domain)
 
 def send_bot_post(instance: str, blocks: dict):
     message = instance + " has blocked the following instances:\n\n"
@@ -316,9 +319,9 @@ def get_mastodon_blocks(domain: str) -> dict:
     # NOISY-DEBUG: print("DEBUG: Fetching mastodon blocks from domain:", domain)
     blocks = {
         "Suspended servers": [],
-        "Filtered media": [],
-        "Limited servers": [],
-        "Silenced servers": [],
+        "Filtered media"   : [],
+        "Limited servers"  : [],
+        "Silenced servers" : [],
     }
 
     translations = {
@@ -518,8 +521,6 @@ def get_misskey_blocks(domain: str) -> dict:
         return {}
 
 def tidyup(string: str) -> str:
-    # NOISY-DEBUG: print("DEBUG: BEFORE string:", string)
-
     # some retards put their blocks in variable case
     string = string.lower().strip()
 
@@ -538,5 +539,4 @@ def tidyup(string: str) -> str:
     # the biggest retards of them all try to block individual users
     string = re.sub("(.+)\@", "", string)
 
-    # NOISY-DEBUG: print("DEBUG: AFTER string:", string)
     return string

diff --git a/fetch_blocks.py b/fetch_blocks.py
index 704176e5dc7228b36c8fc5f0762fe7aee5da9d8f..942068973527a4a47da5afad5d62b2041d4a9cfc 100644 (file)
--- a/fetch_blocks.py
+++ b/fetch_blocks.py
@@ -30,6 +30,9 @@ for blocker, software in fba.cursor.fetchall():
                 print("WARNING: Could not fetch nodeinfo from blocker:", blocker)
                 continue
 
+            print("DEBUG: Updating nodeinfo:", blocker)
+            fba.update_last_nodeinfo(blocker)
+
             federation = json["metadata"]["federation"]
 
             if "enabled" in federation: