Continued:

- added more debug lines
- if zero sized list is being returned, quit the loop, again raised exceptions
  are NOT there for your flow-control!

diff --git a/fba.py b/fba.py
index 4f2e05de7ab5daf9ba4960afc247ad1e090b4a00..f19c1083d9949e1888f4974096f1882c73515862 100644 (file)
--- a/fba.py
+++ b/fba.py
@@ -42,14 +42,15 @@ def get_peers(domain: str) -> str:
     # NOISY-DEBUG: print("DEBUG: Returning peers[]:", type(peers))
     return peers
 
-def post_json_api(domain: str, path: str, data: str) -> str:
-    # NOISY-DEBUG: print("DEBUG: Sending POST to domain,path,data():", domain, path, len(data))
-    doc = reqto.post(f"https://{domain}{path}", data=data, headers=headers, timeout=5).json()
+def post_json_api(domain: str, path: str, data: str) -> list:
+    # NOISY-DEBUG: print("DEBUG: Sending POST to domain,path,data:", domain, path, data)
+    res = reqto.post(f"https://{domain}{path}", data=data, headers=headers, timeout=5)
 
-    if doc == []:
-        print("WARNING: Cannot query JSON API:", domain, path)
+    if not res.ok:
+        print("WARNING: Cannot query JSON API:", domain, path, data, res.status_code)
         raise
 
+    doc = res.json()
     # NOISY-DEBUG: print("DEBUG: Returning doc():", len(doc))
     return doc
 
@@ -58,21 +59,29 @@ def determine_software(domain: str) -> str:
     software = None
     try:
         res = reqto.get(f"https://{domain}/nodeinfo/2.1.json", headers=headers, timeout=5)
+
+        # NOISY-DEBUG: print("DEBUG: domain,res.ok,res.status_code:", domain, res.ok, res.status_code)
         if res.status_code == 404:
             res = reqto.get(f"https://{domain}/nodeinfo/2.0", headers=headers, timeout=5)
+
         if res.status_code == 404:
             res = reqto.get(f"https://{domain}/nodeinfo/2.0.json", headers=headers, timeout=5)
+
         if res.ok and "text/html" in res.headers["content-type"]:
             res = reqto.get(f"https://{domain}/nodeinfo/2.1", headers=headers, timeout=5)
+
         if res.ok:
-            if res.json()["software"]["name"] in ["akkoma", "rebased"]:
+            json = res.json()
+            # NOISY-DEBUG: print("DEBUG: json():", len(json))
+
+            if json["software"]["name"] in ["akkoma", "rebased"]:
                 software = "pleroma"
-            elif res.json()["software"]["name"] in ["hometown", "ecko"]:
+            elif json["software"]["name"] in ["hometown", "ecko"]:
                 software = "mastodon"
-            elif res.json()["software"]["name"] in ["calckey", "groundpolis", "foundkey", "cherrypick"]:
+            elif json["software"]["name"] in ["calckey", "groundpolis", "foundkey", "cherrypick"]:
                 software = "misskey"
             else:
-                software = res.json()["software"]["name"]
+                software = json["software"]["name"]
         elif res.status_code == 404:
             res = reqto.get(f"https://{domain}/api/v1/instance", headers=headers, timeout=5)
         if res.ok:
@@ -286,6 +295,7 @@ def get_misskey_blocks(domain: str) -> dict:
             # sending them all at once
             try:
                 if counter == 0:
+                    # NOISY-DEBUG: print("DEBUG: Sending JSON API request to domain,step,counter:", domain, step, counter)
                     doc = post_json_api(domain, "/api/federation/instances/", json.dumps({
                         "sort": "+caughtAt",
                         "host": None,
@@ -293,6 +303,7 @@ def get_misskey_blocks(domain: str) -> dict:
                         "limit": step
                     }))
                 else:
+                    # NOISY-DEBUG: print("DEBUG: Sending JSON API request to domain,step,counter:", domain, step, counter)
                     doc = post_json_api(domain, "/api/federation/instances/", json.dumps({
                         "sort": "+caughtAt",
                         "host": None,
@@ -301,6 +312,11 @@ def get_misskey_blocks(domain: str) -> dict:
                         "offset": counter-1
                     }))
 
+                # NOISY-DEBUG: print("DEBUG: doc():", len(doc))
+                if len(doc) == 0:
+                    # NOISY-DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
+                    break
+
                 for instance in doc:
                     # just in case
                     if instance["isSuspended"]:
@@ -311,8 +327,12 @@ def get_misskey_blocks(domain: str) -> dict:
                                 "reason": ""
                             }
                         )
+
+                # NOISY-DEBUG: print("DEBUG: Raising counter by step:", step)
                 counter = counter + step
+
             except:
+                print("WARNING: Caught error, exiting loop:", domain)
                 counter = 0
                 break
 
@@ -320,6 +340,7 @@ def get_misskey_blocks(domain: str) -> dict:
             # same shit, different asshole ("blocked" aka full suspend)
             try:
                 if counter == 0:
+                    # NOISY-DEBUG: print("DEBUG: Sending JSON API request to domain,step,counter:", domain, step, counter)
                     doc = post_json_api(domain,"/api/federation/instances", json.dumps({
                         "sort": "+caughtAt",
                         "host": None,
@@ -327,6 +348,7 @@ def get_misskey_blocks(domain: str) -> dict:
                         "limit": step
                     }))
                 else:
+                    # NOISY-DEBUG: print("DEBUG: Sending JSON API request to domain,step,counter:", domain, step, counter)
                     doc = post_json_api(domain,"/api/federation/instances", json.dumps({
                         "sort": "+caughtAt",
                         "host": None,
@@ -335,11 +357,16 @@ def get_misskey_blocks(domain: str) -> dict:
                         "offset": counter-1
                     }))
 
+                # NOISY-DEBUG: print("DEBUG: doc():", len(doc))
+                if len(doc) == 0:
+                    # NOISY-DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
+                    break
+
                 for instance in doc:
                     if instance["isBlocked"]:
                         blocks["blocked"].append({
-                                "domain": instance["host"],
-                                "reason": ""
+                            "domain": instance["host"],
+                            "reason": ""
                         })
                 counter = counter + step
 

diff --git a/fetch_blocks.py b/fetch_blocks.py
index cf9d97c77c70ab7ee8c6b4ea8814e3a000199baa..170e52ad7ab703ad55b183af6be33409b2860f28 100644 (file)
--- a/fetch_blocks.py
+++ b/fetch_blocks.py
@@ -23,8 +23,7 @@ for blocker, software in fba.c.fetchall():
         print("INFO: blocker:", blocker)
         try:
             # Blocks
-            federation = reqto.get(
-                f"https://{blocker}/nodeinfo/2.1.json", headers=fba.headers, timeout=5
+            federation = reqto.get(f"https://{blocker}/nodeinfo/2.1.json", headers=fba.headers, timeout=5
             ).json()["metadata"]["federation"]
 
             if "enabled" in federation:
@@ -173,9 +172,7 @@ for blocker, software in fba.c.fetchall():
                     reqheaders = fba.headers
 
                 # NOISY-DEBUG: print("DEBUG: Quering API domain_blocks:", blocker)
-                blocks = reqto.get(
-                    f"https://{blocker}/api/v1/instance/domain_blocks", headers=reqheaders, timeout=5
-                ).json()
+                blocks = reqto.get(f"https://{blocker}/api/v1/instance/domain_blocks", headers=reqheaders, timeout=5).json()
 
                 # NOISY-DEBUG: print("DEBUG: blocks():", len(blocks))
                 for block in blocks:
@@ -347,15 +344,14 @@ for blocker, software in fba.c.fetchall():
         print("INFO: blocker:", blocker)
         try:
             # Blocks
-            federation = reqto.get(
-                f"https://{blocker}/api/v1/instance/peers?filter=suspended", headers=fba.headers, timeout=5
-            ).json()
+            federation = reqto.get(f"https://{blocker}/api/v1/instance/peers?filter=suspended", headers=fba.headers, timeout=5).json()
 
             if (federation == None):
                 print("WARNING: No valid response:", blocker);
             elif "error" in federation:
                 print("WARNING: API returned error:", federation["error"])
             else:
+                # NOISY-DEBUG: print("DEBUG: Checking fenderation():", len(federation))
                 for peer in federation:
                     blocked = peer["domain"].lower()
                     # NOISY-DEBUG: print("DEBUG: BEFORE-blocked:", blocked)