Continued:

- res.json() can somtimes not being checked (?)
- also need to check each element as it might return 'None' causing error

diff --git a/fba.py b/fba.py
index 4a069023eadfb06032c46583a3fdaa615b8f35c1..e408952d00b9d029f2d49e973117e6133438c8fa 100644 (file)
--- a/fba.py
+++ b/fba.py
@@ -335,7 +335,7 @@ def get_peers(domain: str, software: str) -> list:
     peers = list()
 
     if software == "misskey":
-        print(f"DEBUG: domain='{domain}' is misskey, sending API POST request ...")
+        # DEBUG: print(f"DEBUG: domain='{domain}' is misskey, sending API POST request ...")
 
         counter = 0
         step = config["misskey_offset"]
@@ -373,7 +373,7 @@ def get_peers(domain: str, software: str) -> list:
         try:
             res = reqto.get(f"https://{domain}/api/v3/site", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"]))
 
-            # DEBUG: print(f"DEBUG: res.ok={res.ok},res.json[]={type(res.json())}")
+            # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code='{res.status_code}'")
             if res.ok and isinstance(res.json(), dict):
                 # DEBUG: print("DEBUG: Success, res.json():", len(res.json()))
                 data = res.json()
@@ -399,7 +399,7 @@ def get_peers(domain: str, software: str) -> list:
                 try:
                     res = reqto.get(f"https://{domain}/api/v1/server/{mode}?start={start}&count=100", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"]))
 
-                    # DEBUG: print(f"DEBUG: res.ok={res.ok},res.json[]={type(res.json())}")
+                    # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code='{res.status_code}'")
                     if res.ok and isinstance(res.json(), dict):
                         # DEBUG: print("DEBUG: Success, res.json():", len(res.json()))
                         data = res.json()
@@ -429,12 +429,13 @@ def get_peers(domain: str, software: str) -> list:
             # DEBUG: print("DEBUG: Returning peers[]:", type(peers))
             return peers
 
-    # DEBUG: print(f"DEBUG: Fetching '{get_peers_url}' from '{domain}' ...")
+    # DEBUG: print(f"DEBUG: Fetching get_peers_url='{get_peers_url}' from '{domain}' ...")
     try:
         res = reqto.get(f"https://{domain}{get_peers_url}", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"]))
 
-        # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},res.json[]={type(res.json())}")
+        # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code}")
         if not res.ok or res.status_code >= 400:
+            # DEBUG: print(f"DEBUG: Was not able to fetch '{get_peers_url}', trying alternative ...")
             res = reqto.get(f"https://{domain}/api/v3/site", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"]))
 
             if not res.ok or res.status_code >= 400:
@@ -442,12 +443,18 @@ def get_peers(domain: str, software: str) -> list:
                 update_last_error(domain, res)
             elif "federated_instances" in res.json() and "linked" in res.json()["federated_instances"]:
                 # DEBUG: print("DEBUG: Found federated_instances", domain)
-                peers = res.json()["federated_instances"]["linked"] + res.json()["federated_instances"]["allowed"] + res.json()["federated_instances"]["blocked"]
+                data = res.json()
+
+                for element in ["linked", "allowed", "blocked"]:
+                    # DEBUG: print(f"DEBUG: Checking element='{element}'")
+                    if element in data["federated_instances"] and data["federated_instances"][element] != None:
+                        print(f"DEBUG Adding {len(data['federated_instances'][element])} peer(s) to peers list ...")
+                        peers = peers + data["federated_instances"][element]
             else:
                 print("WARNING: JSON response does not contain 'federated_instances':", domain)
                 update_last_error(domain, res)
         else:
-            # DEBUG: print("DEBUG:Querying API was successful:", domain, len(res.json()))
+            # DEBUG: print("DEBUG: Querying API was successful:", domain, len(res.json()))
             peers = res.json()
             nodeinfos["get_peers_url"][domain] = get_peers_url
 
@@ -461,14 +468,14 @@ def get_peers(domain: str, software: str) -> list:
     return peers
 
 def post_json_api(domain: str, path: str, parameter: str) -> list:
-    print("DEBUG: Sending POST to domain,path,parameter:", domain, path, parameter)
+    # DEBUG: print("DEBUG: Sending POST to domain,path,parameter:", domain, path, parameter)
     data = {}
     try:
         res = reqto.post(f"https://{domain}{path}", data=parameter, headers=headers, timeout=(config["connection_timeout"], config["read_timeout"]))
 
-        print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},res.json[]={type(res.json())}")
+        # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code}")
         if not res.ok or res.status_code >= 400:
-            print("WARNING: Cannot query JSON API:", domain, path, parameter, res.status_code)
+            print(f"WARNING: Cannot query JSON API: domain='{domain}',path='{path}',parameter()={len(parameter)},res.status_code='{res.status_code}'")
             update_last_error(domain, res)
         else:
             update_last_nodeinfo(domain)
@@ -477,7 +484,7 @@ def post_json_api(domain: str, path: str, parameter: str) -> list:
     except BaseException as e:
         print("WARNING: Some error during post():", domain, path, parameter, e)
 
-    print("DEBUG: Returning data():", len(data))
+    # DEBUG: print("DEBUG: Returning data():", len(data))
     return data
 
 def fetch_nodeinfo(domain: str) -> list:
@@ -505,7 +512,7 @@ def fetch_nodeinfo(domain: str) -> list:
             # DEBUG: print("DEBUG: Fetching request:", request)
             res = reqto.get(request, headers=headers, timeout=(config["connection_timeout"], config["read_timeout"]))
 
-            # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},res.json[]={type(res.json())}")
+            # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code}")
             if res.ok and isinstance(res.json(), dict):
                 # DEBUG: print("DEBUG: Success:", request)
                 data = res.json()
@@ -568,7 +575,7 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str:
     software = None
 
     try:
-        # DEBUG: print(f"DEBUG: Fetching '{path}' from '{domain}' ...")
+        # DEBUG: print(f"DEBUG: Fetching path='{path}' from '{domain}' ...")
         res = reqto.get(f"https://{domain}{path}", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"]))
 
         # DEBUG: print("DEBUG: domain,res.ok,res.status_code,res.text[]:", domain, res.ok, res.status_code, type(res.text))
@@ -778,7 +785,7 @@ def is_instance_registered(domain: str) -> bool:
         print(f"ERROR: failed SQL query: last_seen='{last_seen}'blocker='{blocker}',blocked='{blocked}',block_level='{block_level}',first_added='{first_added}',last_seen='{last_seen}',exception:'{e}'")
         sys.exit(255)
 
-    # DEBUG: print("DEBUG: registered='{registered}' - EXIT!")
+    # DEBUG: print(f"DEBUG: registered='{registered}' - EXIT!")
     return registered
 
 def add_instance(domain: str, origin: str, originator: str):
@@ -808,9 +815,9 @@ def add_instance(domain: str, origin: str, originator: str):
         )
 
         for key in nodeinfos:
-            # DEBUG: pprint(f"DEBUG: key='{key}',domain='{domain}',nodeinfos[key]={nodeinfos[key]}")
+            p# DEBUG: print(f"DEBUG: key='{key}',domain='{domain}',nodeinfos[key]={nodeinfos[key]}")
             if domain in nodeinfos[key]:
-                # DEBUG: pprint(f"DEBUG: domain='{domain}' has pending nodeinfo being updated ...")
+                p# DEBUG: print(f"DEBUG: domain='{domain}' has pending nodeinfo being updated ...")
                 update_nodeinfos(domain)
                 remove_pending_error(domain)
                 break
@@ -984,7 +991,7 @@ def get_misskey_blocks(domain: str) -> dict:
                         {
                             "domain": tidyup(instance["host"]),
                             # no reason field, nothing
-                            "reason": ""
+                            "reason": None
                         }
                     )
 
@@ -1031,7 +1038,7 @@ def get_misskey_blocks(domain: str) -> dict:
                 if instance["isBlocked"]:
                     blocks["blocked"].append({
                         "domain": tidyup(instance["host"]),
-                        "reason": ""
+                        "reason": None
                     })
 
             if len(doc) < step: