Continued:

- add_peers() now returns a list of peers, dict doesn't make sense here
- also handled error messages from JSON API POST requests

diff --git a/fba.py b/fba.py
index 9174bba4b805ab4300260aab56afb1ed1f32a717..c294a2d6f9a2559223ca6282b27959650425cd24 100644 (file)
--- a/fba.py
+++ b/fba.py
@@ -126,16 +126,26 @@ patterns = [
     re.compile("^[a-f0-9]{7}$"),
 ]
 
-def add_peers(rows: dict) -> dict:
+def add_peers(rows: dict) -> list:
     # DEBUG: print(f"DEBUG: rows()={len(rows)} - CALLED!")
-    peers = {}
+    peers = list()
     for element in ["linked", "allowed", "blocked"]:
         # DEBUG: print(f"DEBUG: Checking element='{element}'")
         if element in rows and rows[element] != None:
             # DEBUG: print(f"DEBUG: Adding {len(rows[element])} peer(s) to peers list ...")
-            peers = {**peers, **rows[element]}
+            for peer in rows[element]:
+                # DEBUG: print(f"DEBUG: peer='{peer}' - BEFORE!")
+                peer = tidyup(peer)
+
+                # DEBUG: print(f"DEBUG: peer='{peer}' - AFTER!")
+                if is_blacklisted(peer):
+                    # DEBUG: print(f"DEBUG: peer='{peer}' is blacklisted, skipped!")
+                    continue
+
+                # DEBUG: print(f"DEBUG: Adding peer='{peer}' ...")
+                peers.append(peer)
 
-    # DEBUG: print(f"DEBUG: peers()={len(peers)} - CALLED!")
+    # DEBUG: print(f"DEBUG: peers()={len(peers)} - EXIT!")
     return peers
 
 def remove_version(software: str) -> str:
@@ -422,12 +432,17 @@ def get_peers(domain: str, software: str) -> list:
                 offset = offset + step
 
             # Check records
+            # DEBUG: print(f"DEBUG: fetched({len(fetched)})[]={type(fetched)}")
+            if isinstance(fetched, dict) and "error" in fetched and "message" in fetched["error"]:
+                print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}")
+                update_last_error(domain, fetched["error"]["message"])
+                break
             for row in fetched:
                 # DEBUG: print(f"DEBUG: row():{len(row)}")
                 if not "host" in row:
                     print(f"WARNING: row()={len(row)} does not contain element 'host': {row},domain='{domain}'")
                     continue
-                elif "host" in row and is_blacklisted(row["host"]):
+                elif is_blacklisted(row["host"]):
                     # DEBUG: print(f"DEBUG: row[host]='{row['host']}' is blacklisted. domain='{domain}'")
                     continue
 
@@ -443,16 +458,18 @@ def get_peers(domain: str, software: str) -> list:
         try:
             res = reqto.get(f"https://{domain}/api/v3/site", headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"]))
 
-            # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code='{res.status_code}',res.json[]='{type(res.json())}'")
+            data = res.json()
+            # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code='{res.status_code}',data[]='{type(data)}'")
             if not res.ok or res.status_code >= 400:
                 print("WARNING: Could not reach any JSON API:", domain)
                 update_last_error(domain, res)
-            elif res.ok and isinstance(res.json(), list):
-                # DEBUG: print(f"DEBUG: domain='{domain}' returned a list: '{res.json()}'")
+            elif res.ok and isinstance(data, list):
+                # DEBUG: print(f"DEBUG: domain='{domain}' returned a list: '{data}'")
                 sys.exit(255)
-            elif "federated_instances" in res.json():
-                # DEBUG: print("DEBUG: Found federated_instances", domain)
-                peers = peers + add_peers(res.json()["federated_instances"])
+            elif "federated_instances" in data:
+                # DEBUG: print(f"DEBUG: Found federated_instances for domain='{domain}'")
+                peers = peers + add_peers(data["federated_instances"])
+                # DEBUG: print("DEBUG: Added instance(s) to peers")
             else:
                 print("WARNING: JSON response does not contain 'federated_instances':", domain)
                 update_last_error(domain, res)
@@ -474,11 +491,10 @@ def get_peers(domain: str, software: str) -> list:
                 try:
                     res = reqto.get(f"https://{domain}/api/v1/server/{mode}?start={start}&count=100", headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"]))
 
-                    # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code='{res.status_code}',res.json[]='{type(res.json())}'")
-                    if res.ok and isinstance(res.json(), dict):
-                        # DEBUG: print("DEBUG: Success, res.json():", len(res.json()))
-                        data = res.json()
-
+                    data = res.json()
+                    # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code='{res.status_code}',data[]='{type(data)}'")
+                    if res.ok and isinstance(data, dict):
+                        # DEBUG: print("DEBUG: Success, data:", len(data))
                         if "data" in data:
                             # DEBUG: print(f"DEBUG: Found {len(data['data'])} record(s).")
                             for record in data["data"]:
@@ -508,27 +524,30 @@ def get_peers(domain: str, software: str) -> list:
     try:
         res = reqto.get(f"https://{domain}{get_peers_url}", headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"]))
 
-        # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},res.json[]='{type(res.json())}'")
+        data = res.json()
+        # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'")
         if not res.ok or res.status_code >= 400:
             # DEBUG: print(f"DEBUG: Was not able to fetch '{get_peers_url}', trying alternative ...")
             res = reqto.get(f"https://{domain}/api/v3/site", headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"]))
 
-            # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},res.json[]='{type(res.json())}'")
+            data = res.json()
+            # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'")
             if not res.ok or res.status_code >= 400:
                 print("WARNING: Could not reach any JSON API:", domain)
                 update_last_error(domain, res)
-            elif res.ok and isinstance(res.json(), list):
-                # DEBUG: print(f"DEBUG: domain='{domain}' returned a list: '{res.json()}'")
+            elif res.ok and isinstance(data, list):
+                print(f"DEBUG: domain='{domain}' returned a list: '{data}'")
                 sys.exit(255)
-            elif "federated_instances" in res.json():
-                # DEBUG: print("DEBUG: Found federated_instances", domain)
-                peers = peers + add_peers(res.json()["federated_instances"])
+            elif "federated_instances" in data:
+                # DEBUG: print(f"DEBUG: Found federated_instances for domain='{domain}'")
+                peers = peers + add_peers(data["federated_instances"])
+                # DEBUG: print("DEBUG: Added instance(s) to peers")
             else:
                 print("WARNING: JSON response does not contain 'federated_instances':", domain)
                 update_last_error(domain, res)
         else:
-            # DEBUG: print("DEBUG: Querying API was successful:", domain, len(res.json()))
-            peers = res.json()
+            # DEBUG: print("DEBUG: Querying API was successful:", domain, len(data))
+            peers = data
             nodeinfos["get_peers_url"][domain] = get_peers_url
 
     except BaseException as e:
@@ -540,24 +559,24 @@ def get_peers(domain: str, software: str) -> list:
     # DEBUG: print("DEBUG: Returning peers[]:", type(peers))
     return peers
 
-def post_json_api(domain: str, path: str, parameter: str, extra_headers: dict = {}) -> list:
+def post_json_api(domain: str, path: str, parameter: str, extra_headers: dict = {}) -> dict:
     # DEBUG: print("DEBUG: Sending POST to domain,path,parameter:", domain, path, parameter, extra_headers)
-    data = list()
+    data = {}
     try:
         res = reqto.post(f"https://{domain}{path}", data=parameter, headers={**api_headers, **extra_headers}, timeout=(config["connection_timeout"], config["read_timeout"]))
 
-        # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},res.json[]='{type(res.json())}'")
+        data = res.json()
+        # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'")
         if not res.ok or res.status_code >= 400:
-            print(f"WARNING: Cannot query JSON API: domain='{domain}',path='{path}',parameter()={len(parameter)},res.status_code='{res.status_code}',res.json[]='{type(res.json())}'")
+            print(f"WARNING: Cannot query JSON API: domain='{domain}',path='{path}',parameter()={len(parameter)},res.status_code='{res.status_code}',data[]='{type(data)}'")
             update_last_error(domain, res)
         else:
             update_last_nodeinfo(domain)
-            data = res.json()
 
     except BaseException as e:
         print(f"WARNING: Some error during post(): domain='{domain}',path='{path}',parameter()={len(parameter)},exception[{type(e)}]:'{str(e)}'")
 
-    # DEBUG: print("DEBUG: Returning data():", len(data))
+    # DEBUG: print(f"DEBUG: Returning data({len(data)})=[]:{type(data)}")
     return data
 
 def fetch_nodeinfo(domain: str, path: str = None) -> list:
@@ -589,15 +608,15 @@ def fetch_nodeinfo(domain: str, path: str = None) -> list:
             # DEBUG: print("DEBUG: Fetching request:", request)
             res = reqto.get(request, headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"]))
 
-            # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},res.json[]='{type(res.json())}'")
-            if res.ok and isinstance(res.json(), dict):
+            data = res.json()
+            # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'")
+            if res.ok and isinstance(data, dict):
                 # DEBUG: print("DEBUG: Success:", request)
-                data = res.json()
                 nodeinfos["detection_mode"][domain] = "STATIC_CHECK"
                 nodeinfos["nodeinfo_url"][domain] = request
                 break
-            elif res.ok and isinstance(res.json(), list):
-                # DEBUG: print(f"DEBUG: domain='{domain}' returned a list: '{res.json()}'")
+            elif res.ok and isinstance(data, list):
+                # DEBUG: print(f"DEBUG: domain='{domain}' returned a list: '{data}'")
                 sys.exit(255)
             elif not res.ok or res.status_code >= 400:
                 print("WARNING: Failed fetching nodeinfo from domain:", domain)
@@ -618,9 +637,11 @@ def fetch_wellknown_nodeinfo(domain: str) -> list:
 
     try:
         res = reqto.get(f"https://{domain}/.well-known/nodeinfo", headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"]))
-        # DEBUG: print("DEBUG: domain,res.ok,res.json[]:", domain, res.ok, type(res.json()))
-        if res.ok and isinstance(res.json(), dict):
-            nodeinfo = res.json()
+
+        data = res.json()
+        # DEBUG: print("DEBUG: domain,res.ok,data[]:", domain, res.ok, type(data))
+        if res.ok and isinstance(data, dict):
+            nodeinfo = data
             # DEBUG: print("DEBUG: Found entries:", len(nodeinfo), domain)
             if "links" in nodeinfo:
                 # DEBUG: print("DEBUG: Found links in nodeinfo():", len(nodeinfo["links"]))
@@ -630,10 +651,10 @@ def fetch_wellknown_nodeinfo(domain: str) -> list:
                         # DEBUG: print("DEBUG: Fetching nodeinfo from:", link["href"])
                         res = reqto.get(link["href"])
 
+                        data = res.json()
                         # DEBUG: print("DEBUG: href,res.ok,res.status_code:", link["href"], res.ok, res.status_code)
-                        if res.ok and isinstance(res.json(), dict):
-                            # DEBUG: print("DEBUG: Found JSON nodeinfo():", len(res.json()))
-                            data = res.json()
+                        if res.ok and isinstance(data, dict):
+                            # DEBUG: print("DEBUG: Found JSON nodeinfo():", len(data))
                             nodeinfos["detection_mode"][domain] = "AUTO_DISCOVERY"
                             nodeinfos["nodeinfo_url"][domain] = link["href"]
                             break