Continued:

- ops, don't copy whole HTML to last_error_message
- also need to initialize local variable 'peers' first

diff --git a/fba/federation.py b/fba/federation.py
index 888868259cbcecd99ff728718e0c9c07aff04e5f..a912234beb3fd7da9dc87cc6086f5062c306dcf2 100644 (file)
--- a/fba/federation.py
+++ b/fba/federation.py
@@ -120,6 +120,9 @@ def fetch_peers(domain: str, software: str) -> list:
         # DEBUG: print(f"DEBUG: Invoking peertube.fetch_peers({domain}) ...")
         return peertube.fetch_peers(domain)
 
+    # Init peers variable
+    peers = list()
+
     # DEBUG: print(f"DEBUG: Checking CSRF for domain='{domain}'")
     headers = csrf.determine(domain, dict())
 
@@ -130,8 +133,8 @@ def fetch_peers(domain: str, software: str) -> list:
         headers,
         (config.get("connection_timeout"), config.get("read_timeout"))
     )
-    # DEBUG: print(f"DEBUG: data[]='{type(data)}'")
 
+    # DEBUG: print(f"DEBUG: data[]='{type(data)}'")
     if "error_message" in data:
         # DEBUG: print("DEBUG: Was not able to fetch peers, trying alternative ...")
         data = network.get_json_api(

diff --git a/fba/network.py b/fba/network.py
index ec7e4bce4ab26510cf518976d4471f7c55a090a5..7c0bb8a1ab5ddd0741c002b869f920940ffb1b13 100644 (file)
--- a/fba/network.py
+++ b/fba/network.py
@@ -67,7 +67,7 @@ def post_json_api(domain: str, path: str, data: str, headers: dict = {}) -> dict
         if not response.ok or response.status_code >= 400:
             print(f"WARNING: Cannot query JSON API: domain='{domain}',path='{path}',data()={len(data)},response.status_code='{response.status_code}',json_reply[]='{type(json_reply)}'")
             json_reply["status_code"]   = response.status_code
-            json_reply["error_message"] = response.text
+            json_reply["error_message"] = response.reason
             instances.update_last_error(domain, response)
 
     except requests.exceptions.ConnectionError as exception:
@@ -101,7 +101,7 @@ def fetch_api_url(url: str, timeout: tuple) -> dict:
         if not response.ok or response.status_code >= 400:
             print(f"WARNING: Cannot query JSON API: url='{url}',response.status_code='{response.status_code}',json_reply[]='{type(json_reply)}'")
             json_reply["status_code"]   = response.status_code
-            json_reply["error_message"] = response.text
+            json_reply["error_message"] = response.reason
 
     except requests.exceptions.ConnectionError as exception:
         # DEBUG: print(f"DEBUG: Fetching '{url}' failed. exception[{type(exception)}]='{str(exception)}'")
@@ -152,7 +152,7 @@ def get_json_api(domain: str, path: str, headers: dict, timeout: tuple) -> dict:
     if not response.ok or response.status_code >= 400:
         print(f"WARNING: Cannot query JSON API: domain='{domain}',path='{path}',response.status_code='{response.status_code}',json_reply[]='{type(json_reply)}'")
         json_reply["status_code"]   = response.status_code
-        json_reply["error_message"] = response.text
+        json_reply["error_message"] = response.reason
         instances.update_last_error(domain, response)
 
     # DEBUG: print(f"DEBUG: Returning json_reply({len(json_reply)})=[]:{type(json_reply)}")

diff --git a/fba/networks/misskey.py b/fba/networks/misskey.py
index 52c32e3e53aa15985c11f13365b7828e9bde7685..68d3b261c4136a142334b74106da1fc4d0216e7f 100644 (file)
--- a/fba/networks/misskey.py
+++ b/fba/networks/misskey.py
@@ -59,30 +59,33 @@ def fetch_peers(domain: str) -> list:
                 "offset": offset - 1
             }), headers)
 
-        # DEBUG: print(f"DEBUG: fetched()={len(fetched)}")
-        if len(fetched) == 0:
+        # Check records
+        # DEBUG: print(f"DEBUG: fetched[]={type(fetched)}")
+        if "error_message" in fetched:
+            print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}")
+            instances.update_last_error(domain, fetched)
+            break
+        elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
+            print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}")
+            instances.update_last_error(domain, fetched["json"]["error"]["message"])
+            break
+
+        rows = fetched["json"]
+
+        # DEBUG: print(f"DEBUG: rows()={len(rows)}")
+        if len(rows) == 0:
             # DEBUG: print(f"DEBUG: Returned zero bytes, exiting loop, domain='{domain}'")
             break
-        elif len(fetched) != config.get("misskey_limit"):
-            # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'")
-            offset = offset + (config.get("misskey_limit") - len(fetched))
+        elif len(rows) != config.get("misskey_limit"):
+            # DEBUG: print(f"DEBUG: Fetched '{len(rows)}' row(s) but expected: '{config.get('misskey_limit')}'")
+            offset = offset + (config.get("misskey_limit") - len(rows))
         else:
             # DEBUG: print(f"DEBUG: Raising offset by step={step}")
             offset = offset + step
 
-        # Check records
-        # DEBUG: print(f"DEBUG: fetched({len(fetched)})[]={type(fetched)}")
-        if isinstance(fetched, dict) and "error" in fetched and "message" in fetched["error"]:
-            print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}")
-            instances.update_last_error(domain, fetched["error"]["message"])
-            break
-        elif "error_message" in fetched:
-            print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}")
-            instances.update_last_error(domain, fetched)
-            break
-
         already = 0
-        for row in fetched["json"]:
+        # DEBUG: print(f"DEBUG: rows({len(rows)})[]={type(rows)}")
+        for row in rows:
             # DEBUG: print(f"DEBUG: row()={len(row)}")
             if not "host" in row:
                 print(f"WARNING: row()={len(row)} does not contain key 'host': {row},domain='{domain}'")
@@ -101,7 +104,7 @@ def fetch_peers(domain: str) -> list:
             # DEBUG: print(f"DEBUG: Adding peer: '{row['host']}'")
             peers.append(row["host"])
 
-        if already == len(fetched):
+        if already == len(rows):
             # DEBUG: print(f"DEBUG: Host returned same set of '{already}' instances, aborting loop!")
             break
 
@@ -155,19 +158,32 @@ def fetch_blocks(domain: str) -> dict:
                     "offset"   : offset - 1
                 }), headers)
 
-            # DEBUG: print("DEBUG: fetched():", len(fetched))
-            if len(fetched) == 0:
+            # Check records
+            # DEBUG: print(f"DEBUG: fetched[]={type(fetched)}")
+            if "error_message" in fetched:
+                print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}")
+                instances.update_last_error(domain, fetched)
+                break
+            elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
+                print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}")
+                instances.update_last_error(domain, fetched["json"]["error"]["message"])
+                break
+
+            rows = fetched["json"]
+
+            # DEBUG: print("DEBUG: rows():", len(rows))
+            if len(rows) == 0:
                 # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
                 break
-            elif len(fetched) != config.get("misskey_limit"):
-                # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'")
-                offset = offset + (config.get("misskey_limit") - len(fetched))
+            elif len(rows) != config.get("misskey_limit"):
+                # DEBUG: print(f"DEBUG: Fetched '{len(rows)}' row(s) but expected: '{config.get('misskey_limit')}'")
+                offset = offset + (config.get("misskey_limit") - len(rows))
             else:
                 # DEBUG: print("DEBUG: Raising offset by step:", step)
                 offset = offset + step
 
             count = 0
-            for instance in fetched:
+            for instance in rows:
                 # Is it there?
                 if instance["isSuspended"] and not dicts.has_key(blocklist["suspended"], "domain", instance):
                     count = count + 1