Continued:

- maybe str(e) helps here and there?

diff --git a/fba.py b/fba.py
index e4dbe91f884b57d59f73c94f6fbacb593e7e9fee..95d692bc6e17c3570d3ed87bd6d97d9cdfe72cdb 100644 (file)
--- a/fba.py
+++ b/fba.py
@@ -250,7 +250,7 @@ def update_last_blocked(domain: str):
             print("WARNING: Did not update any rows:", domain)
 
     except BaseException as e:
-        print(f"ERROR: failed SQL query: domain='{domain}',exception:'{e}'")
+        print(f"ERROR: failed SQL query: domain='{domain}',exception:'{str(e)}'")
         sys.exit(255)
 
     # DEBUG: print("DEBUG: EXIT!")
@@ -281,7 +281,7 @@ def update_nodeinfos(domain: str):
             print("WARNING: Did not update any rows:", domain)
 
     except BaseException as e:
-        print(f"ERROR: failed SQL query: domain='{domain}',sql='{sql}',exception:'{e}'")
+        print(f"ERROR: failed SQL query: domain='{domain}',sql='{sql}',exception:'{str(e)}'")
         sys.exit(255)
 
     # DEBUG: print("DEBUG: Deleting nodeinfos for domain:", domain)
@@ -323,7 +323,7 @@ def update_last_error(domain: str, res: any):
             pending_errors[domain] = res
 
     except BaseException as e:
-        print(f"ERROR: failed SQL query: domain='{domain}',exception:'{e}'")
+        print(f"ERROR: failed SQL query: domain='{domain}',exception:'{str(e)}'")
         sys.exit(255)
 
     # DEBUG: print("DEBUG: EXIT!")
@@ -341,7 +341,7 @@ def update_last_nodeinfo(domain: str):
             print("WARNING: Did not update any rows:", domain)
 
     except BaseException as e:
-        print(f"ERROR: failed SQL query: domain='{domain}',exception:'{e}'")
+        print(f"ERROR: failed SQL query: domain='{domain}',exception:'{str(e)}'")
         sys.exit(255)
 
     connection.commit()
@@ -380,7 +380,7 @@ def get_peers(domain: str, software: str) -> list:
                 # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
                 break
             elif len(fetched) != config["misskey_offset"]:
-                print(f"WARNING: Fetched '{len(fetched)}' row(s) but expected: '{config['misskey_offset']}'")
+                # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config['misskey_offset']}'")
                 offset = offset + (config["misskey_offset"] - len(fetched))
             else:
                 # DEBUG: print("DEBUG: Raising offset by step:", step)
@@ -406,10 +406,13 @@ def get_peers(domain: str, software: str) -> list:
         try:
             res = reqto.get(f"https://{domain}/api/v3/site", headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"]))
 
-            # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code='{res.status_code}'")
+            # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code='{res.status_code}',res.json[]='{type(res.json())}'")
             if not res.ok or res.status_code >= 400:
                 print("WARNING: Could not reach any JSON API:", domain)
                 update_last_error(domain, res)
+            elif res.ok and isinstance(res.json(), list):
+                # DEBUG: print(f"DEBUG: domain='{domain}' returned a list: '{res.json()}'")
+                sys.exit(255)
             elif "federated_instances" in res.json():
                 # DEBUG: print("DEBUG: Found federated_instances", domain)
                 peers = peers + add_peers(res.json()["federated_instances"])
@@ -418,7 +421,7 @@ def get_peers(domain: str, software: str) -> list:
                 update_last_error(domain, res)
 
         except BaseException as e:
-            print(f"WARNING: Exception during fetching JSON: domain='{domain}',exception:'{e}'")
+            print(f"WARNING: Exception during fetching JSON: domain='{domain}',exception:'{str(e)}'")
 
         update_last_nodeinfo(domain)
 
@@ -434,7 +437,7 @@ def get_peers(domain: str, software: str) -> list:
                 try:
                     res = reqto.get(f"https://{domain}/api/v1/server/{mode}?start={start}&count=100", headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"]))
 
-                    # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code='{res.status_code}'")
+                    # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code='{res.status_code}',res.json[]='{type(res.json())}'")
                     if res.ok and isinstance(res.json(), dict):
                         # DEBUG: print("DEBUG: Success, res.json():", len(res.json()))
                         data = res.json()
@@ -457,7 +460,7 @@ def get_peers(domain: str, software: str) -> list:
                         start = start + 100
 
                 except BaseException as e:
-                    print(f"WARNING: Exception during fetching JSON: domain='{domain}',exception:'{e}'")
+                    print(f"WARNING: Exception during fetching JSON: domain='{domain}',exception:'{str(e)}'")
 
             update_last_nodeinfo(domain)
 
@@ -468,14 +471,18 @@ def get_peers(domain: str, software: str) -> list:
     try:
         res = reqto.get(f"https://{domain}{get_peers_url}", headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"]))
 
-        # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code}")
+        # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},res.json[]='{type(res.json())}'")
         if not res.ok or res.status_code >= 400:
             # DEBUG: print(f"DEBUG: Was not able to fetch '{get_peers_url}', trying alternative ...")
             res = reqto.get(f"https://{domain}/api/v3/site", headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"]))
 
+            # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},res.json[]='{type(res.json())}'")
             if not res.ok or res.status_code >= 400:
                 print("WARNING: Could not reach any JSON API:", domain)
                 update_last_error(domain, res)
+            elif res.ok and isinstance(res.json(), list):
+                # DEBUG: print(f"DEBUG: domain='{domain}' returned a list: '{res.json()}'")
+                sys.exit(255)
             elif "federated_instances" in res.json():
                 # DEBUG: print("DEBUG: Found federated_instances", domain)
                 peers = peers + add_peers(res.json()["federated_instances"])
@@ -498,20 +505,20 @@ def get_peers(domain: str, software: str) -> list:
 
 def post_json_api(domain: str, path: str, parameter: str, extra_headers: dict = {}) -> list:
     # DEBUG: print("DEBUG: Sending POST to domain,path,parameter:", domain, path, parameter, extra_headers)
-    data = {}
+    data = list()
     try:
         res = reqto.post(f"https://{domain}{path}", data=parameter, headers={**api_headers, **extra_headers}, timeout=(config["connection_timeout"], config["read_timeout"]))
 
-        # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code}")
+        # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},res.json[]='{type(res.json())}'")
         if not res.ok or res.status_code >= 400:
-            print(f"WARNING: Cannot query JSON API: domain='{domain}',path='{path}',parameter()={len(parameter)},res.status_code='{res.status_code}'")
+            print(f"WARNING: Cannot query JSON API: domain='{domain}',path='{path}',parameter()={len(parameter)},res.status_code='{res.status_code}',res.json[]='{type(res.json())}'")
             update_last_error(domain, res)
         else:
             update_last_nodeinfo(domain)
             data = res.json()
 
     except BaseException as e:
-        print(f"WARNING: Some error during post(): domain='{domain},path='{path}',parameter()={len(parameter)},exception:'{e}'")
+        print(f"WARNING: Some error during post(): domain='{domain}',path='{path}',parameter()={len(parameter)},exception:'{str(e)}'")
 
     # DEBUG: print("DEBUG: Returning data():", len(data))
     return data
@@ -541,13 +548,16 @@ def fetch_nodeinfo(domain: str) -> list:
             # DEBUG: print("DEBUG: Fetching request:", request)
             res = reqto.get(request, headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"]))
 
-            # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code}")
+            # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},res.json[]='{type(res.json())}'")
             if res.ok and isinstance(res.json(), dict):
                 # DEBUG: print("DEBUG: Success:", request)
                 data = res.json()
                 nodeinfos["detection_mode"][domain] = "STATIC_CHECK"
                 nodeinfos["nodeinfo_url"][domain] = request
                 break
+            elif res.ok and isinstance(res.json(), list):
+                # DEBUG: print(f"DEBUG: domain='{domain}' returned a list: '{res.json()}'")
+                sys.exit(255)
             elif not res.ok or res.status_code >= 400:
                 print("WARNING: Failed fetching nodeinfo from domain:", domain)
                 update_last_error(domain, res)
@@ -742,7 +752,7 @@ def update_block_reason(reason: str, blocker: str, blocked: str, block_level: st
             print("WARNING: Did not update any rows:", domain)
 
     except BaseException as e:
-        print(f"ERROR: failed SQL query: reason='{reason}',blocker='{blocker}',blocked='{blocked}',block_level='{block_level}',sql='{sql}',exception:'{e}'")
+        print(f"ERROR: failed SQL query: reason='{reason}',blocker='{blocker}',blocked='{blocked}',block_level='{block_level}',sql='{sql}',exception:'{str(e)}'")
         sys.exit(255)
 
     # DEBUG: print("DEBUG: EXIT!")
@@ -764,7 +774,7 @@ def update_last_seen(blocker: str, blocked: str, block_level: str):
             print("WARNING: Did not update any rows:", domain)
 
     except BaseException as e:
-        print(f"ERROR: failed SQL query: last_seen='{last_seen}',blocker='{blocker}',blocked='{blocked}',block_level='{block_level}',exception:'{e}'")
+        print(f"ERROR: failed SQL query: last_seen='{last_seen}',blocker='{blocker}',blocked='{blocked}',block_level='{block_level}',exception:'{str(e)}'")
         sys.exit(255)
 
     # DEBUG: print("DEBUG: EXIT!")
@@ -793,7 +803,7 @@ def block_instance(blocker: str, blocked: str, reason: str, block_level: str):
         )
 
     except BaseException as e:
-        print(f"ERROR: failed SQL query: blocker='{blocker}',blocked='{blocked}',reason='{reason}',block_level='{block_level}',first_seen='{first_seen}',last_seen='{last_seen}',exception:'{e}'")
+        print(f"ERROR: failed SQL query: blocker='{blocker}',blocked='{blocked}',reason='{reason}',block_level='{block_level}',first_seen='{first_seen}',last_seen='{last_seen}',exception:'{str(e)}'")
         sys.exit(255)
 
     # DEBUG: print("DEBUG: EXIT!")
@@ -811,7 +821,7 @@ def is_instance_registered(domain: str) -> bool:
         # Check condition
         registered = cursor.fetchone() != None
     except BaseException as e:
-        print(f"ERROR: failed SQL query: last_seen='{last_seen}'blocker='{blocker}',blocked='{blocked}',block_level='{block_level}',first_seen='{first_seen}',last_seen='{last_seen}',exception:'{e}'")
+        print(f"ERROR: failed SQL query: last_seen='{last_seen}'blocker='{blocker}',blocked='{blocked}',block_level='{block_level}',first_seen='{first_seen}',last_seen='{last_seen}',exception:'{str(e)}'")
         sys.exit(255)
 
     # DEBUG: print(f"DEBUG: registered='{registered}' - EXIT!")
@@ -857,7 +867,7 @@ def add_instance(domain: str, origin: str, originator: str):
             remove_pending_error(domain)
 
     except BaseException as e:
-        print(f"ERROR: failed SQL query: domain='{domain}',exception:'{e}'")
+        print(f"ERROR: failed SQL query: domain='{domain}',exception:'{str(e)}'")
         sys.exit(255)
     else:
         # DEBUG: print("DEBUG: Updating nodeinfo for domain:", domain)
@@ -991,7 +1001,7 @@ def get_misskey_blocks(domain: str) -> dict:
         # instances page-by-page, since that troonware doesn't support
         # sending them all at once
         try:
-            print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...")
+            # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...")
             if offset == 0:
                 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
                 fetched = post_json_api(domain, "/api/federation/instances", json.dumps({
@@ -1010,12 +1020,12 @@ def get_misskey_blocks(domain: str) -> dict:
                     "offset"   : offset - 1
                 }), {"Origin": domain})
 
-            print("DEBUG: fetched():", len(fetched))
+            # DEBUG: print("DEBUG: fetched():", len(fetched))
             if len(fetched) == 0:
                 # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
                 break
             elif len(fetched) != config["misskey_offset"]:
-                print(f"WARNING: Fetched '{len(fetched)}' row(s) but expected: '{config['misskey_offset']}'")
+                # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config['misskey_offset']}'")
                 offset = offset + (config["misskey_offset"] - len(fetched))
             else:
                 # DEBUG: print("DEBUG: Raising offset by step:", step)
@@ -1059,12 +1069,12 @@ def get_misskey_blocks(domain: str) -> dict:
                     "offset" : offset-1
                 }), {"Origin": domain})
 
-            print("DEBUG: fetched():", len(fetched))
+            # DEBUG: print("DEBUG: fetched():", len(fetched))
             if len(fetched) == 0:
                 # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
                 break
             elif len(fetched) != config["misskey_offset"]:
-                print(f"WARNING: Fetched '{len(fetched)}' row(s) but expected: '{config['misskey_offset']}'")
+                # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config['misskey_offset']}'")
                 offset = offset + (config["misskey_offset"] - len(fetched))
             else:
                 # DEBUG: print("DEBUG: Raising offset by step:", step)

diff --git a/fetch_blocks.py b/fetch_blocks.py
index 5cda8284cb155e0cecd8fd51d74b1ae1a7ef0658..71cf74f6e82ca5c94a273a7ebdd00277c27253df 100644 (file)
--- a/fetch_blocks.py
+++ b/fetch_blocks.py
@@ -154,7 +154,7 @@ for blocker, software in rows:
 
             fba.connection.commit()
         except Exception as e:
-            print(f"ERROR: blocker='{blocker}',software='{software}',exception='{e}'")
+            print(f"ERROR: blocker='{blocker}',software='{software}',exception='{str(e)}'")
     elif software == "mastodon":
         print("INFO: blocker:", blocker)
         try:
@@ -272,7 +272,7 @@ for blocker, software in rows:
 
             fba.connection.commit()
         except Exception as e:
-            print(f"ERROR: blocker='{blocker}',software='{software}',exception='{e}'")
+            print(f"ERROR: blocker='{blocker}',software='{software}',exception='{str(e)}'")
     elif software == "friendica" or software == "misskey" or software == "bookwyrm" or software == "takahe":
         print("INFO: blocker:", blocker)
         try:
@@ -350,7 +350,7 @@ for blocker, software in rows:
 
             fba.connection.commit()
         except Exception as e:
-            print(f"ERROR: blocker='{blocker}',software='{software}',exception='{e}'")
+            print(f"ERROR: blocker='{blocker}',software='{software}',exception='{str(e)}'")
     elif software == "gotosocial":
         print("INFO: blocker:", blocker)
         try:
@@ -418,7 +418,7 @@ for blocker, software in rows:
 
                 fba.connection.commit()
         except Exception as e:
-            print(f"ERROR: blocker='{blocker}',software='{software}',exception='{e}'")
+            print(f"ERROR: blocker='{blocker}',software='{software}',exception='{str(e)}'")
     else:
         print("WARNING: Unknown software:", blocker, software)
 

diff --git a/fetch_instances.py b/fetch_instances.py
index fbd113afaafc6579a475a0ba3defba4fb79737b6..ceadd47f29eadd69c57a79cffd1c311c27539772 100644 (file)
--- a/fetch_instances.py
+++ b/fetch_instances.py
@@ -43,7 +43,7 @@ def fetch_instances(domain: str, origin: str, software: str):
                 # NOISY-DEBUG: print("DEBUG: Adding new instance:", instance, domain)
                 fba.add_instance(instance, domain, sys.argv[0])
         except BaseException as e:
-            print(f"ERROR: instance='{instance}',exception:'{e}'")
+            print(f"ERROR: instance='{instance}',exception:'{str(e)}'")
             continue
 
 instance = sys.argv[1]