Continued:

- some misskey instances only allow 10 items per fetch
- also used more masked strings

diff --git a/config.defaults.json b/config.defaults.json
index 397365d83adb9ce3ee9c239c66c19ff9c2d1cd96..c5787e6b2369b96bfaf705708b7615931a8abda2 100644 (file)
--- a/config.defaults.json
+++ b/config.defaults.json
@@ -11,5 +11,6 @@
     "bot_visibility"    : "unlisted",
     "slogan"            : "### Your footer slogan ###",
     "recheck_instance"  : 3600,
-    "recheck_block"     : 3600
+    "recheck_block"     : 3600,
+    "misskey_offset"    : 10
 }

diff --git a/fba.py b/fba.py
index 458f9f0d73e683027d2166f5040153500a79178c..4a069023eadfb06032c46583a3fdaa615b8f35c1 100644 (file)
--- a/fba.py
+++ b/fba.py
@@ -233,7 +233,7 @@ def update_last_blocked(domain: str):
             print("WARNING: Did not update any rows:", domain)
 
     except BaseException as e:
-        print("ERROR: failed SQL query:", domain, e)
+        print(f"ERROR: failed SQL query: domain='{domain}',exception:'{e}'")
         sys.exit(255)
 
     # DEBUG: print("DEBUG: EXIT!")
@@ -306,7 +306,7 @@ def update_last_error(domain: str, res: any):
             pending_errors[domain] = res
 
     except BaseException as e:
-        print("ERROR: failed SQL query:", domain, e)
+        print(f"ERROR: failed SQL query: domain='{domain}',exception:'{e}'")
         sys.exit(255)
 
     # DEBUG: print("DEBUG: EXIT!")
@@ -324,7 +324,7 @@ def update_last_nodeinfo(domain: str):
             print("WARNING: Did not update any rows:", domain)
 
     except BaseException as e:
-        print("ERROR: failed SQL query:", domain, e)
+        print(f"ERROR: failed SQL query: domain='{domain}',exception:'{e}'")
         sys.exit(255)
 
     connection.commit()
@@ -338,7 +338,7 @@ def get_peers(domain: str, software: str) -> list:
         print(f"DEBUG: domain='{domain}' is misskey, sending API POST request ...")
 
         counter = 0
-        step = 99
+        step = config["misskey_offset"]
         while True:
             if counter == 0:
                 fetched = post_json_api(domain, "/api/federation/instances", json.dumps({
@@ -367,7 +367,6 @@ def get_peers(domain: str, software: str) -> list:
                     peers.append(row["host"])
 
         # DEBUG: print("DEBUG: Returning peers[]:", type(peers))
-        sys.exit(255)
         return peers
     elif software == "lemmy":
         # DEBUG: print(f"DEBUG: domain='{domain}' is Lemmy, fetching JSON ...")
@@ -434,7 +433,7 @@ def get_peers(domain: str, software: str) -> list:
     try:
         res = reqto.get(f"https://{domain}{get_peers_url}", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"]))
 
-        # DEBUG: print("DEBUG: res.ok,res.json[]:", res.ok, type(res.json()))
+        # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},res.json[]={type(res.json())}")
         if not res.ok or res.status_code >= 400:
             res = reqto.get(f"https://{domain}/api/v3/site", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"]))
 
@@ -461,24 +460,24 @@ def get_peers(domain: str, software: str) -> list:
     # DEBUG: print("DEBUG: Returning peers[]:", type(peers))
     return peers
 
-def post_json_api(domain: str, path: str, data: str) -> list:
-    # DEBUG: print("DEBUG: Sending POST to domain,path,data:", domain, path, data)
+def post_json_api(domain: str, path: str, parameter: str) -> list:
+    print("DEBUG: Sending POST to domain,path,parameter:", domain, path, parameter)
     data = {}
     try:
-        res = reqto.post(f"https://{domain}{path}", data=data, headers=headers, timeout=(config["connection_timeout"], config["read_timeout"]))
+        res = reqto.post(f"https://{domain}{path}", data=parameter, headers=headers, timeout=(config["connection_timeout"], config["read_timeout"]))
 
-        # DEBUG: print("DEBUG: res.ok,res.json[]:", res.ok, type(res.json()))
+        print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},res.json[]={type(res.json())}")
         if not res.ok or res.status_code >= 400:
-            print("WARNING: Cannot query JSON API:", domain, path, data, res.status_code)
+            print("WARNING: Cannot query JSON API:", domain, path, parameter, res.status_code)
             update_last_error(domain, res)
         else:
             update_last_nodeinfo(domain)
             data = res.json()
 
     except BaseException as e:
-        print("WARNING: Some error during post():", domain, path, data, e)
+        print("WARNING: Some error during post():", domain, path, parameter, e)
 
-    # DEBUG: print("DEBUG: Returning data():", len(data))
+    print("DEBUG: Returning data():", len(data))
     return data
 
 def fetch_nodeinfo(domain: str) -> list:
@@ -506,7 +505,7 @@ def fetch_nodeinfo(domain: str) -> list:
             # DEBUG: print("DEBUG: Fetching request:", request)
             res = reqto.get(request, headers=headers, timeout=(config["connection_timeout"], config["read_timeout"]))
 
-            # DEBUG: print("DEBUG: res.ok,res.json[]:", res.ok, type(res.json()))
+            # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},res.json[]={type(res.json())}")
             if res.ok and isinstance(res.json(), dict):
                 # DEBUG: print("DEBUG: Success:", request)
                 data = res.json()
@@ -707,7 +706,7 @@ def update_block_reason(reason: str, blocker: str, blocked: str, block_level: st
             print("WARNING: Did not update any rows:", domain)
 
     except BaseException as e:
-        print("ERROR: failed SQL query:", reason, blocker, blocked, block_level, e)
+        print(f"ERROR: failed SQL query: reason='{reason}',blocker='{blocker}',blocked='{blocked}',block_level='{block_level}',sql='{sql}',exception:'{e}'")
         sys.exit(255)
 
     # DEBUG: print("DEBUG: EXIT!")
@@ -729,7 +728,7 @@ def update_last_seen(blocker: str, blocked: str, block_level: str):
             print("WARNING: Did not update any rows:", domain)
 
     except BaseException as e:
-        print("ERROR: failed SQL query:", last_seen, blocker, blocked, block_level, e)
+        print(f"ERROR: failed SQL query: last_seen='{last_seen}',blocker='{blocker}',blocked='{blocked}',block_level='{block_level}',exception:'{e}'")
         sys.exit(255)
 
     # DEBUG: print("DEBUG: EXIT!")
@@ -758,7 +757,7 @@ def block_instance(blocker: str, blocked: str, reason: str, block_level: str):
         )
 
     except BaseException as e:
-        print("ERROR: failed SQL query:", blocker, blocked, reason, block_level, first_added, last_seen, e)
+        print(f"ERROR: failed SQL query: blocker='{blocker}',blocked='{blocked}',reason='{reason}',block_level='{block_level}',first_added='{first_added}',last_seen='{last_seen}',exception:'{e}'")
         sys.exit(255)
 
     # DEBUG: print("DEBUG: EXIT!")
@@ -776,7 +775,7 @@ def is_instance_registered(domain: str) -> bool:
         # Check condition
         registered = cursor.fetchone() != None
     except BaseException as e:
-        print("ERROR: failed SQL query:", blocker, blocked, reason, block_level, first_added, last_seen, e)
+        print(f"ERROR: failed SQL query: last_seen='{last_seen}'blocker='{blocker}',blocked='{blocked}',block_level='{block_level}',first_added='{first_added}',last_seen='{last_seen}',exception:'{e}'")
         sys.exit(255)
 
     # DEBUG: print("DEBUG: registered='{registered}' - EXIT!")
@@ -822,7 +821,7 @@ def add_instance(domain: str, origin: str, originator: str):
             remove_pending_error(domain)
 
     except BaseException as e:
-        print("ERROR: failed SQL query:", domain, e)
+        print(f"ERROR: failed SQL query: domain='{domain}',exception:'{e}'")
         sys.exit(255)
     else:
         # DEBUG: print("DEBUG: Updating nodeinfo for domain:", domain)
@@ -949,7 +948,7 @@ def get_misskey_blocks(domain: str) -> dict:
     }
 
     counter = 0
-    step = 99
+    step = config["misskey_offset"]
     while True:
         # iterating through all "suspended" (follow-only in its terminology)
         # instances page-by-page, since that troonware doesn't support