From eb80198afdcd1b42ea23684c6c4d1272cea6fd46 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Sun, 28 May 2023 12:15:19 +0200 Subject: [PATCH] Continued: - some misskey instances only allow 10 items per fetch - also used more masked strings --- config.defaults.json | 3 ++- fba.py | 39 +++++++++++++++++++-------------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/config.defaults.json b/config.defaults.json index 397365d..c5787e6 100644 --- a/config.defaults.json +++ b/config.defaults.json @@ -11,5 +11,6 @@ "bot_visibility" : "unlisted", "slogan" : "### Your footer slogan ###", "recheck_instance" : 3600, - "recheck_block" : 3600 + "recheck_block" : 3600, + "misskey_offset" : 10 } diff --git a/fba.py b/fba.py index 458f9f0..4a06902 100644 --- a/fba.py +++ b/fba.py @@ -233,7 +233,7 @@ def update_last_blocked(domain: str): print("WARNING: Did not update any rows:", domain) except BaseException as e: - print("ERROR: failed SQL query:", domain, e) + print(f"ERROR: failed SQL query: domain='{domain}',exception:'{e}'") sys.exit(255) # DEBUG: print("DEBUG: EXIT!") @@ -306,7 +306,7 @@ def update_last_error(domain: str, res: any): pending_errors[domain] = res except BaseException as e: - print("ERROR: failed SQL query:", domain, e) + print(f"ERROR: failed SQL query: domain='{domain}',exception:'{e}'") sys.exit(255) # DEBUG: print("DEBUG: EXIT!") @@ -324,7 +324,7 @@ def update_last_nodeinfo(domain: str): print("WARNING: Did not update any rows:", domain) except BaseException as e: - print("ERROR: failed SQL query:", domain, e) + print(f"ERROR: failed SQL query: domain='{domain}',exception:'{e}'") sys.exit(255) connection.commit() @@ -338,7 +338,7 @@ def get_peers(domain: str, software: str) -> list: print(f"DEBUG: domain='{domain}' is misskey, sending API POST request ...") counter = 0 - step = 99 + step = config["misskey_offset"] while True: if counter == 0: fetched = post_json_api(domain, "/api/federation/instances", json.dumps({ @@ -367,7 +367,6 @@ def get_peers(domain: str, software: str) -> list: peers.append(row["host"]) # DEBUG: print("DEBUG: Returning peers[]:", type(peers)) - sys.exit(255) return peers elif software == "lemmy": # DEBUG: print(f"DEBUG: domain='{domain}' is Lemmy, fetching JSON ...") @@ -434,7 +433,7 @@ def get_peers(domain: str, software: str) -> list: try: res = reqto.get(f"https://{domain}{get_peers_url}", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])) - # DEBUG: print("DEBUG: res.ok,res.json[]:", res.ok, type(res.json())) + # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},res.json[]={type(res.json())}") if not res.ok or res.status_code >= 400: res = reqto.get(f"https://{domain}/api/v3/site", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])) @@ -461,24 +460,24 @@ def get_peers(domain: str, software: str) -> list: # DEBUG: print("DEBUG: Returning peers[]:", type(peers)) return peers -def post_json_api(domain: str, path: str, data: str) -> list: - # DEBUG: print("DEBUG: Sending POST to domain,path,data:", domain, path, data) +def post_json_api(domain: str, path: str, parameter: str) -> list: + print("DEBUG: Sending POST to domain,path,parameter:", domain, path, parameter) data = {} try: - res = reqto.post(f"https://{domain}{path}", data=data, headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])) + res = reqto.post(f"https://{domain}{path}", data=parameter, headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])) - # DEBUG: print("DEBUG: res.ok,res.json[]:", res.ok, type(res.json())) + print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},res.json[]={type(res.json())}") if not res.ok or res.status_code >= 400: - print("WARNING: Cannot query JSON API:", domain, path, data, res.status_code) + print("WARNING: Cannot query JSON API:", domain, path, parameter, res.status_code) update_last_error(domain, res) else: update_last_nodeinfo(domain) data = res.json() except BaseException as e: - print("WARNING: Some error during post():", domain, path, data, e) + print("WARNING: Some error during post():", domain, path, parameter, e) - # DEBUG: print("DEBUG: Returning data():", len(data)) + print("DEBUG: Returning data():", len(data)) return data def fetch_nodeinfo(domain: str) -> list: @@ -506,7 +505,7 @@ def fetch_nodeinfo(domain: str) -> list: # DEBUG: print("DEBUG: Fetching request:", request) res = reqto.get(request, headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])) - # DEBUG: print("DEBUG: res.ok,res.json[]:", res.ok, type(res.json())) + # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},res.json[]={type(res.json())}") if res.ok and isinstance(res.json(), dict): # DEBUG: print("DEBUG: Success:", request) data = res.json() @@ -707,7 +706,7 @@ def update_block_reason(reason: str, blocker: str, blocked: str, block_level: st print("WARNING: Did not update any rows:", domain) except BaseException as e: - print("ERROR: failed SQL query:", reason, blocker, blocked, block_level, e) + print(f"ERROR: failed SQL query: reason='{reason}',blocker='{blocker}',blocked='{blocked}',block_level='{block_level}',sql='{sql}',exception:'{e}'") sys.exit(255) # DEBUG: print("DEBUG: EXIT!") @@ -729,7 +728,7 @@ def update_last_seen(blocker: str, blocked: str, block_level: str): print("WARNING: Did not update any rows:", domain) except BaseException as e: - print("ERROR: failed SQL query:", last_seen, blocker, blocked, block_level, e) + print(f"ERROR: failed SQL query: last_seen='{last_seen}',blocker='{blocker}',blocked='{blocked}',block_level='{block_level}',exception:'{e}'") sys.exit(255) # DEBUG: print("DEBUG: EXIT!") @@ -758,7 +757,7 @@ def block_instance(blocker: str, blocked: str, reason: str, block_level: str): ) except BaseException as e: - print("ERROR: failed SQL query:", blocker, blocked, reason, block_level, first_added, last_seen, e) + print(f"ERROR: failed SQL query: blocker='{blocker}',blocked='{blocked}',reason='{reason}',block_level='{block_level}',first_added='{first_added}',last_seen='{last_seen}',exception:'{e}'") sys.exit(255) # DEBUG: print("DEBUG: EXIT!") @@ -776,7 +775,7 @@ def is_instance_registered(domain: str) -> bool: # Check condition registered = cursor.fetchone() != None except BaseException as e: - print("ERROR: failed SQL query:", blocker, blocked, reason, block_level, first_added, last_seen, e) + print(f"ERROR: failed SQL query: last_seen='{last_seen}'blocker='{blocker}',blocked='{blocked}',block_level='{block_level}',first_added='{first_added}',last_seen='{last_seen}',exception:'{e}'") sys.exit(255) # DEBUG: print("DEBUG: registered='{registered}' - EXIT!") @@ -822,7 +821,7 @@ def add_instance(domain: str, origin: str, originator: str): remove_pending_error(domain) except BaseException as e: - print("ERROR: failed SQL query:", domain, e) + print(f"ERROR: failed SQL query: domain='{domain}',exception:'{e}'") sys.exit(255) else: # DEBUG: print("DEBUG: Updating nodeinfo for domain:", domain) @@ -949,7 +948,7 @@ def get_misskey_blocks(domain: str) -> dict: } counter = 0 - step = 99 + step = config["misskey_offset"] while True: # iterating through all "suspended" (follow-only in its terminology) # instances page-by-page, since that troonware doesn't support -- 2.39.5