From 5f4d330d4244fc19cc3dc2647f737bfd8153f9e7 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Sun, 28 May 2023 17:52:55 +0200 Subject: [PATCH] Continued: - maybe str(e) helps here and there? --- fba.py | 60 +++++++++++++++++++++++++++------------------- fetch_blocks.py | 8 +++---- fetch_instances.py | 2 +- 3 files changed, 40 insertions(+), 30 deletions(-) diff --git a/fba.py b/fba.py index e4dbe91..95d692b 100644 --- a/fba.py +++ b/fba.py @@ -250,7 +250,7 @@ def update_last_blocked(domain: str): print("WARNING: Did not update any rows:", domain) except BaseException as e: - print(f"ERROR: failed SQL query: domain='{domain}',exception:'{e}'") + print(f"ERROR: failed SQL query: domain='{domain}',exception:'{str(e)}'") sys.exit(255) # DEBUG: print("DEBUG: EXIT!") @@ -281,7 +281,7 @@ def update_nodeinfos(domain: str): print("WARNING: Did not update any rows:", domain) except BaseException as e: - print(f"ERROR: failed SQL query: domain='{domain}',sql='{sql}',exception:'{e}'") + print(f"ERROR: failed SQL query: domain='{domain}',sql='{sql}',exception:'{str(e)}'") sys.exit(255) # DEBUG: print("DEBUG: Deleting nodeinfos for domain:", domain) @@ -323,7 +323,7 @@ def update_last_error(domain: str, res: any): pending_errors[domain] = res except BaseException as e: - print(f"ERROR: failed SQL query: domain='{domain}',exception:'{e}'") + print(f"ERROR: failed SQL query: domain='{domain}',exception:'{str(e)}'") sys.exit(255) # DEBUG: print("DEBUG: EXIT!") @@ -341,7 +341,7 @@ def update_last_nodeinfo(domain: str): print("WARNING: Did not update any rows:", domain) except BaseException as e: - print(f"ERROR: failed SQL query: domain='{domain}',exception:'{e}'") + print(f"ERROR: failed SQL query: domain='{domain}',exception:'{str(e)}'") sys.exit(255) connection.commit() @@ -380,7 +380,7 @@ def get_peers(domain: str, software: str) -> list: # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain) break elif len(fetched) != config["misskey_offset"]: - print(f"WARNING: Fetched '{len(fetched)}' row(s) but expected: '{config['misskey_offset']}'") + # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config['misskey_offset']}'") offset = offset + (config["misskey_offset"] - len(fetched)) else: # DEBUG: print("DEBUG: Raising offset by step:", step) @@ -406,10 +406,13 @@ def get_peers(domain: str, software: str) -> list: try: res = reqto.get(f"https://{domain}/api/v3/site", headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"])) - # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code='{res.status_code}'") + # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code='{res.status_code}',res.json[]='{type(res.json())}'") if not res.ok or res.status_code >= 400: print("WARNING: Could not reach any JSON API:", domain) update_last_error(domain, res) + elif res.ok and isinstance(res.json(), list): + # DEBUG: print(f"DEBUG: domain='{domain}' returned a list: '{res.json()}'") + sys.exit(255) elif "federated_instances" in res.json(): # DEBUG: print("DEBUG: Found federated_instances", domain) peers = peers + add_peers(res.json()["federated_instances"]) @@ -418,7 +421,7 @@ def get_peers(domain: str, software: str) -> list: update_last_error(domain, res) except BaseException as e: - print(f"WARNING: Exception during fetching JSON: domain='{domain}',exception:'{e}'") + print(f"WARNING: Exception during fetching JSON: domain='{domain}',exception:'{str(e)}'") update_last_nodeinfo(domain) @@ -434,7 +437,7 @@ def get_peers(domain: str, software: str) -> list: try: res = reqto.get(f"https://{domain}/api/v1/server/{mode}?start={start}&count=100", headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"])) - # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code='{res.status_code}'") + # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code='{res.status_code}',res.json[]='{type(res.json())}'") if res.ok and isinstance(res.json(), dict): # DEBUG: print("DEBUG: Success, res.json():", len(res.json())) data = res.json() @@ -457,7 +460,7 @@ def get_peers(domain: str, software: str) -> list: start = start + 100 except BaseException as e: - print(f"WARNING: Exception during fetching JSON: domain='{domain}',exception:'{e}'") + print(f"WARNING: Exception during fetching JSON: domain='{domain}',exception:'{str(e)}'") update_last_nodeinfo(domain) @@ -468,14 +471,18 @@ def get_peers(domain: str, software: str) -> list: try: res = reqto.get(f"https://{domain}{get_peers_url}", headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"])) - # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code}") + # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},res.json[]='{type(res.json())}'") if not res.ok or res.status_code >= 400: # DEBUG: print(f"DEBUG: Was not able to fetch '{get_peers_url}', trying alternative ...") res = reqto.get(f"https://{domain}/api/v3/site", headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"])) + # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},res.json[]='{type(res.json())}'") if not res.ok or res.status_code >= 400: print("WARNING: Could not reach any JSON API:", domain) update_last_error(domain, res) + elif res.ok and isinstance(res.json(), list): + # DEBUG: print(f"DEBUG: domain='{domain}' returned a list: '{res.json()}'") + sys.exit(255) elif "federated_instances" in res.json(): # DEBUG: print("DEBUG: Found federated_instances", domain) peers = peers + add_peers(res.json()["federated_instances"]) @@ -498,20 +505,20 @@ def get_peers(domain: str, software: str) -> list: def post_json_api(domain: str, path: str, parameter: str, extra_headers: dict = {}) -> list: # DEBUG: print("DEBUG: Sending POST to domain,path,parameter:", domain, path, parameter, extra_headers) - data = {} + data = list() try: res = reqto.post(f"https://{domain}{path}", data=parameter, headers={**api_headers, **extra_headers}, timeout=(config["connection_timeout"], config["read_timeout"])) - # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code}") + # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},res.json[]='{type(res.json())}'") if not res.ok or res.status_code >= 400: - print(f"WARNING: Cannot query JSON API: domain='{domain}',path='{path}',parameter()={len(parameter)},res.status_code='{res.status_code}'") + print(f"WARNING: Cannot query JSON API: domain='{domain}',path='{path}',parameter()={len(parameter)},res.status_code='{res.status_code}',res.json[]='{type(res.json())}'") update_last_error(domain, res) else: update_last_nodeinfo(domain) data = res.json() except BaseException as e: - print(f"WARNING: Some error during post(): domain='{domain},path='{path}',parameter()={len(parameter)},exception:'{e}'") + print(f"WARNING: Some error during post(): domain='{domain}',path='{path}',parameter()={len(parameter)},exception:'{str(e)}'") # DEBUG: print("DEBUG: Returning data():", len(data)) return data @@ -541,13 +548,16 @@ def fetch_nodeinfo(domain: str) -> list: # DEBUG: print("DEBUG: Fetching request:", request) res = reqto.get(request, headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"])) - # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code}") + # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},res.json[]='{type(res.json())}'") if res.ok and isinstance(res.json(), dict): # DEBUG: print("DEBUG: Success:", request) data = res.json() nodeinfos["detection_mode"][domain] = "STATIC_CHECK" nodeinfos["nodeinfo_url"][domain] = request break + elif res.ok and isinstance(res.json(), list): + # DEBUG: print(f"DEBUG: domain='{domain}' returned a list: '{res.json()}'") + sys.exit(255) elif not res.ok or res.status_code >= 400: print("WARNING: Failed fetching nodeinfo from domain:", domain) update_last_error(domain, res) @@ -742,7 +752,7 @@ def update_block_reason(reason: str, blocker: str, blocked: str, block_level: st print("WARNING: Did not update any rows:", domain) except BaseException as e: - print(f"ERROR: failed SQL query: reason='{reason}',blocker='{blocker}',blocked='{blocked}',block_level='{block_level}',sql='{sql}',exception:'{e}'") + print(f"ERROR: failed SQL query: reason='{reason}',blocker='{blocker}',blocked='{blocked}',block_level='{block_level}',sql='{sql}',exception:'{str(e)}'") sys.exit(255) # DEBUG: print("DEBUG: EXIT!") @@ -764,7 +774,7 @@ def update_last_seen(blocker: str, blocked: str, block_level: str): print("WARNING: Did not update any rows:", domain) except BaseException as e: - print(f"ERROR: failed SQL query: last_seen='{last_seen}',blocker='{blocker}',blocked='{blocked}',block_level='{block_level}',exception:'{e}'") + print(f"ERROR: failed SQL query: last_seen='{last_seen}',blocker='{blocker}',blocked='{blocked}',block_level='{block_level}',exception:'{str(e)}'") sys.exit(255) # DEBUG: print("DEBUG: EXIT!") @@ -793,7 +803,7 @@ def block_instance(blocker: str, blocked: str, reason: str, block_level: str): ) except BaseException as e: - print(f"ERROR: failed SQL query: blocker='{blocker}',blocked='{blocked}',reason='{reason}',block_level='{block_level}',first_seen='{first_seen}',last_seen='{last_seen}',exception:'{e}'") + print(f"ERROR: failed SQL query: blocker='{blocker}',blocked='{blocked}',reason='{reason}',block_level='{block_level}',first_seen='{first_seen}',last_seen='{last_seen}',exception:'{str(e)}'") sys.exit(255) # DEBUG: print("DEBUG: EXIT!") @@ -811,7 +821,7 @@ def is_instance_registered(domain: str) -> bool: # Check condition registered = cursor.fetchone() != None except BaseException as e: - print(f"ERROR: failed SQL query: last_seen='{last_seen}'blocker='{blocker}',blocked='{blocked}',block_level='{block_level}',first_seen='{first_seen}',last_seen='{last_seen}',exception:'{e}'") + print(f"ERROR: failed SQL query: last_seen='{last_seen}'blocker='{blocker}',blocked='{blocked}',block_level='{block_level}',first_seen='{first_seen}',last_seen='{last_seen}',exception:'{str(e)}'") sys.exit(255) # DEBUG: print(f"DEBUG: registered='{registered}' - EXIT!") @@ -857,7 +867,7 @@ def add_instance(domain: str, origin: str, originator: str): remove_pending_error(domain) except BaseException as e: - print(f"ERROR: failed SQL query: domain='{domain}',exception:'{e}'") + print(f"ERROR: failed SQL query: domain='{domain}',exception:'{str(e)}'") sys.exit(255) else: # DEBUG: print("DEBUG: Updating nodeinfo for domain:", domain) @@ -991,7 +1001,7 @@ def get_misskey_blocks(domain: str) -> dict: # instances page-by-page, since that troonware doesn't support # sending them all at once try: - print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...") + # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...") if offset == 0: # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset) fetched = post_json_api(domain, "/api/federation/instances", json.dumps({ @@ -1010,12 +1020,12 @@ def get_misskey_blocks(domain: str) -> dict: "offset" : offset - 1 }), {"Origin": domain}) - print("DEBUG: fetched():", len(fetched)) + # DEBUG: print("DEBUG: fetched():", len(fetched)) if len(fetched) == 0: # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain) break elif len(fetched) != config["misskey_offset"]: - print(f"WARNING: Fetched '{len(fetched)}' row(s) but expected: '{config['misskey_offset']}'") + # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config['misskey_offset']}'") offset = offset + (config["misskey_offset"] - len(fetched)) else: # DEBUG: print("DEBUG: Raising offset by step:", step) @@ -1059,12 +1069,12 @@ def get_misskey_blocks(domain: str) -> dict: "offset" : offset-1 }), {"Origin": domain}) - print("DEBUG: fetched():", len(fetched)) + # DEBUG: print("DEBUG: fetched():", len(fetched)) if len(fetched) == 0: # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain) break elif len(fetched) != config["misskey_offset"]: - print(f"WARNING: Fetched '{len(fetched)}' row(s) but expected: '{config['misskey_offset']}'") + # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config['misskey_offset']}'") offset = offset + (config["misskey_offset"] - len(fetched)) else: # DEBUG: print("DEBUG: Raising offset by step:", step) diff --git a/fetch_blocks.py b/fetch_blocks.py index 5cda828..71cf74f 100644 --- a/fetch_blocks.py +++ b/fetch_blocks.py @@ -154,7 +154,7 @@ for blocker, software in rows: fba.connection.commit() except Exception as e: - print(f"ERROR: blocker='{blocker}',software='{software}',exception='{e}'") + print(f"ERROR: blocker='{blocker}',software='{software}',exception='{str(e)}'") elif software == "mastodon": print("INFO: blocker:", blocker) try: @@ -272,7 +272,7 @@ for blocker, software in rows: fba.connection.commit() except Exception as e: - print(f"ERROR: blocker='{blocker}',software='{software}',exception='{e}'") + print(f"ERROR: blocker='{blocker}',software='{software}',exception='{str(e)}'") elif software == "friendica" or software == "misskey" or software == "bookwyrm" or software == "takahe": print("INFO: blocker:", blocker) try: @@ -350,7 +350,7 @@ for blocker, software in rows: fba.connection.commit() except Exception as e: - print(f"ERROR: blocker='{blocker}',software='{software}',exception='{e}'") + print(f"ERROR: blocker='{blocker}',software='{software}',exception='{str(e)}'") elif software == "gotosocial": print("INFO: blocker:", blocker) try: @@ -418,7 +418,7 @@ for blocker, software in rows: fba.connection.commit() except Exception as e: - print(f"ERROR: blocker='{blocker}',software='{software}',exception='{e}'") + print(f"ERROR: blocker='{blocker}',software='{software}',exception='{str(e)}'") else: print("WARNING: Unknown software:", blocker, software) diff --git a/fetch_instances.py b/fetch_instances.py index fbd113a..ceadd47 100644 --- a/fetch_instances.py +++ b/fetch_instances.py @@ -43,7 +43,7 @@ def fetch_instances(domain: str, origin: str, software: str): # NOISY-DEBUG: print("DEBUG: Adding new instance:", instance, domain) fba.add_instance(instance, domain, sys.argv[0]) except BaseException as e: - print(f"ERROR: instance='{instance}',exception:'{e}'") + print(f"ERROR: instance='{instance}',exception:'{str(e)}'") continue instance = sys.argv[1] -- 2.39.5