From: Roland Häder Date: Sun, 28 May 2023 11:12:14 +0000 (+0200) Subject: Continued: X-Git-Url: https://git.mxchange.org/?a=commitdiff_plain;h=4adb426894b599ae739c67149b269614705712dd;p=fba.git Continued: - res.json() can somtimes not being checked (?) - also need to check each element as it might return 'None' causing error --- diff --git a/fba.py b/fba.py index 4a06902..e408952 100644 --- a/fba.py +++ b/fba.py @@ -335,7 +335,7 @@ def get_peers(domain: str, software: str) -> list: peers = list() if software == "misskey": - print(f"DEBUG: domain='{domain}' is misskey, sending API POST request ...") + # DEBUG: print(f"DEBUG: domain='{domain}' is misskey, sending API POST request ...") counter = 0 step = config["misskey_offset"] @@ -373,7 +373,7 @@ def get_peers(domain: str, software: str) -> list: try: res = reqto.get(f"https://{domain}/api/v3/site", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])) - # DEBUG: print(f"DEBUG: res.ok={res.ok},res.json[]={type(res.json())}") + # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code='{res.status_code}'") if res.ok and isinstance(res.json(), dict): # DEBUG: print("DEBUG: Success, res.json():", len(res.json())) data = res.json() @@ -399,7 +399,7 @@ def get_peers(domain: str, software: str) -> list: try: res = reqto.get(f"https://{domain}/api/v1/server/{mode}?start={start}&count=100", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])) - # DEBUG: print(f"DEBUG: res.ok={res.ok},res.json[]={type(res.json())}") + # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code='{res.status_code}'") if res.ok and isinstance(res.json(), dict): # DEBUG: print("DEBUG: Success, res.json():", len(res.json())) data = res.json() @@ -429,12 +429,13 @@ def get_peers(domain: str, software: str) -> list: # DEBUG: print("DEBUG: Returning peers[]:", type(peers)) return peers - # DEBUG: print(f"DEBUG: Fetching '{get_peers_url}' from '{domain}' ...") + # DEBUG: print(f"DEBUG: Fetching get_peers_url='{get_peers_url}' from '{domain}' ...") try: res = reqto.get(f"https://{domain}{get_peers_url}", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])) - # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},res.json[]={type(res.json())}") + # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code}") if not res.ok or res.status_code >= 400: + # DEBUG: print(f"DEBUG: Was not able to fetch '{get_peers_url}', trying alternative ...") res = reqto.get(f"https://{domain}/api/v3/site", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])) if not res.ok or res.status_code >= 400: @@ -442,12 +443,18 @@ def get_peers(domain: str, software: str) -> list: update_last_error(domain, res) elif "federated_instances" in res.json() and "linked" in res.json()["federated_instances"]: # DEBUG: print("DEBUG: Found federated_instances", domain) - peers = res.json()["federated_instances"]["linked"] + res.json()["federated_instances"]["allowed"] + res.json()["federated_instances"]["blocked"] + data = res.json() + + for element in ["linked", "allowed", "blocked"]: + # DEBUG: print(f"DEBUG: Checking element='{element}'") + if element in data["federated_instances"] and data["federated_instances"][element] != None: + print(f"DEBUG Adding {len(data['federated_instances'][element])} peer(s) to peers list ...") + peers = peers + data["federated_instances"][element] else: print("WARNING: JSON response does not contain 'federated_instances':", domain) update_last_error(domain, res) else: - # DEBUG: print("DEBUG:Querying API was successful:", domain, len(res.json())) + # DEBUG: print("DEBUG: Querying API was successful:", domain, len(res.json())) peers = res.json() nodeinfos["get_peers_url"][domain] = get_peers_url @@ -461,14 +468,14 @@ def get_peers(domain: str, software: str) -> list: return peers def post_json_api(domain: str, path: str, parameter: str) -> list: - print("DEBUG: Sending POST to domain,path,parameter:", domain, path, parameter) + # DEBUG: print("DEBUG: Sending POST to domain,path,parameter:", domain, path, parameter) data = {} try: res = reqto.post(f"https://{domain}{path}", data=parameter, headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])) - print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},res.json[]={type(res.json())}") + # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code}") if not res.ok or res.status_code >= 400: - print("WARNING: Cannot query JSON API:", domain, path, parameter, res.status_code) + print(f"WARNING: Cannot query JSON API: domain='{domain}',path='{path}',parameter()={len(parameter)},res.status_code='{res.status_code}'") update_last_error(domain, res) else: update_last_nodeinfo(domain) @@ -477,7 +484,7 @@ def post_json_api(domain: str, path: str, parameter: str) -> list: except BaseException as e: print("WARNING: Some error during post():", domain, path, parameter, e) - print("DEBUG: Returning data():", len(data)) + # DEBUG: print("DEBUG: Returning data():", len(data)) return data def fetch_nodeinfo(domain: str) -> list: @@ -505,7 +512,7 @@ def fetch_nodeinfo(domain: str) -> list: # DEBUG: print("DEBUG: Fetching request:", request) res = reqto.get(request, headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])) - # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},res.json[]={type(res.json())}") + # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code}") if res.ok and isinstance(res.json(), dict): # DEBUG: print("DEBUG: Success:", request) data = res.json() @@ -568,7 +575,7 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str: software = None try: - # DEBUG: print(f"DEBUG: Fetching '{path}' from '{domain}' ...") + # DEBUG: print(f"DEBUG: Fetching path='{path}' from '{domain}' ...") res = reqto.get(f"https://{domain}{path}", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])) # DEBUG: print("DEBUG: domain,res.ok,res.status_code,res.text[]:", domain, res.ok, res.status_code, type(res.text)) @@ -778,7 +785,7 @@ def is_instance_registered(domain: str) -> bool: print(f"ERROR: failed SQL query: last_seen='{last_seen}'blocker='{blocker}',blocked='{blocked}',block_level='{block_level}',first_added='{first_added}',last_seen='{last_seen}',exception:'{e}'") sys.exit(255) - # DEBUG: print("DEBUG: registered='{registered}' - EXIT!") + # DEBUG: print(f"DEBUG: registered='{registered}' - EXIT!") return registered def add_instance(domain: str, origin: str, originator: str): @@ -808,9 +815,9 @@ def add_instance(domain: str, origin: str, originator: str): ) for key in nodeinfos: - # DEBUG: pprint(f"DEBUG: key='{key}',domain='{domain}',nodeinfos[key]={nodeinfos[key]}") + p# DEBUG: print(f"DEBUG: key='{key}',domain='{domain}',nodeinfos[key]={nodeinfos[key]}") if domain in nodeinfos[key]: - # DEBUG: pprint(f"DEBUG: domain='{domain}' has pending nodeinfo being updated ...") + p# DEBUG: print(f"DEBUG: domain='{domain}' has pending nodeinfo being updated ...") update_nodeinfos(domain) remove_pending_error(domain) break @@ -984,7 +991,7 @@ def get_misskey_blocks(domain: str) -> dict: { "domain": tidyup(instance["host"]), # no reason field, nothing - "reason": "" + "reason": None } ) @@ -1031,7 +1038,7 @@ def get_misskey_blocks(domain: str) -> dict: if instance["isBlocked"]: blocks["blocked"].append({ "domain": tidyup(instance["host"]), - "reason": "" + "reason": None }) if len(doc) < step: