From: Roland Häder Date: Mon, 29 May 2023 00:48:40 +0000 (+0200) Subject: Continued: X-Git-Url: https://git.mxchange.org/?a=commitdiff_plain;h=bdf44782e24a91a45cdb752f87e39dd40209fbd0;p=fba.git Continued: - add_peers() now returns a list of peers, dict doesn't make sense here - also handled error messages from JSON API POST requests --- diff --git a/fba.py b/fba.py index 9174bba..c294a2d 100644 --- a/fba.py +++ b/fba.py @@ -126,16 +126,26 @@ patterns = [ re.compile("^[a-f0-9]{7}$"), ] -def add_peers(rows: dict) -> dict: +def add_peers(rows: dict) -> list: # DEBUG: print(f"DEBUG: rows()={len(rows)} - CALLED!") - peers = {} + peers = list() for element in ["linked", "allowed", "blocked"]: # DEBUG: print(f"DEBUG: Checking element='{element}'") if element in rows and rows[element] != None: # DEBUG: print(f"DEBUG: Adding {len(rows[element])} peer(s) to peers list ...") - peers = {**peers, **rows[element]} + for peer in rows[element]: + # DEBUG: print(f"DEBUG: peer='{peer}' - BEFORE!") + peer = tidyup(peer) + + # DEBUG: print(f"DEBUG: peer='{peer}' - AFTER!") + if is_blacklisted(peer): + # DEBUG: print(f"DEBUG: peer='{peer}' is blacklisted, skipped!") + continue + + # DEBUG: print(f"DEBUG: Adding peer='{peer}' ...") + peers.append(peer) - # DEBUG: print(f"DEBUG: peers()={len(peers)} - CALLED!") + # DEBUG: print(f"DEBUG: peers()={len(peers)} - EXIT!") return peers def remove_version(software: str) -> str: @@ -422,12 +432,17 @@ def get_peers(domain: str, software: str) -> list: offset = offset + step # Check records + # DEBUG: print(f"DEBUG: fetched({len(fetched)})[]={type(fetched)}") + if isinstance(fetched, dict) and "error" in fetched and "message" in fetched["error"]: + print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}") + update_last_error(domain, fetched["error"]["message"]) + break for row in fetched: # DEBUG: print(f"DEBUG: row():{len(row)}") if not "host" in row: print(f"WARNING: row()={len(row)} does not contain element 'host': {row},domain='{domain}'") continue - elif "host" in row and is_blacklisted(row["host"]): + elif is_blacklisted(row["host"]): # DEBUG: print(f"DEBUG: row[host]='{row['host']}' is blacklisted. domain='{domain}'") continue @@ -443,16 +458,18 @@ def get_peers(domain: str, software: str) -> list: try: res = reqto.get(f"https://{domain}/api/v3/site", headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"])) - # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code='{res.status_code}',res.json[]='{type(res.json())}'") + data = res.json() + # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code='{res.status_code}',data[]='{type(data)}'") if not res.ok or res.status_code >= 400: print("WARNING: Could not reach any JSON API:", domain) update_last_error(domain, res) - elif res.ok and isinstance(res.json(), list): - # DEBUG: print(f"DEBUG: domain='{domain}' returned a list: '{res.json()}'") + elif res.ok and isinstance(data, list): + # DEBUG: print(f"DEBUG: domain='{domain}' returned a list: '{data}'") sys.exit(255) - elif "federated_instances" in res.json(): - # DEBUG: print("DEBUG: Found federated_instances", domain) - peers = peers + add_peers(res.json()["federated_instances"]) + elif "federated_instances" in data: + # DEBUG: print(f"DEBUG: Found federated_instances for domain='{domain}'") + peers = peers + add_peers(data["federated_instances"]) + # DEBUG: print("DEBUG: Added instance(s) to peers") else: print("WARNING: JSON response does not contain 'federated_instances':", domain) update_last_error(domain, res) @@ -474,11 +491,10 @@ def get_peers(domain: str, software: str) -> list: try: res = reqto.get(f"https://{domain}/api/v1/server/{mode}?start={start}&count=100", headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"])) - # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code='{res.status_code}',res.json[]='{type(res.json())}'") - if res.ok and isinstance(res.json(), dict): - # DEBUG: print("DEBUG: Success, res.json():", len(res.json())) - data = res.json() - + data = res.json() + # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code='{res.status_code}',data[]='{type(data)}'") + if res.ok and isinstance(data, dict): + # DEBUG: print("DEBUG: Success, data:", len(data)) if "data" in data: # DEBUG: print(f"DEBUG: Found {len(data['data'])} record(s).") for record in data["data"]: @@ -508,27 +524,30 @@ def get_peers(domain: str, software: str) -> list: try: res = reqto.get(f"https://{domain}{get_peers_url}", headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"])) - # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},res.json[]='{type(res.json())}'") + data = res.json() + # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'") if not res.ok or res.status_code >= 400: # DEBUG: print(f"DEBUG: Was not able to fetch '{get_peers_url}', trying alternative ...") res = reqto.get(f"https://{domain}/api/v3/site", headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"])) - # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},res.json[]='{type(res.json())}'") + data = res.json() + # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'") if not res.ok or res.status_code >= 400: print("WARNING: Could not reach any JSON API:", domain) update_last_error(domain, res) - elif res.ok and isinstance(res.json(), list): - # DEBUG: print(f"DEBUG: domain='{domain}' returned a list: '{res.json()}'") + elif res.ok and isinstance(data, list): + print(f"DEBUG: domain='{domain}' returned a list: '{data}'") sys.exit(255) - elif "federated_instances" in res.json(): - # DEBUG: print("DEBUG: Found federated_instances", domain) - peers = peers + add_peers(res.json()["federated_instances"]) + elif "federated_instances" in data: + # DEBUG: print(f"DEBUG: Found federated_instances for domain='{domain}'") + peers = peers + add_peers(data["federated_instances"]) + # DEBUG: print("DEBUG: Added instance(s) to peers") else: print("WARNING: JSON response does not contain 'federated_instances':", domain) update_last_error(domain, res) else: - # DEBUG: print("DEBUG: Querying API was successful:", domain, len(res.json())) - peers = res.json() + # DEBUG: print("DEBUG: Querying API was successful:", domain, len(data)) + peers = data nodeinfos["get_peers_url"][domain] = get_peers_url except BaseException as e: @@ -540,24 +559,24 @@ def get_peers(domain: str, software: str) -> list: # DEBUG: print("DEBUG: Returning peers[]:", type(peers)) return peers -def post_json_api(domain: str, path: str, parameter: str, extra_headers: dict = {}) -> list: +def post_json_api(domain: str, path: str, parameter: str, extra_headers: dict = {}) -> dict: # DEBUG: print("DEBUG: Sending POST to domain,path,parameter:", domain, path, parameter, extra_headers) - data = list() + data = {} try: res = reqto.post(f"https://{domain}{path}", data=parameter, headers={**api_headers, **extra_headers}, timeout=(config["connection_timeout"], config["read_timeout"])) - # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},res.json[]='{type(res.json())}'") + data = res.json() + # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'") if not res.ok or res.status_code >= 400: - print(f"WARNING: Cannot query JSON API: domain='{domain}',path='{path}',parameter()={len(parameter)},res.status_code='{res.status_code}',res.json[]='{type(res.json())}'") + print(f"WARNING: Cannot query JSON API: domain='{domain}',path='{path}',parameter()={len(parameter)},res.status_code='{res.status_code}',data[]='{type(data)}'") update_last_error(domain, res) else: update_last_nodeinfo(domain) - data = res.json() except BaseException as e: print(f"WARNING: Some error during post(): domain='{domain}',path='{path}',parameter()={len(parameter)},exception[{type(e)}]:'{str(e)}'") - # DEBUG: print("DEBUG: Returning data():", len(data)) + # DEBUG: print(f"DEBUG: Returning data({len(data)})=[]:{type(data)}") return data def fetch_nodeinfo(domain: str, path: str = None) -> list: @@ -589,15 +608,15 @@ def fetch_nodeinfo(domain: str, path: str = None) -> list: # DEBUG: print("DEBUG: Fetching request:", request) res = reqto.get(request, headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"])) - # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},res.json[]='{type(res.json())}'") - if res.ok and isinstance(res.json(), dict): + data = res.json() + # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'") + if res.ok and isinstance(data, dict): # DEBUG: print("DEBUG: Success:", request) - data = res.json() nodeinfos["detection_mode"][domain] = "STATIC_CHECK" nodeinfos["nodeinfo_url"][domain] = request break - elif res.ok and isinstance(res.json(), list): - # DEBUG: print(f"DEBUG: domain='{domain}' returned a list: '{res.json()}'") + elif res.ok and isinstance(data, list): + # DEBUG: print(f"DEBUG: domain='{domain}' returned a list: '{data}'") sys.exit(255) elif not res.ok or res.status_code >= 400: print("WARNING: Failed fetching nodeinfo from domain:", domain) @@ -618,9 +637,11 @@ def fetch_wellknown_nodeinfo(domain: str) -> list: try: res = reqto.get(f"https://{domain}/.well-known/nodeinfo", headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"])) - # DEBUG: print("DEBUG: domain,res.ok,res.json[]:", domain, res.ok, type(res.json())) - if res.ok and isinstance(res.json(), dict): - nodeinfo = res.json() + + data = res.json() + # DEBUG: print("DEBUG: domain,res.ok,data[]:", domain, res.ok, type(data)) + if res.ok and isinstance(data, dict): + nodeinfo = data # DEBUG: print("DEBUG: Found entries:", len(nodeinfo), domain) if "links" in nodeinfo: # DEBUG: print("DEBUG: Found links in nodeinfo():", len(nodeinfo["links"])) @@ -630,10 +651,10 @@ def fetch_wellknown_nodeinfo(domain: str) -> list: # DEBUG: print("DEBUG: Fetching nodeinfo from:", link["href"]) res = reqto.get(link["href"]) + data = res.json() # DEBUG: print("DEBUG: href,res.ok,res.status_code:", link["href"], res.ok, res.status_code) - if res.ok and isinstance(res.json(), dict): - # DEBUG: print("DEBUG: Found JSON nodeinfo():", len(res.json())) - data = res.json() + if res.ok and isinstance(data, dict): + # DEBUG: print("DEBUG: Found JSON nodeinfo():", len(data)) nodeinfos["detection_mode"][domain] = "AUTO_DISCOVERY" nodeinfos["nodeinfo_url"][domain] = link["href"] break