From: Roland Häder Date: Sat, 10 Jun 2023 14:29:43 +0000 (+0200) Subject: WIP: X-Git-Url: https://git.mxchange.org/?a=commitdiff_plain;h=aded0c0e72448f59bfe3e1f49b6f5c003fe874b7;p=fba.git WIP: - introduced network.fetch_api_url() which takes a whole URL (not domain/path separated) and returns it's reply in a json_reply dictionary as both other API functions are handling it --- diff --git a/fba/fba.py b/fba/fba.py index ab413e6..27c678b 100644 --- a/fba/fba.py +++ b/fba/fba.py @@ -177,8 +177,8 @@ def get_hash(domain: str) -> str: return hashlib.sha256(domain.encode("utf-8")).hexdigest() -def log_error(domain: str, response: requests.models.Response): - # DEBUG: print("DEBUG: domain,response[]:", domain, type(response)) +def log_error(domain: str, error: dict): + # DEBUG: print("DEBUG: domain,error[]:", domain, type(error)) if not isinstance(domain, str): raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": @@ -187,22 +187,22 @@ def log_error(domain: str, response: requests.models.Response): # DEBUG: print(f"DEBUG: Writing to error_log is disabled in configuruation file - EXIT!") return - # DEBUG: print("DEBUG: BEFORE response[]:", type(response)) - if isinstance(response, BaseException) or isinstance(response, json.decoder.JSONDecodeError): - response = f"response[{type(response)}]='{str(response)}'" + # DEBUG: print("DEBUG: BEFORE error[]:", type(error)) + if isinstance(error, BaseException) or isinstance(error, json.decoder.JSONDecodeError): + error = f"error[{type(error)}]='{str(error)}'" - # DEBUG: print("DEBUG: AFTER response[]:", type(response)) - if isinstance(response, str): + # DEBUG: print("DEBUG: AFTER error[]:", type(error)) + if isinstance(error, str): cursor.execute("INSERT INTO error_log (domain, error_code, error_message, created) VALUES (?, 999, ?, ?)",[ domain, - response, + error, time.time() ]) else: cursor.execute("INSERT INTO error_log (domain, error_code, error_message, created) VALUES (?, ?, ?, ?)",[ domain, - response.status_code, - response.reason, + error["status_code"], + error["error_message"], time.time() ]) diff --git a/fba/federation.py b/fba/federation.py index 4db6e5c..8d5515e 100644 --- a/fba/federation.py +++ b/fba/federation.py @@ -233,15 +233,13 @@ def fetch_wellknown_nodeinfo(domain: str) -> list: # DEBUG: print("DEBUG: rel,href:", link["rel"], link["href"]) if link["rel"] in nodeinfo_identifier: # DEBUG: print("DEBUG: Fetching nodeinfo from:", link["href"]) - response = fba.fetch_url( + data = network.fetch_api_url( link["href"], - network.api_headers, (config.get("connection_timeout"), config.get("read_timeout")) ) - data = network.json_from_response(response) - # DEBUG: print("DEBUG: href,response.ok,response.status_code:", link["href"], response.ok, response.status_code) - if response.ok and len(data) > 0: + # DEBUG: print("DEBUG: href,data[]:", link["href"], type(data)) + if "json" in data: # DEBUG: print("DEBUG: Found JSON nodeinfo():", len(data)) instances.set_data("detection_mode", domain, "AUTO_DISCOVERY") instances.set_data("nodeinfo_url" , domain, link["href"]) diff --git a/fba/instances.py b/fba/instances.py index c872c2b..f41d049 100644 --- a/fba/instances.py +++ b/fba/instances.py @@ -257,32 +257,32 @@ def update_last_nodeinfo(domain: str): # DEBUG: print("DEBUG: EXIT!") -def update_last_error(domain: str, response: requests.models.Response): - print("DEBUG: domain,response[]:", domain, type(response)) +def update_last_error(domain: str, error: dict): + print("DEBUG: domain,error[]:", domain, type(error)) if not isinstance(domain, str): raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") - print("DEBUG: BEFORE response[]:", type(response)) - if isinstance(response, BaseException) or isinstance(response, json.decoder.JSONDecodeError): - response = f"response[{type(response)}]='{str(response)}'" + print("DEBUG: BEFORE error[]:", type(error)) + if isinstance(error, BaseException) or isinstance(error, json.decoder.JSONDecodeError): + error = f"error[{type(error)}]='{str(error)}'" - print("DEBUG: AFTER response[]:", type(response)) - if isinstance(response, str): - print(f"DEBUG: Setting last_error_details='{response}'") + print("DEBUG: AFTER error[]:", type(error)) + if isinstance(error, str): + print(f"DEBUG: Setting last_error_details='{error}'") set_data("last_status_code" , domain, 999) - set_data("last_error_details", domain, response) + set_data("last_error_details", domain, error) else: - print(f"DEBUG: Setting last_error_details='{response.reason}'") - set_data("last_status_code" , domain, response.status_code) - set_data("last_error_details", domain, response.reason) + print(f"DEBUG: Setting last_error_details='{error['error_message']}'") + set_data("last_status_code" , domain, error["status_code"]) + set_data("last_error_details", domain, error["reason"]) # Running pending updated print(f"DEBUG: Invoking update_data({domain}) ...") update_data(domain) - fba.log_error(domain, response) + fba.log_error(domain, error) print("DEBUG: EXIT!") diff --git a/fba/network.py b/fba/network.py index 9a30d58..e0520a1 100644 --- a/fba/network.py +++ b/fba/network.py @@ -20,6 +20,7 @@ import requests from fba import config from fba import csrf +from fba import fba from fba import instances # HTTP headers for non-API requests @@ -83,6 +84,38 @@ def post_json_api(domain: str, path: str, data: str, headers: dict = {}) -> dict # DEBUG: print(f"DEBUG: Returning json_reply({len(json_reply)})=[]:{type(json_reply)}") return json_reply +def fetch_api_url(url: str, timeout: tuple) -> dict: + # DEBUG: print(f"DEBUG: url='{url}',timeout()={len(timeout)} - CALLED!") + if not isinstance(url, str): + raise ValueError(f"Parameter url[]='{type(url)}' is not 'str'") + elif not isinstance(timeout, tuple): + raise ValueError(f"timeout[]={type(timeout)} is not 'tuple'") + + json_reply = { + "status_code": 200, + } + + try: + # DEBUG: print(f"DEBUG: Fetching url='{url}' ...") + response = fba.fetch_url(url, api_headers, timeout) + + json_reply["json"] = json_from_response(response) + + # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},json_reply[]='{type(json_reply)}'") + if not response.ok or response.status_code >= 400: + print(f"WARNING: Cannot query JSON API: url='{url}',response.status_code='{response.status_code}',json_reply[]='{type(json_reply)}'") + json_reply["status_code"] = response.status_code + json_reply["error_message"] = response.text + + except requests.exceptions.ConnectionError as exception: + # DEBUG: print(f"DEBUG: Fetching '{url}' failed. exception[{type(exception)}]='{str(exception)}'") + json_reply["status_code"] = 999 + json_reply["error_message"] = f"exception['{type(exception)}']='{str(exception)}'" + raise exception + + # DEBUG: print(f"DEBUG: Returning json_reply({len(json_reply)})=[]:{type(json_reply)}") + return json_reply + def get_json_api(domain: str, path: str, timeout: tuple) -> dict: # DEBUG: print(f"DEBUG: domain='{domain}',path='{path}',data='{data}',timeout()={len(timeout)} - CALLED!") if not isinstance(domain, str):