From ecb4223bbdb74aba8a638ff16d06fdacd628e35f Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Fri, 9 Jun 2023 23:29:18 +0200 Subject: [PATCH] Continued: - removed remains of code around pending_errors dictionary - rewrote it towards instances.update_last_error() - also rewrote/removed some try/except blocks --- fba/fba.py | 67 ++++++++++++++++------------------------------- fba/federation.py | 14 ++++------ fba/instances.py | 14 +++------- fba/network.py | 18 +++++++------ 4 files changed, 41 insertions(+), 72 deletions(-) diff --git a/fba/fba.py b/fba/fba.py index 1dada8c..ab413e6 100644 --- a/fba/fba.py +++ b/fba/fba.py @@ -27,10 +27,6 @@ import requests from fba import config from fba import network -# Array with pending errors needed to be written to database -pending_errors = { -} - # Connect to database connection = sqlite3.connect("blocks.db") cursor = connection.cursor() @@ -173,21 +169,6 @@ def strip_until(software: str, until: str) -> str: # DEBUG: print(f"DEBUG: software='{software}' - EXIT!") return software -def remove_pending_error(domain: str): - if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError("Parameter 'domain' is empty") - - try: - # Prevent updating any pending errors, nodeinfo was found - del pending_errors[domain] - - except: - pass - - # DEBUG: print("DEBUG: EXIT!") - def get_hash(domain: str) -> str: if not isinstance(domain, str): raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") @@ -206,32 +187,28 @@ def log_error(domain: str, response: requests.models.Response): # DEBUG: print(f"DEBUG: Writing to error_log is disabled in configuruation file - EXIT!") return - try: - # DEBUG: print("DEBUG: BEFORE response[]:", type(response)) - if isinstance(response, BaseException) or isinstance(response, json.decoder.JSONDecodeError): - response = f"response[{type(response)}]='{str(response)}'" - - # DEBUG: print("DEBUG: AFTER response[]:", type(response)) - if isinstance(response, str): - cursor.execute("INSERT INTO error_log (domain, error_code, error_message, created) VALUES (?, 999, ?, ?)",[ - domain, - response, - time.time() - ]) - else: - cursor.execute("INSERT INTO error_log (domain, error_code, error_message, created) VALUES (?, ?, ?, ?)",[ - domain, - response.status_code, - response.reason, - time.time() - ]) - - # Cleanup old entries - # DEBUG: print(f"DEBUG: Purging old records (distance: {config.get('error_log_cleanup')})") - cursor.execute("DELETE FROM error_log WHERE created < ?", [time.time() - config.get("error_log_cleanup")]) - except BaseException as exception: - print(f"ERROR: failed SQL query: domain='{domain}',exception[{type(exception)}]:'{str(exception)}'") - sys.exit(255) + # DEBUG: print("DEBUG: BEFORE response[]:", type(response)) + if isinstance(response, BaseException) or isinstance(response, json.decoder.JSONDecodeError): + response = f"response[{type(response)}]='{str(response)}'" + + # DEBUG: print("DEBUG: AFTER response[]:", type(response)) + if isinstance(response, str): + cursor.execute("INSERT INTO error_log (domain, error_code, error_message, created) VALUES (?, 999, ?, ?)",[ + domain, + response, + time.time() + ]) + else: + cursor.execute("INSERT INTO error_log (domain, error_code, error_message, created) VALUES (?, ?, ?, ?)",[ + domain, + response.status_code, + response.reason, + time.time() + ]) + + # Cleanup old entries + # DEBUG: print(f"DEBUG: Purging old records (distance: {config.get('error_log_cleanup')})") + cursor.execute("DELETE FROM error_log WHERE created < ?", [time.time() - config.get("error_log_cleanup")]) # DEBUG: print("DEBUG: EXIT!") diff --git a/fba/federation.py b/fba/federation.py index acc58e2..fac06c0 100644 --- a/fba/federation.py +++ b/fba/federation.py @@ -78,7 +78,7 @@ def fetch_instances(domain: str, origin: str, software: str, script: str, path: print(f"INFO: Checking {len(peerlist)} instances from {domain} ...") for instance in peerlist: if instance is None: - # Skip "None" types as tidup() cannot parse them + # Skip "None" types as tidup.domain() cannot parse them continue # DEBUG: print(f"DEBUG: instance='{instance}' - BEFORE") @@ -96,13 +96,9 @@ def fetch_instances(domain: str, origin: str, software: str, script: str, path: continue # DEBUG: print("DEBUG: Handling instance:", instance) - try: - if not instances.is_registered(instance): - # DEBUG: print("DEBUG: Adding new instance:", instance, domain) - instances.add(instance, domain, script) - except BaseException as exception: - print(f"ERROR: instance='{instance}',exception[{type(exception)}]:'{str(exception)}'") - continue + if not instances.is_registered(instance): + # DEBUG: print("DEBUG: Adding new instance:", instance, domain) + instances.add(instance, domain, script) # DEBUG: print("DEBUG: EXIT!") @@ -278,7 +274,7 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str: doc = bs4.BeautifulSoup(response.text, "html.parser") # DEBUG: print("DEBUG: doc[]:", type(doc)) - generator = doc.find("meta", {"name": "generator"}) + generator = doc.find("meta", {"name" : "generator"}) site_name = doc.find("meta", {"property": "og:site_name"}) # DEBUG: print(f"DEBUG: generator='{generator}',site_name='{site_name}'") diff --git a/fba/instances.py b/fba/instances.py index 6ac8bcb..a345e13 100644 --- a/fba/instances.py +++ b/fba/instances.py @@ -227,16 +227,10 @@ def add(domain: str, origin: str, command: str, path: str = None): set_data("last_status_code" , domain, None) set_data("last_error_details", domain, None) update_data(domain) - fba.remove_pending_error(domain) - - if domain in fba.pending_errors: - # DEBUG: print("DEBUG: domain has pending error being updated:", domain) - update_last_error(domain, fba.pending_errors[domain]) - fba.remove_pending_error(domain) except BaseException as exception: - print(f"ERROR: failed SQL query: domain='{domain}',exception[{type(exception)}]:'{str(exception)}'") - sys.exit(255) + update_last_error(domain, exception) + raise Exception(f"ERROR: failed SQL query: domain='{domain}',exception[{type(exception)}]:'{str(exception)}'") from exception else: # DEBUG: print("DEBUG: Updating nodeinfo for domain:", domain) update_last_nodeinfo(domain) @@ -305,8 +299,8 @@ def is_registered(domain: str) -> bool: # Check Set all cache.set_all("is_registered", fba.cursor.fetchall(), True) except BaseException as exception: - print(f"ERROR: failed SQL query: domain='{domain}',exception[{type(exception)}]:'{str(exception)}'") - sys.exit(255) + update_last_error(domain, exception) + raise Exception(f"ERROR: failed SQL query: domain='{domain}',exception[{type(exception)}]:'{str(exception)}'") from exception # Is cache found? registered = cache.sub_key_exists("is_registered", domain) diff --git a/fba/network.py b/fba/network.py index 261d266..751c7d1 100644 --- a/fba/network.py +++ b/fba/network.py @@ -60,14 +60,16 @@ def post_json_api(domain: str, path: str, parameter: str, extra_headers: dict = timeout=(config.get("connection_timeout"), config.get("read_timeout")) ) - data = json_from_response(response) - # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'") - if not response.ok or response.status_code >= 400: - print(f"WARNING: Cannot query JSON API: domain='{domain}',path='{path}',parameter()={len(parameter)},response.status_code='{response.status_code}',data[]='{type(data)}'") - instances.update_last_error(domain, response) - - except BaseException as exception: - print(f"WARNING: Some error during post(): domain='{domain}',path='{path}',parameter()={len(parameter)},exception[{type(exception)}]:'{str(exception)}'") + except requests.exceptions.ConnectionError as exception: + # DEBUG: print(f"DEBUG: Fetching '{path}' from '{domain}' failed. exception[{type(exception)}]='{str(exception)}'") + instances.update_last_error(domain, exception) + raise exception + + data = json_from_response(response) + # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'") + if not response.ok or response.status_code >= 400: + print(f"WARNING: Cannot query JSON API: domain='{domain}',path='{path}',parameter()={len(parameter)},response.status_code='{response.status_code}',data[]='{type(data)}'") + instances.update_last_error(domain, response) # DEBUG: print(f"DEBUG: Returning data({len(data)})=[]:{type(data)}") return data -- 2.39.5