From: Roland Häder Date: Sun, 4 Jun 2023 11:06:23 +0000 (+0200) Subject: Continued: X-Git-Url: https://git.mxchange.org/?a=commitdiff_plain;h=f001be92b5776b5f250db7cd03588e8b6ca0fbf9;p=fba.git Continued: - renamed functions in 'cache' module as they are already in proper module - tried to prevent error if an empty (0 Byte) result is returned --- diff --git a/fba/cache.py b/fba/cache.py index 843e353..25adde2 100644 --- a/fba/cache.py +++ b/fba/cache.py @@ -14,20 +14,19 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . - # Cache for redundant SQL queries _cache = {} ##### Cache ##### -def is_cache_initialized(key: str) -> bool: +def key_exists(key: str) -> bool: return key in _cache -def set_all_cache_key(key: str, rows: list, value: any): +def set_all(key: str, rows: list, value: any): # NOISY-DEBUG: print(f"DEBUG: key='{key}',rows()={len(rows)},value[]={type(value)} - CALLED!") if type(key) != str: raise ValueError("Parameter key[]='{type(key)}' is not 'str'") - elif not is_cache_initialized(key): + elif not key_exists(key): # NOISY-DEBUG: print(f"DEBUG: Cache for key='{key}' not initialized.") _cache[key] = {} @@ -41,23 +40,23 @@ def set_all_cache_key(key: str, rows: list, value: any): # NOISY-DEBUG: print("DEBUG: EXIT!") -def set_cache_key(key: str, sub: str, value: any): +def set_sub_key(key: str, sub: str, value: any): if type(key) != str: raise ValueError("Parameter key[]='{type(key)}' is not 'str'") elif type(sub) != str: raise ValueError("Parameter sub[]='{type(sub)}' is not 'str'") - elif not is_cache_initialized(key): + elif not key_exists(key): print(f"WARNING: Bad method call, key='{key}' is not initialized yet.") raise Exception(f"Cache for key='{key}' is not initialized, but function called") _cache[key][sub] = value -def is_cache_key_set(key: str, sub: str) -> bool: +def sub_key_exists(key: str, sub: str) -> bool: if type(key) != str: raise ValueError("Parameter key[]='{type(key)}' is not 'str'") elif type(sub) != str: raise ValueError("Parameter sub[]='{type(sub)}' is not 'str'") - elif not is_cache_initialized(key): + elif not key_exists(key): print(f"WARNING: Bad method call, key='{key}' is not initialized yet.") raise Exception(f"Cache for key='{key}' is not initialized, but function called") diff --git a/fba/fba.py b/fba/fba.py index 7d24b55..2025397 100644 --- a/fba/fba.py +++ b/fba/fba.py @@ -362,7 +362,7 @@ def update_last_blocked(domain: str): raise ValueError(f"Parameter 'domain' cannot be empty") # DEBUG: print("DEBUG: Updating last_blocked for domain", domain) - instances.set_instance_data("last_blocked", domain, time.time()) + instances.set("last_blocked", domain, time.time()) # Running pending updated # DEBUG: print(f"DEBUG: Invoking instances.update_instance_data({domain}) ...") @@ -415,17 +415,17 @@ def update_last_error(domain: str, res: any): # DEBUG: print("DEBUG: BEFORE res[]:", type(res)) if isinstance(res, BaseException) or isinstance(res, json.JSONDecodeError): - res = str(res) + res = f"{type}:str(res)" # DEBUG: print("DEBUG: AFTER res[]:", type(res)) if type(res) is str: # DEBUG: print(f"DEBUG: Setting last_error_details='{res}'"); - instances.set_instance_data("last_status_code" , domain, 999) - instances.set_instance_data("last_error_details", domain, res) + instances.set("last_status_code" , domain, 999) + instances.set("last_error_details", domain, res) else: # DEBUG: print(f"DEBUG: Setting last_error_details='{res.reason}'"); - instances.set_instance_data("last_status_code" , domain, res.status_code) - instances.set_instance_data("last_error_details", domain, res.reason) + instances.set("last_status_code" , domain, res.status_code) + instances.set("last_error_details", domain, res.reason) # Running pending updated # DEBUG: print(f"DEBUG: Invoking instances.update_instance_data({domain}) ...") @@ -443,7 +443,7 @@ def update_last_instance_fetch(domain: str): raise ValueError(f"Parameter 'domain' cannot be empty") # DEBUG: print("DEBUG: Updating last_instance_fetch for domain:", domain) - instances.set_instance_data("last_instance_fetch", domain, time.time()) + instances.set("last_instance_fetch", domain, time.time()) # Running pending updated # DEBUG: print(f"DEBUG: Invoking instances.update_instance_data({domain}) ...") @@ -459,8 +459,8 @@ def update_last_nodeinfo(domain: str): raise ValueError(f"Parameter 'domain' cannot be empty") # DEBUG: print("DEBUG: Updating last_nodeinfo for domain:", domain) - instances.set_instance_data("last_nodeinfo", domain, time.time()) - instances.set_instance_data("last_updated" , domain, time.time()) + instances.set("last_nodeinfo", domain, time.time()) + instances.set("last_updated" , domain, time.time()) # Running pending updated # DEBUG: print(f"DEBUG: Invoking instances.update_instance_data({domain}) ...") @@ -546,7 +546,7 @@ def get_peers(domain: str, software: str) -> list: break # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'") - instances.set_instance_data("total_peers", domain, len(peers)) + instances.set("total_peers", domain, len(peers)) # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") update_last_instance_fetch(domain) @@ -578,7 +578,7 @@ def get_peers(domain: str, software: str) -> list: print(f"WARNING: Exception during fetching JSON: domain='{domain}',exception[{type(e)}]:'{str(e)}'") # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'") - instances.set_instance_data("total_peers", domain, len(peers)) + instances.set("total_peers", domain, len(peers)) # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") update_last_instance_fetch(domain) @@ -620,7 +620,7 @@ def get_peers(domain: str, software: str) -> list: print(f"WARNING: Exception during fetching JSON: domain='{domain}',exception[{type(e)}]:'{str(e)}'") # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'") - instances.set_instance_data("total_peers", domain, len(peers)) + instances.set("total_peers", domain, len(peers)) # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") update_last_instance_fetch(domain) @@ -638,7 +638,10 @@ def get_peers(domain: str, software: str) -> list: # DEBUG: print(f"DEBUG: Was not able to fetch '{get_peers_url}', trying alternative ...") res = reqto.get(f"https://{domain}/api/v3/site", headers=api_headers, timeout=(config.get("connection_timeout"), config.get("read_timeout"))) - data = res.json() + data = json.dumps({}) + if res.text.strip() != "": + data = res.json() + # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'") if not res.ok or res.status_code >= 400: print("WARNING: Could not reach any JSON API:", domain) @@ -662,7 +665,7 @@ def get_peers(domain: str, software: str) -> list: update_last_error(domain, e) # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'") - instances.set_instance_data("total_peers", domain, len(peers)) + instances.set("total_peers", domain, len(peers)) # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") update_last_instance_fetch(domain) @@ -678,7 +681,7 @@ def post_json_api(domain: str, path: str, parameter: str, extra_headers: dict = elif type(path) != str: raise ValueError(f"path[]={type(path)} is not 'str'") elif path == "": - raise ValueError(f"path cannot be empty") + raise ValueError("Parameter 'path' cannot be empty") elif type(parameter) != str: raise ValueError(f"parameter[]={type(parameter)} is not 'str'") @@ -739,8 +742,8 @@ def fetch_nodeinfo(domain: str, path: str = None) -> list: # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'") if res.ok and isinstance(data, dict): # DEBUG: print("DEBUG: Success:", request) - instances.set_instance_data("detection_mode", domain, "STATIC_CHECK") - instances.set_instance_data("nodeinfo_url" , domain, request) + instances.set("detection_mode", domain, "STATIC_CHECK") + instances.set("nodeinfo_url" , domain, request) break elif res.ok and isinstance(data, list): # DEBUG: print(f"DEBUG: domain='{domain}' returned a list: '{data}'") @@ -788,8 +791,8 @@ def fetch_wellknown_nodeinfo(domain: str) -> list: # DEBUG: print("DEBUG: href,res.ok,res.status_code:", link["href"], res.ok, res.status_code) if res.ok and isinstance(data, dict): # DEBUG: print("DEBUG: Found JSON nodeinfo():", len(data)) - instances.set_instance_data("detection_mode", domain, "AUTO_DISCOVERY") - instances.set_instance_data("nodeinfo_url" , domain, link["href"]) + instances.set("detection_mode", domain, "AUTO_DISCOVERY") + instances.set("nodeinfo_url" , domain, link["href"]) break else: print("WARNING: Unknown 'rel' value:", domain, link["rel"]) @@ -836,13 +839,13 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str: # DEBUG: print("DEBUG: Found generator meta tag:", domain) software = tidyup_domain(generator.get("content")) print(f"INFO: domain='{domain}' is generated by '{software}'") - instances.set_instance_data("detection_mode", domain, "GENERATOR") + instances.set("detection_mode", domain, "GENERATOR") remove_pending_error(domain) elif isinstance(site_name, bs4.element.Tag): # DEBUG: print("DEBUG: Found property=og:site_name:", domain) sofware = tidyup_domain(site_name.get("content")) print(f"INFO: domain='{domain}' has og:site_name='{software}'") - instances.set_instance_data("detection_mode", domain, "SITE_NAME") + instances.set("detection_mode", domain, "SITE_NAME") remove_pending_error(domain) except BaseException as e: @@ -1094,19 +1097,19 @@ def is_instance_registered(domain: str) -> bool: raise ValueError(f"Parameter 'domain' cannot be empty") # NOISY-DEBUG: # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") - if not cache.is_cache_initialized("is_registered"): + if not cache.key_exists("is_registered"): # NOISY-DEBUG: # DEBUG: print(f"DEBUG: Cache for 'is_registered' not initialized, fetching all rows ...") try: cursor.execute("SELECT domain FROM instances") # Check Set all - cache.set_all_cache_key("is_registered", cursor.fetchall(), True) + cache.set_all("is_registered", cursor.fetchall(), True) except BaseException as e: print(f"ERROR: failed SQL query: domain='{domain}',exception[{type(e)}]:'{str(e)}'") sys.exit(255) # Is cache found? - registered = cache.is_cache_key_set("is_registered", domain) + registered = cache.sub_key_exists("is_registered", domain) # NOISY-DEBUG: # DEBUG: print(f"DEBUG: registered='{registered}' - EXIT!") return registered @@ -1148,12 +1151,12 @@ def add_instance(domain: str, origin: str, originator: str, path: str = None): ), ) - cache.set_cache_key("is_registered", domain, True) + cache.set_sub_key("is_registered", domain, True) if instances.has_pending_instance_data(domain): # DEBUG: print(f"DEBUG: domain='{domain}' has pending nodeinfo being updated ...") - instances.set_instance_data("last_status_code" , domain, None) - instances.set_instance_data("last_error_details", domain, None) + instances.set("last_status_code" , domain, None) + instances.set("last_error_details", domain, None) instances.update_instance_data(domain) remove_pending_error(domain) diff --git a/fba/instances.py b/fba/instances.py index 16b0e33..3312fa8 100644 --- a/fba/instances.py +++ b/fba/instances.py @@ -44,7 +44,7 @@ _pending = { "last_error_details" : {}, } -def set_instance_data(key: str, domain: str, value: any): +def set(key: str, domain: str, value: any): # NOISY-DEBUG: print(f"DEBUG: key='{key}',domain='{domain}',value[]='{type(value)}' - CALLED!") if type(key) != str: raise ValueError("Parameter key[]='{type(key)}' is not 'str'")