X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;f=fba%2Finstances.py;h=988401474b0d73dd81c08fb7842a343a89de6bfb;hb=b4b7a362cf34916fccecded268a717f6263c2d84;hp=c872c2b96832c47bf74ef85ac969ff27f0028983;hpb=a993efe50352f30ef4fb2d5e3a2aab640a129019;p=fba.git diff --git a/fba/instances.py b/fba/instances.py index c872c2b..9884014 100644 --- a/fba/instances.py +++ b/fba/instances.py @@ -22,9 +22,12 @@ import requests import validators from fba import blacklist -from fba import cache +from fba import config from fba import fba from fba import federation +from fba import network + +from fba.helpers import cache # Found info from node, such as nodeinfo URL, detection mode that needs to be # written to database. Both arrays must be filled at the same time or else @@ -49,8 +52,6 @@ _pending = { "last_status_code" : {}, # Last error details "last_error_details" : {}, - # Whether CSRF tokens are present - "has_csrf" : {}, } def set_data(key: str, domain: str, value: any): @@ -73,30 +74,30 @@ def set_data(key: str, domain: str, value: any): # DEBUG: print("DEBUG: EXIT!") -def has_pending_instance_data(domain: str) -> bool: +def has_pending(domain: str) -> bool: # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") - has_pending = False + has = False for key in _pending: # DEBUG: print(f"DEBUG: key='{key}',domain='{domain}',_pending[key]()='{len(_pending[key])}'") if domain in _pending[key]: - has_pending = True + has = True break - # DEBUG: print(f"DEBUG: has_pending='{has_pending}' - EXIT!") - return has_pending + # DEBUG: print(f"DEBUG: has='{has}' - EXIT!") + return has def update_data(domain: str): # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") - elif not has_pending_instance_data(domain): + elif not has_pending(domain): raise Exception(f"Domain '{domain}' has no pending instance data, but function invoked") # DEBUG: print(f"DEBUG: Updating instance data for domain='{domain}' ...") @@ -131,13 +132,11 @@ def update_data(domain: str): # DEBUG: print("DEBUG: Committing changes ...") fba.connection.commit() - # DEBUG: print("DEBUG: Deleting _pending for domain:", domain) + # DEBUG: print(f"DEBUG: Deleting _pending for domain='{domain}'") for key in _pending: - try: - # DEBUG: print("DEBUG: Deleting key:", key) + # DEBUG: print(f"DEBUG: domain='{domain}',key='{key}'") + if domain in _pending[key]: del _pending[key][domain] - except: - pass except BaseException as exception: print(f"ERROR: failed SQL query: domain='{domain}',sql_string='{sql_string}',exception[{type(exception)}]:'{str(exception)}'") @@ -148,7 +147,7 @@ def update_data(domain: str): def update_last_instance_fetch(domain: str): # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") @@ -162,8 +161,9 @@ def update_last_instance_fetch(domain: str): # DEBUG: print("DEBUG: EXIT!") def update_last_blocked(domain: str): + # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") @@ -179,18 +179,18 @@ def update_last_blocked(domain: str): def add(domain: str, origin: str, command: str, path: str = None): # DEBUG: print(f"DEBUG: domain='{domain}',origin='{origin}',command='{command}',path='{path}' - CALLED!") if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") elif not isinstance(origin, str) and origin is not None: - raise ValueError(f"origin[]={type(origin)} is not 'str'") + raise ValueError(f"origin[]='{type(origin)}' is not 'str'") elif origin == "": raise ValueError("Parameter 'origin' is empty") elif not isinstance(command, str): - raise ValueError(f"command[]={type(command)} is not 'str'") + raise ValueError(f"command[]='{type(command)}' is not 'str'") elif command == "": raise ValueError("Parameter 'command' is empty") - elif not validators.domain(domain.split("/")[0]): + elif not validators.domain(domain.split("/")[0]) or domain.split(".")[-1] == "arpa": raise ValueError(f"Bad domain name='{domain}'") elif origin is not None and not validators.domain(origin.split("/")[0]): raise ValueError(f"Bad origin name='{origin}'") @@ -199,8 +199,12 @@ def add(domain: str, origin: str, command: str, path: str = None): elif domain.find("/profile/") > 0 or domain.find("/users/") > 0: raise Exception(f"domain='{domain}' is a single user") - # DEBUG: print("DEBUG: domain,origin,command,path:", domain, origin, command, path) - software = federation.determine_software(domain, path) + software = None + try: + # DEBUG: print("DEBUG: domain,origin,command,path:", domain, origin, command, path) + software = federation.determine_software(domain, path) + except network.exceptions as exception: + print(f"WARNING Exception '{type(exception)}' during determining software type") # DEBUG: print("DEBUG: Determined software:", software) if software == "lemmy" and domain.find("/c/") > 0: @@ -210,40 +214,35 @@ def add(domain: str, origin: str, command: str, path: str = None): return print(f"INFO: Adding instance domain='{domain}' (origin='{origin}',software='{software}')") - try: - fba.cursor.execute( - "INSERT INTO instances (domain, origin, command, hash, software, first_seen) VALUES (?, ?, ?, ?, ?, ?)", - ( - domain, - origin, - command, - fba.get_hash(domain), - software, - time.time() - ), - ) - - cache.set_sub_key("is_registered", domain, True) - - if has_pending_instance_data(domain): - # DEBUG: print(f"DEBUG: domain='{domain}' has pending nodeinfo being updated ...") - set_data("last_status_code" , domain, None) - set_data("last_error_details", domain, None) - update_data(domain) - - except BaseException as exception: - update_last_error(domain, exception) - raise Exception(f"ERROR: failed SQL query: domain='{domain}',exception[{type(exception)}]:'{str(exception)}'") from exception - else: - # DEBUG: print("DEBUG: Updating nodeinfo for domain:", domain) - update_last_nodeinfo(domain) + fba.cursor.execute( + "INSERT INTO instances (domain, origin, command, hash, software, first_seen) VALUES (?, ?, ?, ?, ?, ?)", + ( + domain, + origin, + command, + fba.get_hash(domain), + software, + time.time() + ), + ) + + cache.set_sub_key("is_registered", domain, True) + + if has_pending(domain): + # DEBUG: print(f"DEBUG: domain='{domain}' has pending nodeinfo being updated ...") + set_data("last_status_code" , domain, None) + set_data("last_error_details", domain, None) + update_data(domain) + + # DEBUG: print(f"DEBUG: Updating nodeinfo for domain='{domain}'") + update_last_nodeinfo(domain) # DEBUG: print("DEBUG: EXIT!") def update_last_nodeinfo(domain: str): # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") @@ -257,56 +256,114 @@ def update_last_nodeinfo(domain: str): # DEBUG: print("DEBUG: EXIT!") -def update_last_error(domain: str, response: requests.models.Response): - print("DEBUG: domain,response[]:", domain, type(response)) +def update_last_error(domain: str, error: dict): + # DEBUG: print("DEBUG: domain,error[]:", domain, type(error)) if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") - print("DEBUG: BEFORE response[]:", type(response)) - if isinstance(response, BaseException) or isinstance(response, json.decoder.JSONDecodeError): - response = f"response[{type(response)}]='{str(response)}'" + # DEBUG: print("DEBUG: BEFORE error[]:", type(error)) + if isinstance(error, (BaseException, json.decoder.JSONDecodeError)): + error = f"error[{type(error)}]='{str(error)}'" + # DEBUG: print("DEBUG: AFTER error[]:", type(error)) - print("DEBUG: AFTER response[]:", type(response)) - if isinstance(response, str): - print(f"DEBUG: Setting last_error_details='{response}'") + if isinstance(error, str): + # DEBUG: print(f"DEBUG: Setting last_error_details='{error}'") set_data("last_status_code" , domain, 999) - set_data("last_error_details", domain, response) - else: - print(f"DEBUG: Setting last_error_details='{response.reason}'") - set_data("last_status_code" , domain, response.status_code) - set_data("last_error_details", domain, response.reason) + set_data("last_error_details", domain, error if error != "" else None) + elif isinstance(error, requests.models.Response): + # DEBUG: print(f"DEBUG: Setting last_error_details='{error.reason}'") + set_data("last_status_code" , domain, error.status_code) + set_data("last_error_details", domain, error.reason if error.reason != "" else None) + elif not isinstance(error, dict): + raise KeyError(f"Cannot handle keys in error[{type(error)}]='{error}'") + elif "status_code" in error and "error_message" in error: + # DEBUG: print(f"DEBUG: Setting last_error_details='{error['error_message']}'") + set_data("last_status_code" , domain, error["status_code"]) + set_data("last_error_details", domain, error["error_message"] if error["error_message"] != "" else None) + elif "json" in error and "error" in error["json"]: + set_data("last_status_code" , domain, error["status_code"]) + set_data("last_error_details", domain, error["json"]["error"] if error["json"]["error"] != "" else None) # Running pending updated - print(f"DEBUG: Invoking update_data({domain}) ...") + # DEBUG: print(f"DEBUG: Invoking update_data({domain}) ...") update_data(domain) - fba.log_error(domain, response) + fba.log_error(domain, error) - print("DEBUG: EXIT!") + # DEBUG: print("DEBUG: EXIT!") def is_registered(domain: str) -> bool: # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") if not cache.key_exists("is_registered"): # DEBUG: print("DEBUG: Cache for 'is_registered' not initialized, fetching all rows ...") - try: - fba.cursor.execute("SELECT domain FROM instances") + fba.cursor.execute("SELECT domain FROM instances") - # Check Set all - cache.set_all("is_registered", fba.cursor.fetchall(), True) - except BaseException as exception: - update_last_error(domain, exception) - raise Exception(f"ERROR: failed SQL query: domain='{domain}',exception[{type(exception)}]:'{str(exception)}'") from exception + # Check Set all + cache.set_all("is_registered", fba.cursor.fetchall(), True) # Is cache found? registered = cache.sub_key_exists("is_registered", domain) # DEBUG: print(f"DEBUG: registered='{registered}' - EXIT!") return registered + +def is_recent(domain: str) -> bool: + # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") + if not isinstance(domain, str): + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") + elif domain == "": + raise ValueError("Parameter 'domain' is empty") + elif not is_registered(domain): + # DEBUG: print(f"DEBUG: domain='{domain}' is not registered, returning False - EXIT!") + return False + + # Query database + fba.cursor.execute("SELECT last_instance_fetch FROM instances WHERE domain = ? LIMIT 1", [domain]) + + # Fetch row + fetched = fba.cursor.fetchone()[0] + + # DEBUG: print(f"DEBUG: fetched[{type(fetched)}]='{fetched}'") + recently = isinstance(fetched, float) and time.time() - fetched <= config.get("recheck_instance") + + # DEBUG: print(f"DEBUG: recently='{recently}' - EXIT!") + return recently + +def deobscure(char: str, domain: str, blocked_hash: str = None) -> tuple: + #print(f"DEBUG: char='{char}',domain='{domain}',blocked_hash='{blocked_hash}' - CALLED!") + if not isinstance(char, str): + raise ValueError(f"Parameter char[]='{type(char)}' is not 'str'") + elif char == "": + raise ValueError("Parameter 'char' is empty") + elif not isinstance(domain, str): + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") + elif domain == "": + raise ValueError("Parameter 'domain' is empty") + elif not isinstance(blocked_hash, str) and blocked_hash is not None: + raise ValueError(f"Parameter blocked_hash[]='{type(blocked_hash)}' is not 'str'") + + if isinstance(blocked_hash, str): + fba.cursor.execute( + "SELECT domain, origin, nodeinfo_url FROM instances WHERE hash = ? LIMIT 1", [blocked_hash] + ) + + if fba.cursor.fetchone() is None: + #print(f"DEBUG: blocked_hash='{blocked_hash}' not found, trying domain='{domain}' ...") + return deobscure(char, domain) + else: + fba.cursor.execute( + "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [domain.replace(char, "_")] + ) + + row = fba.cursor.fetchone() + + #print(f"DEBUG: row[]='{type(row)}' - EXIT!") + return row