From: Roland Häder Date: Fri, 2 Jun 2023 17:57:12 +0000 (+0200) Subject: Continued: X-Git-Url: https://git.mxchange.org/?a=commitdiff_plain;h=6f1e8e8fd5dbd0e2bc7e19698326b380be289895;p=fba.git Continued: - moved fetch_instances() to fba --- diff --git a/fba.py b/fba.py index a31bd74..bec2f99 100644 --- a/fba.py +++ b/fba.py @@ -194,6 +194,60 @@ def is_primitive(var: any) -> bool: # NOISY-DEBUG: print(f"DEBUG: var[]='{type(var)}' - CALLED!") return type(var) in {int, str, float, bool} or var == None +def fetch_instances(domain: str, origin: str, software: str, path: str = None): + if type(domain) != str: + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError(f"Parameter 'domain' cannot be empty") + elif type(origin) != str and origin != None: + raise ValueError(f"Parameter origin[]={type(origin)} is not 'str'") + + # DEBUG: print("DEBUG: domain,origin,software,path:", domain, origin, software, path) + if not is_instance_registered(domain): + # DEBUG: print("DEBUG: Adding new domain:", domain, origin) + add_instance(domain, origin, sys.argv[0], path) + + # DEBUG: print("DEBUG: Fetching instances for domain:", domain, software) + peerlist = get_peers(domain, software) + + if (peerlist is None): + print("ERROR: Cannot fetch peers:", domain) + return + elif has_pending_instance_data(domain): + # DEBUG: print(f"DEBUG: domain='{domain}' has pending nodeinfo data, flushing ...") + update_instance_data(domain) + + print(f"INFO: Checking {len(peerlist)} instances from {domain} ...") + for instance in peerlist: + if instance == None: + # Skip "None" types as tidup() cannot parse them + continue + + # DEBUG: print(f"DEBUG: instance[{type(instance}]={instance} - BEFORE") + instance = tidyup(instance) + # DEBUG: print(f"DEBUG: instance[{type(instance}]={instance} - AFTER") + + if instance == "": + print("WARNING: Empty instance after tidyup(), domain:", domain) + continue + elif not validators.domain(instance.split("/")[0]): + print(f"WARNING: Bad instance='{instance}' from domain='{domain}',origin='{origin}',software='{software}'") + continue + elif is_blacklisted(instance): + # DEBUG: print("DEBUG: instance is blacklisted:", instance) + continue + + # DEBUG: print("DEBUG: Handling instance:", instance) + try: + if not is_instance_registered(instance): + # DEBUG: print("DEBUG: Adding new instance:", instance, domain) + add_instance(instance, domain, sys.argv[0]) + except BaseException as e: + print(f"ERROR: instance='{instance}',exception[{type(e)}]:'{str(e)}'") + continue + + # DEBUG: print("DEBUG: EXIT!") + def set_instance_data(key: str, domain: str, value: any): # NOISY-DEBUG: print(f"DEBUG: key='{key}',domain='{domain}',value[]='{type(value)}' - CALLED!") if type(key) != str: @@ -212,6 +266,8 @@ def set_instance_data(key: str, domain: str, value: any): # Set it instance_data[key][domain] = value + # DEBUG: print("DEBUG: EXIT!") + def add_peers(rows: dict) -> list: # DEBUG: print(f"DEBUG: rows()={len(rows)} - CALLED!") peers = list() @@ -376,6 +432,8 @@ def remove_pending_error(domain: str): except: pass + # DEBUG: print("DEBUG: EXIT!") + def get_hash(domain: str) -> str: if type(domain) != str: raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") diff --git a/fetch_instances.py b/fetch_instances.py index a0f89f9..438142a 100755 --- a/fetch_instances.py +++ b/fetch_instances.py @@ -24,55 +24,10 @@ import time import validators import fba -def fetch_instances(domain: str, origin: str, software: str, path: str = None): - # DEBUG: print("DEBUG: domain,origin,software,path:", domain, origin, software, path) - if not fba.is_instance_registered(domain): - # DEBUG: print("DEBUG: Adding new domain:", domain, origin) - fba.add_instance(domain, origin, sys.argv[0], path) - - # DEBUG: print("DEBUG: Fetching instances for domain:", domain, software) - peerlist = fba.get_peers(domain, software) - - if (peerlist is None): - print("ERROR: Cannot fetch peers:", domain) - return - elif fba.has_pending_instance_data(domain): - # DEBUG: print(f"DEBUG: domain='{domain}' has pending nodeinfo data, flushing ...") - fba.update_instance_data(domain) - - print(f"INFO: Checking {len(peerlist)} instances from {domain} ...") - for instance in peerlist: - if instance == None: - # Skip "None" types as tidup() cannot parse them - continue - - # DEBUG: print(f"DEBUG: instance[{type(instance}]={instance} - BEFORE") - instance = fba.tidyup(instance) - # DEBUG: print(f"DEBUG: instance[{type(instance}]={instance} - AFTER") - - if instance == "": - print("WARNING: Empty instance after tidyup(), domain:", domain) - continue - elif not validators.domain(instance.split("/")[0]): - print(f"WARNING: Bad instance='{instance}' from domain='{domain}',origin='{origin}',software='{software}'") - continue - elif fba.is_blacklisted(instance): - # DEBUG: print("DEBUG: instance is blacklisted:", instance) - continue - - # DEBUG: print("DEBUG: Handling instance:", instance) - try: - if not fba.is_instance_registered(instance): - # DEBUG: print("DEBUG: Adding new instance:", instance, domain) - fba.add_instance(instance, domain, sys.argv[0]) - except BaseException as e: - print(f"ERROR: instance='{instance}',exception[{type(e)}]:'{str(e)}'") - continue - instance = sys.argv[1] # Initial fetch -fetch_instances(instance, None, None) +fba.fetch_instances(instance, None, None) # Loop through some instances fba.cursor.execute( @@ -88,6 +43,6 @@ for row in rows: continue print(f"INFO: Fetching instances for instance '{row[0]}' ('{row[2]}') of origin '{row[1]}',nodeinfo_url='{row[3]}'") - fetch_instances(row[0], row[1], row[2], row[3]) + fba.fetch_instances(row[0], row[1], row[2], row[3]) fba.connection.close()