From: Roland Häder Date: Mon, 5 Jun 2023 07:31:18 +0000 (+0200) Subject: Continued: X-Git-Url: https://git.mxchange.org/?a=commitdiff_plain;ds=inline;h=75fa3f40dff62d1b312cb4bb29470a4df7605440;p=fba.git Continued: - some APIs return the same result at the end all over again - so we need to check if the domain is already part of the created list - for that purpose has_element() is introduced --- diff --git a/fba/cache.py b/fba/cache.py index 790d88f..22c2b00 100644 --- a/fba/cache.py +++ b/fba/cache.py @@ -23,22 +23,22 @@ def key_exists(key: str) -> bool: return key in _cache def set_all(key: str, rows: list, value: any): - # NOISY-DEBUG: print(f"DEBUG: key='{key}',rows()={len(rows)},value[]={type(value)} - CALLED!") + # DEBUG: print(f"DEBUG: key='{key}',rows()={len(rows)},value[]={type(value)} - CALLED!") if type(key) != str: raise ValueError("Parameter key[]='{type(key)}' is not 'str'") elif not key_exists(key): - # NOISY-DEBUG: print(f"DEBUG: Cache for key='{key}' not initialized.") + # DEBUG: print(f"DEBUG: Cache for key='{key}' not initialized.") _cache[key] = {} for sub in rows: - # NOISY-DEBUG: print(f"DEBUG: Setting key='{key}',sub[{type(sub)}]='{sub}'") + # DEBUG: print(f"DEBUG: Setting key='{key}',sub[{type(sub)}]='{sub}'") if isinstance(sub, tuple): _cache[key][sub[0]] = value else: print(f"WARNING: Unsupported type row[]='{type(row)}'") - # NOISY-DEBUG: print("DEBUG: EXIT!") + # DEBUG: print("DEBUG: EXIT!") def set_sub_key(key: str, sub: str, value: any): if type(key) != str: diff --git a/fba/fba.py b/fba/fba.py index d8abc9e..5350a0e 100644 --- a/fba/fba.py +++ b/fba/fba.py @@ -125,6 +125,7 @@ def is_primitive(var: any) -> bool: return type(var) in {int, str, float, bool} or var == None def fetch_instances(domain: str, origin: str, software: str, script: str, path: str = None): + # DEBUG: print(f"DEBUG: domain={domain},origin={origin},software={software},path={path} - CALLED!") if type(domain) != str: raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": @@ -136,7 +137,6 @@ def fetch_instances(domain: str, origin: str, software: str, script: str, path: elif domain == "": raise ValueError(f"Parameter 'domain' cannot be empty") - # DEBUG: print("DEBUG: domain,origin,software,path:", domain, origin, software, path) if not is_instance_registered(domain): # DEBUG: print("DEBUG: Adding new domain:", domain, origin) add_instance(domain, origin, script, path) @@ -157,9 +157,9 @@ def fetch_instances(domain: str, origin: str, software: str, script: str, path: # Skip "None" types as tidup() cannot parse them continue - # DEBUG: print(f"DEBUG: instance[{type(instance}]={instance} - BEFORE") + # DEBUG: print(f"DEBUG: instance='{instance}' - BEFORE") instance = tidyup_domain(instance) - # DEBUG: print(f"DEBUG: instance[{type(instance}]={instance} - AFTER") + # DEBUG: print(f"DEBUG: instance='{instance}' - AFTER") if instance == "": print("WARNING: Empty instance after tidyup_domain(), domain:", domain) @@ -508,7 +508,7 @@ def get_peers(domain: str, software: str) -> list: "Origin": domain }) - # DEBUG: print(f"DEBUG: fetched()={len(fetched))}") + # DEBUG: print(f"DEBUG: fetched()={len(fetched)}") if len(fetched) == 0: # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain) break @@ -1100,9 +1100,9 @@ def is_instance_registered(domain: str) -> bool: elif domain == "": raise ValueError(f"Parameter 'domain' cannot be empty") - # NOISY-DEBUG: # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") + # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") if not cache.key_exists("is_registered"): - # NOISY-DEBUG: # DEBUG: print(f"DEBUG: Cache for 'is_registered' not initialized, fetching all rows ...") + # DEBUG: print(f"DEBUG: Cache for 'is_registered' not initialized, fetching all rows ...") try: cursor.execute("SELECT domain FROM instances") @@ -1115,7 +1115,7 @@ def is_instance_registered(domain: str) -> bool: # Is cache found? registered = cache.sub_key_exists("is_registered", domain) - # NOISY-DEBUG: # DEBUG: print(f"DEBUG: registered='{registered}' - EXIT!") + # DEBUG: print(f"DEBUG: registered='{registered}' - EXIT!") return registered def add_instance(domain: str, origin: str, originator: str, path: str = None): @@ -1359,7 +1359,7 @@ def get_misskey_blocks(domain: str) -> dict: for instance in fetched: # just in case - if instance["isSuspended"]: + if instance["isSuspended"] and not has_element(blocks["suspended"], "domain", instance): blocks["suspended"].append( { "domain": tidyup_domain(instance["host"]), @@ -1411,7 +1411,7 @@ def get_misskey_blocks(domain: str) -> dict: offset = offset + step for instance in fetched: - if instance["isBlocked"]: + if instance["isBlocked"] and not has_element(blocks["blocked"], "domain", instance): blocks["blocked"].append({ "domain": tidyup_domain(instance["host"]), "reason": None @@ -1478,8 +1478,12 @@ def get_response(domain: str, path: str, headers: dict, timeout: list) -> reques # DEBUG: print(f"DEBUG: domain='{domain}',path='{path}',headers()={len(headers)},timeout={timeout} - CALLED!") if type(domain) != str: raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") + elif domain == "": + raise ValueError("Parameter 'domain' cannot be empty") elif type(path) != str: raise ValueError(f"Parameter path[]='{type(path)}' is not 'str'") + elif path == "": + raise ValueError("Parameter 'path' cannot be empty") try: # DEBUG: print(f"DEBUG: Sending request to '{domain}{path}' ...") @@ -1491,3 +1495,25 @@ def get_response(domain: str, path: str, headers: dict, timeout: list) -> reques # DEBUG: print(f"DEBUG: response[]='{type(response)}' - EXXIT!") return response + +def has_element(elements: list, key: str, value: any) -> bool: + # DEBUG: print(f"DEBUG: element()={len(element)},key='{key}',value[]='{type(value)}' - CALLED!") + if type(key) != str: + raise ValueError(f"Parameter key[]='{type(key)}' is not 'str'") + elif key == "": + raise ValueError("Parameter 'key' cannot be empty") + + has = False + # DEBUG: print(f"DEBUG: Checking elements()={len(elements)} ...") + for element in elements: + # DEBUG: print(f"DEBUG: element[]='{type(element)}'") + if type(element) != dict: + raise ValueError(f"element[]='{type(element)}' is not 'dict'") + elif not key in element: + raise KeyError(f"Cannot find key='{key}'") + elif element[key] == value: + has = True + break + + # DEBUG: print(f"DEBUG: has={has} - EXIT!") + return has diff --git a/fba/instances.py b/fba/instances.py index b21e85b..f900bb1 100644 --- a/fba/instances.py +++ b/fba/instances.py @@ -45,7 +45,7 @@ _pending = { } def set(key: str, domain: str, value: any): - # NOISY-DEBUG: print(f"DEBUG: key='{key}',domain='{domain}',value[]='{type(value)}' - CALLED!") + # DEBUG: print(f"DEBUG: key='{key}',domain='{domain}',value[]='{type(value)}' - CALLED!") if type(key) != str: raise ValueError("Parameter key[]='{type(key)}' is not 'str'") elif key == "": @@ -62,7 +62,7 @@ def set(key: str, domain: str, value: any): # Set it _pending[key][domain] = value - # DEBUG: print("DEBUG: EXIT!") + # DEBUG: # DEBUG: print("DEBUG: EXIT!") def has_pending_instance_data(domain: str) -> bool: # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") @@ -91,7 +91,7 @@ def update_instance_data(domain: str): raise Exception(f"Domain '{domain}' has no pending instance data, but function invoked") # DEBUG: print(f"DEBUG: Updating instance data for domain='{domain}' ...") - sql_string = '' + sql_string = "" fields = list() for key in _pending: # DEBUG: print("DEBUG: key:", key) @@ -103,7 +103,7 @@ def update_instance_data(domain: str): fields.append(time.time()) fields.append(domain) - if sql_string == '': + if sql_string == "": raise ValueError(f"No fields have been set, but method invoked, domain='{domain}'") # DEBUG: print(f"DEBUG: sql_string='{sql_string}',fields()={len(fields)}") diff --git a/fetch_blocks.py b/fetch_blocks.py index bb03061..c12c26f 100755 --- a/fetch_blocks.py +++ b/fetch_blocks.py @@ -138,7 +138,7 @@ for blocker, software, origin, nodeinfo_url in rows: "reason" : None }) else: - print(f"DEBUG: Updating block last seen for blocker='{blocker}',blocked='{blocked}' ...") + # DEBUG: print(f"DEBUG: Updating block last seen for blocker='{blocker}',blocked='{blocked}' ...") fba.update_last_seen(blocker, blocked, block_level) # DEBUG: print("DEBUG: Committing changes ...") @@ -331,11 +331,10 @@ for blocker, software, origin, nodeinfo_url in rows: fba.block_instance(blocker, blocking, reason, block_level) if block_level == "reject": - blockdict.append( - { - "blocked": blocked, - "reason" : reason - }) + blockdict.append({ + "blocked": blocked, + "reason" : reason + }) else: # DEBUG: print(f"DEBUG: Updating block last seen and reason for blocker='{blocker}',blocking='{blocking}' ...") fba.update_last_seen(blocker, blocking, block_level) @@ -429,13 +428,12 @@ for blocker, software, origin, nodeinfo_url in rows: fba.block_instance(blocker, blocked, reason, block_level) if block_level == "reject": - blockdict.append( - { - "blocked": blocked, - "reason" : reason - }) + blockdict.append({ + "blocked": blocked, + "reason" : reason + }) else: - print(f"DEBUG: Updating block last seen and reason for blocker='{blocker}',blocked='{blocked}' ...") + # DEBUG: print(f"DEBUG: Updating block last seen and reason for blocker='{blocker}',blocked='{blocked}' ...") fba.update_last_seen(blocker, blocked, block_level) fba.update_block_reason(reason, blocker, blocked, block_level) @@ -497,13 +495,12 @@ for blocker, software, origin, nodeinfo_url in rows: # DEBUG: print(f"DEBUG: blocker='{blocker}' is blocking '{blocked}' for unknown reason at this point") fba.block_instance(blocker, blocked, "unknown", "reject") - blockdict.append( - { - "blocked": blocked, - "reason" : None - }) + blockdict.append({ + "blocked": blocked, + "reason" : None + }) else: - print(f"DEBUG: Updating block last seen for blocker='{blocker}',blocked='{blocked}' ...") + # DEBUG: print(f"DEBUG: Updating block last seen for blocker='{blocker}',blocked='{blocked}' ...") fba.update_last_seen(blocker, blocked, "reject") if "public_comment" in peer: