From 476f1e56794a02d5f0953a6aecfdae7539f3b647 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Mon, 29 May 2023 18:25:43 +0200 Subject: [PATCH] Continued: - introduced has_pending_nodeinfos() - also log exception name (aka. type) --- fba.py | 54 +++++++++++++++++++++++++--------------------- fetch_instances.py | 2 +- 2 files changed, 31 insertions(+), 25 deletions(-) diff --git a/fba.py b/fba.py index 56514ad..3c62ba7 100644 --- a/fba.py +++ b/fba.py @@ -80,8 +80,6 @@ nodeinfos = { "detection_mode": {}, # Found nodeinfo URL "nodeinfo_url": {}, - # Where to fetch peers (other instances) - "get_peers_url": {}, } language_mapping = { @@ -316,11 +314,23 @@ def update_last_blocked(domain: str): print("WARNING: Did not update any rows:", domain) except BaseException as e: - print(f"ERROR: failed SQL query: domain='{domain}',exception:'{str(e)}'") + print(f"ERROR: failed SQL query: domain='{domain}',exception[{type(e)}]:'{str(e)}'") sys.exit(255) # DEBUG: print("DEBUG: EXIT!") +def has_pending_nodeinfos(domain: str) -> bool: + # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") + has_pending = False + for key in nodeinfos: + # DEBUG: print(f"DEBUG: key='{key}',domain='{domain}',nodeinfos[key]='{nodeinfos[key]}'") + if domain in nodeinfos[key]: + has_pending = True + break + + # DEBUG: print(f"DEBUG: has_pending='{has_pending}' - EXIT!") + return has_pending + def update_nodeinfos(domain: str): # DEBUG: print("DEBUG: Updating nodeinfo for domain:", domain) sql_string = '' @@ -347,7 +357,7 @@ def update_nodeinfos(domain: str): print("WARNING: Did not update any rows:", domain) except BaseException as e: - print(f"ERROR: failed SQL query: domain='{domain}',sql='{sql}',exception:'{str(e)}'") + print(f"ERROR: failed SQL query: domain='{domain}',sql='{sql}',exception[{type(e)}]:'{str(e)}'") sys.exit(255) # DEBUG: print("DEBUG: Deleting nodeinfos for domain:", domain) @@ -389,7 +399,7 @@ def update_last_error(domain: str, res: any): pending_errors[domain] = res except BaseException as e: - print(f"ERROR: failed SQL query: domain='{domain}',exception:'{str(e)}'") + print(f"ERROR: failed SQL query: domain='{domain}',exception[{type(e)}]:'{str(e)}'") sys.exit(255) # DEBUG: print("DEBUG: EXIT!") @@ -407,7 +417,7 @@ def update_last_instance_fetch(domain: str): print("WARNING: Did not update any rows:", domain) except BaseException as e: - print(f"ERROR: failed SQL query: domain='{domain}',exception:'{str(e)}'") + print(f"ERROR: failed SQL query: domain='{domain}',exception[{type(e)}]:'{str(e)}'") sys.exit(255) connection.commit() @@ -426,21 +436,21 @@ def update_last_nodeinfo(domain: str): print("WARNING: Did not update any rows:", domain) except BaseException as e: - print(f"ERROR: failed SQL query: domain='{domain}',exception:'{str(e)}'") + print(f"ERROR: failed SQL query: domain='{domain}',exception[{type(e)}]:'{str(e)}'") sys.exit(255) connection.commit() # DEBUG: print("DEBUG: EXIT!") def get_peers(domain: str, software: str) -> list: - # DEBUG: print("DEBUG: Getting peers for domain:", domain, software) + # DEBUG: print(f"DEBUG: domain='{domain}',software='{software}' - CALLED!") peers = list() if software == "misskey": # DEBUG: print(f"DEBUG: domain='{domain}' is misskey, sending API POST request ...") - offset = 0 step = config["misskey_offset"] + # iterating through all "suspended" (follow-only in its terminology) # instances page-by-page, since that troonware doesn't support # sending them all at once @@ -515,7 +525,7 @@ def get_peers(domain: str, software: str) -> list: update_last_error(domain, res) except BaseException as e: - print(f"WARNING: Exception during fetching JSON: domain='{domain}',exception:'{str(e)}'") + print(f"WARNING: Exception during fetching JSON: domain='{domain}',exception[{type(e)}]:'{str(e)}'") update_last_instance_fetch(domain) @@ -553,7 +563,7 @@ def get_peers(domain: str, software: str) -> list: start = start + 100 except BaseException as e: - print(f"WARNING: Exception during fetching JSON: domain='{domain}',exception:'{str(e)}'") + print(f"WARNING: Exception during fetching JSON: domain='{domain}',exception[{type(e)}]:'{str(e)}'") update_last_instance_fetch(domain) @@ -588,7 +598,6 @@ def get_peers(domain: str, software: str) -> list: else: # DEBUG: print("DEBUG: Querying API was successful:", domain, len(data)) peers = data - nodeinfos["get_peers_url"][domain] = get_peers_url except BaseException as e: print("WARNING: Some error during get():", domain, e) @@ -854,7 +863,7 @@ def update_block_reason(reason: str, blocker: str, blocked: str, block_level: st print("WARNING: Did not update any rows:", domain) except BaseException as e: - print(f"ERROR: failed SQL query: reason='{reason}',blocker='{blocker}',blocked='{blocked}',block_level='{block_level}',sql='{sql}',exception:'{str(e)}'") + print(f"ERROR: failed SQL query: reason='{reason}',blocker='{blocker}',blocked='{blocked}',block_level='{block_level}',sql='{sql}',exception[{type(e)}]:'{str(e)}'") sys.exit(255) # DEBUG: print("DEBUG: EXIT!") @@ -876,7 +885,7 @@ def update_last_seen(blocker: str, blocked: str, block_level: str): print("WARNING: Did not update any rows:", domain) except BaseException as e: - print(f"ERROR: failed SQL query: last_seen='{last_seen}',blocker='{blocker}',blocked='{blocked}',block_level='{block_level}',exception:'{str(e)}'") + print(f"ERROR: failed SQL query: last_seen='{last_seen}',blocker='{blocker}',blocked='{blocked}',block_level='{block_level}',exception[{type(e)}]:'{str(e)}'") sys.exit(255) # DEBUG: print("DEBUG: EXIT!") @@ -905,7 +914,7 @@ def block_instance(blocker: str, blocked: str, reason: str, block_level: str): ) except BaseException as e: - print(f"ERROR: failed SQL query: blocker='{blocker}',blocked='{blocked}',reason='{reason}',block_level='{block_level}',first_seen='{first_seen}',last_seen='{last_seen}',exception:'{str(e)}'") + print(f"ERROR: failed SQL query: blocker='{blocker}',blocked='{blocked}',reason='{reason}',block_level='{block_level}',first_seen='{first_seen}',last_seen='{last_seen}',exception[{type(e)}]:'{str(e)}'") sys.exit(255) # DEBUG: print("DEBUG: EXIT!") @@ -913,7 +922,7 @@ def block_instance(blocker: str, blocked: str, reason: str, block_level: str): def is_instance_registered(domain: str) -> bool: # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") if not is_cache_initialized("is_registered"): - print(f"DEBUG: Cache for {__name__} not initialized, fetching all rows ...") + # DEBUG: print(f"DEBUG: Cache for 'is_registered' not initialized, fetching all rows ...") try: cursor.execute("SELECT domain FROM instances") @@ -957,13 +966,10 @@ def add_instance(domain: str, origin: str, originator: str, path: str = None): set_cache_key("is_registered", domain, True) - for key in nodeinfos: - # DEBUG: print(f"DEBUG: key='{key}',domain='{domain}',nodeinfos[key]={nodeinfos[key]}") - if domain in nodeinfos[key]: - # DEBUG: print(f"DEBUG: domain='{domain}' has pending nodeinfo being updated ...") - update_nodeinfos(domain) - remove_pending_error(domain) - break + if has_pending_nodeinfos(domain): + # DEBUG: print(f"DEBUG: domain='{domain}' has pending nodeinfo being updated ...") + update_nodeinfos(domain) + remove_pending_error(domain) if domain in pending_errors: # DEBUG: print("DEBUG: domain has pending error being updated:", domain) @@ -971,7 +977,7 @@ def add_instance(domain: str, origin: str, originator: str, path: str = None): remove_pending_error(domain) except BaseException as e: - print(f"ERROR: failed SQL query: domain='{domain}',exception:'{str(e)}'") + print(f"ERROR: failed SQL query: domain='{domain}',exception[{type(e)}]:'{str(e)}'") sys.exit(255) else: # DEBUG: print("DEBUG: Updating nodeinfo for domain:", domain) diff --git a/fetch_instances.py b/fetch_instances.py index c63803b..a453fd9 100644 --- a/fetch_instances.py +++ b/fetch_instances.py @@ -33,7 +33,7 @@ def fetch_instances(domain: str, origin: str, software: str, path: str = None): if (peerlist is None): print("ERROR: Cannot fetch peers:", domain) return - elif domain in fba.nodeinfos["get_peers_url"]: + elif fba.has_pending_nodeinfos(domain): # NOISY-DEBUG: print(f"DEBUG: domain='{domain}' has pending nodeinfo data, flushing ...") fba.update_nodeinfos(domain) -- 2.39.5