From e4bf463a9330f10c8deac38f1f7ff6cc92822388 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Tue, 23 May 2023 09:01:41 +0200 Subject: [PATCH] Continued: - flexible handling of SQL columns and values --- fba.py | 65 +++++++++++++++++++++++++++++++++++++------------ fetch_blocks.py | 2 +- 2 files changed, 50 insertions(+), 17 deletions(-) diff --git a/fba.py b/fba.py index 32a0d23..b1a1fd8 100644 --- a/fba.py +++ b/fba.py @@ -46,14 +46,20 @@ headers = { # Found info from node, such as nodeinfo URL, detection mode that needs to be # written to database. Both arrays must be filled at the same time or else -# update_nodeinfo() will fail +# update_nodeinfos() will fail nodeinfos = { # Detection mode: AUTO_DISCOVERY or STATIC_CHECKS "detection_mode": {}, # Found nodeinfo URL "nodeinfo_url": {}, + # Where to fetch peers (other instances) + "get_peers_url": {}, } +# URL for fetching peers +get_peers_url = "/api/v1/instance/peers" + +# Connect to database connection = sqlite3.connect("blocks.db") cursor = connection.cursor() @@ -84,29 +90,47 @@ def update_last_blocked(domain: str): print("ERROR: failed SQL query:", domain, e) sys.exit(255) + # NOISY-DEBUG: print("DEBUG: EXIT!") + def update_nodeinfos(domain: str): - #print("DEBUG: Updating nodeinfo for domain:", domain) - if domain not in nodeinfos["detection_mode"] or domain not in nodeinfos["nodeinfo_url"]: - print(f"WARNING: domain {domain} has no pending nodeinfo!") - raise + # NOISY-DEBUG: print("DEBUG: Updating nodeinfo for domain:", domain) + sql_string = '' + fields = list() + for key in nodeinfos: + # NOISY-DEBUG: print("DEBUG: key:", key) + if domain in nodeinfos[key]: + # NOISY-DEBUG: print(f"DEBUG: Adding '{nodeinfos[key][domain]}' for key='{key}' ...") + fields.append(nodeinfos[key][domain]) + sql_string += f" {key} = ?," + + sql_string = sql_string[:-1] + fields.append(domain) + # NOISY-DEBUG: print(f"DEBUG: sql_string='{sql_string}',fields()={len(fields)}") + + sql = "UPDATE instances SET" + sql_string + " WHERE domain = ? LIMIT 1" + # NOISY-DEBUG: print("DEBUG: sql:", sql) try: - cursor.execute("UPDATE instances SET detection_mode = ?, nodeinfo_url = ? WHERE domain = ? LIMIT 1", [ - nodeinfos["detection_mode"][domain], - nodeinfos["nodeinfo_url"][domain], - domain - ]) + # NOISY-DEBUG: print("DEBUG: Executing SQL:", sql) + cursor.execute(sql, fields) + # NOISY-DEBUG: print(f"DEBUG: Success! (rowcount={cursor.rowcount })") if cursor.rowcount == 0: print("WARNING: Did not update any rows:", domain) except BaseException as e: - print("ERROR: failed SQL query:", domain, e) + print(f"ERROR: failed SQL query: domain='{domain}',sql='{sql}',exception='{e}'") sys.exit(255) - # NOISY-DEBUG: print("DEBUG: Deleting nodeinfos for domain:", domain) - del nodeinfos["detection_mode"][domain] - del nodeinfos["nodeinfo_url"][domain] + # NOISY-DEBUG: # NOISY-DEBUG: print("DEBUG: Deleting nodeinfos for domain:", domain) + for key in nodeinfos: + try: + # NOISY-DEBUG: print("DEBUG: Deleting key:", key) + del nodeinfos[key][domain] + except: + pass + + # NOISY-DEBUG: print("DEBUG: EXIT!") def update_last_error(domain: str, res: any): # NOISY-DEBUG: print("DEBUG: domain,res.status_code:", domain, res.status_code, res.reason) @@ -137,8 +161,9 @@ def update_last_error(domain: str, res: any): print("ERROR: failed SQL query:", domain, e) sys.exit(255) - # NOISY-DEBUG: print("DEBUG: Deleting pending_errors for domain:", domain) + # NOISY-DEBUG: print(f"DEBUG: Deleting domain='{domain}' from pending_errors") del pending_errors[domain] + # NOISY-DEBUG: print("DEBUG: EXIT!") def update_last_nodeinfo(domain: str): # NOISY-DEBUG: print("DEBUG: Updating last_nodeinfo for domain:", domain) @@ -157,13 +182,14 @@ def update_last_nodeinfo(domain: str): sys.exit(255) connection.commit() + # NOISY-DEBUG: print("DEBUG: EXIT!") def get_peers(domain: str) -> list: # NOISY-DEBUG: print("DEBUG: Getting peers for domain:", domain) peers = None try: - res = reqto.get(f"https://{domain}/api/v1/instance/peers", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])) + res = reqto.get(f"https://{domain}{get_peers_url}", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])) if not res.ok or res.status_code >= 400: print("WARNING: Cannot fetch peers:", domain) @@ -171,6 +197,7 @@ def get_peers(domain: str) -> list: else: # NOISY-DEBUG: print("DEBUG: Querying API was successful:", domain, len(res.json())) peers = res.json() + nodeinfos["get_peers_url"][domain] = get_peers_url except: print("WARNING: Some error during get():", domain) @@ -379,6 +406,8 @@ def update_last_seen(blocker: str, blocked: str, block_level: str): print("ERROR: failed SQL query:", last_seen, blocker, blocked, block_level) sys.exit(255) + # NOISY-DEBUG: print("DEBUG: EXIT!") + def block_instance(blocker: str, blocked: str, reason: str, block_level: str): # NOISY-DEBUG: print("DEBUG: blocker,blocked,reason,block_level:", blocker, blocked, reason, block_level) if blocker.find("@") > 0: @@ -406,6 +435,8 @@ def block_instance(blocker: str, blocked: str, reason: str, block_level: str): print("ERROR: failed SQL query:", blocker, blocked, reason, block_level, first_added, last_seen) sys.exit(255) + # NOISY-DEBUG: print("DEBUG: EXIT!") + def add_instance(domain: str, origin: str, originator: str): # NOISY-DEBUG: print("DEBUG: domain,origin:", domain, origin, originator) if domain.find("@") > 0: @@ -448,6 +479,8 @@ def add_instance(domain: str, origin: str, originator: str): # NOISY-DEBUG: print("DEBUG: Updating nodeinfo for domain:", domain) update_last_nodeinfo(domain) + # NOISY-DEBUG: print("DEBUG: EXIT!") + def send_bot_post(instance: str, blocks: dict): message = instance + " has blocked the following instances:\n\n" truncated = False diff --git a/fetch_blocks.py b/fetch_blocks.py index ab9849f..59c4415 100644 --- a/fetch_blocks.py +++ b/fetch_blocks.py @@ -356,7 +356,7 @@ for blocker, software in fba.cursor.fetchall(): print("INFO: blocker:", blocker) try: # Blocks - federation = reqto.get(f"https://{blocker}/api/v1/instance/peers?filter=suspended", headers=fba.headers, timeout=(fba.config["connection_timeout"], config["read_timeout"])).json() + federation = reqto.get(f"https://{blocker}{get_peers_url}?filter=suspended", headers=fba.headers, timeout=(fba.config["connection_timeout"], config["read_timeout"])).json() if (federation == None): print("WARNING: No valid response:", blocker); -- 2.39.5