From a4e77d848a97d3c14332e4b3aecfc2135f25c5aa Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Tue, 30 May 2023 03:26:24 +0200 Subject: [PATCH] Continued: - also fetch origin and nodeinfo_url here - also add instance if not found - improved some log messages --- fetch_blocks.py | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/fetch_blocks.py b/fetch_blocks.py index ebf3697..4702736 100755 --- a/fetch_blocks.py +++ b/fetch_blocks.py @@ -100,7 +100,7 @@ for blocker, software, origin, nodeinfo_url in rows: # NOISY-DEBUG: print("DEBUG: Looking up instance by domain:", blocked) if not fba.is_instance_registered(blocked): - # NOISY-DEBUG: print("DEBUG: Domain wasn't found, adding:", blocked, blocker) + # NOISY-DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., blocker='{blocker}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'") fba.add_instance(blocked, blocker, origin, nodeinfo_url) fba.cursor.execute( @@ -157,12 +157,19 @@ for blocker, software, origin, nodeinfo_url in rows: elif blocked.count("*") > 1: # same domain guess as above, but for reasons field fba.cursor.execute( - "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")] + "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")] ) searchres = fba.cursor.fetchone() if searchres != None: blocked = searchres[0] + origin = searchres[1] + nodeinfo_url = searchres[2] + + # NOISY-DEBUG: print("DEBUG: Looking up instance by domain:", blocked) + if not fba.is_instance_registered(blocked): + # NOISY-DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., blocker='{blocker}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'") + fba.add_instance(blocked, blocker, origin, nodeinfo_url) # NOISY-DEBUG: print("DEBUG: Updating block reason:", blocker, blocked, reason["reason"]) fba.update_block_reason(reason["reason"], blocker, blocked, block_level) @@ -253,17 +260,24 @@ for blocker, software, origin, nodeinfo_url in rows: if fba.cursor.fetchone() == None: # NOISY-DEBUG: print("DEBUG: Hash wasn't found, adding:", blocked, blocker) - fba.add_instance(blocked, blocker, origin) + fba.add_instance(blocked, blocker, origin, nodeinfo_url) else: # Doing the hash search for instance names as well to tidy up DB fba.cursor.execute( - "SELECT domain FROM instances WHERE hash = ? LIMIT 1", [blocked_hash] + "SELECT domain, origin, nodeinfo_url FROM instances WHERE hash = ? LIMIT 1", [blocked_hash] ) searchres = fba.cursor.fetchone() if searchres != None: # NOISY-DEBUG: print("DEBUG: Updating domain: ", searchres[0]) blocked = searchres[0] + origin = searchres[1] + nodeinfo_url = searchres[2] + + # NOISY-DEBUG: print("DEBUG: Looking up instance by domain:", blocked) + if not fba.is_instance_registered(blocked): + # NOISY-DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., blocker='{blocker}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'") + fba.add_instance(blocked, blocker, origin, nodeinfo_url) fba.cursor.execute( "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ? LIMIT 1", @@ -324,14 +338,18 @@ for blocker, software, origin, nodeinfo_url in rows: if blocked == "": print("WARNING: blocked is empty:", blocker) continue - if blocked.count("*") > 0: + elif blocked.count("*") > 0: # Some friendica servers also obscure domains without hash fba.cursor.execute( - "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")] + "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")] ) + searchres = fba.cursor.fetchone() + if searchres != None: blocked = searchres[0] + origin = searchres[1] + nodeinfo_url = searchres[2] if blocked.count("?") > 0: # Some obscure them with question marks, not sure if that's dependent on version or not @@ -407,7 +425,7 @@ for blocker, software, origin, nodeinfo_url in rows: nodeinfo_url = searchres[2] if not fba.is_instance_registered(blocked): - # NOISY-DEBUG: print("DEBUG: Domain wasn't found, adding:", blocked, blocker) + # NOISY-DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., blocker='{blocker}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'") fba.add_instance(blocked, blocker, origin, nodeinfo_url) fba.cursor.execute( -- 2.39.5