From 1da20dd5f63ca2f55bba4c16d46b6d4dd8bb58eb Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Fri, 2 Jun 2023 16:04:42 +0200 Subject: [PATCH] Continued: - better warn about not deobsfucated domain names --- fetch_blocks.py | 96 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 64 insertions(+), 32 deletions(-) diff --git a/fetch_blocks.py b/fetch_blocks.py index c72d56e..612b84f 100755 --- a/fetch_blocks.py +++ b/fetch_blocks.py @@ -101,11 +101,14 @@ for blocker, software, origin, nodeinfo_url in rows: nodeinfo_url = searchres[1] # DEBUG: print("DEBUG: Looked up domain:", blocked) elif not validators.domain(blocked): - print(f"WARNING: blocked='{blocked}' is not a valid domai name - skipped!") + print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domai name - skipped!") continue # DEBUG: print("DEBUG: Looking up instance by domain:", blocked) - if not fba.is_instance_registered(blocked): + if not validators.domain(blocked): + print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!") + continue + elif not fba.is_instance_registered(blocked): # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., blocker='{blocker}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'") fba.add_instance(blocked, blocker, origin, nodeinfo_url) @@ -175,11 +178,14 @@ for blocker, software, origin, nodeinfo_url in rows: origin = searchres[1] nodeinfo_url = searchres[2] elif not validators.domain(blocked): - print(f"WARNING: blocked='{blocked}' is not a valid domai name - skipped!") + print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domai name - skipped!") continue # DEBUG: print("DEBUG: Looking up instance by domain:", blocked) - if not fba.is_instance_registered(blocked): + if not validators.domain(blocked): + print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!") + continue + elif not fba.is_instance_registered(blocked): # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., blocker='{blocker}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'") fba.add_instance(blocked, blocker, origin, nodeinfo_url) @@ -223,7 +229,7 @@ for blocker, software, origin, nodeinfo_url in rows: # DEBUG: print("DEBUG: Querying API domain_blocks:", blocker) blocks = reqto.get(f"https://{blocker}/api/v1/instance/domain_blocks", headers=reqheaders, timeout=(fba.config["connection_timeout"], fba.config["read_timeout"])).json() - print(f"INFO: Checking {len(blocks)} entries from blocker='{blocker}' ...") + print(f"INFO: Checking {len(blocks)} entries from blocker='{blocker}',software='{software}' ...") for block in blocks: entry = { 'domain': block['domain'], @@ -246,7 +252,7 @@ for blocker, software, origin, nodeinfo_url in rows: # DEBUG: print(f"DEBUG: Failed, trying mastodon-specific fetches: blocker='{blocker}',exception[{type(e)}]={str(e)}") json = fba.get_mastodon_blocks(blocker) - print(f"INFO: Checking {len(json.items())} entries from blocker='{blocker}' ...") + print(f"INFO: Checking {len(json.items())} entries from blocker='{blocker}',software='{software}' ...") for block_level, blocks in json.items(): # DEBUG: print("DEBUG: blocker,block_level,blocks():", blocker, block_level, len(blocks)) block_level = fba.tidyup(block_level) @@ -274,22 +280,31 @@ for blocker, software, origin, nodeinfo_url in rows: ) searchres = fba.cursor.fetchone() - if searchres != None: - # DEBUG: print("DEBUG: Updating domain: ", searchres[0]) - blocked = searchres[0] - origin = searchres[1] - nodeinfo_url = searchres[2] + if searchres == None: + print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!") + continue + + # DEBUG: print("DEBUG: Updating domain: ", searchres[0]) + blocked = searchres[0] + origin = searchres[1] + nodeinfo_url = searchres[2] # DEBUG: print("DEBUG: Looking up instance by domain:", blocked) - if not fba.is_instance_registered(blocked): + if not validators.domain(blocked): + print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!") + continue + elif not fba.is_instance_registered(blocked): # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., blocker='{blocker}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'") fba.add_instance(blocked, blocker, origin, nodeinfo_url) elif not validators.domain(blocked): - print(f"WARNING: blocked='{blocked}' is not a valid domai name - skipped!") + print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domai name - skipped!") continue # DEBUG: print("DEBUG: Looking up instance by domain:", blocked) - if not fba.is_instance_registered(blocked): + if not validators.domain(blocked): + print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!") + continue + elif not fba.is_instance_registered(blocked): # DEBUG: print("DEBUG: Hash wasn't found, adding:", blocked, blocker) fba.add_instance(blocked, blocker, origin, nodeinfo_url) @@ -336,7 +351,7 @@ for blocker, software, origin, nodeinfo_url in rows: print("WARNING: takahe is not fully supported for fetching blacklist!", blocker) #json = fba.get_takahe_blocks(blocker) - print(f"INFO: Checking {len(json.items())} entries from blocker='{blocker}' ...") + print(f"INFO: Checking {len(json.items())} entries from blocker='{blocker}',software='{software}' ...") for block_level, blocks in json.items(): # DEBUG: print("DEBUG: blocker,block_level,blocks():", blocker, block_level, len(blocks)) block_level = fba.tidyup(block_level) @@ -365,26 +380,37 @@ for blocker, software, origin, nodeinfo_url in rows: searchres = fba.cursor.fetchone() - if searchres != None: - blocked = searchres[0] - origin = searchres[1] - nodeinfo_url = searchres[2] + if searchres == None: + print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!") + continue + + blocked = searchres[0] + origin = searchres[1] + nodeinfo_url = searchres[2] elif blocked.count("?") > 0: # Some obscure them with question marks, not sure if that's dependent on version or not fba.cursor.execute( "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("?", "_")] ) + searchres = fba.cursor.fetchone() - if searchres != None: - blocked = searchres[0] - origin = searchres[1] - nodeinfo_url = searchres[2] + + if searchres == None: + print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!") + continue + + blocked = searchres[0] + origin = searchres[1] + nodeinfo_url = searchres[2] elif not validators.domain(blocked): - print(f"WARNING: blocked='{blocked}' is not a valid domai name - skipped!") + print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domai name - skipped!") continue # DEBUG: print("DEBUG: Looking up instance by domain:", blocked) - if not fba.is_instance_registered(blocked): + if not validators.domain(blocked): + print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!") + continue + elif not fba.is_instance_registered(blocked): # DEBUG: print("DEBUG: Hash wasn't found, adding:", blocked, blocker) fba.add_instance(blocked, blocker, origin, nodeinfo_url) @@ -423,7 +449,7 @@ for blocker, software, origin, nodeinfo_url in rows: elif "error" in federation: print("WARNING: API returned error:", federation["error"]) else: - print(f"INFO: Checking {len(federation)} entries from blocker='{blocker}' ...") + print(f"INFO: Checking {len(federation)} entries from blocker='{blocker}',software='{software}' ...") for peer in federation: blocked = peer["domain"].lower() # DEBUG: print("DEBUG: BEFORE blocked:", blocked) @@ -443,16 +469,22 @@ for blocker, software, origin, nodeinfo_url in rows: ) searchres = fba.cursor.fetchone() - if searchres != None: - blocked = searchres[0] - origin = searchres[1] - nodeinfo_url = searchres[2] + if searchres == None: + print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!") + continue + + blocked = searchres[0] + origin = searchres[1] + nodeinfo_url = searchres[2] elif not validators.domain(blocked): - print(f"WARNING: blocked='{blocked}' is not a valid domai name - skipped!") + print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domai name - skipped!") continue # DEBUG: print("DEBUG: Looking up instance by domain:", blocked) - if not fba.is_instance_registered(blocked): + if not validators.domain(blocked): + print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!") + continue + elif not fba.is_instance_registered(blocked): # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., blocker='{blocker}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'") fba.add_instance(blocked, blocker, origin, nodeinfo_url) -- 2.39.5