From 41fce91ff83d84b3f9ddd341e18cd89e1e2b031a Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Thu, 18 May 2023 22:33:09 +0200 Subject: [PATCH] Continued: - do not include anything that is empty after being cleared (exception reason) --- fetch_blocks.py | 71 ++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 62 insertions(+), 9 deletions(-) diff --git a/fetch_blocks.py b/fetch_blocks.py index 7cc6fda..2ea4a86 100644 --- a/fetch_blocks.py +++ b/fetch_blocks.py @@ -10,9 +10,15 @@ fba.c.execute( ) for blocker, software in fba.c.fetchall(): - # NOISY-DEBUG: print("DEBUG: blocker,software:", blocker, software) + # NOISY-DEBUG: print("DEBUG: BEFORE-blocker,software:", blocker, software) blockdict = [] blocker = fba.tidyup(blocker) + # NOISY-DEBUG: print("DEBUG: AFTER-blocker,software:", blocker, software) + + if blocker == "": + print("WARNING: blocker is now empty!") + continue + if software == "pleroma": print("INFO: blocker:", blocker) try: @@ -25,6 +31,14 @@ for blocker, software in fba.c.fetchall(): {**federation["mrf_simple"], **{"quarantined_instances": federation["quarantined_instances"]}} ).items(): + # NOISY-DEBUG: print("DEBUG: block_level, blocks.items():", block_level, len(blocks.items())) + block_level = fba.tidyup(block_level) + # NOISY-DEBUG: print("DEBUG: BEFORE block_level:", block_level) + + if block_level == "": + print("WARNING: block_level is now empty!") + continue + for blocked in blocks: # NOISY-DEBUG: print("DEBUG: BEFORE blocked:", blocked) blocked = fba.tidyup(blocked) @@ -43,6 +57,7 @@ for blocker, software in fba.c.fetchall(): # NOISY-DEBUG: print("DEBUG: searchres[]:", type(searchres)) if searchres != None: blocked = searchres[0] + # NOISY-DEBUG: print("DEBUG: Looked up domain:", blocked) # NOISY-DEBUG: print("DEBUG: Looking up instance by domain:", blocked) fba.c.execute( @@ -64,15 +79,18 @@ for blocker, software in fba.c.fetchall(): ) if fba.c.fetchone() == None: + # NOISY-DEBUG: print("DEBUG: Blocking:", blocker, blocked, block_level) fba.block_instance(blocker, blocked, "unknown", block_level, timestamp, timestamp) if block_level == "reject": + # NOISY-DEBUG: print("DEBUG: Adding to blockdict:", blocked) blockdict.append( { "blocked": blocked, "reason": None }) else: + # NOISY-DEBUG: print("DEBUG: Updating last_seen:", blocker, blocked, block_level) fba.update_last_seen(timestamp, blocker, blocked, block_level) fba.conn.commit() @@ -87,6 +105,13 @@ for blocker, software in fba.c.fetchall(): else {})} ).items(): # NOISY-DEBUG: print("DEBUG: block_level, info.items():", block_level, len(info.items())) + block_level = fba.tidyup(block_level) + # NOISY-DEBUG: print("DEBUG: BEFORE block_level:", block_level) + + if block_level == "": + print("WARNING: block_level is now empty!") + continue + for blocked, reason in info.items(): # NOISY-DEBUG: print("DEBUG: BEFORE blocked:", blocked) blocked = fba.tidyup(blocked) @@ -95,8 +120,7 @@ for blocker, software in fba.c.fetchall(): if blocked == "": print("WARNING: blocked is empty after fba.tidyup():", blocker, block_level) continue - - if blocked.count("*") > 1: + elif blocked.count("*") > 1: # same domain guess as above, but for reasons field fba.c.execute( "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", (blocked.replace("*", "_"),) @@ -150,7 +174,11 @@ for blocker, software in fba.c.fetchall(): # NOISY-DEBUG: print("DEBUG: blocks():", len(blocks)) for block in blocks: - entry = {'domain': block['domain'], 'hash': block['digest'], 'reason': block['comment']} + entry = { + 'domain': block['domain'], + 'hash': block['digest'], + 'reason': block['comment'] + } # NOISY-DEBUG: print("DEBUG: severity,domain,hash,comment:", block['severity'], block['domain'], block['digest'], block['comment']) if block['severity'] == 'suspend': @@ -170,14 +198,22 @@ for blocker, software in fba.c.fetchall(): # NOISY-DEBUG: print("DEBUG: json.items():", blocker, len(json.items())) for block_level, blocks in json.items(): # NOISY-DEBUG: print("DEBUG: blocker,block_level,blocks():", blocker, block_level, len(blocks)) + block_level = fba.tidyup(block_level) + # NOISY-DEBUG: print("DEBUG: AFTER-block_level:", block_level) + if block_level == "": + print("WARNING: block_level is empty, blocker:", blocker) + continue + for instance in blocks: blocked, blocked_hash, reason = instance.values() # NOISY-DEBUG: print("DEBUG: blocked,hash,reason:", blocked, blocked_hash, reason) - blocked = fba.tidyup(blocked) - # NOISY-DEBUG: print("DEBUG: blocked:", blocked) + # NOISY-DEBUG: print("DEBUG: AFTER-blocked:", blocked) - if blocked.count("*") < 1: + if blocked == "": + print("WARNING: blocked is empty:", blocker) + continue + elif blocked.count("*") < 1: # No obsfucation for this instance fba.c.execute( "SELECT hash FROM instances WHERE domain = ? LIMIT 1", (blocked,) @@ -233,12 +269,24 @@ for blocker, software in fba.c.fetchall(): json = fba.get_friendica_blocks(blocker) elif software == "misskey": json = fba.get_misskey_blocks(blocker) + for block_level, blocks in json.items(): + # NOISY-DEBUG: print("DEBUG: blocker,block_level,blocks():", blocker, block_level, len(blocks)) + block_level = fba.tidyup(block_level) + # NOISY-DEBUG: print("DEBUG: AFTER-block_level:", block_level) + if block_level == "": + print("WARNING: block_level is empty, blocker:", blocker) + continue + for instance in blocks: blocked, reason = instance.values() + # NOISY-DEBUG: print("DEBUG: BEFORE-blocked:", blocked) blocked = fba.tidyup(blocked) + # NOISY-DEBUG: print("DEBUG: AFTER-blocked:", blocked) - # NOISY-DEBUG: print("DEBUG: BEFORE-blocked:", blocked) + if blocked == "": + print("WARNING: blocked is empty:", blocker) + continue if blocked.count("*") > 0: # Some friendica servers also obscure domains without hash fba.c.execute( @@ -308,8 +356,13 @@ for blocker, software in fba.c.fetchall(): break blocked = peer["domain"].lower() - # NOISY-DEBUG: print("DEBUG: blocked:", blocked) + # NOISY-DEBUG: print("DEBUG: BEFORE-blocked:", blocked) + blocked = fba.tidyup(blocked) + # NOISY-DEBUG: print("DEBUG: AFTER-blocked:", blocked) + if blocked == "": + print("WARNING: blocked is empty:", blocker) + continue if blocked.count("*") > 0: # GTS does not have hashes for obscured domains, so we have to guess it fba.c.execute( -- 2.39.5