From 96129f482ce5c0a128ac0cd88f5d9df5afbd7dd6 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Sun, 21 May 2023 17:18:38 +0200 Subject: [PATCH] Continued: - don't shorten variable names, wrong lazyness --- api.py | 32 ++++++++++----------- fba.py | 20 +++++++------ fetch_blocks.py | 72 +++++++++++++++++++++++----------------------- fetch_instances.py | 16 +++++------ 4 files changed, 71 insertions(+), 69 deletions(-) diff --git a/api.py b/api.py index e595f61..7ad7c33 100644 --- a/api.py +++ b/api.py @@ -14,8 +14,8 @@ templates = Jinja2Templates(directory=".") @app.get(fba.config["base_url"] + "/info") def info(): - fba.c.execute("SELECT (SELECT COUNT(domain) FROM instances), (SELECT COUNT(domain) FROM instances WHERE software IN ('pleroma', 'mastodon', 'misskey', 'gotosocial', 'friendica')), (SELECT COUNT(blocker) FROM blocks)") - known, indexed, blocks = fba.c.fetchone() + fba.cursor.execute("SELECT (SELECT COUNT(domain) FROM instances), (SELECT COUNT(domain) FROM instances WHERE software IN ('pleroma', 'mastodon', 'misskey', 'gotosocial', 'friendica')), (SELECT COUNT(blocker) FROM blocks)") + known, indexed, blocks = fba.cursor.fetchone() return { "known_instances" : known, @@ -29,23 +29,23 @@ def top(blocked: int = None, blockers: int = None, reference: int = None, softwa if blocked != None: if blocked > 500: raise HTTPException(status_code=400, detail="Too many results") - fba.c.execute("SELECT blocked, COUNT(blocked) FROM blocks WHERE block_level = 'reject' GROUP BY blocked ORDER BY COUNT(blocked) DESC LIMIT ?", [blocked]) + fba.cursor.execute("SELECT blocked, COUNT(blocked) FROM blocks WHERE block_level = 'reject' GROUP BY blocked ORDER BY COUNT(blocked) DESC LIMIT ?", [blocked]) elif blockers != None: if blockers > 500: raise HTTPException(status_code=400, detail="Too many results") - fba.c.execute("SELECT blocker, COUNT(blocker) FROM blocks WHERE block_level = 'reject' GROUP BY blocker ORDER BY COUNT(blocker) DESC LIMIT ?", [blockers]) + fba.cursor.execute("SELECT blocker, COUNT(blocker) FROM blocks WHERE block_level = 'reject' GROUP BY blocker ORDER BY COUNT(blocker) DESC LIMIT ?", [blockers]) elif reference != None: if reference > 500: raise HTTPException(status_code=400, detail="Too many results") - fba.c.execute("SELECT origin, COUNT(domain) FROM instances GROUP BY origin ORDER BY COUNT(domain) DESC LIMIT ?", [reference]) + fba.cursor.execute("SELECT origin, COUNT(domain) FROM instances GROUP BY origin ORDER BY COUNT(domain) DESC LIMIT ?", [reference]) elif software != None: if software > 500: raise HTTPException(status_code=400, detail="Too many results") - fba.c.execute("SELECT software, COUNT(domain) FROM instances GROUP BY software ORDER BY COUNT(domain) DESC LIMIT ?", [software]) + fba.cursor.execute("SELECT software, COUNT(domain) FROM instances GROUP BY software ORDER BY COUNT(domain) DESC LIMIT ?", [software]) else: raise HTTPException(status_code=400, detail="No filter specified") - scores = fba.c.fetchall() + scores = fba.cursor.fetchall() scoreboard = [] @@ -65,13 +65,13 @@ def blocked(domain: str = None, reason: str = None, reverse: str = None): if domain != None: wildchar = "*." + ".".join(domain.split(".")[-domain.count("."):]) punycode = domain.encode('idna').decode('utf-8') - fba.c.execute("SELECT blocker, blocked, block_level, reason, first_added, last_seen FROM blocks WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? ORDER BY first_added ASC", + fba.cursor.execute("SELECT blocker, blocked, block_level, reason, first_added, last_seen FROM blocks WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? ORDER BY first_added ASC", (domain, "*." + domain, wildchar, fba.get_hash(domain), punycode, "*." + punycode)) elif reverse != None: - fba.c.execute("SELECT blocker, blocked, block_level, reason, first_added, last_seen FROM blocks WHERE blocker = ? ORDER BY first_added ASC", [reverse]) + fba.cursor.execute("SELECT blocker, blocked, block_level, reason, first_added, last_seen FROM blocks WHERE blocker = ? ORDER BY first_added ASC", [reverse]) else: - fba.c.execute("SELECT blocker, blocked, block_level, reason, first_added, last_seen FROM blocks WHERE reason like ? AND reason != '' ORDER BY first_added ASC", ["%" + reason + "%"]) - blocks = fba.c.fetchall() + fba.cursor.execute("SELECT blocker, blocked, block_level, reason, first_added, last_seen FROM blocks WHERE reason like ? AND reason != '' ORDER BY first_added ASC", ["%" + reason + "%"]) + blocks = fba.cursor.fetchall() result = {} for blocker, blocked, block_level, reason, first_added, last_seen in blocks: @@ -156,7 +156,7 @@ def index(request: Request, domain: str = None, reason: str = None, reverse: str @app.get(fba.config["base_url"] + "/api/mutual") def mutual(domains: list[str] = Query()): """Return 200 if federation is open between the two, 4xx otherwise""" - fba.c.execute( + fba.cursor.execute( "SELECT block_level FROM blocks " \ "WHERE ((blocker = :a OR blocker = :b) AND (blocked = :b OR blocked = :a OR blocked = :aw OR blocked = :bw)) " \ "AND block_level = 'reject' " \ @@ -168,7 +168,7 @@ def mutual(domains: list[str] = Query()): "bw": "*." + domains[1], }, ) - res = fba.c.fetchone() + res = fba.cursor.fetchone() if res is not None: # Blocks found @@ -182,12 +182,12 @@ def rss(request: Request, domain: str = None): if domain != None: wildchar = "*." + ".".join(domain.split(".")[-domain.count("."):]) punycode = domain.encode('idna').decode('utf-8') - fba.c.execute("SELECT blocker, blocked, block_level, reason, first_added, last_seen FROM blocks WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? ORDER BY first_added DESC LIMIT 50", + fba.cursor.execute("SELECT blocker, blocked, block_level, reason, first_added, last_seen FROM blocks WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? ORDER BY first_added DESC LIMIT 50", (domain, "*." + domain, wildchar, fba.get_hash(domain), punycode, "*." + punycode)) else: - fba.c.execute("SELECT blocker, blocked, block_level, reason, first_added, last_seen FROM blocks ORDER BY first_added DESC LIMIT 50") + fba.cursor.execute("SELECT blocker, blocked, block_level, reason, first_added, last_seen FROM blocks ORDER BY first_added DESC LIMIT 50") - blocks = fba.c.fetchall() + blocks = fba.cursor.fetchall() result = [] for blocker, blocked, block_level, reason, first_added, last_seen in blocks: diff --git a/fba.py b/fba.py index c48ac88..4505d44 100644 --- a/fba.py +++ b/fba.py @@ -24,8 +24,8 @@ headers = { "user-agent": config["useragent"] } -conn = sqlite3.connect("blocks.db") -c = conn.cursor() +connection = sqlite3.connect("blocks.db") +cursor = connection.cursor() def is_blacklisted(domain: str) -> bool: # NOISY-DEBUG: print("DEBUG: Checking blacklist for domain:", domain) @@ -46,7 +46,7 @@ def update_last_blocked(domain: str): # NOISY-DEBUG: print("DEBUG: Updating last_blocked for domain", domain) try: - c.execute("UPDATE instances SET last_blocked = ?, last_updated = ? WHERE domain = ?", [ + cursor.execute("UPDATE instances SET last_blocked = ?, last_updated = ? WHERE domain = ?", [ time.time(), time.time(), domain @@ -59,7 +59,7 @@ def update_last_blocked(domain: str): def update_last_error(domain: str, res: any): # NOISY-DEBUG: print("DEBUG: domain,res.status_code:", domain, res.status_code, res.reason) try: - c.execute("UPDATE instances SET last_status_code = ?, last_error_details = ?, last_updated = ? WHERE domain = ?", [ + cursor.execute("UPDATE instances SET last_status_code = ?, last_error_details = ?, last_updated = ? WHERE domain = ?", [ res.status_code, res.reason, time.time(), @@ -73,7 +73,7 @@ def update_last_error(domain: str, res: any): def update_last_nodeinfo(domain: str): # NOISY-DEBUG: print("DEBUG: Updating last_nodeinfo for domain:", domain) try: - c.execute("UPDATE instances SET last_nodeinfo = ?, last_updated = ? WHERE domain = ?", [ + cursor.execute("UPDATE instances SET last_nodeinfo = ?, last_updated = ? WHERE domain = ?", [ time.time(), time.time(), domain @@ -83,6 +83,8 @@ def update_last_nodeinfo(domain: str): print("ERROR: failed SQL query:", domain) sys.exit(255) + connection.commit() + def get_peers(domain: str) -> list: # NOISY-DEBUG: print("DEBUG: Getting peers for domain:", domain) peers = None @@ -188,7 +190,7 @@ def determine_software(domain: str) -> str: def update_block_reason(reason: str, blocker: str, blocked: str, block_level: str): # NOISY: print("--- Updating block reason:", reason, blocker, blocked, block_level) try: - c.execute( + cursor.execute( "UPDATE blocks SET reason = ?, last_seen = ? WHERE blocker = ? AND blocked = ? AND block_level = ? AND reason = ''", ( reason, @@ -206,7 +208,7 @@ def update_block_reason(reason: str, blocker: str, blocked: str, block_level: st def update_last_seen(blocker: str, blocked: str, block_level: str): # NOISY: print("--- Updating last_seen for:", blocker, blocked, block_level) try: - c.execute( + cursor.execute( "UPDATE blocks SET last_seen = ? WHERE blocker = ? AND blocked = ? AND block_level = ?", ( time.time(), @@ -231,7 +233,7 @@ def block_instance(blocker: str, blocked: str, reason: str, block_level: str): print("--- New block:", blocker, blocked, reason, block_level, first_added, last_seen) try: - c.execute( + cursor.execute( "INSERT INTO blocks (blocker, blocked, reason, block_level, first_added, last_seen) VALUES(?, ?, ?, ?, ?, ?)", ( blocker, @@ -258,7 +260,7 @@ def add_instance(domain: str, origin: str, originator: str): print(f"--- Adding new instance {domain} (origin: {origin})") try: - c.execute( + cursor.execute( "INSERT INTO instances (domain, origin, originator, hash, software, first_seen) VALUES (?, ?, ?, ?, ?, ?)", ( domain, diff --git a/fetch_blocks.py b/fetch_blocks.py index ba02a3e..6363971 100644 --- a/fetch_blocks.py +++ b/fetch_blocks.py @@ -5,11 +5,11 @@ import itertools import re import fba -fba.c.execute( +fba.cursor.execute( "SELECT domain, software FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial') AND (last_blocked IS NULL OR last_blocked < ?) ORDER BY rowid DESC", [time.time() - fba.config["recheck_block"]] ) -for blocker, software in fba.c.fetchall(): +for blocker, software in fba.cursor.fetchall(): # NOISY-DEBUG: print("DEBUG: BEFORE-blocker,software:", blocker, software) blockdict = [] blocker = fba.tidyup(blocker) @@ -60,25 +60,25 @@ for blocker, software in fba.c.fetchall(): if blocked.count("*") > 1: # -ACK!-oma also started obscuring domains without hash - fba.c.execute( + fba.cursor.execute( "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")] ) - searchres = fba.c.fetchone() + searchres = fba.cursor.fetchone() # NOISY-DEBUG: print("DEBUG: searchres[]:", type(searchres)) if searchres != None: blocked = searchres[0] # NOISY-DEBUG: print("DEBUG: Looked up domain:", blocked) # NOISY-DEBUG: print("DEBUG: Looking up instance by domain:", blocked) - fba.c.execute( + fba.cursor.execute( "SELECT domain FROM instances WHERE domain = ?", [blocked] ) - if fba.c.fetchone() == None: + if fba.cursor.fetchone() == None: # NOISY-DEBUG: print("DEBUG: Domain wasn't found, adding:", blocked, blocker) fba.add_instance(blocked, blocker, argv[0]) - fba.c.execute( + fba.cursor.execute( "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ?", ( blocker, @@ -87,7 +87,7 @@ for blocker, software in fba.c.fetchall(): ), ) - if fba.c.fetchone() == None: + if fba.cursor.fetchone() == None: # NOISY-DEBUG: print("DEBUG: Blocking:", blocker, blocked, block_level) fba.block_instance(blocker, blocked, "unknown", block_level) @@ -102,7 +102,7 @@ for blocker, software in fba.c.fetchall(): # NOISY-DEBUG: print("DEBUG: Updating last_seen:", blocker, blocked, block_level) fba.update_last_seen(blocker, blocked, block_level) - fba.conn.commit() + fba.connection.commit() # Reasons if "mrf_simple_info" in federation: @@ -131,10 +131,10 @@ for blocker, software in fba.c.fetchall(): continue elif blocked.count("*") > 1: # same domain guess as above, but for reasons field - fba.c.execute( + fba.cursor.execute( "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")] ) - searchres = fba.c.fetchone() + searchres = fba.cursor.fetchone() if searchres != None: blocked = searchres[0] @@ -147,7 +147,7 @@ for blocker, software in fba.c.fetchall(): # NOISY-DEBUG: print("DEBUG: Updating entry reason:", blocked) entry["reason"] = reason["reason"] - fba.conn.commit() + fba.connection.commit() except Exception as e: print("error:", e, blocker, software) elif software == "mastodon": @@ -222,25 +222,25 @@ for blocker, software in fba.c.fetchall(): continue elif blocked.count("*") < 1: # No obsfucation for this instance - fba.c.execute( + fba.cursor.execute( "SELECT hash FROM instances WHERE domain = ? LIMIT 1", [blocked] ) - if fba.c.fetchone() == None: + if fba.cursor.fetchone() == None: # NOISY-DEBUG: print("DEBUG: Hash wasn't found, adding:", blocked, blocker) fba.add_instance(blocked, blocker, argv[0]) else: # Doing the hash search for instance names as well to tidy up DB - fba.c.execute( + fba.cursor.execute( "SELECT domain FROM instances WHERE hash = ? LIMIT 1", [blocked_hash] ) - searchres = fba.c.fetchone() + searchres = fba.cursor.fetchone() if searchres != None: # NOISY-DEBUG: print("DEBUG: Updating domain: ", searchres[0]) blocked = searchres[0] - fba.c.execute( + fba.cursor.execute( "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ?", ( blocker, @@ -249,7 +249,7 @@ for blocker, software in fba.c.fetchall(): ), ) - if fba.c.fetchone() == None: + if fba.cursor.fetchone() == None: fba.block_instance(blocker, blocked if blocked.count("*") <= 1 else blocked_hash, reason, block_level) if block_level == "reject": @@ -265,7 +265,7 @@ for blocker, software in fba.c.fetchall(): # NOISY-DEBUG: print("DEBUG: Updating block reason:", blocker, blocked, reason) fba.update_block_reason(reason, blocker, blocked if blocked.count("*") <= 1 else blocked_hash, block_level) - fba.conn.commit() + fba.connection.commit() except Exception as e: print("error:", e, blocker, software) elif software == "friendica" or software == "misskey": @@ -295,36 +295,36 @@ for blocker, software in fba.c.fetchall(): continue if blocked.count("*") > 0: # Some friendica servers also obscure domains without hash - fba.c.execute( + fba.cursor.execute( "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")] ) - searchres = fba.c.fetchone() + searchres = fba.cursor.fetchone() if searchres != None: blocked = searchres[0] if blocked.count("?") > 0: # Some obscure them with question marks, not sure if that's dependent on version or not - fba.c.execute( + fba.cursor.execute( "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("?", "_")] ) - searchres = fba.c.fetchone() + searchres = fba.cursor.fetchone() if searchres != None: blocked = searchres[0] # NOISY-DEBUG: print("DEBUG: AFTER-blocked:", blocked) - fba.c.execute( + fba.cursor.execute( "SELECT domain FROM instances WHERE domain = ?", [blocked] ) - if fba.c.fetchone() == None: + if fba.cursor.fetchone() == None: # NOISY-DEBUG: print("DEBUG: Hash wasn't found, adding:", blocked, blocker) fba.add_instance(blocked, blocker) - fba.c.execute( + fba.cursor.execute( "SELECT * FROM blocks WHERE blocker = ? AND blocked = ?", (blocker, blocked), ) - if fba.c.fetchone() == None: + if fba.cursor.fetchone() == None: fba.block_instance(blocker, blocked, reason, block_level) if block_level == "reject": @@ -340,7 +340,7 @@ for blocker, software in fba.c.fetchall(): # NOISY-DEBUG: print("DEBUG: Updating block reason:", blocker, blocked, reason) fba.update_block_reason(reason, blocker, blocked, block_level) - fba.conn.commit() + fba.connection.commit() except Exception as e: print("error:", e, blocker, software) elif software == "gotosocial": @@ -366,23 +366,23 @@ for blocker, software in fba.c.fetchall(): continue if blocked.count("*") > 0: # GTS does not have hashes for obscured domains, so we have to guess it - fba.c.execute( + fba.cursor.execute( "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")] ) - searchres = fba.c.fetchone() + searchres = fba.cursor.fetchone() if searchres != None: blocked = searchres[0] - fba.c.execute( + fba.cursor.execute( "SELECT domain FROM instances WHERE domain = ?", [blocked] ) - if fba.c.fetchone() == None: + if fba.cursor.fetchone() == None: # NOISY-DEBUG: print("DEBUG: Updating block reason:", blocker, blocked, reason) fba.update_block_reason(reason, blocker, blocked, block_level) - fba.c.execute( + fba.cursor.execute( "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ?", ( blocker, @@ -391,7 +391,7 @@ for blocker, software in fba.c.fetchall(): ), ) - if fba.c.fetchone() == None: + if fba.cursor.fetchone() == None: fba.block_instance(blocker, blocked, "unknown", "reject") blockdict.append( @@ -411,7 +411,7 @@ for blocker, software in fba.c.fetchall(): if entry["blocked"] == blocked: entry["reason"] = reason - fba.conn.commit() + fba.connection.commit() except Exception as e: print("error:", e, blocker, software) else: @@ -422,4 +422,4 @@ for blocker, software in fba.c.fetchall(): blockdict = [] -fba.conn.close() +fba.connection.close() diff --git a/fetch_instances.py b/fetch_instances.py index 5a9f534..0975ff4 100644 --- a/fetch_instances.py +++ b/fetch_instances.py @@ -12,11 +12,11 @@ def fetch_instances(domain: str, origin: str): print("ERROR: Cannot fetch peers:", domain) return - fba.c.execute( + fba.cursor.execute( "SELECT domain FROM instances WHERE domain = ? LIMIT 1", [domain] ) - if fba.c.fetchone() == None: + if fba.cursor.fetchone() == None: # NOISY-DEBUG: print("DEBUG: Adding new domain:", domain, origin) fba.add_instance(domain, origin, sys.argv[0]) @@ -34,15 +34,15 @@ def fetch_instances(domain: str, origin: str): # NOISY-DEBUG: print("DEBUG: Handling instance:", instance) try: - fba.c.execute( + fba.cursor.execute( "SELECT domain FROM instances WHERE domain = ? LIMIT 1", [instance] ) - if fba.c.fetchone() == None: + if fba.cursor.fetchone() == None: # NOISY-DEBUG: print("DEBUG: Adding new instance:", instance, domain) fba.add_instance(instance, domain, sys.argv[0]) - fba.conn.commit() + fba.connection.commit() except Exception as e: print("ERROR:", e, instance) @@ -54,11 +54,11 @@ instance = sys.argv[1] fetch_instances(instance, None) # Loop through some instances -fba.c.execute( +fba.cursor.execute( "SELECT domain FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial') AND (last_nodeinfo IS NULL OR last_nodeinfo < ?) ORDER BY rowid DESC", [time.time() - fba.config["recheck_instance"]] ) -for instance in fba.c.fetchall(): +for instance in fba.cursor.fetchall(): if fba.is_blacklisted(instance[0]): # NOISY-DEBUG: print("DEBUG: domain is blacklisted:", instance) continue @@ -66,4 +66,4 @@ for instance in fba.c.fetchall(): print("INFO: Fetching instances for instance:", instance[0]) fetch_instances(instance[0], None) -fba.conn.close() +fba.connection.close() -- 2.39.5