From 294d8f587ca7b60f75e2f6675d0c5e4140289c60 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Wed, 17 May 2023 15:12:42 +0200 Subject: [PATCH] Continued: - since functions had been moved to module "fba", also the names must be changed --- api.py | 109 +++++++++++++++++++++------------------------ fba.py | 7 +++ fetch_blocks.py | 88 ++++++++++++++++++------------------ fetch_instances.py | 7 ++- 4 files changed, 105 insertions(+), 106 deletions(-) diff --git a/api.py b/api.py index 917101b..02de37d 100644 --- a/api.py +++ b/api.py @@ -5,51 +5,40 @@ from hashlib import sha256 from fastapi.templating import Jinja2Templates from requests import get from json import loads -from re import sub from datetime import datetime from email import utils +import re +import fba -with open("config.json") as f: - config = loads(f.read()) - base_url = config["base_url"] - port = config["port"] - log_level = config["log_level"] -app = FastAPI(docs_url=base_url+"/docs", redoc_url=base_url+"/redoc") +app = FastAPI(docs_url=fba.config["base_url"] + "/docs", redoc_url=fba.config["base_url"] + "/redoc") templates = Jinja2Templates(directory=".") -def get_hash(domain: str) -> str: - return sha256(domain.encode("utf-8")).hexdigest() - -@app.get(base_url+"/info") +@app.get(fba.config["base_url"] + "/info") def info(): - conn = sqlite3.connect("blocks.db") - c = conn.cursor() - c.execute("SELECT (SELECT count(domain) FROM instances), (SELECT count(domain) FROM instances WHERE software IN ('pleroma', 'mastodon', 'misskey', 'gotosocial', 'friendica')), (SELECT count(blocker) FROM blocks)") - known, indexed, blocks = c.fetchone() - c.close() + fba.c.execute("SELECT (SELECT count(domain) FROM instances), (SELECT count(domain) FROM instances WHERE software IN ('pleroma', 'mastodon', 'misskey', 'gotosocial', 'friendica')), (SELECT count(blocker) FROM blocks)") + known, indexed, blocks = fba.c.fetchone() + fba.c.close() return { "known_instances": known, "indexed_instances": indexed, "blocks_recorded": blocks, - "slogan": config["slogan"] + "slogan": fba.config["slogan"] } -@app.get(base_url+"/top") +@app.get(fba.config["base_url"] + "/top") def top(blocked: int = None, blockers: int = None): - conn = sqlite3.connect("blocks.db") - c = conn.cursor() if blocked == None and blockers == None: raise HTTPException(status_code=400, detail="No filter specified") elif blocked != None: if blocked > 500: raise HTTPException(status_code=400, detail="Too many results") - c.execute("SELECT blocked, count(blocked) FROM blocks WHERE block_level = 'reject' group by blocked ORDER BY count(blocked) DESC LIMIT ?", (blocked,)) + fba.c.execute("SELECT blocked, count(blocked) FROM blocks WHERE block_level = 'reject' group by blocked ORDER BY count(blocked) DESC LIMIT ?", (blocked,)) elif blockers != None: if blockers > 500: raise HTTPException(status_code=400, detail="Too many results") - c.execute("SELECT blocker, count(blocker) FROM blocks WHERE block_level = 'reject' group by blocker ORDER BY count(blocker) DESC LIMIT ?", (blockers,)) - scores = c.fetchall() - c.close() + fba.c.execute("SELECT blocker, count(blocker) FROM blocks WHERE block_level = 'reject' group by blocker ORDER BY count(blocker) DESC LIMIT ?", (blockers,)) + scores = fba.c.fetchall() + fba.c.close() scoreboard = [] print(scores) @@ -58,27 +47,25 @@ def top(blocked: int = None, blockers: int = None): return scoreboard -@app.get(base_url+"/api") +@app.get(fba.config["base_url"] + "/api") def blocked(domain: str = None, reason: str = None, reverse: str = None): if domain == None and reason == None and reverse == None: raise HTTPException(status_code=400, detail="No filter specified") if reason != None: - reason = sub("(%|_)", "", reason) + reason = re.sub("(%|_)", "", reason) if len(reason) < 3: raise HTTPException(status_code=400, detail="Keyword is shorter than three characters") - conn = sqlite3.connect("blocks.db") - c = conn.cursor() if domain != None: wildchar = "*." + ".".join(domain.split(".")[-domain.count("."):]) punycode = domain.encode('idna').decode('utf-8') - c.execute("SELECT blocker, blocked, block_level, reason, first_added, last_seen FROM blocks WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? ORDER BY first_added asc", - (domain, "*." + domain, wildchar, get_hash(domain), punycode, "*." + punycode)) + fba.c.execute("SELECT blocker, blocked, block_level, reason, first_added, last_seen FROM blocks WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? ORDER BY first_added asc", + (domain, "*." + domain, wildchar, fba.get_hash(domain), punycode, "*." + punycode)) elif reverse != None: - c.execute("SELECT blocker, blocked, block_level, reason, first_added, last_seen FROM blocks WHERE blocker = ? ORDER BY first_added asc", (reverse,)) + fba.c.execute("SELECT blocker, blocked, block_level, reason, first_added, last_seen FROM blocks WHERE blocker = ? ORDER BY first_added asc", (reverse,)) else: - c.execute("SELECT blocker, blocked, block_level, reason, first_added, last_seen FROM blocks WHERE reason like ? AND reason != '' ORDER BY first_added asc", ("%"+reason+"%",)) - blocks = c.fetchall() - c.close() + fba.c.execute("SELECT blocker, blocked, block_level, reason, first_added, last_seen FROM blocks WHERE reason like ? AND reason != '' ORDER BY first_added asc", ("%"+reason+"%",)) + blocks = fba.c.fetchall() + fba.c.close() result = {} for blocker, blocked, block_level, reason, first_added, last_seen in blocks: @@ -90,37 +77,37 @@ def blocked(domain: str = None, reason: str = None, reverse: str = None): return result -@app.get(base_url+"/scoreboard") +@app.get(fba.config["base_url"] + "/scoreboard") def index(request: Request, blockers: int = None, blocked: int = None): if blockers == None and blocked == None: raise HTTPException(status_code=400, detail="No filter specified") elif blockers != None: - scores = get(f"http://127.0.0.1:{port}{base_url}/top?blockers={blockers}") + scores = get(f"http://127.0.0.1:{fba.config['base_url']}{fba.config['base_url']}/top?blockers={blockers}") elif blocked != None: - scores = get(f"http://127.0.0.1:{port}{base_url}/top?blocked={blocked}") + scores = get(f"http://127.0.0.1:{fba.config['base_url']}{fba.config['base_url']}/top?blocked={blocked}") if scores != None: if not scores.ok: raise HTTPException(status_code=blocks.status_code, detail=blocks.text) scores = scores.json() return templates.TemplateResponse("index.html", {"request": request, "scoreboard": True, "blockers": blockers, "blocked": blocked, "scores": scores}) -@app.get(base_url+"/") +@app.get(fba.config["base_url"] + "/") def index(request: Request, domain: str = None, reason: str = None, reverse: str = None): if domain == "" or reason == "" or reverse == "": return responses.RedirectResponse("/") info = None blocks = None if domain == None and reason == None and reverse == None: - info = get(f"http://127.0.0.1:{port}{base_url}/info") + info = get(f"http://127.0.0.1:{fba.config['base_url']}{fba.config['base_url']}/info") if not info.ok: raise HTTPException(status_code=info.status_code, detail=info.text) info = info.json() elif domain != None: - blocks = get(f"http://127.0.0.1:{port}{base_url}/api?domain={domain}") + blocks = get(f"http://127.0.0.1:{fba.config['base_url']}{fba.config['base_url']}/api?domain={domain}") elif reason != None: - blocks = get(f"http://127.0.0.1:{port}{base_url}/api?reason={reason}") + blocks = get(f"http://127.0.0.1:{fba.config['base_url']}{fba.config['base_url']}/api?reason={reason}") elif reverse != None: - blocks = get(f"http://127.0.0.1:{port}{base_url}/api?reverse={reverse}") + blocks = get(f"http://127.0.0.1:{fba.config['base_url']}{fba.config['base_url']}/api?reverse={reverse}") if blocks != None: if not blocks.ok: raise HTTPException(status_code=blocks.status_code, detail=blocks.text) @@ -132,12 +119,10 @@ def index(request: Request, domain: str = None, reason: str = None, reverse: str return templates.TemplateResponse("index.html", {"request": request, "domain": domain, "blocks": blocks, "reason": reason, "reverse": reverse, "info": info}) -@app.get(base_url+"/api/mutual") +@app.get(fba.config["base_url"] + "/api/mutual") def mutual(domains: list[str] = Query()): """Return 200 if federation is open between the two, 4xx otherwise""" - conn = sqlite3.connect('blocks.db') - c = conn.cursor() - c.execute( + fba.c.execute( "SELECT block_level FROM blocks " \ "WHERE ((blocker = :a OR blocker = :b) AND (blocked = :b OR blocked = :a OR blocked = :aw OR blocked = :bw)) " \ "AND block_level = 'reject' " \ @@ -149,27 +134,26 @@ def mutual(domains: list[str] = Query()): "bw": "*." + domains[1], }, ) - res = c.fetchone() - c.close() + res = fba.c.fetchone() + fba.c.close() if res is not None: # Blocks found return responses.JSONResponse(status_code=418, content={}) # No known blocks return responses.JSONResponse(status_code=200, content={}) -@app.get(base_url+"/rss") +@app.get(fba.config["base_url"] + "/rss") def rss(request: Request, domain: str = None): - conn = sqlite3.connect("blocks.db") - c = conn.cursor() if domain != None: wildchar = "*." + ".".join(domain.split(".")[-domain.count("."):]) punycode = domain.encode('idna').decode('utf-8') - c.execute("SELECT blocker, blocked, block_level, reason, first_added, last_seen FROM blocks WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? ORDER BY first_added DESC LIMIT 50", - (domain, "*." + domain, wildchar, get_hash(domain), punycode, "*." + punycode)) + fba.c.execute("SELECT blocker, blocked, block_level, reason, first_added, last_seen FROM blocks WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? ORDER BY first_added DESC LIMIT 50", + (domain, "*." + domain, wildchar, fba.get_hash(domain), punycode, "*." + punycode)) else: - c.execute("SELECT blocker, blocked, block_level, reason, first_added, last_seen FROM blocks ORDER BY first_added DESC LIMIT 50") - blocks = c.fetchall() - c.close() + fba.c.execute("SELECT blocker, blocked, block_level, reason, first_added, last_seen FROM blocks ORDER BY first_added DESC LIMIT 50") + + blocks = fba.c.fetchall() + fba.c.close() result = [] for blocker, blocked, block_level, reason, first_added, last_seen in blocks: @@ -178,11 +162,20 @@ def rss(request: Request, domain: str = None): reason = "No reason provided." else: reason = "Provided reason: '" + reason + "'" - result.append({"blocker": blocker, "blocked": blocked, "block_level": block_level, "reason": reason, "first_added": first_added}) + + result.append( + { + "blocker": blocker, + "blocked": blocked, + "block_level": block_level, + "reason": reason, + "first_added": first_added + } + ) timestamp = utils.format_datetime(datetime.now()) return templates.TemplateResponse("rss.xml", {"request": request, "timestamp": timestamp, "domain": domain, "blocks": result}, headers={"Content-Type": "application/rss+xml"}) if __name__ == "__main__": - uvicorn.run("api:app", host="127.0.0.1", port=port, log_level=log_level) + uvicorn.run("api:app", host="127.0.0.1", port=fba.config["port"], log_level=fba.config["log_level"]) diff --git a/fba.py b/fba.py index c93996d..8614e32 100644 --- a/fba.py +++ b/fba.py @@ -3,6 +3,7 @@ from reqto import post from bs4 import BeautifulSoup from reqto import get from hashlib import sha256 +import re import sqlite3 import json import sys @@ -301,14 +302,20 @@ def get_misskey_blocks(domain: str) -> dict: def tidyup(domain: str) -> str: # some retards put their blocks in variable case domain = domain.lower() + # other retards put the port domain = re.sub("\:\d+$", "", domain) + # bigger retards put the schema in their blocklist, sometimes even without slashes domain = re.sub("^https?\:(\/*)", "", domain) + # and trailing slash domain = re.sub("\/$", "", domain) + # and the @ domain = re.sub("^\@", "", domain) + # the biggest retards of them all try to block individual users domain = re.sub("(.+)\@", "", domain) + return domain diff --git a/fetch_blocks.py b/fetch_blocks.py index 5ded601..bcf4576 100644 --- a/fetch_blocks.py +++ b/fetch_blocks.py @@ -7,15 +7,14 @@ from json import loads import re from time import time import itertools -from fba import c import fba -c.execute( +fba.c.execute( "SELECT domain, software FROM instances WHERE domain='tooting.intensifi.es'" #"SELECT domain, software FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial')" ) -for blocker, software in c.fetchall(): +for blocker, software in fba.c.fetchall(): blockdict = [] blocker = fba.tidyup(blocker) if software == "pleroma": @@ -39,22 +38,22 @@ for blocker, software in c.fetchall(): if blocked.count("*") > 1: # -ACK!-oma also started obscuring domains without hash - c.execute( + fba.c.execute( "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", (blocked.replace("*", "_"),) ) - searchres = c.fetchone() + searchres = fba.c.fetchone() if searchres != None: blocked = searchres[0] - c.execute( + fba.c.execute( "SELECT domain FROM instances WHERE domain = ?", (blocked) ) - if c.fetchone() == None: - add_instance(blocked) + if fba.c.fetchone() == None: + fba.add_instance(blocked) timestamp = int(time()) - c.execute( + fba.c.execute( "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ?", ( blocker, @@ -62,7 +61,7 @@ for blocker, software in c.fetchall(): block_level ), ) - if c.fetchone() == None: + if fba.c.fetchone() == None: block_instance(blocker, blocked, reason, block_level, timestamp, timestamp) if block_level == "reject": @@ -72,7 +71,8 @@ for blocker, software in c.fetchall(): "reason": None }) else: - update_last_seen(timestamp, blocker, blocked, block_level) + fba.update_last_seen(timestamp, blocker, blocked, block_level) + fba.conn.commit() # Reasons if "mrf_simple_info" in federation: @@ -91,15 +91,15 @@ for blocker, software in c.fetchall(): if blocked.count("*") > 1: # same domain guess as above, but for reasons field - c.execute( + fba.c.execute( "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", (blocked.replace("*", "_"),) ) - searchres = c.fetchone() + searchres = fba.c.fetchone() if searchres != None: blocked = searchres[0] - update_block_reason(reason["reason"], blocker, blocked, block_level) + fba.update_block_reason(reason["reason"], blocker, blocked, block_level) for entry in blockdict: if entry["blocked"] == blocked: @@ -156,23 +156,23 @@ for blocker, software in c.fetchall(): blocked = fba.tidyup(blocked) if blocked.count("*") <= 1: - c.execute( + fba.c.execute( "SELECT hash FROM instances WHERE hash = ?", (blocked_hash,) ) - if c.fetchone() == None: - add_instance(blocked) + if fba.c.fetchone() == None: + fba.add_instance(blocked) else: # Doing the hash search for instance names as well to tidy up DB - c.execute( + fba.c.execute( "SELECT domain FROM instances WHERE hash = ?", (blocked_hash,) ) - searchres = c.fetchone() + searchres = fba.c.fetchone() if searchres != None: blocked = searchres[0] timestamp = int(time()) - c.execute( + fba.c.execute( "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ?", ( blocker, @@ -180,7 +180,7 @@ for blocker, software in c.fetchall(): block_level ), ) - if c.fetchone() == None: + if fba.c.fetchone() == None: block_instance(blocker, blocked if blocked.count("*") <= 1 else blocked_hash, reason, block_level, timestamp, timestamp) if block_level == "reject": @@ -190,10 +190,10 @@ for blocker, software in c.fetchall(): "reason": reason }) else: - update_last_seen(timestamp, blocker, blocked if blocked.count("*") <= 1 else blocked_hash, block_level) + fba.update_last_seen(timestamp, blocker, blocked if blocked.count("*") <= 1 else blocked_hash, block_level) if reason != '': - update_block_reason(reason, blocker, blocked if blocked.count("*") <= 1 else blocked_hash, block_level) + fba.update_block_reason(reason, blocker, blocked if blocked.count("*") <= 1 else blocked_hash, block_level) fba.conn.commit() except Exception as e: @@ -213,36 +213,36 @@ for blocker, software in c.fetchall(): print("BEFORE-blocked:", blocked) if blocked.count("*") > 0: # Some friendica servers also obscure domains without hash - c.execute( + fba.c.execute( "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", (blocked.replace("*", "_"),) ) - searchres = c.fetchone() + searchres = fba.c.fetchone() if searchres != None: blocked = searchres[0] if blocked.count("?") > 0: # Some obscure them with question marks, not sure if that's dependent on version or not - c.execute( + fba.c.execute( "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", (blocked.replace("?", "_"),) ) - searchres = c.fetchone() + searchres = fba.c.fetchone() if searchres != None: blocked = searchres[0] print("AFTER-blocked:", blocked) - c.execute( + fba.c.execute( "SELECT domain FROM instances WHERE domain = ?", (blocked,) ) - if c.fetchone() == None: - add_instance(blocked) + if fba.c.fetchone() == None: + fba.add_instance(blocked) timestamp = int(time()) - c.execute( + fba.c.execute( "SELECT * FROM blocks WHERE blocker = ? AND blocked = ?", (blocker, blocked), ) - if c.fetchone() == None: + if fba.c.fetchone() == None: block_instance(blocker, blocked, reason, block_level, timestamp, timestamp) if block_level == "reject": @@ -252,10 +252,10 @@ for blocker, software in c.fetchall(): "reason": reason }) else: - update_last_seen(timestamp, blocker, blocked, block_level) + fba.update_last_seen(timestamp, blocker, blocked, block_level) if reason != '': - update_block_reason(reason, blocker, blocked, block_level) + fba.update_block_reason(reason, blocker, blocked, block_level) fba.conn.commit() except Exception as e: @@ -276,28 +276,28 @@ for blocker, software in c.fetchall(): if blocked.count("*") > 0: # GTS does not have hashes for obscured domains, so we have to guess it - c.execute( + fba.c.execute( "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", (blocked.replace("*", "_"),) ) - searchres = c.fetchone() + searchres = fba.c.fetchone() if searchres != None: blocked = searchres[0] - c.execute( + fba.c.execute( "SELECT domain FROM instances WHERE domain = ?", (blocked,) ) - if c.fetchone() == None: - add_instance(blocked) + if fba.c.fetchone() == None: + fba.add_instance(blocked) - c.execute( + fba.c.execute( "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ?", (blocker, blocked, "reject"), ) timestamp = int(time()) - if c.fetchone() == None: + if fba.c.fetchone() == None: block_instance(blocker, blocked, "", "reject", timestamp, timestamp) blockdict.append( @@ -306,11 +306,11 @@ for blocker, software in c.fetchall(): "reason": None }) else: - update_last_seen(timestamp, blocker, blocked, "reject") + fba.update_last_seen(timestamp, blocker, blocked, "reject") if "public_comment" in peer: reason = peer["public_comment"] - update_block_reason(reason, blocker, blocked, "reject") + fba.update_block_reason(reason, blocker, blocked, "reject") for entry in blockdict: if entry["blocked"] == blocked: @@ -321,7 +321,7 @@ for blocker, software in c.fetchall(): else: print("WARNING: Unknown software:", software) - if config["bot_enabled"] and len(blockdict) > 0: + if fba.config["bot_enabled"] and len(blockdict) > 0: send_bot_post(blocker, blockdict) blockdict = [] diff --git a/fetch_instances.py b/fetch_instances.py index f25b440..346ece0 100644 --- a/fetch_instances.py +++ b/fetch_instances.py @@ -1,7 +1,6 @@ import sqlite3 import sys import json -from fba import c import fba domain = sys.argv[1] @@ -12,7 +11,7 @@ if (peerlist is None): print("FATAL: CANNOT FETCH PEERS:", domain) sys.exit(255) -c.execute( +fba.c.execute( "SELECT domain FROM instances WHERE 1" ) @@ -30,11 +29,11 @@ for instance in peerlist: print("Handling instance:", instance) try: - c.execute( + fba.c.execute( "SELECT domain FROM instances WHERE domain = ?", (instance,) ) - if c.fetchone() == None: + if fba.c.fetchone() == None: fba.add_instance(instance) fba.conn.commit() -- 2.39.5