X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;f=api.py;h=5ba8937f4e990092c67cc6ada82caa0231dba4f6;hb=34ad7a718cfd8370f978b5a085087f140f1b0a70;hp=d89cf3f8ce9dfdecaf81150f74e358df2c91bbba;hpb=bf22fd754cd661f7260dbf599e525433c0bed81f;p=fba.git diff --git a/api.py b/api.py index d89cf3f..5ba8937 100644 --- a/api.py +++ b/api.py @@ -14,96 +14,156 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import re + +from datetime import datetime +from email.utils import format_datetime from fastapi import Request, HTTPException, Query from fastapi.responses import JSONResponse from fastapi.responses import PlainTextResponse from fastapi.templating import Jinja2Templates -from datetime import datetime -from email import utils import fastapi import uvicorn import requests -import re -from fba import * +import validators -router = fastapi.FastAPI(docs_url=fba.config["base_url"] + "/docs", redoc_url=fba.config["base_url"] + "/redoc") +from fba import database +from fba import utils + +from fba.helpers import config +from fba.helpers import tidyup + +from fba.http import network + +from fba.models import blocks + +router = fastapi.FastAPI(docs_url=config.get("base_url") + "/docs", redoc_url=config.get("base_url") + "/redoc") templates = Jinja2Templates(directory="templates") -@router.get(fba.config["base_url"] + "/api/info.json", response_class=JSONResponse) -def info(): - fba.cursor.execute("SELECT (SELECT COUNT(domain) FROM instances), (SELECT COUNT(domain) FROM instances WHERE software IN ('pleroma', 'mastodon', 'misskey', 'gotosocial', 'friendica', 'bookwyrm', 'takahe', 'peertube')), (SELECT COUNT(blocker) FROM blocks), (SELECT COUNT(domain) FROM instances WHERE last_status_code IS NOT NULL)") - known, indexed, blocks, errorous = fba.cursor.fetchone() +@router.get(config.get("base_url") + "/api/info.json", response_class=JSONResponse) +def api_info(): + database.cursor.execute("SELECT (SELECT COUNT(domain) FROM instances), (SELECT COUNT(domain) FROM instances WHERE software IN ('pleroma', 'mastodon', 'lemmy', 'friendica', 'misskey', 'peertube')), (SELECT COUNT(blocker) FROM blocks), (SELECT COUNT(domain) FROM instances WHERE last_error_details IS NOT NULL)") + row = database.cursor.fetchone() return { - "known_instances" : known, - "indexed_instances" : indexed, - "blocks_recorded" : blocks, - "errorous_instances": errorous, - "slogan" : fba.config["slogan"] + "known_instances" : row[0], + "supported_instances": row[1], + "blocks_recorded" : row[2], + "erroneous_instances": row[3], + "slogan" : config.get("slogan"), } -@router.get(fba.config["base_url"] + "/api/top.json", response_class=JSONResponse) -def top(blocked: int = None, blockers: int = None, reference: int = None, software: int = None, originator: int = None): - if blocked != None: - if blocked > 500: - raise HTTPException(status_code=400, detail="Too many results") - fba.cursor.execute("SELECT blocked, COUNT(blocked) FROM blocks WHERE block_level = 'reject' GROUP BY blocked ORDER BY COUNT(blocked) DESC LIMIT ?", [blocked]) - elif blockers != None: - if blockers > 500: - raise HTTPException(status_code=400, detail="Too many results") - fba.cursor.execute("SELECT blocker, COUNT(blocker) FROM blocks WHERE block_level = 'reject' GROUP BY blocker ORDER BY COUNT(blocker) DESC LIMIT ?", [blockers]) - elif reference != None: - if reference > 500: - raise HTTPException(status_code=400, detail="Too many results") - fba.cursor.execute("SELECT origin, COUNT(domain) FROM instances WHERE software IS NOT NULL GROUP BY origin ORDER BY COUNT(domain) DESC LIMIT ?", [reference]) - elif software != None: - if software > 500: - raise HTTPException(status_code=400, detail="Too many results") - fba.cursor.execute("SELECT software, COUNT(domain) FROM instances WHERE software IS NOT NULL GROUP BY software ORDER BY COUNT(domain) DESC, software ASC LIMIT ?", [software]) - elif originator != None: - if originator > 500: - raise HTTPException(status_code=400, detail="Too many results") - fba.cursor.execute("SELECT originator, COUNT(domain) FROM instances WHERE originator IS NOT NULL GROUP BY originator ORDER BY COUNT(domain) DESC, originator ASC LIMIT ?", [originator]) +@router.get(config.get("base_url") + "/api/scoreboard.json", response_class=JSONResponse) +def api_scoreboard(mode: str, amount: int): + if amount > 500: + raise HTTPException(status_code=400, detail="Too many results") + + if mode == "blocked": + database.cursor.execute("SELECT blocked, COUNT(blocked) AS score FROM blocks GROUP BY blocked ORDER BY score DESC LIMIT ?", [amount]) + elif mode == "blocker": + database.cursor.execute("SELECT blocker, COUNT(blocker) AS score FROM blocks GROUP BY blocker ORDER BY score DESC LIMIT ?", [amount]) + elif mode == "reference": + database.cursor.execute("SELECT origin, COUNT(domain) AS score FROM instances WHERE software IS NOT NULL GROUP BY origin ORDER BY score DESC LIMIT ?", [amount]) + elif mode == "software": + database.cursor.execute("SELECT software, COUNT(domain) AS score FROM instances WHERE software IS NOT NULL GROUP BY software ORDER BY score DESC, software ASC LIMIT ?", [amount]) + elif mode == "command": + database.cursor.execute("SELECT command, COUNT(domain) AS score FROM instances WHERE command IS NOT NULL GROUP BY command ORDER BY score DESC, command ASC LIMIT ?", [amount]) + elif mode == "error_code": + database.cursor.execute("SELECT last_status_code, COUNT(domain) AS score FROM instances WHERE last_status_code IS NOT NULL AND last_status_code != '200' GROUP BY last_status_code ORDER BY score DESC LIMIT ?", [amount]) + elif mode == "detection_mode": + database.cursor.execute("SELECT detection_mode, COUNT(domain) AS cnt FROM instances GROUP BY detection_mode ORDER BY cnt DESC LIMIT ?", [amount]) + elif mode == "avg_peers": + database.cursor.execute("SELECT software, AVG(total_peers) AS average FROM instances WHERE software IS NOT NULL GROUP BY software HAVING average>0 ORDER BY average DESC LIMIT ?", [amount]) + elif mode == "obfuscator": + database.cursor.execute("SELECT software, COUNT(domain) AS cnt FROM instances WHERE has_obfuscation = 1 GROUP BY software ORDER BY cnt DESC LIMIT ?", [amount]) + elif mode == "obfuscation": + database.cursor.execute("SELECT has_obfuscation, COUNT(domain) AS cnt FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica') GROUP BY has_obfuscation ORDER BY cnt DESC LIMIT ?", [amount]) + elif mode == "block_level": + database.cursor.execute("SELECT block_level, COUNT(rowid) AS cnt FROM blocks GROUP BY block_level ORDER BY cnt DESC LIMIT ?", [amount]) else: raise HTTPException(status_code=400, detail="No filter specified") - scores = fba.cursor.fetchall() - - scoreboard = [] + scores = list() - for domain, highscore in scores: - scoreboard.append({ - "domain" : domain, - "highscore": highscore + for domain, score in database.cursor.fetchall(): + scores.append({ + "domain": domain, + "score" : round(score) }) - return scoreboard + return scores -@router.get(fba.config["base_url"] + "/api/index.json", response_class=JSONResponse) -def blocked(domain: str = None, reason: str = None, reverse: str = None): - if domain == None and reason == None and reverse == None: - raise HTTPException(status_code=400, detail="No filter specified") +@router.get(config.get("base_url") + "/api/index.json", response_class=JSONResponse) +def api_index(request: Request, mode: str, value: str, amount: int): + if mode is None or value is None or amount is None: + raise HTTPException(status_code=500, detail="No filter specified") + elif amount > 500: + raise HTTPException(status_code=500, detail=f"amount={amount} is to big") - if reason != None: - reason = re.sub("(%|_)", "", reason) - if len(reason) < 3: - raise HTTPException(status_code=400, detail="Keyword is shorter than three characters") + domain = whildchar = punycode = reason = None + + if mode == "block_level": + database.cursor.execute( + "SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks WHERE block_level = ? LIMIT ?", [value, amount] + ) + elif mode in ["domain", "reverse"]: + domain = tidyup.domain(value) + if not utils.is_domain_wanted(domain): + raise HTTPException(status_code=500, detail=f"domain='{domain}' is not wanted") - if domain != None: wildchar = "*." + ".".join(domain.split(".")[-domain.count("."):]) punycode = domain.encode('idna').decode('utf-8') - fba.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? ORDER BY first_seen ASC", - (domain, "*." + domain, wildchar, fba.get_hash(domain), punycode, "*." + punycode)) - elif reverse != None: - fba.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks WHERE blocker = ? ORDER BY first_seen ASC", [reverse]) - else: - fba.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks WHERE reason like ? AND reason != '' ORDER BY first_seen ASC", ["%" + reason + "%"]) + elif mode == "reason": + reason = re.sub("(%|_)", "", tidyup.reason(value)) + if len(reason) < 3: + raise HTTPException(status_code=400, detail="Keyword is shorter than three characters") - blocks = fba.cursor.fetchall() + if mode == "domain": + database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \ +FROM blocks \ +WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? ORDER BY first_seen ASC LIMIT ?", + [ + domain, + "*." + domain, + wildchar, + utils.get_hash(domain), + punycode, + "*." + punycode, + amount + ] + ) + elif mode == "reverse": + database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \ +FROM blocks \ +WHERE blocker = ? OR blocker = ? OR blocker = ? OR blocker = ? OR blocker = ? OR blocker = ? \ +ORDER BY first_seen ASC \ +LIMIT ?", [ + domain, + "*." + domain, + wildchar, + utils.get_hash(domain), + punycode, + "*." + punycode, + amount + ]) + elif mode == "reason": + database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \ +FROM blocks \ +WHERE reason LIKE ? AND reason != '' \ +ORDER BY first_seen ASC \ +LIMIT ?", [ + "%" + reason + "%", + amount + ]) + + blocklist = database.cursor.fetchall() result = {} - for blocker, blocked, block_level, reason, first_seen, last_seen in blocks: + for blocker, blocked, block_level, reason, first_seen, last_seen in blocklist: + if reason is not None and reason != "": + reason = reason.replace(",", " ").replace(" ", " ") + entry = { "blocker" : blocker, "blocked" : blocked, @@ -111,6 +171,7 @@ def blocked(domain: str = None, reason: str = None, reverse: str = None): "first_seen": first_seen, "last_seen" : last_seen } + if block_level in result: result[block_level].append(entry) else: @@ -118,10 +179,10 @@ def blocked(domain: str = None, reason: str = None, reverse: str = None): return result -@router.get(fba.config["base_url"] + "/api/mutual.json", response_class=JSONResponse) -def mutual(domains: list[str] = Query()): +@router.get(config.get("base_url") + "/api/mutual.json", response_class=JSONResponse) +def api_mutual(domains: list[str] = Query()): """Return 200 if federation is open between the two, 4xx otherwise""" - fba.cursor.execute( + database.cursor.execute( "SELECT block_level FROM blocks " \ "WHERE ((blocker = :a OR blocker = :b) AND (blocked = :b OR blocked = :a OR blocked = :aw OR blocked = :bw)) " \ "AND block_level = 'reject' " \ @@ -133,135 +194,162 @@ def mutual(domains: list[str] = Query()): "bw": "*." + domains[1], }, ) - res = fba.cursor.fetchone() + response = database.cursor.fetchone() - if res is not None: + if response is not None: # Blocks found return JSONResponse(status_code=418, content={}) # No known blocks return JSONResponse(status_code=200, content={}) -@router.get(fba.config["base_url"] + "/scoreboard") -def index(request: Request, blockers: int = None, blocked: int = None, reference: int = None, software: int = None, originator: int = None): - scores = None - - if blockers != None and blockers > 0: - res = requests.get(f"http://{fba.config['host']}:{fba.config['port']}{fba.config['base_url']}/api/top.json?blockers={blockers}") - elif blocked != None and blocked > 0: - res = requests.get(f"http://{fba.config['host']}:{fba.config['port']}{fba.config['base_url']}/api/top.json?blocked={blocked}") - elif reference != None and reference > 0: - res = requests.get(f"http://{fba.config['host']}:{fba.config['port']}{fba.config['base_url']}/api/top.json?reference={reference}") - elif software != None and software > 0: - res = requests.get(f"http://{fba.config['host']}:{fba.config['port']}{fba.config['base_url']}/api/top.json?software={software}") - elif originator != None and originator > 0: - res = requests.get(f"http://{fba.config['host']}:{fba.config['port']}{fba.config['base_url']}/api/top.json?originator={originator}") +@router.get(config.get("base_url") + "/scoreboard") +def scoreboard(request: Request, mode: str, amount: int): + response = None + + if mode == "blocker" and amount > 0: + response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=blocker&amount={amount}") + elif mode == "blocked" and amount > 0: + response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=blocked&amount={amount}") + elif mode == "reference" and amount > 0: + response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=reference&amount={amount}") + elif mode == "software" and amount > 0: + response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=software&amount={amount}") + elif mode == "command" and amount > 0: + response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=command&amount={amount}") + elif mode == "error_code" and amount > 0: + response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=error_code&amount={amount}") + elif mode == "detection_mode" and amount > 0: + response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=detection_mode&amount={amount}") + elif mode == "avg_peers" and amount > 0: + response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=avg_peers&amount={amount}") + elif mode == "obfuscator" and amount > 0: + response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=obfuscator&amount={amount}") + elif mode == "obfuscation" and amount > 0: + response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=obfuscation&amount={amount}") + elif mode == "block_level" and amount > 0: + response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=block_level&amount={amount}") else: raise HTTPException(status_code=400, detail="No filter specified") - if res == None: + if response is None: raise HTTPException(status_code=500, detail="Could not determine scores") - elif not res.ok: - raise HTTPException(status_code=res.status_code, detail=res.text) + elif not response.ok: + raise HTTPException(status_code=response.status_code, detail=response.text) - return templates.TemplateResponse("scoreboard.html", { - "base_url" : fba.config["base_url"], - "slogan" : fba.config["slogan"], + return templates.TemplateResponse("views/scoreboard.html", { + "base_url" : config.get("base_url"), + "slogan" : config.get("slogan"), "request" : request, "scoreboard": True, - "blockers" : blockers, - "blocked" : blocked, - "reference" : reference, - "software" : software, - "originator": originator, - "scores" : res.json() + "mode" : mode, + "amount" : amount, + "scores" : network.json_from_response(response) }) -@router.get(fba.config["base_url"] + "/") -def index(request: Request, domain: str = None, reason: str = None, reverse: str = None): - if domain == "" or reason == "" or reverse == "": - return fastapi.responses.RedirectResponse("/") - - info = None - blocks = None - - if domain == None and reason == None and reverse == None: - info = requests.get(f"http://{fba.config['host']}:{fba.config['port']}{fba.config['base_url']}/api/info.json") - - if not info.ok: - raise HTTPException(status_code=info.status_code, detail=info.text) - - info = info.json() - elif domain != None: - blocks = requests.get(f"http://{fba.config['host']}:{fba.config['port']}{fba.config['base_url']}/api/index.json?domain={domain}") - elif reason != None: - blocks = requests.get(f"http://{fba.config['host']}:{fba.config['port']}{fba.config['base_url']}/api/index.json?reason={reason}") - elif reverse != None: - blocks = requests.get(f"http://{fba.config['host']}:{fba.config['port']}{fba.config['base_url']}/api/index.json?reverse={reverse}") - - if blocks != None: - if not blocks.ok: - raise HTTPException(status_code=blocks.status_code, detail=blocks.text) - blocks = blocks.json() - for block_level in blocks: - for block in blocks[block_level]: - block["first_seen"] = datetime.utcfromtimestamp(block["first_seen"]).strftime('%Y-%m-%d %H:%M') - block["last_seen"] = datetime.utcfromtimestamp(block["last_seen"]).strftime('%Y-%m-%d %H:%M') - - return templates.TemplateResponse("index.html", { +@router.get(config.get("base_url") + "/") +def index(request: Request): + # Get info + response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/info.json") + + if not response.ok: + raise HTTPException(status_code=response.status_code, detail=response.text) + + return templates.TemplateResponse("views/index.html", { "request": request, - "domain" : domain, - "blocks" : blocks, - "reason" : reason, - "reverse": reverse, - "info" : info + "info" : response.json() + }) + +@router.get(config.get("base_url") + "/top") +def top(request: Request, mode: str, value: str, amount: int = 500): + response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/info.json") + + if not response.ok: + raise HTTPException(status_code=response.status_code, detail=response.text) + elif mode == "" or value == "" or amount == 0: + raise HTTPException(status_code=500, detail="Parameter mode, value and amount must always be set") + elif amount > 500: + raise HTTPException(status_code=500, detail=f"amount='{amount}' is to big") + + info = response.json() + response = None + blocklist = list() + + if mode == "block_level" and not blocks.is_valid_level(value): + raise HTTPException(status_code=500, detail="Invalid block level provided") + elif mode in ["domain", "reverse"] and not utils.is_domain_wanted(value): + raise HTTPException(status_code=500, detail="Invalid or blocked domain specified") + + response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/index.json?mode={mode}&value={value}&amount={amount}") + + if response is not None: + blocklist = response.json() + + found = 0 + for block_level in blocklist: + for block in blocklist[block_level]: + block["first_seen"] = datetime.utcfromtimestamp(block["first_seen"]).strftime(config.get("timestamp_format")) + block["last_seen"] = datetime.utcfromtimestamp(block["last_seen"]).strftime(config.get("timestamp_format")) + found = found + 1 + + return templates.TemplateResponse("views/top.html", { + "request" : request, + "mode" : mode if response is not None else None, + "value" : value if response is not None else None, + "amount" : amount if response is not None else None, + "found" : found, + "blocklist": blocklist, + "info" : info, }) -@router.get(fba.config["base_url"] + "/rss") +@router.get(config.get("base_url") + "/rss") def rss(request: Request, domain: str = None): - if domain != None: + if domain is not None: + domain = tidyup.domain(domain) + wildchar = "*." + ".".join(domain.split(".")[-domain.count("."):]) - punycode = domain.encode('idna').decode('utf-8') - fba.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? ORDER BY first_seen DESC LIMIT 50", - (domain, "*." + domain, wildchar, fba.get_hash(domain), punycode, "*." + punycode)) + punycode = domain.encode("idna").decode("utf-8") + + database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? ORDER BY first_seen DESC LIMIT ?", [ + domain, + "*." + domain, wildchar, + utils.get_hash(domain), + punycode, + "*." + punycode, + config.get("rss_limit") + ]) else: - fba.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks ORDER BY first_seen DESC LIMIT 50") - - blocks = fba.cursor.fetchall() - - result = [] - for blocker, blocked, block_level, reason, first_seen, last_seen in blocks: - first_seen = utils.format_datetime(datetime.fromtimestamp(first_seen)) - if reason == None or reason == '': - reason = "No reason provided." - else: - reason = "Provided reason: '" + reason + "'" - - result.append({ - "blocker" : blocker, - "blocked" : blocked, - "block_level": block_level, - "reason" : reason, - "first_seen" : first_seen + database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks ORDER BY first_seen DESC LIMIT ?", [config.get("rss_limit")]) + + result = database.cursor.fetchall() + blocklist = [] + + for row in result: + blocklist.append({ + "blocker" : row[0], + "blocked" : row[1], + "block_level": row[2], + "reason" : "Provided reason: '" + row[3] + "'" if row[3] is not None and row[3] != "" else "No reason provided.", + "first_seen" : format_datetime(datetime.fromtimestamp(row[4])), + "last_seen" : format_datetime(datetime.fromtimestamp(row[5])), }) - timestamp = utils.format_datetime(datetime.now()) - - return templates.TemplateResponse("rss.xml", { + return templates.TemplateResponse("views/rss.xml", { "request" : request, - "timestamp": timestamp, + "timestamp": format_datetime(datetime.now()), "domain" : domain, - "blocks" : result + "hostname" : config.get("hostname"), + "blocks" : blocklist }, headers={ "Content-Type": "routerlication/rss+xml" }) -@router.get(fba.config["base_url"] + "/robots.txt", response_class=PlainTextResponse) +@router.get(config.get("base_url") + "/robots.txt", response_class=PlainTextResponse) def robots(request: Request): - return templates.TemplateResponse("robots.txt", { + return templates.TemplateResponse("views/robots.txt", { "request" : request, - "base_url": fba.config["base_url"] + "base_url": config.get("base_url") }) if __name__ == "__main__": - uvicorn.run("api:router", host=fba.config["host"], port=fba.config["port"], log_level=fba.config["log_level"]) + uvicorn.run("api:router", host=config.get("host"), port=config.get("port"), log_level=config.get("log_level"))