X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;f=api.py;h=1033a01ce80d1e8b1b8e62c981683022f9191a5a;hb=18857b03ac26793984c03a212eb5b3dd834b988f;hp=9931c25e94a5964100692e6b2ac3922bcc6a5ee8;hpb=f5f7007080cbbaad7f6cdea2a9a1089899b8e485;p=fba.git diff --git a/api.py b/api.py index 9931c25..1033a01 100644 --- a/api.py +++ b/api.py @@ -1,96 +1,310 @@ -import uvicorn -from fastapi import FastAPI, Request, HTTPException, responses -import sqlite3 -from hashlib import sha256 +# Fedi API Block - An aggregator for fetching blocking data from fediverse nodes +# Copyright (C) 2023 Free Software Foundation +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from datetime import datetime +from email import utils + +import re + +from fastapi import Request, HTTPException, Query +from fastapi.responses import JSONResponse +from fastapi.responses import PlainTextResponse from fastapi.templating import Jinja2Templates -from requests import get -from json import loads - -with open("config.json") as f: - config = loads(f.read()) - base_url = config["base_url"] - port = config["port"] -app = FastAPI(docs_url=base_url+"/docs", redoc_url=base_url+"/redoc") -templates = Jinja2Templates(directory=".") - -def get_hash(domain: str) -> str: - return sha256(domain.encode("utf-8")).hexdigest() - -@app.get(base_url+"/info") -def info(): - conn = sqlite3.connect("blocks.db") - c = conn.cursor() - c.execute("select (select count(domain) from instances), (select count(domain) from instances where software in ('pleroma', 'mastodon', 'misskey', 'gotosocial', 'friendica')), (select count(blocker) from blocks)") - known, indexed, blocks = c.fetchone() - c.close() + +import fastapi +import uvicorn +import requests +import validators + +from fba import config +from fba import fba +from fba import network + +from fba.helpers import tidyup + +router = fastapi.FastAPI(docs_url=config.get("base_url") + "/docs", redoc_url=config.get("base_url") + "/redoc") +templates = Jinja2Templates(directory="templates") + +@router.get(config.get("base_url") + "/api/info.json", response_class=JSONResponse) +def api_info(): + fba.cursor.execute("SELECT (SELECT COUNT(domain) FROM instances), (SELECT COUNT(domain) FROM instances WHERE software IN ('pleroma', 'mastodon', 'lemmy', 'friendica', 'misskey', 'bookwyrm', 'takahe', 'peertube')), (SELECT COUNT(blocker) FROM blocks), (SELECT COUNT(domain) FROM instances WHERE last_error_details IS NOT NULL)") + row = fba.cursor.fetchone() + return { - "known_instances": known, - "indexed_instances": indexed, - "blocks_recorded": blocks, - "source_code": "https://git.kiwifarms.net/mint/fedi-block-api", + "known_instances" : row[0], + "indexed_instances" : row[1], + "blocks_recorded" : row[2], + "errorous_instances": row[3], + "slogan" : config.get("slogan") } -@app.get(base_url+"/api") -def blocked(domain: str = None, reason: str = None): - if domain == None and reason == None: +@router.get(config.get("base_url") + "/api/top.json", response_class=JSONResponse) +def api_top(mode: str, amount: int): + if amount > 500: + raise HTTPException(status_code=400, detail="Too many results") + + if mode == "blocked": + fba.cursor.execute("SELECT blocked, COUNT(blocked) AS score FROM blocks WHERE block_level = 'reject' GROUP BY blocked ORDER BY score DESC LIMIT ?", [amount]) + elif mode == "blocker": + fba.cursor.execute("SELECT blocker, COUNT(blocker) AS score FROM blocks WHERE block_level = 'reject' GROUP BY blocker ORDER BY score DESC LIMIT ?", [amount]) + elif mode == "reference": + fba.cursor.execute("SELECT origin, COUNT(domain) AS score FROM instances WHERE software IS NOT NULL GROUP BY origin ORDER BY score DESC LIMIT ?", [amount]) + elif mode == "software": + fba.cursor.execute("SELECT software, COUNT(domain) AS score FROM instances WHERE software IS NOT NULL GROUP BY software ORDER BY score DESC, software ASC LIMIT ?", [amount]) + elif mode == "command": + fba.cursor.execute("SELECT command, COUNT(domain) AS score FROM instances WHERE command IS NOT NULL GROUP BY command ORDER BY score DESC, command ASC LIMIT ?", [amount]) + elif mode == "error_code": + fba.cursor.execute("SELECT last_status_code, COUNT(domain) AS score FROM instances WHERE last_status_code IS NOT NULL AND last_status_code != '200' GROUP BY last_status_code ORDER BY score DESC LIMIT ?", [amount]) + else: raise HTTPException(status_code=400, detail="No filter specified") - conn = sqlite3.connect("blocks.db") - c = conn.cursor() - if domain != None: + + scores = list() + + for domain, score in fba.cursor.fetchall(): + scores.append({ + "domain": domain, + "score" : score + }) + + return scores + +@router.get(config.get("base_url") + "/api/index.json", response_class=JSONResponse) +def api_blocked(domain: str = None, reason: str = None, reverse: str = None): + if domain is None and reason is None and reverse is None: + raise HTTPException(status_code=400, detail="No filter specified") + + if reason is not None: + reason = re.sub("(%|_)", "", tidyup.reason(reason)) + if len(reason) < 3: + raise HTTPException(status_code=400, detail="Keyword is shorter than three characters") + + if domain is not None: + domain = tidyup.domain(domain) + if not validators.domain(domain): + raise HTTPException(status_code=500, detail="Invalid domain") + wildchar = "*." + ".".join(domain.split(".")[-domain.count("."):]) punycode = domain.encode('idna').decode('utf-8') - c.execute("select blocker, blocked, block_level, reason from blocks where blocked = ? or blocked = ? or blocked = ? or blocked = ? or blocked = ? or blocked = ?", - (domain, "*." + domain, wildchar, get_hash(domain), punycode, "*." + punycode)) + + fba.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? ORDER BY first_seen ASC", + (domain, "*." + domain, wildchar, fba.get_hash(domain), punycode, "*." + punycode)) + elif reverse is not None: + reverse = tidyup.domain(reverse) + if not validators.domain(reverse): + raise HTTPException(status_code=500, detail="Invalid domain") + + fba.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks WHERE blocker = ? ORDER BY first_seen ASC", [reverse]) else: - c.execute("select * from blocks where reason like ? and reason != ''", ("%"+reason+"%",)) - blocks = c.fetchall() - conn.close() + fba.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks WHERE reason like ? AND reason != '' ORDER BY first_seen ASC", ["%" + reason + "%"]) + + blocklist = fba.cursor.fetchall() result = {} - reasons = {} - wildcards = [] - if domain != None: - for domain, blocked, block_level, reason in blocks: - if block_level in result: - result[block_level].append(domain) - else: - result[block_level] = [domain] - if blocked == "*." + ".".join(blocked.split(".")[-blocked.count("."):]): - wildcards.append(domain) - if reason != "": - if block_level in reasons: - reasons[block_level][domain] = reason - else: - reasons[block_level] = {domain: reason} - return {"blocks": result, "reasons": reasons, "wildcards": wildcards} - - for blocker, blocked, reason, block_level in blocks: + for blocker, blocked, block_level, reason, first_seen, last_seen in blocklist: + if reason is not None and reason != "": + reason = reason.replace(",", " ").replace(" ", " ") + + entry = { + "blocker" : blocker, + "blocked" : blocked, + "reason" : reason, + "first_seen": first_seen, + "last_seen" : last_seen + } + if block_level in result: - result[block_level].append({"blocker": blocker, "blocked": blocked, "reason": reason}) + result[block_level].append(entry) else: - result[block_level] = [{"blocker": blocker, "blocked": blocked, "reason": reason}] - return {"blocks": result} - -@app.get(base_url+"/") -def index(request: Request, domain: str = None, reason: str = None): - if domain == "" or reason == "": - return responses.RedirectResponse("/") - info = None - blocks = None - if domain == None and reason == None: - info = get(f"http://127.0.0.1:{port}{base_url}/info") - if not info.ok: - raise HTTPException(status_code=info.status_code, detail=info.text) - info = info.json() - elif domain != None: - blocks = get(f"http://127.0.0.1:{port}{base_url}/api?domain={domain}") - elif reason != None: - blocks = get(f"http://127.0.0.1:{port}{base_url}/api?reason={reason}") - if blocks != None: - if not blocks.ok: - raise HTTPException(status_code=blocks.status_code, detail=blocks.text) - blocks = blocks.json() - return templates.TemplateResponse("index.html", {"request": request, "domain": domain, "blocks": blocks, "reason": reason, "info": info}) + result[block_level] = [entry] + + return result + +@router.get(config.get("base_url") + "/api/mutual.json", response_class=JSONResponse) +def api_mutual(domains: list[str] = Query()): + """Return 200 if federation is open between the two, 4xx otherwise""" + fba.cursor.execute( + "SELECT block_level FROM blocks " \ + "WHERE ((blocker = :a OR blocker = :b) AND (blocked = :b OR blocked = :a OR blocked = :aw OR blocked = :bw)) " \ + "AND block_level = 'reject' " \ + "LIMIT 1", + { + "a" : domains[0], + "b" : domains[1], + "aw": "*." + domains[0], + "bw": "*." + domains[1], + }, + ) + response = fba.cursor.fetchone() + + if response is not None: + # Blocks found + return JSONResponse(status_code=418, content={}) + + # No known blocks + return JSONResponse(status_code=200, content={}) + +@router.get(config.get("base_url") + "/scoreboard") +def scoreboard(request: Request, mode: str, amount: int): + response = None + + if mode == "blocker" and amount > 0: + response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/top.json?mode=blocker&amount={amount}") + elif mode == "blocked" and amount > 0: + response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/top.json?mode=blocked&amount={amount}") + elif mode == "reference" and amount > 0: + response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/top.json?mode=reference&amount={amount}") + elif mode == "software" and amount > 0: + response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/top.json?mode=software&amount={amount}") + elif mode == "command" and amount > 0: + response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/top.json?mode=command&amount={amount}") + elif mode == "error_code" and amount > 0: + response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/top.json?mode=error_code&amount={amount}") + else: + raise HTTPException(status_code=400, detail="No filter specified") + + if response is None: + raise HTTPException(status_code=500, detail="Could not determine scores") + elif not response.ok: + raise HTTPException(status_code=response.status_code, detail=response.text) + + return templates.TemplateResponse("views/scoreboard.html", { + "base_url" : config.get("base_url"), + "slogan" : config.get("slogan"), + "request" : request, + "scoreboard": True, + "mode" : mode, + "amount" : amount, + "scores" : network.json_from_response(response) + }) + +@router.get(config.get("base_url") + "/") +def index(request: Request): + # Get info + response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/info.json") + + if not response.ok: + raise HTTPException(status_code=response.status_code, detail=response.text) + + return templates.TemplateResponse("views/index.html", { + "request": request, + "info" : response.json() + }) + +@router.get(config.get("base_url") + "/top") +def top(request: Request, domain: str = None, reason: str = None, reverse: str = None): + if domain == "" or reason == "" or reverse == "": + raise HTTPException(status_code=500, detail="Insufficient parameter provided") + + response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/info.json") + + if not response.ok: + raise HTTPException(status_code=response.status_code, detail=response.text) + + info = response.json() + response = None + + if domain is not None: + domain = tidyup.domain(domain) + if not validators.domain(domain): + raise HTTPException(status_code=500, detail="Invalid domain") + + response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/index.json?domain={domain}") + elif reason is not None: + response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/index.json?reason={reason}") + elif reverse is not None: + reverse = tidyup.domain(reverse) + if not validators.domain(reverse): + raise HTTPException(status_code=500, detail="Invalid domain") + + response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/index.json?reverse={reverse}") + + if response is not None: + if not response.ok: + raise HTTPException(status_code=response.status_code, detail=response.text) + + blocklist = response.json() + + for block_level in blocklist: + for block in blocklist[block_level]: + block["first_seen"] = datetime.utcfromtimestamp(block["first_seen"]).strftime(config.get("timestamp_format")) + block["last_seen"] = datetime.utcfromtimestamp(block["last_seen"]).strftime(config.get("timestamp_format")) + + return templates.TemplateResponse("views/top.html", { + "request": request, + "domain" : domain, + "blocks" : blocklist, + "reason" : reason, + "reverse": reverse, + "info" : info + }) + +@router.get(config.get("base_url") + "/rss") +def rss(request: Request, domain: str = None): + if domain is not None: + domain = tidyup.domain(domain) + + wildchar = "*." + ".".join(domain.split(".")[-domain.count("."):]) + punycode = domain.encode("idna").decode("utf-8") + + fba.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? ORDER BY first_seen DESC LIMIT ?", [ + domain, + "*." + domain, wildchar, + fba.get_hash(domain), + punycode, + "*." + punycode, + config.get("rss_limit") + ]) + else: + fba.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks ORDER BY first_seen DESC LIMIT ?", [config.get("rss_limit")]) + + result = fba.cursor.fetchall() + + blocklist = [] + for blocker, blocked, block_level, reason, first_seen, last_seen in result: + first_seen = utils.format_datetime(datetime.fromtimestamp(first_seen)) + if reason is None or reason == "": + reason = "No reason provided." + else: + reason = "Provided reason: '" + reason + "'" + + blocklist.append({ + "blocker" : blocker, + "blocked" : blocked, + "block_level": block_level, + "reason" : reason, + "first_seen" : first_seen, + "last_seen" : last_seen, + }) + + return templates.TemplateResponse("rss.xml", { + "request" : request, + "timestamp": utils.format_datetime(datetime.now()), + "domain" : domain, + "hostname" : config.get("hostname"), + "blocks" : blocklist + }, headers={ + "Content-Type": "routerlication/rss+xml" + }) + +@router.get(config.get("base_url") + "/robots.txt", response_class=PlainTextResponse) +def robots(request: Request): + return templates.TemplateResponse("robots.txt", { + "request" : request, + "base_url": config.get("base_url") + }) if __name__ == "__main__": - uvicorn.run("api:app", host="127.0.0.1", port=port, log_level="info") + uvicorn.run("api:router", host=config.get("host"), port=config.get("port"), log_level=config.get("log_level"))