-#!/usr/bin/python3
+#!venv/bin/python3
# -*- coding: utf-8 -*-
# Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
-import re
-
from datetime import datetime
from email.utils import format_datetime
from pathlib import Path
+import re
+import validators
+
import fastapi
from fastapi import Request, HTTPException, Query
from fastapi.responses import JSONResponse
from fba.models import blocks
from fba.models import instances
-router = fastapi.FastAPI(docs_url=config.get("base_url") + "/docs", redoc_url=config.get("base_url") + "/redoc")
+# Base URL for all our own requests
+_base_url = config.get("base_url")
+
+router = fastapi.FastAPI(docs_url=_base_url + "/docs", redoc_url=_base_url + "/redoc")
router.mount(
"/static",
StaticFiles(directory=Path(__file__).parent.absolute() / "static"),
templates = Jinja2Templates(directory="templates")
-@router.get(config.get("base_url") + "/api/info.json", response_class=JSONResponse)
-def api_info():
- database.cursor.execute("SELECT (SELECT COUNT(domain) FROM instances), (SELECT COUNT(domain) FROM instances WHERE software IN ('pleroma', 'mastodon', 'lemmy', 'friendica', 'misskey', 'peertube', 'takahe', 'gotosocial', 'brighteon', 'wildebeest', 'bookwyrm', 'mitra', 'areionskey', 'mammuthus', 'neodb')), (SELECT COUNT(blocker) FROM blocks), (SELECT COUNT(domain) FROM instances WHERE last_error_details IS NOT NULL)")
+@router.get(_base_url + "/api/info.json", response_class=JSONResponse)
+def api_info() -> None:
+ database.cursor.execute("SELECT ( \
+SELECT COUNT(domain) FROM instances) AS total_websites, \
+(SELECT COUNT(domain) FROM instances WHERE software IN ( \
+ 'pleroma' , 'mastodon', 'lemmy' , 'friendica', 'misskey' , \
+ 'peertube' , 'takahe' , 'gotosocial', 'brighteon', 'wildebeest' , \
+ 'bookwyrm' , 'mitra' , 'areionskey', 'mammuthus', 'neodb' , \
+ 'smithereen', 'vebinet' , 'toki' , 'snac' , 'biblioreads', \
+ 'wordpress' , 'oolong' , 'diaspora' , 'appy' , 'neko' \
+)) AS supported_instances, \
+(SELECT COUNT(blocker) FROM blocks) AS total_blocks, \
+(SELECT COUNT(domain) FROM instances WHERE last_error_details IS NOT NULL) AS erroneous_instances")
+
row = database.cursor.fetchone()
return JSONResponse(status_code=200, content={
- "known_instances" : row[0],
- "supported_instances": row[1],
- "blocks_recorded" : row[2],
- "erroneous_instances": row[3],
+ "total_websites" : row["total_websites"],
+ "supported_instances": row["supported_instances"],
+ "total_blocks" : row["total_blocks"],
+ "erroneous_instances": row["erroneous_instances"],
})
-@router.get(config.get("base_url") + "/api/scoreboard.json", response_class=JSONResponse)
-def api_scoreboard(mode: str, amount: int):
- if amount > config.get("api_limit"):
+
+@router.get(_base_url + "/api/scoreboard.json", response_class=JSONResponse)
+def api_scoreboard(mode: str, amount: int) -> None:
+ if mode is None or amount is None:
+ raise HTTPException(status_code=500, detail="No filter specified")
+ elif amount <= 0:
+ raise HTTPException(status_code=500, detail=f"amount={amount} is to small")
+ elif amount > config.get("api_limit"):
raise HTTPException(status_code=400, detail="Too many results")
if mode == "blocked":
- database.cursor.execute("SELECT blocked, COUNT(blocked) AS score FROM blocks GROUP BY blocked ORDER BY score DESC LIMIT ?", [amount])
+ database.cursor.execute("SELECT blocked AS data, COUNT(blocked) AS score FROM blocks GROUP BY blocked ORDER BY score DESC LIMIT ?", [amount])
elif mode == "blocker":
- database.cursor.execute("SELECT blocker, COUNT(blocker) AS score FROM blocks GROUP BY blocker ORDER BY score DESC LIMIT ?", [amount])
+ database.cursor.execute("SELECT blocker AS data, COUNT(blocker) AS score FROM blocks GROUP BY blocker ORDER BY score DESC LIMIT ?", [amount])
elif mode == "reference":
- database.cursor.execute("SELECT origin, COUNT(domain) AS score FROM instances WHERE origin IS NOT NULL GROUP BY origin ORDER BY score DESC LIMIT ?", [amount])
+ database.cursor.execute("SELECT origin AS data, COUNT(domain) AS score FROM instances WHERE origin IS NOT NULL GROUP BY origin ORDER BY score DESC LIMIT ?", [amount])
+ elif mode == "original_software":
+ database.cursor.execute("SELECT original_software AS data, COUNT(domain) AS score FROM instances WHERE original_software IS NOT NULL GROUP BY original_software ORDER BY score DESC, original_software ASC LIMIT ?", [amount])
elif mode == "software":
- database.cursor.execute("SELECT software, COUNT(domain) AS score FROM instances WHERE software IS NOT NULL GROUP BY software ORDER BY score DESC, software ASC LIMIT ?", [amount])
+ database.cursor.execute("SELECT software AS data, COUNT(domain) AS score FROM instances WHERE software IS NOT NULL GROUP BY software ORDER BY score DESC, software ASC LIMIT ?", [amount])
elif mode == "command":
- database.cursor.execute("SELECT command, COUNT(domain) AS score FROM instances WHERE command IS NOT NULL GROUP BY command ORDER BY score DESC, command ASC LIMIT ?", [amount])
+ database.cursor.execute("SELECT command AS data, COUNT(domain) AS score FROM instances WHERE command IS NOT NULL GROUP BY command ORDER BY score DESC, command ASC LIMIT ?", [amount])
elif mode == "error_code":
- database.cursor.execute("SELECT last_status_code, COUNT(domain) AS score FROM instances WHERE last_status_code IS NOT NULL AND last_status_code != '200' GROUP BY last_status_code ORDER BY score DESC LIMIT ?", [amount])
+ database.cursor.execute("SELECT last_status_code AS data, COUNT(domain) AS score FROM instances WHERE last_status_code IS NOT NULL AND last_status_code != '200' GROUP BY last_status_code ORDER BY score DESC LIMIT ?", [amount])
elif mode == "detection_mode":
- database.cursor.execute("SELECT detection_mode, COUNT(domain) AS cnt FROM instances GROUP BY detection_mode ORDER BY cnt DESC LIMIT ?", [amount])
+ database.cursor.execute("SELECT detection_mode AS data, COUNT(domain) AS score FROM instances GROUP BY detection_mode ORDER BY score DESC LIMIT ?", [amount])
elif mode == "avg_peers":
- database.cursor.execute("SELECT software, AVG(total_peers) AS average FROM instances WHERE software IS NOT NULL AND total_peers IS NOT NULL GROUP BY software HAVING average > 0 ORDER BY average DESC LIMIT ?", [amount])
+ database.cursor.execute("SELECT software AS data, AVG(total_peers) AS score FROM instances WHERE software IS NOT NULL AND total_peers IS NOT NULL GROUP BY software HAVING score > 0 ORDER BY score DESC LIMIT ?", [amount])
elif mode == "avg_blocks":
- database.cursor.execute("SELECT software, AVG(total_blocks) AS average FROM instances WHERE software IS NOT NULL AND total_blocks IS NOT NULL GROUP BY software HAVING average > 0 ORDER BY average DESC LIMIT ?", [amount])
+ database.cursor.execute("SELECT software AS data, AVG(total_blocks) AS score FROM instances WHERE software IS NOT NULL AND total_blocks IS NOT NULL GROUP BY software HAVING score > 0 ORDER BY score DESC LIMIT ?", [amount])
elif mode == "obfuscator":
- database.cursor.execute("SELECT software, COUNT(domain) AS cnt FROM instances WHERE has_obfuscation = 1 GROUP BY software ORDER BY cnt DESC LIMIT ?", [amount])
+ database.cursor.execute("SELECT software AS data, COUNT(domain) AS score FROM instances WHERE has_obfuscation = 1 GROUP BY software ORDER BY score DESC LIMIT ?", [amount])
elif mode == "obfuscation":
- database.cursor.execute("SELECT has_obfuscation, COUNT(domain) AS cnt FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica') GROUP BY has_obfuscation ORDER BY cnt DESC LIMIT ?", [amount])
+ database.cursor.execute("SELECT has_obfuscation AS data, COUNT(domain) AS score FROM instances WHERE has_obfuscation = 1 GROUP BY has_obfuscation ORDER BY score DESC LIMIT ?", [amount])
elif mode == "block_level":
- database.cursor.execute("SELECT block_level, COUNT(rowid) AS cnt FROM blocks GROUP BY block_level ORDER BY cnt DESC LIMIT ?", [amount])
+ database.cursor.execute("SELECT block_level AS data, COUNT(rowid) AS score FROM blocks GROUP BY block_level ORDER BY score DESC LIMIT ?", [amount])
else:
- raise HTTPException(status_code=400, detail="No filter specified")
+ raise HTTPException(status_code=400, detail=f"mode='{mode}' is not supported")
- scores = list()
+ scores = []
for row in database.cursor.fetchall():
scores.append({
- "domain": row[0],
- "score" : round(row[1]),
+ "data" : row["data"],
+ "score": round(row["score"]),
})
return JSONResponse(status_code=200, content=scores)
-@router.get(config.get("base_url") + "/api/list.json", response_class=JSONResponse)
-def api_list(request: Request, mode: str, value: str, amount: int):
+@router.get(_base_url + "/api/list.json", response_class=JSONResponse)
+def api_list(request: Request, mode: str, value: str, amount: int) -> None:
if mode is None or value is None or amount is None:
raise HTTPException(status_code=500, detail="No filter specified")
+ elif amount <= 0:
+ raise HTTPException(status_code=500, detail=f"amount={amount} is to small")
elif amount > config.get("api_limit"):
raise HTTPException(status_code=500, detail=f"amount={amount} is to big")
- if mode in ("detection_mode", "software", "command", "origin"):
+ if mode in ["detection_mode", "original_software", "software", "command", "origin"]:
database.cursor.execute(
f"SELECT * \
FROM instances \
ORDER BY domain \
LIMIT ?", [value, amount]
)
+ elif mode == "added":
+ database.cursor.execute(
+ "SELECT * \
+FROM instances \
+ORDER BY first_seen DESC \
+LIMIT ?", [amount]
+ )
+ elif mode == "updated":
+ database.cursor.execute(
+ "SELECT * \
+FROM instances \
+ORDER BY last_updated DESC \
+LIMIT ?", [amount]
+ )
+ else:
+ raise HTTPException(status_code=500, detail=f"mode='{mode}' is unsupported")
domainlist = database.cursor.fetchall()
return domainlist
-@router.get(config.get("base_url") + "/api/top.json", response_class=JSONResponse)
-def api_index(request: Request, mode: str, value: str, amount: int):
+@router.get(_base_url + "/api/top.json", response_class=JSONResponse)
+def api_top(request: Request, mode: str, value: str, amount: int) -> None:
if mode is None or value is None or amount is None:
raise HTTPException(status_code=500, detail="No filter specified")
+ elif amount <= 0:
+ raise HTTPException(status_code=500, detail=f"amount={amount} is to small")
elif amount > config.get("api_limit"):
raise HTTPException(status_code=500, detail=f"amount={amount} is to big")
domain,
"*." + domain,
wildchar,
- utils.get_hash(domain),
+ utils.hash_domain(domain),
punycode,
"*." + punycode,
amount
domain,
"*." + domain,
wildchar,
- utils.get_hash(domain),
+ utils.hash_domain(domain),
punycode,
"*." + punycode,
amount
amount
])
- blocklist = database.cursor.fetchall()
+ rows = database.cursor.fetchall()
result = {}
- for blocker, blocked, block_level, reason, first_seen, last_seen in blocklist:
- if reason is not None and reason != "":
- reason = reason.replace(",", " ").replace(" ", " ")
-
- entry = {
- "blocker" : blocker,
- "blocked" : blocked,
- "reason" : reason,
- "first_seen": first_seen,
- "last_seen" : last_seen
- }
-
- if block_level in result:
- result[block_level].append(entry)
+ for row in list(rows):
+ entry = dict(row)
+ entry["reason"] = entry["reason"].replace(",", " ").replace(" ", " ") if entry["reason"] not in [None, ""] else None
+
+ if entry["block_level"] in result:
+ result[entry["block_level"]].append(entry)
else:
- result[block_level] = [entry]
+ result[entry["block_level"]] = [entry]
return result
-@router.get(config.get("base_url") + "/api/domain.json", response_class=JSONResponse)
-def api_domain(domain: str):
+@router.get(_base_url + "/api/domain.json", response_class=JSONResponse)
+def api_domain(domain: str) -> None:
if domain is None:
raise HTTPException(status_code=400, detail="Invalid request, parameter 'domain' missing")
return JSONResponse(status_code=200, content=dict(domain_data))
-@router.get(config.get("base_url") + "/api/mutual.json", response_class=JSONResponse)
-def api_mutual(domains: list[str] = Query()):
+@router.get(_base_url + "/api/mutual.json", response_class=JSONResponse)
+def api_mutual(domains: list[str] = Query()) -> None:
"""Return 200 if federation is open between the two, 4xx otherwise"""
- database.cursor.execute(
- "SELECT block_level FROM blocks " \
- "WHERE ((blocker = :a OR blocker = :b) AND (blocked = :b OR blocked = :a OR blocked = :aw OR blocked = :bw)) " \
- "AND block_level = 'reject' " \
- "LIMIT 1",
+ database.cursor.execute("SELECT block_level FROM blocks \
+WHERE ((blocker = :a OR blocker = :b) AND (blocked = :b OR blocked = :a OR blocked = :aw OR blocked = :bw)) \
+AND block_level = 'reject' \
+LIMIT 1",
{
"a" : domains[0],
"b" : domains[1],
# No known blocks
return JSONResponse(status_code=200, content={})
-@router.get(config.get("base_url") + "/.well-known/nodeinfo", response_class=JSONResponse)
-def wellknown_nodeinfo(request: Request):
+@router.get(_base_url + "/.well-known/nodeinfo", response_class=JSONResponse)
+def wellknown_nodeinfo(request: Request) -> None:
return JSONResponse(status_code=200, content={
"links": ({
"rel" : "http://nodeinfo.diaspora.software/ns/schema/1.0",
})
})
-@router.get(config.get("base_url") + "/nodeinfo/1.0", response_class=JSONResponse)
-def nodeinfo_1_0(request: Request):
+@router.get(_base_url + "/nodeinfo/1.0", response_class=JSONResponse)
+def nodeinfo_1_0(request: Request) -> None:
return JSONResponse(status_code=200, content={
"version": "1.0",
"software": {
},
})
-@router.get(config.get("base_url") + "/api/v1/instance/domain_blocks", response_class=JSONResponse)
-def api_domain_blocks(request: Request):
+@router.get(_base_url + "/api/v1/instance/domain_blocks", response_class=JSONResponse)
+def api_domain_blocks(request: Request) -> None:
+ blocking = []
blocked = blacklist.get_all()
- blocking = list()
for block in blocked:
blocking.append({
"domain" : block,
- "digest" : utils.get_hash(block),
+ "digest" : utils.hash_domain(block) if validators.domain(block, rfc_2782=True) else None,
"severity": "suspend",
"comment" : blocked[block],
})
return JSONResponse(status_code=200, content=blocking)
-@router.get(config.get("base_url") + "/api/v1/instance/peers", response_class=JSONResponse)
-def api_peers(request: Request):
+@router.get(_base_url + "/api/v1/instance/peers", response_class=JSONResponse)
+def api_peers(request: Request) -> None:
database.cursor.execute("SELECT domain FROM instances WHERE nodeinfo_url IS NOT NULL")
- peers = list()
+ peers = []
for row in database.cursor.fetchall():
peers.append(row["domain"])
return JSONResponse(status_code=200, content=peers)
-@router.get(config.get("base_url") + "/scoreboard")
-def scoreboard(request: Request, mode: str, amount: int):
+@router.get(_base_url + "/scoreboard")
+def scoreboard(request: Request, mode: str, amount: int) -> None:
if mode == "":
raise HTTPException(status_code=400, detail="No mode specified")
elif amount <= 0:
raise HTTPException(status_code=500, detail="Invalid amount specified")
- response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode={mode}&amount={amount}")
+ response = requests.get(
+ f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode={mode}&amount={amount}",
+ timeout=config.timeout
+ )
if response is None:
raise HTTPException(status_code=500, detail="Could not determine scores")
"scores" : json_helper.from_response(response)
})
-@router.get(config.get("base_url") + "/list")
-def list_domains(request: Request, mode: str, value: str, amount: int = config.get("api_limit")):
+@router.get(_base_url + "/list")
+def list_domains(request: Request, mode: str, value: str, amount: int = config.get("api_limit")) -> None:
if mode == "detection_mode" and not instances.valid(value, "detection_mode"):
raise HTTPException(status_code=500, detail="Invalid detection mode provided")
- response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/list.json?mode={mode}&value={value}&amount={amount}")
+ response = requests.get(
+ f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/list.json?mode={mode}&value={value}&amount={amount}",
+ timeout=config.timeout
+ )
- domainlist = list()
+ domainlist = []
if response is not None and response.ok:
domainlist = response.json()
tformat = config.get("timestamp_format")
"theme" : config.get("theme"),
})
-@router.get(config.get("base_url") + "/top")
-def top(request: Request, mode: str, value: str, amount: int = config.get("api_limit")):
+@router.get(_base_url + "/top")
+def top(request: Request, mode: str, value: str, amount: int = config.get("api_limit")) -> None:
if mode == "block_level" and not blocks.valid(value, "block_level"):
raise HTTPException(status_code=500, detail="Invalid block level provided")
elif mode in ["domain", "reverse"] and not domain_helper.is_wanted(value):
raise HTTPException(status_code=500, detail="Invalid or blocked domain specified")
- response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/top.json?mode={mode}&value={value}&amount={amount}")
+ response = requests.get(
+ f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/top.json?mode={mode}&value={value}&amount={amount}",
+ timeout=config.timeout
+ )
found = 0
- blocklist = list()
+ blocklist = []
if response.ok and response.status_code == 200 and len(response.text) > 0:
blocklist = response.json()
"theme" : config.get("theme"),
})
-@router.get(config.get("base_url") + "/infos")
-def infos(request: Request, domain: str):
+@router.get(_base_url + "/infos")
+def infos(request: Request, domain: str) -> None:
if domain is None:
raise HTTPException(status_code=400, detail="Invalid request, parameter 'domain' missing")
if not domain_helper.is_wanted(domain):
raise HTTPException(status_code=500, detail=f"domain='{domain}' is not wanted")
- response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/domain.json?domain={domain}")
+ response = requests.get(
+ f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/domain.json?domain={domain}",
+ timeout=config.timeout
+ )
if not response.ok or response.status_code > 200 or response.text.strip() == "":
raise HTTPException(status_code=response.status_code, detail=response.reason)
# Format timestamps
tformat = config.get("timestamp_format")
- instance = dict()
+ instance = {}
for key in domain_data.keys():
if key in ["last_nodeinfo", "last_blocked", "first_seen", "last_updated", "last_instance_fetch"] and isinstance(domain_data[key], float):
# Timestamps
"slogan" : config.get("slogan"),
})
-@router.get(config.get("base_url") + "/rss")
-def rss(request: Request, domain: str = None):
+@router.get(_base_url + "/rss")
+def rss(request: Request, domain: str = None) -> None:
if domain is not None:
domain = tidyup.domain(domain).encode("idna").decode("utf-8")
wildchar = "*." + ".".join(domain.split(".")[-domain.count("."):])
punycode = domain.encode("idna").decode("utf-8")
- database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? ORDER BY first_seen DESC LIMIT ?", [
+ database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
+FROM blocks \
+WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? \
+ORDER BY first_seen DESC \
+LIMIT ?", [
domain,
"*." + domain, wildchar,
- utils.get_hash(domain),
+ utils.hash_domain(domain),
punycode,
"*." + punycode,
config.get("rss_limit")
])
else:
- database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks ORDER BY first_seen DESC LIMIT ?", [config.get("rss_limit")])
+ database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
+FROM blocks \
+ORDER BY first_seen DESC \
+LIMIT ?", [config.get("rss_limit")])
result = database.cursor.fetchall()
blocklist = []
for row in result:
blocklist.append({
- "blocker" : row[0],
- "blocked" : row[1],
- "block_level": row[2],
- "reason" : "Provided reason: '" + row[3] + "'" if row[3] is not None and row[3] != "" else "No reason provided.",
- "first_seen" : format_datetime(datetime.fromtimestamp(row[4])),
- "last_seen" : format_datetime(datetime.fromtimestamp(row[5])),
+ "blocker" : row["blocker"],
+ "blocked" : row["blocked"],
+ "block_level": row["block_level"],
+ "reason" : "Provided reason: '" + row["reason"] + "'" if row["reason"] is not None and row["reason"] != "" else "No reason provided.",
+ "first_seen" : format_datetime(datetime.fromtimestamp(row["first_seen"])),
+ "last_seen" : format_datetime(datetime.fromtimestamp(row["last_seen"])),
})
return templates.TemplateResponse("views/rss.xml", {
"Content-Type": "routerlication/rss+xml"
})
-@router.get(config.get("base_url") + "/robots.txt", response_class=PlainTextResponse)
-def robots(request: Request):
+@router.get(_base_url + "/robots.txt", response_class=PlainTextResponse)
+def robots(request: Request) -> None:
return templates.TemplateResponse("views/robots.txt", {
"request" : request,
- "base_url": config.get("base_url")
+ "base_url": _base_url
})
-@router.get(config.get("base_url") + "/")
-def index(request: Request):
+@router.get(_base_url + "/")
+def index(request: Request) -> None:
# Get info
- response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/info.json")
+ response = requests.get(
+ f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/info.json",
+ timeout=config.timeout
+ )
if not response.ok:
raise HTTPException(status_code=response.status_code, detail=response.text)
})
if __name__ == "__main__":
- uvicorn.run("daemon:router", host=config.get("host"), port=config.get("port"), log_level=config.get("log_level"), proxy_headers=True)
+ uvicorn.run(
+ "daemon:router",
+ host=config.get("host"),
+ port=config.get("port"),
+ log_level=config.get("log_level"),
+ proxy_headers=True
+ )
projects / fba.git / blobdiff