2 # -*- coding: utf-8 -*-
4 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
5 # Copyright (C) 2023 Free Software Foundation
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published
9 # by the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <https://www.gnu.org/licenses/>.
22 from datetime import datetime
23 from email.utils import format_datetime
24 from pathlib import Path
27 from fastapi import Request, HTTPException, Query
28 from fastapi.responses import JSONResponse
29 from fastapi.responses import PlainTextResponse
30 from fastapi.staticfiles import StaticFiles
31 from fastapi.templating import Jinja2Templates
36 from fba import database
39 from fba.helpers import blacklist
40 from fba.helpers import config
41 from fba.helpers import domain as domain_helper
42 from fba.helpers import json as json_helper
43 from fba.helpers import tidyup
45 from fba.models import blocks
46 from fba.models import instances
48 router = fastapi.FastAPI(docs_url=config.get("base_url") + "/docs", redoc_url=config.get("base_url") + "/redoc")
51 StaticFiles(directory=Path(__file__).parent.absolute() / "static"),
55 templates = Jinja2Templates(directory="templates")
57 @router.get(config.get("base_url") + "/api/info.json", response_class=JSONResponse)
59 database.cursor.execute("SELECT (SELECT COUNT(domain) FROM instances), (SELECT COUNT(domain) FROM instances WHERE software IN ('pleroma', 'mastodon', 'lemmy', 'friendica', 'misskey', 'peertube', 'takahe', 'gotosocial', 'brighteon', 'wildebeest', 'bookwyrm', 'mitra', 'areionskey', 'mammuthus', 'neodb')), (SELECT COUNT(blocker) FROM blocks), (SELECT COUNT(domain) FROM instances WHERE last_error_details IS NOT NULL)")
60 row = database.cursor.fetchone()
62 return JSONResponse(status_code=200, content={
63 "known_instances" : row[0],
64 "supported_instances": row[1],
65 "blocks_recorded" : row[2],
66 "erroneous_instances": row[3],
69 @router.get(config.get("base_url") + "/api/scoreboard.json", response_class=JSONResponse)
70 def api_scoreboard(mode: str, amount: int):
71 if amount > config.get("api_limit"):
72 raise HTTPException(status_code=400, detail="Too many results")
75 database.cursor.execute("SELECT blocked, COUNT(blocked) AS score FROM blocks GROUP BY blocked ORDER BY score DESC LIMIT ?", [amount])
76 elif mode == "blocker":
77 database.cursor.execute("SELECT blocker, COUNT(blocker) AS score FROM blocks GROUP BY blocker ORDER BY score DESC LIMIT ?", [amount])
78 elif mode == "reference":
79 database.cursor.execute("SELECT origin, COUNT(domain) AS score FROM instances WHERE origin IS NOT NULL GROUP BY origin ORDER BY score DESC LIMIT ?", [amount])
80 elif mode == "software":
81 database.cursor.execute("SELECT software, COUNT(domain) AS score FROM instances WHERE software IS NOT NULL GROUP BY software ORDER BY score DESC, software ASC LIMIT ?", [amount])
82 elif mode == "command":
83 database.cursor.execute("SELECT command, COUNT(domain) AS score FROM instances WHERE command IS NOT NULL GROUP BY command ORDER BY score DESC, command ASC LIMIT ?", [amount])
84 elif mode == "error_code":
85 database.cursor.execute("SELECT last_status_code, COUNT(domain) AS score FROM instances WHERE last_status_code IS NOT NULL AND last_status_code != '200' GROUP BY last_status_code ORDER BY score DESC LIMIT ?", [amount])
86 elif mode == "detection_mode":
87 database.cursor.execute("SELECT detection_mode, COUNT(domain) AS cnt FROM instances GROUP BY detection_mode ORDER BY cnt DESC LIMIT ?", [amount])
88 elif mode == "avg_peers":
89 database.cursor.execute("SELECT software, AVG(total_peers) AS average FROM instances WHERE software IS NOT NULL AND total_peers IS NOT NULL GROUP BY software HAVING average > 0 ORDER BY average DESC LIMIT ?", [amount])
90 elif mode == "avg_blocks":
91 database.cursor.execute("SELECT software, AVG(total_blocks) AS average FROM instances WHERE software IS NOT NULL AND total_blocks IS NOT NULL GROUP BY software HAVING average > 0 ORDER BY average DESC LIMIT ?", [amount])
92 elif mode == "obfuscator":
93 database.cursor.execute("SELECT software, COUNT(domain) AS cnt FROM instances WHERE has_obfuscation = 1 GROUP BY software ORDER BY cnt DESC LIMIT ?", [amount])
94 elif mode == "obfuscation":
95 database.cursor.execute("SELECT has_obfuscation, COUNT(domain) AS cnt FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica') GROUP BY has_obfuscation ORDER BY cnt DESC LIMIT ?", [amount])
96 elif mode == "block_level":
97 database.cursor.execute("SELECT block_level, COUNT(rowid) AS cnt FROM blocks GROUP BY block_level ORDER BY cnt DESC LIMIT ?", [amount])
99 raise HTTPException(status_code=400, detail="No filter specified")
103 for row in database.cursor.fetchall():
106 "score" : round(row[1]),
109 return JSONResponse(status_code=200, content=scores)
111 @router.get(config.get("base_url") + "/api/list.json", response_class=JSONResponse)
112 def api_list(request: Request, mode: str, value: str, amount: int):
113 if mode is None or value is None or amount is None:
114 raise HTTPException(status_code=500, detail="No filter specified")
115 elif amount > config.get("api_limit"):
116 raise HTTPException(status_code=500, detail=f"amount={amount} is to big")
118 if mode in ("detection_mode", "software", "command", "origin"):
119 database.cursor.execute(
124 LIMIT ?", [value, amount]
127 domainlist = database.cursor.fetchall()
130 @router.get(config.get("base_url") + "/api/top.json", response_class=JSONResponse)
131 def api_index(request: Request, mode: str, value: str, amount: int):
132 if mode is None or value is None or amount is None:
133 raise HTTPException(status_code=500, detail="No filter specified")
134 elif amount > config.get("api_limit"):
135 raise HTTPException(status_code=500, detail=f"amount={amount} is to big")
137 domain = wildchar = punycode = reason = None
139 if mode == "block_level":
140 database.cursor.execute(
141 "SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks WHERE block_level = ? LIMIT ?", [value, amount]
143 elif mode in ["domain", "reverse"]:
144 domain = tidyup.domain(value)
145 if not domain_helper.is_wanted(domain):
146 raise HTTPException(status_code=500, detail=f"domain='{domain}' is not wanted")
148 wildchar = "*." + ".".join(domain.split(".")[-domain.count("."):])
149 punycode = domain.encode("idna").decode("utf-8")
150 elif mode == "reason":
151 reason = re.sub("(%|_)", "", tidyup.reason(value))
153 raise HTTPException(status_code=400, detail="Keyword is shorter than three characters")
156 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
158 WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? \
159 ORDER BY block_level ASC, first_seen ASC \
165 utils.get_hash(domain),
171 elif mode == "reverse":
172 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
174 WHERE blocker = ? OR blocker = ? OR blocker = ? OR blocker = ? OR blocker = ? OR blocker = ? \
175 ORDER BY first_seen ASC \
180 utils.get_hash(domain),
185 elif mode == "reason":
186 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
188 WHERE reason LIKE ? AND reason != '' \
189 ORDER BY first_seen ASC \
195 blocklist = database.cursor.fetchall()
198 for blocker, blocked, block_level, reason, first_seen, last_seen in blocklist:
199 if reason is not None and reason != "":
200 reason = reason.replace(",", " ").replace(" ", " ")
206 "first_seen": first_seen,
207 "last_seen" : last_seen
210 if block_level in result:
211 result[block_level].append(entry)
213 result[block_level] = [entry]
217 @router.get(config.get("base_url") + "/api/domain.json", response_class=JSONResponse)
218 def api_domain(domain: str):
220 raise HTTPException(status_code=400, detail="Invalid request, parameter 'domain' missing")
222 # Tidy up domain name
223 domain = tidyup.domain(domain).encode("idna").decode("utf-8")
225 if not domain_helper.is_wanted(domain):
226 raise HTTPException(status_code=500, detail=f"domain='{domain}' is not wanted")
229 database.cursor.execute("SELECT * FROM instances WHERE domain = ? LIMIT 1", [domain])
230 domain_data = database.cursor.fetchone()
232 if domain_data is None:
233 raise HTTPException(status_code=404, detail=f"domain='{domain}' not found")
235 return JSONResponse(status_code=200, content=dict(domain_data))
237 @router.get(config.get("base_url") + "/api/mutual.json", response_class=JSONResponse)
238 def api_mutual(domains: list[str] = Query()):
239 """Return 200 if federation is open between the two, 4xx otherwise"""
240 database.cursor.execute(
241 "SELECT block_level FROM blocks " \
242 "WHERE ((blocker = :a OR blocker = :b) AND (blocked = :b OR blocked = :a OR blocked = :aw OR blocked = :bw)) " \
243 "AND block_level = 'reject' " \
248 "aw": "*." + domains[0],
249 "bw": "*." + domains[1],
253 if database.cursor.fetchone() is not None:
255 return JSONResponse(status_code=418, content={})
258 return JSONResponse(status_code=200, content={})
260 @router.get(config.get("base_url") + "/.well-known/nodeinfo", response_class=JSONResponse)
261 def wellknown_nodeinfo(request: Request):
262 return JSONResponse(status_code=200, content={
264 "rel" : "http://nodeinfo.diaspora.software/ns/schema/1.0",
265 "href": f"{config.get('scheme')}://{config.get('hostname')}{config.get('base_url')}/nodeinfo/1.0"
269 @router.get(config.get("base_url") + "/nodeinfo/1.0", response_class=JSONResponse)
270 def nodeinfo_1_0(request: Request):
271 return JSONResponse(status_code=200, content={
292 "openRegistrations": False,
294 "nodeName": "Fedi Block API",
307 "explicitContent": False,
311 @router.get(config.get("base_url") + "/api/v1/instance/domain_blocks", response_class=JSONResponse)
312 def api_domain_blocks(request: Request):
313 blocked = blacklist.get_all()
316 for block in blocked:
319 "digest" : utils.get_hash(block),
320 "severity": "suspend",
321 "comment" : blocked[block],
324 return JSONResponse(status_code=200, content=blocking)
326 @router.get(config.get("base_url") + "/api/v1/instance/peers", response_class=JSONResponse)
327 def api_peers(request: Request):
328 database.cursor.execute("SELECT domain FROM instances WHERE nodeinfo_url IS NOT NULL")
331 for row in database.cursor.fetchall():
332 peers.append(row["domain"])
334 return JSONResponse(status_code=200, content=peers)
336 @router.get(config.get("base_url") + "/scoreboard")
337 def scoreboard(request: Request, mode: str, amount: int):
339 raise HTTPException(status_code=400, detail="No mode specified")
341 raise HTTPException(status_code=500, detail="Invalid amount specified")
343 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode={mode}&amount={amount}")
346 raise HTTPException(status_code=500, detail="Could not determine scores")
347 elif not response.ok:
348 raise HTTPException(status_code=response.status_code, detail=response.text)
350 return templates.TemplateResponse("views/scoreboard.html", {
351 "base_url" : utils.base_url(),
352 "slogan" : config.get("slogan"),
353 "theme" : config.get("theme"),
358 "scores" : json_helper.from_response(response)
361 @router.get(config.get("base_url") + "/list")
362 def list_domains(request: Request, mode: str, value: str, amount: int = config.get("api_limit")):
363 if mode == "detection_mode" and not instances.valid(value, "detection_mode"):
364 raise HTTPException(status_code=500, detail="Invalid detection mode provided")
366 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/list.json?mode={mode}&value={value}&amount={amount}")
369 if response is not None and response.ok:
370 domainlist = response.json()
371 tformat = config.get("timestamp_format")
372 for row in domainlist:
373 row["first_seen"] = datetime.utcfromtimestamp(row["first_seen"]).strftime(tformat)
374 row["last_updated"] = datetime.utcfromtimestamp(row["last_updated"]).strftime(tformat) if isinstance(row["last_updated"], float) else None
376 return templates.TemplateResponse("views/list.html", {
378 "mode" : mode if response is not None else None,
379 "value" : value if response is not None else None,
380 "amount" : amount if response is not None else None,
381 "found" : len(domainlist),
382 "domainlist": domainlist,
383 "slogan" : config.get("slogan"),
384 "theme" : config.get("theme"),
387 @router.get(config.get("base_url") + "/top")
388 def top(request: Request, mode: str, value: str, amount: int = config.get("api_limit")):
389 if mode == "block_level" and not blocks.valid(value, "block_level"):
390 raise HTTPException(status_code=500, detail="Invalid block level provided")
391 elif mode in ["domain", "reverse"] and not domain_helper.is_wanted(value):
392 raise HTTPException(status_code=500, detail="Invalid or blocked domain specified")
394 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/top.json?mode={mode}&value={value}&amount={amount}")
398 if response.ok and response.status_code == 200 and len(response.text) > 0:
399 blocklist = response.json()
401 tformat = config.get("timestamp_format")
402 for block_level in blocklist:
403 for row in blocklist[block_level]:
404 row["first_seen"] = datetime.utcfromtimestamp(row["first_seen"]).strftime(tformat)
405 row["last_seen"] = datetime.utcfromtimestamp(row["last_seen"]).strftime(tformat) if isinstance(row["last_seen"], float) else None
408 return templates.TemplateResponse("views/top.html", {
410 "mode" : mode if response is not None else None,
411 "value" : value if response is not None else None,
412 "amount" : amount if response is not None else None,
414 "blocklist": blocklist,
415 "slogan" : config.get("slogan"),
416 "theme" : config.get("theme"),
419 @router.get(config.get("base_url") + "/infos")
420 def infos(request: Request, domain: str):
422 raise HTTPException(status_code=400, detail="Invalid request, parameter 'domain' missing")
424 # Tidy up domain name
425 domain = tidyup.domain(domain).encode("idna").decode("utf-8")
427 if not domain_helper.is_wanted(domain):
428 raise HTTPException(status_code=500, detail=f"domain='{domain}' is not wanted")
430 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/domain.json?domain={domain}")
432 if not response.ok or response.status_code > 200 or response.text.strip() == "":
433 raise HTTPException(status_code=response.status_code, detail=response.reason)
435 domain_data = response.json()
438 tformat = config.get("timestamp_format")
440 for key in domain_data.keys():
441 if key in ["last_nodeinfo", "last_blocked", "first_seen", "last_updated", "last_instance_fetch"] and isinstance(domain_data[key], float):
443 instance[key] = datetime.utcfromtimestamp(domain_data[key]).strftime(tformat)
446 instance[key] = domain_data[key]
448 return templates.TemplateResponse("views/infos.html", {
451 "instance": instance,
452 "theme" : config.get("theme"),
453 "slogan" : config.get("slogan"),
456 @router.get(config.get("base_url") + "/rss")
457 def rss(request: Request, domain: str = None):
458 if domain is not None:
459 domain = tidyup.domain(domain).encode("idna").decode("utf-8")
461 wildchar = "*." + ".".join(domain.split(".")[-domain.count("."):])
462 punycode = domain.encode("idna").decode("utf-8")
464 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? ORDER BY first_seen DESC LIMIT ?", [
466 "*." + domain, wildchar,
467 utils.get_hash(domain),
470 config.get("rss_limit")
473 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks ORDER BY first_seen DESC LIMIT ?", [config.get("rss_limit")])
475 result = database.cursor.fetchall()
482 "block_level": row[2],
483 "reason" : "Provided reason: '" + row[3] + "'" if row[3] is not None and row[3] != "" else "No reason provided.",
484 "first_seen" : format_datetime(datetime.fromtimestamp(row[4])),
485 "last_seen" : format_datetime(datetime.fromtimestamp(row[5])),
488 return templates.TemplateResponse("views/rss.xml", {
490 "timestamp": format_datetime(datetime.now()),
492 "scheme" : config.get("scheme"),
493 "hostname" : config.get("hostname"),
496 "Content-Type": "routerlication/rss+xml"
499 @router.get(config.get("base_url") + "/robots.txt", response_class=PlainTextResponse)
500 def robots(request: Request):
501 return templates.TemplateResponse("views/robots.txt", {
503 "base_url": config.get("base_url")
506 @router.get(config.get("base_url") + "/")
507 def index(request: Request):
509 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/info.json")
512 raise HTTPException(status_code=response.status_code, detail=response.text)
514 return templates.TemplateResponse("views/index.html", {
516 "theme" : config.get("theme"),
517 "info" : response.json(),
518 "slogan" : config.get("slogan"),
521 if __name__ == "__main__":
522 uvicorn.run("daemon:router", host=config.get("host"), port=config.get("port"), log_level=config.get("log_level"), proxy_headers=True)