2 # -*- coding: utf-8 -*-
4 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
5 # Copyright (C) 2023 Free Software Foundation
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published
9 # by the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <https://www.gnu.org/licenses/>.
22 from datetime import datetime
23 from email.utils import format_datetime
24 from pathlib import Path
27 from fastapi import Request, HTTPException, Query
28 from fastapi.responses import JSONResponse
29 from fastapi.responses import PlainTextResponse
30 from fastapi.staticfiles import StaticFiles
31 from fastapi.templating import Jinja2Templates
36 from fba import database
39 from fba.helpers import blacklist
40 from fba.helpers import config
41 from fba.helpers import domain as domain_helper
42 from fba.helpers import json as json_helper
43 from fba.helpers import tidyup
45 from fba.models import blocks
46 from fba.models import instances
48 router = fastapi.FastAPI(docs_url=config.get("base_url") + "/docs", redoc_url=config.get("base_url") + "/redoc")
51 StaticFiles(directory=Path(__file__).parent.absolute() / "static"),
55 templates = Jinja2Templates(directory="templates")
57 @router.get(config.get("base_url") + "/api/info.json", response_class=JSONResponse)
59 database.cursor.execute("SELECT (SELECT COUNT(domain) FROM instances) AS total_websites, (SELECT COUNT(domain) FROM instances WHERE software IN ('pleroma', 'mastodon', 'lemmy', 'friendica', 'misskey', 'peertube', 'takahe', 'gotosocial', 'brighteon', 'wildebeest', 'bookwyrm', 'mitra', 'areionskey', 'mammuthus', 'neodb', 'smithereen', 'vebinet', 'hugo', 'toki', 'snac')) AS supported_instances, (SELECT COUNT(blocker) FROM blocks) AS total_blocks, (SELECT COUNT(domain) FROM instances WHERE last_error_details IS NOT NULL) AS erroneous_instances")
61 row = database.cursor.fetchone()
63 return JSONResponse(status_code=200, content={
64 "total_websites" : row["total_websites"],
65 "supported_instances": row["supported_instances"],
66 "total_blocks" : row["total_blocks"],
67 "erroneous_instances": row["erroneous_instances"],
71 @router.get(config.get("base_url") + "/api/scoreboard.json", response_class=JSONResponse)
72 def api_scoreboard(mode: str, amount: int):
73 if amount > config.get("api_limit"):
74 raise HTTPException(status_code=400, detail="Too many results")
77 database.cursor.execute("SELECT blocked, COUNT(blocked) AS score FROM blocks GROUP BY blocked ORDER BY score DESC LIMIT ?", [amount])
78 elif mode == "blocker":
79 database.cursor.execute("SELECT blocker, COUNT(blocker) AS score FROM blocks GROUP BY blocker ORDER BY score DESC LIMIT ?", [amount])
80 elif mode == "reference":
81 database.cursor.execute("SELECT origin, COUNT(domain) AS score FROM instances WHERE origin IS NOT NULL GROUP BY origin ORDER BY score DESC LIMIT ?", [amount])
82 elif mode == "original_software":
83 database.cursor.execute("SELECT original_software, COUNT(domain) AS score FROM instances WHERE original_software IS NOT NULL GROUP BY original_software ORDER BY score DESC, original_software ASC LIMIT ?", [amount])
84 elif mode == "software":
85 database.cursor.execute("SELECT software, COUNT(domain) AS score FROM instances WHERE software IS NOT NULL GROUP BY software ORDER BY score DESC, software ASC LIMIT ?", [amount])
86 elif mode == "command":
87 database.cursor.execute("SELECT command, COUNT(domain) AS score FROM instances WHERE command IS NOT NULL GROUP BY command ORDER BY score DESC, command ASC LIMIT ?", [amount])
88 elif mode == "error_code":
89 database.cursor.execute("SELECT last_status_code, COUNT(domain) AS score FROM instances WHERE last_status_code IS NOT NULL AND last_status_code != '200' GROUP BY last_status_code ORDER BY score DESC LIMIT ?", [amount])
90 elif mode == "detection_mode":
91 database.cursor.execute("SELECT detection_mode, COUNT(domain) AS cnt FROM instances GROUP BY detection_mode ORDER BY cnt DESC LIMIT ?", [amount])
92 elif mode == "avg_peers":
93 database.cursor.execute("SELECT software, AVG(total_peers) AS average FROM instances WHERE software IS NOT NULL AND total_peers IS NOT NULL GROUP BY software HAVING average > 0 ORDER BY average DESC LIMIT ?", [amount])
94 elif mode == "avg_blocks":
95 database.cursor.execute("SELECT software, AVG(total_blocks) AS average FROM instances WHERE software IS NOT NULL AND total_blocks IS NOT NULL GROUP BY software HAVING average > 0 ORDER BY average DESC LIMIT ?", [amount])
96 elif mode == "obfuscator":
97 database.cursor.execute("SELECT software, COUNT(domain) AS cnt FROM instances WHERE has_obfuscation = 1 GROUP BY software ORDER BY cnt DESC LIMIT ?", [amount])
98 elif mode == "obfuscation":
99 database.cursor.execute("SELECT has_obfuscation, COUNT(domain) AS cnt FROM instances WHERE software IN ('pleroma', 'lemmy', 'mastodon', 'misskey', 'friendica') GROUP BY has_obfuscation ORDER BY cnt DESC LIMIT ?", [amount])
100 elif mode == "block_level":
101 database.cursor.execute("SELECT block_level, COUNT(rowid) AS cnt FROM blocks GROUP BY block_level ORDER BY cnt DESC LIMIT ?", [amount])
103 raise HTTPException(status_code=400, detail="No filter specified")
107 for row in database.cursor.fetchall():
110 "score" : round(row[1]),
113 return JSONResponse(status_code=200, content=scores)
115 @router.get(config.get("base_url") + "/api/list.json", response_class=JSONResponse)
116 def api_list(request: Request, mode: str, value: str, amount: int):
117 if mode is None or value is None or amount is None:
118 raise HTTPException(status_code=500, detail="No filter specified")
119 elif amount > config.get("api_limit"):
120 raise HTTPException(status_code=500, detail=f"amount={amount} is to big")
122 if mode in ("detection_mode", "original_software", "software", "command", "origin"):
123 database.cursor.execute(
128 LIMIT ?", [value, amount]
130 elif mode == "recently":
131 database.cursor.execute(
134 ORDER BY first_seen DESC \
138 raise HTTPException(status_code=500, detail=f"mode='{mode}' is unsupported")
140 domainlist = database.cursor.fetchall()
143 @router.get(config.get("base_url") + "/api/top.json", response_class=JSONResponse)
144 def api_index(request: Request, mode: str, value: str, amount: int):
145 if mode is None or value is None or amount is None:
146 raise HTTPException(status_code=500, detail="No filter specified")
147 elif amount > config.get("api_limit"):
148 raise HTTPException(status_code=500, detail=f"amount={amount} is to big")
150 domain = wildchar = punycode = reason = None
152 if mode == "block_level":
153 database.cursor.execute(
154 "SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks WHERE block_level = ? LIMIT ?", [value, amount]
156 elif mode in ["domain", "reverse"]:
157 domain = tidyup.domain(value)
158 if not domain_helper.is_wanted(domain):
159 raise HTTPException(status_code=500, detail=f"domain='{domain}' is not wanted")
161 wildchar = "*." + ".".join(domain.split(".")[-domain.count("."):])
162 punycode = domain.encode("idna").decode("utf-8")
163 elif mode == "reason":
164 reason = re.sub("(%|_)", "", tidyup.reason(value))
166 raise HTTPException(status_code=400, detail="Keyword is shorter than three characters")
169 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
171 WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? \
172 ORDER BY block_level ASC, first_seen ASC \
178 utils.get_hash(domain),
184 elif mode == "reverse":
185 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
187 WHERE blocker = ? OR blocker = ? OR blocker = ? OR blocker = ? OR blocker = ? OR blocker = ? \
188 ORDER BY first_seen ASC \
193 utils.get_hash(domain),
198 elif mode == "reason":
199 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
201 WHERE reason LIKE ? AND reason != '' \
202 ORDER BY first_seen ASC \
208 blocklist = database.cursor.fetchall()
211 for blocker, blocked, block_level, reason, first_seen, last_seen in blocklist:
212 if reason is not None and reason != "":
213 reason = reason.replace(",", " ").replace(" ", " ")
219 "first_seen": first_seen,
220 "last_seen" : last_seen
223 if block_level in result:
224 result[block_level].append(entry)
226 result[block_level] = [entry]
230 @router.get(config.get("base_url") + "/api/domain.json", response_class=JSONResponse)
231 def api_domain(domain: str):
233 raise HTTPException(status_code=400, detail="Invalid request, parameter 'domain' missing")
235 # Tidy up domain name
236 domain = tidyup.domain(domain).encode("idna").decode("utf-8")
238 if not domain_helper.is_wanted(domain):
239 raise HTTPException(status_code=500, detail=f"domain='{domain}' is not wanted")
242 database.cursor.execute("SELECT * FROM instances WHERE domain = ? LIMIT 1", [domain])
243 domain_data = database.cursor.fetchone()
245 if domain_data is None:
246 raise HTTPException(status_code=404, detail=f"domain='{domain}' not found")
248 return JSONResponse(status_code=200, content=dict(domain_data))
250 @router.get(config.get("base_url") + "/api/mutual.json", response_class=JSONResponse)
251 def api_mutual(domains: list[str] = Query()):
252 """Return 200 if federation is open between the two, 4xx otherwise"""
253 database.cursor.execute(
254 "SELECT block_level FROM blocks " \
255 "WHERE ((blocker = :a OR blocker = :b) AND (blocked = :b OR blocked = :a OR blocked = :aw OR blocked = :bw)) " \
256 "AND block_level = 'reject' " \
261 "aw": "*." + domains[0],
262 "bw": "*." + domains[1],
266 if database.cursor.fetchone() is not None:
268 return JSONResponse(status_code=418, content={})
271 return JSONResponse(status_code=200, content={})
273 @router.get(config.get("base_url") + "/.well-known/nodeinfo", response_class=JSONResponse)
274 def wellknown_nodeinfo(request: Request):
275 return JSONResponse(status_code=200, content={
277 "rel" : "http://nodeinfo.diaspora.software/ns/schema/1.0",
278 "href": f"{config.get('scheme')}://{config.get('hostname')}{config.get('base_url')}/nodeinfo/1.0"
282 @router.get(config.get("base_url") + "/nodeinfo/1.0", response_class=JSONResponse)
283 def nodeinfo_1_0(request: Request):
284 return JSONResponse(status_code=200, content={
305 "openRegistrations": False,
307 "nodeName": "Fedi Block API",
320 "explicitContent": False,
324 @router.get(config.get("base_url") + "/api/v1/instance/domain_blocks", response_class=JSONResponse)
325 def api_domain_blocks(request: Request):
326 blocked = blacklist.get_all()
329 for block in blocked:
332 "digest" : utils.get_hash(block),
333 "severity": "suspend",
334 "comment" : blocked[block],
337 return JSONResponse(status_code=200, content=blocking)
339 @router.get(config.get("base_url") + "/api/v1/instance/peers", response_class=JSONResponse)
340 def api_peers(request: Request):
341 database.cursor.execute("SELECT domain FROM instances WHERE nodeinfo_url IS NOT NULL")
344 for row in database.cursor.fetchall():
345 peers.append(row["domain"])
347 return JSONResponse(status_code=200, content=peers)
349 @router.get(config.get("base_url") + "/scoreboard")
350 def scoreboard(request: Request, mode: str, amount: int):
352 raise HTTPException(status_code=400, detail="No mode specified")
354 raise HTTPException(status_code=500, detail="Invalid amount specified")
356 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode={mode}&amount={amount}")
359 raise HTTPException(status_code=500, detail="Could not determine scores")
360 elif not response.ok:
361 raise HTTPException(status_code=response.status_code, detail=response.text)
363 return templates.TemplateResponse("views/scoreboard.html", {
364 "base_url" : utils.base_url(),
365 "slogan" : config.get("slogan"),
366 "theme" : config.get("theme"),
371 "scores" : json_helper.from_response(response)
374 @router.get(config.get("base_url") + "/list")
375 def list_domains(request: Request, mode: str, value: str, amount: int = config.get("api_limit")):
376 if mode == "detection_mode" and not instances.valid(value, "detection_mode"):
377 raise HTTPException(status_code=500, detail="Invalid detection mode provided")
379 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/list.json?mode={mode}&value={value}&amount={amount}")
382 if response is not None and response.ok:
383 domainlist = response.json()
384 tformat = config.get("timestamp_format")
385 for row in domainlist:
386 row["first_seen"] = datetime.utcfromtimestamp(row["first_seen"]).strftime(tformat)
387 row["last_updated"] = datetime.utcfromtimestamp(row["last_updated"]).strftime(tformat) if isinstance(row["last_updated"], float) else None
389 return templates.TemplateResponse("views/list.html", {
391 "mode" : mode if response is not None else None,
392 "value" : value if response is not None else None,
393 "amount" : amount if response is not None else None,
394 "found" : len(domainlist),
395 "domainlist": domainlist,
396 "slogan" : config.get("slogan"),
397 "theme" : config.get("theme"),
400 @router.get(config.get("base_url") + "/top")
401 def top(request: Request, mode: str, value: str, amount: int = config.get("api_limit")):
402 if mode == "block_level" and not blocks.valid(value, "block_level"):
403 raise HTTPException(status_code=500, detail="Invalid block level provided")
404 elif mode in ["domain", "reverse"] and not domain_helper.is_wanted(value):
405 raise HTTPException(status_code=500, detail="Invalid or blocked domain specified")
407 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/top.json?mode={mode}&value={value}&amount={amount}")
411 if response.ok and response.status_code == 200 and len(response.text) > 0:
412 blocklist = response.json()
414 tformat = config.get("timestamp_format")
415 for block_level in blocklist:
416 for row in blocklist[block_level]:
417 row["first_seen"] = datetime.utcfromtimestamp(row["first_seen"]).strftime(tformat)
418 row["last_seen"] = datetime.utcfromtimestamp(row["last_seen"]).strftime(tformat) if isinstance(row["last_seen"], float) else None
421 return templates.TemplateResponse("views/top.html", {
423 "mode" : mode if response is not None else None,
424 "value" : value if response is not None else None,
425 "amount" : amount if response is not None else None,
427 "blocklist": blocklist,
428 "slogan" : config.get("slogan"),
429 "theme" : config.get("theme"),
432 @router.get(config.get("base_url") + "/infos")
433 def infos(request: Request, domain: str):
435 raise HTTPException(status_code=400, detail="Invalid request, parameter 'domain' missing")
437 # Tidy up domain name
438 domain = tidyup.domain(domain).encode("idna").decode("utf-8")
440 if not domain_helper.is_wanted(domain):
441 raise HTTPException(status_code=500, detail=f"domain='{domain}' is not wanted")
443 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/domain.json?domain={domain}")
445 if not response.ok or response.status_code > 200 or response.text.strip() == "":
446 raise HTTPException(status_code=response.status_code, detail=response.reason)
448 domain_data = response.json()
451 tformat = config.get("timestamp_format")
453 for key in domain_data.keys():
454 if key in ["last_nodeinfo", "last_blocked", "first_seen", "last_updated", "last_instance_fetch"] and isinstance(domain_data[key], float):
456 instance[key] = datetime.utcfromtimestamp(domain_data[key]).strftime(tformat)
459 instance[key] = domain_data[key]
461 return templates.TemplateResponse("views/infos.html", {
464 "instance": instance,
465 "theme" : config.get("theme"),
466 "slogan" : config.get("slogan"),
469 @router.get(config.get("base_url") + "/rss")
470 def rss(request: Request, domain: str = None):
471 if domain is not None:
472 domain = tidyup.domain(domain).encode("idna").decode("utf-8")
474 wildchar = "*." + ".".join(domain.split(".")[-domain.count("."):])
475 punycode = domain.encode("idna").decode("utf-8")
477 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
479 WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? \
480 ORDER BY first_seen DESC \
483 "*." + domain, wildchar,
484 utils.get_hash(domain),
487 config.get("rss_limit")
490 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
492 ORDER BY first_seen DESC \
493 LIMIT ?", [config.get("rss_limit")])
495 result = database.cursor.fetchall()
502 "block_level": row[2],
503 "reason" : "Provided reason: '" + row[3] + "'" if row[3] is not None and row[3] != "" else "No reason provided.",
504 "first_seen" : format_datetime(datetime.fromtimestamp(row[4])),
505 "last_seen" : format_datetime(datetime.fromtimestamp(row[5])),
508 return templates.TemplateResponse("views/rss.xml", {
510 "timestamp": format_datetime(datetime.now()),
512 "scheme" : config.get("scheme"),
513 "hostname" : config.get("hostname"),
516 "Content-Type": "routerlication/rss+xml"
519 @router.get(config.get("base_url") + "/robots.txt", response_class=PlainTextResponse)
520 def robots(request: Request):
521 return templates.TemplateResponse("views/robots.txt", {
523 "base_url": config.get("base_url")
526 @router.get(config.get("base_url") + "/")
527 def index(request: Request):
529 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/info.json")
532 raise HTTPException(status_code=response.status_code, detail=response.text)
534 return templates.TemplateResponse("views/index.html", {
536 "theme" : config.get("theme"),
537 "info" : response.json(),
538 "slogan" : config.get("slogan"),
541 if __name__ == "__main__":
544 host=config.get("host"),
545 port=config.get("port"),
546 log_level=config.get("log_level"),