2 # -*- coding: utf-8 -*-
4 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
5 # Copyright (C) 2023 Free Software Foundation
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published
9 # by the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <https://www.gnu.org/licenses/>.
22 from datetime import datetime
23 from email.utils import format_datetime
24 from pathlib import Path
27 from fastapi import Request, HTTPException, Query
28 from fastapi.responses import JSONResponse
29 from fastapi.responses import PlainTextResponse
30 from fastapi.staticfiles import StaticFiles
31 from fastapi.templating import Jinja2Templates
36 from fba import database
39 from fba.helpers import blacklist
40 from fba.helpers import config
41 from fba.helpers import domain as domain_helper
42 from fba.helpers import json as json_helper
43 from fba.helpers import tidyup
45 from fba.models import blocks
46 from fba.models import instances
48 router = fastapi.FastAPI(docs_url=config.get("base_url") + "/docs", redoc_url=config.get("base_url") + "/redoc")
51 StaticFiles(directory=Path(__file__).parent.absolute() / "static"),
55 templates = Jinja2Templates(directory="templates")
57 @router.get(config.get("base_url") + "/api/info.json", response_class=JSONResponse)
59 database.cursor.execute("SELECT (SELECT COUNT(domain) AS total_websites FROM instances), (SELECT COUNT(domain) AS supported_instances FROM instances WHERE software IN ('pleroma', 'mastodon', 'lemmy', 'friendica', 'misskey', 'peertube', 'takahe', 'gotosocial', 'brighteon', 'wildebeest', 'bookwyrm', 'mitra', 'areionskey', 'mammuthus', 'neodb', 'smithereen', 'vebinet')), (SELECT COUNT(blocker) FROM blocks) AS total_blocks, (SELECT COUNT(domain) AS erroneous_instances FROM instances WHERE last_error_details IS NOT NULL)")
60 row = database.cursor.fetchone()
62 return JSONResponse(status_code=200, content=row)
64 @router.get(config.get("base_url") + "/api/scoreboard.json", response_class=JSONResponse)
65 def api_scoreboard(mode: str, amount: int):
66 if amount > config.get("api_limit"):
67 raise HTTPException(status_code=400, detail="Too many results")
70 database.cursor.execute("SELECT blocked, COUNT(blocked) AS score FROM blocks GROUP BY blocked ORDER BY score DESC LIMIT ?", [amount])
71 elif mode == "blocker":
72 database.cursor.execute("SELECT blocker, COUNT(blocker) AS score FROM blocks GROUP BY blocker ORDER BY score DESC LIMIT ?", [amount])
73 elif mode == "reference":
74 database.cursor.execute("SELECT origin, COUNT(domain) AS score FROM instances WHERE origin IS NOT NULL GROUP BY origin ORDER BY score DESC LIMIT ?", [amount])
75 elif mode == "original_software":
76 database.cursor.execute("SELECT original_software, COUNT(domain) AS score FROM instances WHERE original_software IS NOT NULL GROUP BY original_software ORDER BY score DESC, original_software ASC LIMIT ?", [amount])
77 elif mode == "software":
78 database.cursor.execute("SELECT software, COUNT(domain) AS score FROM instances WHERE software IS NOT NULL GROUP BY software ORDER BY score DESC, software ASC LIMIT ?", [amount])
79 elif mode == "command":
80 database.cursor.execute("SELECT command, COUNT(domain) AS score FROM instances WHERE command IS NOT NULL GROUP BY command ORDER BY score DESC, command ASC LIMIT ?", [amount])
81 elif mode == "error_code":
82 database.cursor.execute("SELECT last_status_code, COUNT(domain) AS score FROM instances WHERE last_status_code IS NOT NULL AND last_status_code != '200' GROUP BY last_status_code ORDER BY score DESC LIMIT ?", [amount])
83 elif mode == "detection_mode":
84 database.cursor.execute("SELECT detection_mode, COUNT(domain) AS cnt FROM instances GROUP BY detection_mode ORDER BY cnt DESC LIMIT ?", [amount])
85 elif mode == "avg_peers":
86 database.cursor.execute("SELECT software, AVG(total_peers) AS average FROM instances WHERE software IS NOT NULL AND total_peers IS NOT NULL GROUP BY software HAVING average > 0 ORDER BY average DESC LIMIT ?", [amount])
87 elif mode == "avg_blocks":
88 database.cursor.execute("SELECT software, AVG(total_blocks) AS average FROM instances WHERE software IS NOT NULL AND total_blocks IS NOT NULL GROUP BY software HAVING average > 0 ORDER BY average DESC LIMIT ?", [amount])
89 elif mode == "obfuscator":
90 database.cursor.execute("SELECT software, COUNT(domain) AS cnt FROM instances WHERE has_obfuscation = 1 GROUP BY software ORDER BY cnt DESC LIMIT ?", [amount])
91 elif mode == "obfuscation":
92 database.cursor.execute("SELECT has_obfuscation, COUNT(domain) AS cnt FROM instances WHERE software IN ('pleroma', 'lemmy', 'mastodon', 'misskey', 'friendica') GROUP BY has_obfuscation ORDER BY cnt DESC LIMIT ?", [amount])
93 elif mode == "block_level":
94 database.cursor.execute("SELECT block_level, COUNT(rowid) AS cnt FROM blocks GROUP BY block_level ORDER BY cnt DESC LIMIT ?", [amount])
96 raise HTTPException(status_code=400, detail="No filter specified")
100 for row in database.cursor.fetchall():
103 "score" : round(row[1]),
106 return JSONResponse(status_code=200, content=scores)
108 @router.get(config.get("base_url") + "/api/list.json", response_class=JSONResponse)
109 def api_list(request: Request, mode: str, value: str, amount: int):
110 if mode is None or value is None or amount is None:
111 raise HTTPException(status_code=500, detail="No filter specified")
112 elif amount > config.get("api_limit"):
113 raise HTTPException(status_code=500, detail=f"amount={amount} is to big")
115 if mode in ("detection_mode", "original_software", "software", "command", "origin"):
116 database.cursor.execute(
121 LIMIT ?", [value, amount]
123 elif mode == "recently":
124 database.cursor.execute(
127 ORDER BY first_seen DESC \
131 raise HTTPException(status_code=500, detail=f"mode='{mode}' is unsupported")
133 domainlist = database.cursor.fetchall()
136 @router.get(config.get("base_url") + "/api/top.json", response_class=JSONResponse)
137 def api_index(request: Request, mode: str, value: str, amount: int):
138 if mode is None or value is None or amount is None:
139 raise HTTPException(status_code=500, detail="No filter specified")
140 elif amount > config.get("api_limit"):
141 raise HTTPException(status_code=500, detail=f"amount={amount} is to big")
143 domain = wildchar = punycode = reason = None
145 if mode == "block_level":
146 database.cursor.execute(
147 "SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks WHERE block_level = ? LIMIT ?", [value, amount]
149 elif mode in ["domain", "reverse"]:
150 domain = tidyup.domain(value)
151 if not domain_helper.is_wanted(domain):
152 raise HTTPException(status_code=500, detail=f"domain='{domain}' is not wanted")
154 wildchar = "*." + ".".join(domain.split(".")[-domain.count("."):])
155 punycode = domain.encode("idna").decode("utf-8")
156 elif mode == "reason":
157 reason = re.sub("(%|_)", "", tidyup.reason(value))
159 raise HTTPException(status_code=400, detail="Keyword is shorter than three characters")
162 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
164 WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? \
165 ORDER BY block_level ASC, first_seen ASC \
171 utils.get_hash(domain),
177 elif mode == "reverse":
178 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
180 WHERE blocker = ? OR blocker = ? OR blocker = ? OR blocker = ? OR blocker = ? OR blocker = ? \
181 ORDER BY first_seen ASC \
186 utils.get_hash(domain),
191 elif mode == "reason":
192 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
194 WHERE reason LIKE ? AND reason != '' \
195 ORDER BY first_seen ASC \
201 blocklist = database.cursor.fetchall()
204 for blocker, blocked, block_level, reason, first_seen, last_seen in blocklist:
205 if reason is not None and reason != "":
206 reason = reason.replace(",", " ").replace(" ", " ")
212 "first_seen": first_seen,
213 "last_seen" : last_seen
216 if block_level in result:
217 result[block_level].append(entry)
219 result[block_level] = [entry]
223 @router.get(config.get("base_url") + "/api/domain.json", response_class=JSONResponse)
224 def api_domain(domain: str):
226 raise HTTPException(status_code=400, detail="Invalid request, parameter 'domain' missing")
228 # Tidy up domain name
229 domain = tidyup.domain(domain).encode("idna").decode("utf-8")
231 if not domain_helper.is_wanted(domain):
232 raise HTTPException(status_code=500, detail=f"domain='{domain}' is not wanted")
235 database.cursor.execute("SELECT * FROM instances WHERE domain = ? LIMIT 1", [domain])
236 domain_data = database.cursor.fetchone()
238 if domain_data is None:
239 raise HTTPException(status_code=404, detail=f"domain='{domain}' not found")
241 return JSONResponse(status_code=200, content=dict(domain_data))
243 @router.get(config.get("base_url") + "/api/mutual.json", response_class=JSONResponse)
244 def api_mutual(domains: list[str] = Query()):
245 """Return 200 if federation is open between the two, 4xx otherwise"""
246 database.cursor.execute(
247 "SELECT block_level FROM blocks " \
248 "WHERE ((blocker = :a OR blocker = :b) AND (blocked = :b OR blocked = :a OR blocked = :aw OR blocked = :bw)) " \
249 "AND block_level = 'reject' " \
254 "aw": "*." + domains[0],
255 "bw": "*." + domains[1],
259 if database.cursor.fetchone() is not None:
261 return JSONResponse(status_code=418, content={})
264 return JSONResponse(status_code=200, content={})
266 @router.get(config.get("base_url") + "/.well-known/nodeinfo", response_class=JSONResponse)
267 def wellknown_nodeinfo(request: Request):
268 return JSONResponse(status_code=200, content={
270 "rel" : "http://nodeinfo.diaspora.software/ns/schema/1.0",
271 "href": f"{config.get('scheme')}://{config.get('hostname')}{config.get('base_url')}/nodeinfo/1.0"
275 @router.get(config.get("base_url") + "/nodeinfo/1.0", response_class=JSONResponse)
276 def nodeinfo_1_0(request: Request):
277 return JSONResponse(status_code=200, content={
298 "openRegistrations": False,
300 "nodeName": "Fedi Block API",
313 "explicitContent": False,
317 @router.get(config.get("base_url") + "/api/v1/instance/domain_blocks", response_class=JSONResponse)
318 def api_domain_blocks(request: Request):
319 blocked = blacklist.get_all()
322 for block in blocked:
325 "digest" : utils.get_hash(block),
326 "severity": "suspend",
327 "comment" : blocked[block],
330 return JSONResponse(status_code=200, content=blocking)
332 @router.get(config.get("base_url") + "/api/v1/instance/peers", response_class=JSONResponse)
333 def api_peers(request: Request):
334 database.cursor.execute("SELECT domain FROM instances WHERE nodeinfo_url IS NOT NULL")
337 for row in database.cursor.fetchall():
338 peers.append(row["domain"])
340 return JSONResponse(status_code=200, content=peers)
342 @router.get(config.get("base_url") + "/scoreboard")
343 def scoreboard(request: Request, mode: str, amount: int):
345 raise HTTPException(status_code=400, detail="No mode specified")
347 raise HTTPException(status_code=500, detail="Invalid amount specified")
349 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode={mode}&amount={amount}")
352 raise HTTPException(status_code=500, detail="Could not determine scores")
353 elif not response.ok:
354 raise HTTPException(status_code=response.status_code, detail=response.text)
356 return templates.TemplateResponse("views/scoreboard.html", {
357 "base_url" : utils.base_url(),
358 "slogan" : config.get("slogan"),
359 "theme" : config.get("theme"),
364 "scores" : json_helper.from_response(response)
367 @router.get(config.get("base_url") + "/list")
368 def list_domains(request: Request, mode: str, value: str, amount: int = config.get("api_limit")):
369 if mode == "detection_mode" and not instances.valid(value, "detection_mode"):
370 raise HTTPException(status_code=500, detail="Invalid detection mode provided")
372 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/list.json?mode={mode}&value={value}&amount={amount}")
375 if response is not None and response.ok:
376 domainlist = response.json()
377 tformat = config.get("timestamp_format")
378 for row in domainlist:
379 row["first_seen"] = datetime.utcfromtimestamp(row["first_seen"]).strftime(tformat)
380 row["last_updated"] = datetime.utcfromtimestamp(row["last_updated"]).strftime(tformat) if isinstance(row["last_updated"], float) else None
382 return templates.TemplateResponse("views/list.html", {
384 "mode" : mode if response is not None else None,
385 "value" : value if response is not None else None,
386 "amount" : amount if response is not None else None,
387 "found" : len(domainlist),
388 "domainlist": domainlist,
389 "slogan" : config.get("slogan"),
390 "theme" : config.get("theme"),
393 @router.get(config.get("base_url") + "/top")
394 def top(request: Request, mode: str, value: str, amount: int = config.get("api_limit")):
395 if mode == "block_level" and not blocks.valid(value, "block_level"):
396 raise HTTPException(status_code=500, detail="Invalid block level provided")
397 elif mode in ["domain", "reverse"] and not domain_helper.is_wanted(value):
398 raise HTTPException(status_code=500, detail="Invalid or blocked domain specified")
400 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/top.json?mode={mode}&value={value}&amount={amount}")
404 if response.ok and response.status_code == 200 and len(response.text) > 0:
405 blocklist = response.json()
407 tformat = config.get("timestamp_format")
408 for block_level in blocklist:
409 for row in blocklist[block_level]:
410 row["first_seen"] = datetime.utcfromtimestamp(row["first_seen"]).strftime(tformat)
411 row["last_seen"] = datetime.utcfromtimestamp(row["last_seen"]).strftime(tformat) if isinstance(row["last_seen"], float) else None
414 return templates.TemplateResponse("views/top.html", {
416 "mode" : mode if response is not None else None,
417 "value" : value if response is not None else None,
418 "amount" : amount if response is not None else None,
420 "blocklist": blocklist,
421 "slogan" : config.get("slogan"),
422 "theme" : config.get("theme"),
425 @router.get(config.get("base_url") + "/infos")
426 def infos(request: Request, domain: str):
428 raise HTTPException(status_code=400, detail="Invalid request, parameter 'domain' missing")
430 # Tidy up domain name
431 domain = tidyup.domain(domain).encode("idna").decode("utf-8")
433 if not domain_helper.is_wanted(domain):
434 raise HTTPException(status_code=500, detail=f"domain='{domain}' is not wanted")
436 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/domain.json?domain={domain}")
438 if not response.ok or response.status_code > 200 or response.text.strip() == "":
439 raise HTTPException(status_code=response.status_code, detail=response.reason)
441 domain_data = response.json()
444 tformat = config.get("timestamp_format")
446 for key in domain_data.keys():
447 if key in ["last_nodeinfo", "last_blocked", "first_seen", "last_updated", "last_instance_fetch"] and isinstance(domain_data[key], float):
449 instance[key] = datetime.utcfromtimestamp(domain_data[key]).strftime(tformat)
452 instance[key] = domain_data[key]
454 return templates.TemplateResponse("views/infos.html", {
457 "instance": instance,
458 "theme" : config.get("theme"),
459 "slogan" : config.get("slogan"),
462 @router.get(config.get("base_url") + "/rss")
463 def rss(request: Request, domain: str = None):
464 if domain is not None:
465 domain = tidyup.domain(domain).encode("idna").decode("utf-8")
467 wildchar = "*." + ".".join(domain.split(".")[-domain.count("."):])
468 punycode = domain.encode("idna").decode("utf-8")
470 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
472 WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? \
473 ORDER BY first_seen DESC \
476 "*." + domain, wildchar,
477 utils.get_hash(domain),
480 config.get("rss_limit")
483 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
485 ORDER BY first_seen DESC \
486 LIMIT ?", [config.get("rss_limit")])
488 result = database.cursor.fetchall()
495 "block_level": row[2],
496 "reason" : "Provided reason: '" + row[3] + "'" if row[3] is not None and row[3] != "" else "No reason provided.",
497 "first_seen" : format_datetime(datetime.fromtimestamp(row[4])),
498 "last_seen" : format_datetime(datetime.fromtimestamp(row[5])),
501 return templates.TemplateResponse("views/rss.xml", {
503 "timestamp": format_datetime(datetime.now()),
505 "scheme" : config.get("scheme"),
506 "hostname" : config.get("hostname"),
509 "Content-Type": "routerlication/rss+xml"
512 @router.get(config.get("base_url") + "/robots.txt", response_class=PlainTextResponse)
513 def robots(request: Request):
514 return templates.TemplateResponse("views/robots.txt", {
516 "base_url": config.get("base_url")
519 @router.get(config.get("base_url") + "/")
520 def index(request: Request):
522 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/info.json")
525 raise HTTPException(status_code=response.status_code, detail=response.text)
527 return templates.TemplateResponse("views/index.html", {
529 "theme" : config.get("theme"),
530 "info" : response.json(),
531 "slogan" : config.get("slogan"),
534 if __name__ == "__main__":
537 host=config.get("host"),
538 port=config.get("port"),
539 log_level=config.get("log_level"),