2 # -*- coding: utf-8 -*-
4 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
5 # Copyright (C) 2023 Free Software Foundation
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published
9 # by the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <https://www.gnu.org/licenses/>.
22 from datetime import datetime
23 from email.utils import format_datetime
24 from pathlib import Path
27 from fastapi import Request, HTTPException, Query
28 from fastapi.responses import JSONResponse
29 from fastapi.responses import PlainTextResponse
30 from fastapi.staticfiles import StaticFiles
31 from fastapi.templating import Jinja2Templates
36 from fba import database
39 from fba.helpers import blacklist
40 from fba.helpers import config
41 from fba.helpers import domain as domain_helper
42 from fba.helpers import json as json_helper
43 from fba.helpers import tidyup
45 from fba.models import blocks
46 from fba.models import instances
48 router = fastapi.FastAPI(docs_url=config.get("base_url") + "/docs", redoc_url=config.get("base_url") + "/redoc")
51 StaticFiles(directory=Path(__file__).parent.absolute() / "static"),
55 templates = Jinja2Templates(directory="templates")
57 @router.get(config.get("base_url") + "/api/info.json", response_class=JSONResponse)
59 database.cursor.execute("SELECT (SELECT COUNT(domain) FROM instances), (SELECT COUNT(domain) FROM instances WHERE software IN ('pleroma', 'mastodon', 'lemmy', 'friendica', 'misskey', 'peertube', 'takahe', 'gotosocial', 'brighteon', 'wildebeest', 'bookwyrm', 'mitra', 'areionskey', 'mammuthus', 'neodb')), (SELECT COUNT(blocker) FROM blocks), (SELECT COUNT(domain) FROM instances WHERE last_error_details IS NOT NULL)")
60 row = database.cursor.fetchone()
62 return JSONResponse(status_code=200, content={
63 "known_instances" : row[0],
64 "supported_instances": row[1],
65 "blocks_recorded" : row[2],
66 "erroneous_instances": row[3],
69 @router.get(config.get("base_url") + "/api/scoreboard.json", response_class=JSONResponse)
70 def api_scoreboard(mode: str, amount: int):
71 if amount > config.get("api_limit"):
72 raise HTTPException(status_code=400, detail="Too many results")
75 database.cursor.execute("SELECT blocked, COUNT(blocked) AS score FROM blocks GROUP BY blocked ORDER BY score DESC LIMIT ?", [amount])
76 elif mode == "blocker":
77 database.cursor.execute("SELECT blocker, COUNT(blocker) AS score FROM blocks GROUP BY blocker ORDER BY score DESC LIMIT ?", [amount])
78 elif mode == "reference":
79 database.cursor.execute("SELECT origin, COUNT(domain) AS score FROM instances WHERE origin IS NOT NULL GROUP BY origin ORDER BY score DESC LIMIT ?", [amount])
80 elif mode == "original_software":
81 database.cursor.execute("SELECT original_software, COUNT(domain) AS score FROM instances WHERE original_software IS NOT NULL GROUP BY original_software ORDER BY score DESC, original_software ASC LIMIT ?", [amount])
82 elif mode == "software":
83 database.cursor.execute("SELECT software, COUNT(domain) AS score FROM instances WHERE software IS NOT NULL GROUP BY software ORDER BY score DESC, software ASC LIMIT ?", [amount])
84 elif mode == "command":
85 database.cursor.execute("SELECT command, COUNT(domain) AS score FROM instances WHERE command IS NOT NULL GROUP BY command ORDER BY score DESC, command ASC LIMIT ?", [amount])
86 elif mode == "error_code":
87 database.cursor.execute("SELECT last_status_code, COUNT(domain) AS score FROM instances WHERE last_status_code IS NOT NULL AND last_status_code != '200' GROUP BY last_status_code ORDER BY score DESC LIMIT ?", [amount])
88 elif mode == "detection_mode":
89 database.cursor.execute("SELECT detection_mode, COUNT(domain) AS cnt FROM instances GROUP BY detection_mode ORDER BY cnt DESC LIMIT ?", [amount])
90 elif mode == "avg_peers":
91 database.cursor.execute("SELECT software, AVG(total_peers) AS average FROM instances WHERE software IS NOT NULL AND total_peers IS NOT NULL GROUP BY software HAVING average > 0 ORDER BY average DESC LIMIT ?", [amount])
92 elif mode == "avg_blocks":
93 database.cursor.execute("SELECT software, AVG(total_blocks) AS average FROM instances WHERE software IS NOT NULL AND total_blocks IS NOT NULL GROUP BY software HAVING average > 0 ORDER BY average DESC LIMIT ?", [amount])
94 elif mode == "obfuscator":
95 database.cursor.execute("SELECT software, COUNT(domain) AS cnt FROM instances WHERE has_obfuscation = 1 GROUP BY software ORDER BY cnt DESC LIMIT ?", [amount])
96 elif mode == "obfuscation":
97 database.cursor.execute("SELECT has_obfuscation, COUNT(domain) AS cnt FROM instances WHERE software IN ('pleroma', 'lemmy', 'mastodon', 'misskey', 'friendica') GROUP BY has_obfuscation ORDER BY cnt DESC LIMIT ?", [amount])
98 elif mode == "block_level":
99 database.cursor.execute("SELECT block_level, COUNT(rowid) AS cnt FROM blocks GROUP BY block_level ORDER BY cnt DESC LIMIT ?", [amount])
101 raise HTTPException(status_code=400, detail="No filter specified")
105 for row in database.cursor.fetchall():
108 "score" : round(row[1]),
111 return JSONResponse(status_code=200, content=scores)
113 @router.get(config.get("base_url") + "/api/list.json", response_class=JSONResponse)
114 def api_list(request: Request, mode: str, value: str, amount: int):
115 if mode is None or value is None or amount is None:
116 raise HTTPException(status_code=500, detail="No filter specified")
117 elif amount > config.get("api_limit"):
118 raise HTTPException(status_code=500, detail=f"amount={amount} is to big")
120 if mode in ("detection_mode", "original_software", "software", "command", "origin"):
121 database.cursor.execute(
126 LIMIT ?", [value, amount]
128 elif mode == "recently":
129 database.cursor.execute(
132 ORDER BY first_seen DESC \
136 raise HTTPException(status_code=500, detail=f"mode='{mode}' is unsupported")
138 domainlist = database.cursor.fetchall()
141 @router.get(config.get("base_url") + "/api/top.json", response_class=JSONResponse)
142 def api_index(request: Request, mode: str, value: str, amount: int):
143 if mode is None or value is None or amount is None:
144 raise HTTPException(status_code=500, detail="No filter specified")
145 elif amount > config.get("api_limit"):
146 raise HTTPException(status_code=500, detail=f"amount={amount} is to big")
148 domain = wildchar = punycode = reason = None
150 if mode == "block_level":
151 database.cursor.execute(
152 "SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks WHERE block_level = ? LIMIT ?", [value, amount]
154 elif mode in ["domain", "reverse"]:
155 domain = tidyup.domain(value)
156 if not domain_helper.is_wanted(domain):
157 raise HTTPException(status_code=500, detail=f"domain='{domain}' is not wanted")
159 wildchar = "*." + ".".join(domain.split(".")[-domain.count("."):])
160 punycode = domain.encode("idna").decode("utf-8")
161 elif mode == "reason":
162 reason = re.sub("(%|_)", "", tidyup.reason(value))
164 raise HTTPException(status_code=400, detail="Keyword is shorter than three characters")
167 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
169 WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? \
170 ORDER BY block_level ASC, first_seen ASC \
176 utils.get_hash(domain),
182 elif mode == "reverse":
183 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
185 WHERE blocker = ? OR blocker = ? OR blocker = ? OR blocker = ? OR blocker = ? OR blocker = ? \
186 ORDER BY first_seen ASC \
191 utils.get_hash(domain),
196 elif mode == "reason":
197 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
199 WHERE reason LIKE ? AND reason != '' \
200 ORDER BY first_seen ASC \
206 blocklist = database.cursor.fetchall()
209 for blocker, blocked, block_level, reason, first_seen, last_seen in blocklist:
210 if reason is not None and reason != "":
211 reason = reason.replace(",", " ").replace(" ", " ")
217 "first_seen": first_seen,
218 "last_seen" : last_seen
221 if block_level in result:
222 result[block_level].append(entry)
224 result[block_level] = [entry]
228 @router.get(config.get("base_url") + "/api/domain.json", response_class=JSONResponse)
229 def api_domain(domain: str):
231 raise HTTPException(status_code=400, detail="Invalid request, parameter 'domain' missing")
233 # Tidy up domain name
234 domain = tidyup.domain(domain).encode("idna").decode("utf-8")
236 if not domain_helper.is_wanted(domain):
237 raise HTTPException(status_code=500, detail=f"domain='{domain}' is not wanted")
240 database.cursor.execute("SELECT * FROM instances WHERE domain = ? LIMIT 1", [domain])
241 domain_data = database.cursor.fetchone()
243 if domain_data is None:
244 raise HTTPException(status_code=404, detail=f"domain='{domain}' not found")
246 return JSONResponse(status_code=200, content=dict(domain_data))
248 @router.get(config.get("base_url") + "/api/mutual.json", response_class=JSONResponse)
249 def api_mutual(domains: list[str] = Query()):
250 """Return 200 if federation is open between the two, 4xx otherwise"""
251 database.cursor.execute(
252 "SELECT block_level FROM blocks " \
253 "WHERE ((blocker = :a OR blocker = :b) AND (blocked = :b OR blocked = :a OR blocked = :aw OR blocked = :bw)) " \
254 "AND block_level = 'reject' " \
259 "aw": "*." + domains[0],
260 "bw": "*." + domains[1],
264 if database.cursor.fetchone() is not None:
266 return JSONResponse(status_code=418, content={})
269 return JSONResponse(status_code=200, content={})
271 @router.get(config.get("base_url") + "/.well-known/nodeinfo", response_class=JSONResponse)
272 def wellknown_nodeinfo(request: Request):
273 return JSONResponse(status_code=200, content={
275 "rel" : "http://nodeinfo.diaspora.software/ns/schema/1.0",
276 "href": f"{config.get('scheme')}://{config.get('hostname')}{config.get('base_url')}/nodeinfo/1.0"
280 @router.get(config.get("base_url") + "/nodeinfo/1.0", response_class=JSONResponse)
281 def nodeinfo_1_0(request: Request):
282 return JSONResponse(status_code=200, content={
303 "openRegistrations": False,
305 "nodeName": "Fedi Block API",
318 "explicitContent": False,
322 @router.get(config.get("base_url") + "/api/v1/instance/domain_blocks", response_class=JSONResponse)
323 def api_domain_blocks(request: Request):
324 blocked = blacklist.get_all()
327 for block in blocked:
330 "digest" : utils.get_hash(block),
331 "severity": "suspend",
332 "comment" : blocked[block],
335 return JSONResponse(status_code=200, content=blocking)
337 @router.get(config.get("base_url") + "/api/v1/instance/peers", response_class=JSONResponse)
338 def api_peers(request: Request):
339 database.cursor.execute("SELECT domain FROM instances WHERE nodeinfo_url IS NOT NULL")
342 for row in database.cursor.fetchall():
343 peers.append(row["domain"])
345 return JSONResponse(status_code=200, content=peers)
347 @router.get(config.get("base_url") + "/scoreboard")
348 def scoreboard(request: Request, mode: str, amount: int):
350 raise HTTPException(status_code=400, detail="No mode specified")
352 raise HTTPException(status_code=500, detail="Invalid amount specified")
354 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode={mode}&amount={amount}")
357 raise HTTPException(status_code=500, detail="Could not determine scores")
358 elif not response.ok:
359 raise HTTPException(status_code=response.status_code, detail=response.text)
361 return templates.TemplateResponse("views/scoreboard.html", {
362 "base_url" : utils.base_url(),
363 "slogan" : config.get("slogan"),
364 "theme" : config.get("theme"),
369 "scores" : json_helper.from_response(response)
372 @router.get(config.get("base_url") + "/list")
373 def list_domains(request: Request, mode: str, value: str, amount: int = config.get("api_limit")):
374 if mode == "detection_mode" and not instances.valid(value, "detection_mode"):
375 raise HTTPException(status_code=500, detail="Invalid detection mode provided")
377 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/list.json?mode={mode}&value={value}&amount={amount}")
380 if response is not None and response.ok:
381 domainlist = response.json()
382 tformat = config.get("timestamp_format")
383 for row in domainlist:
384 row["first_seen"] = datetime.utcfromtimestamp(row["first_seen"]).strftime(tformat)
385 row["last_updated"] = datetime.utcfromtimestamp(row["last_updated"]).strftime(tformat) if isinstance(row["last_updated"], float) else None
387 return templates.TemplateResponse("views/list.html", {
389 "mode" : mode if response is not None else None,
390 "value" : value if response is not None else None,
391 "amount" : amount if response is not None else None,
392 "found" : len(domainlist),
393 "domainlist": domainlist,
394 "slogan" : config.get("slogan"),
395 "theme" : config.get("theme"),
398 @router.get(config.get("base_url") + "/top")
399 def top(request: Request, mode: str, value: str, amount: int = config.get("api_limit")):
400 if mode == "block_level" and not blocks.valid(value, "block_level"):
401 raise HTTPException(status_code=500, detail="Invalid block level provided")
402 elif mode in ["domain", "reverse"] and not domain_helper.is_wanted(value):
403 raise HTTPException(status_code=500, detail="Invalid or blocked domain specified")
405 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/top.json?mode={mode}&value={value}&amount={amount}")
409 if response.ok and response.status_code == 200 and len(response.text) > 0:
410 blocklist = response.json()
412 tformat = config.get("timestamp_format")
413 for block_level in blocklist:
414 for row in blocklist[block_level]:
415 row["first_seen"] = datetime.utcfromtimestamp(row["first_seen"]).strftime(tformat)
416 row["last_seen"] = datetime.utcfromtimestamp(row["last_seen"]).strftime(tformat) if isinstance(row["last_seen"], float) else None
419 return templates.TemplateResponse("views/top.html", {
421 "mode" : mode if response is not None else None,
422 "value" : value if response is not None else None,
423 "amount" : amount if response is not None else None,
425 "blocklist": blocklist,
426 "slogan" : config.get("slogan"),
427 "theme" : config.get("theme"),
430 @router.get(config.get("base_url") + "/infos")
431 def infos(request: Request, domain: str):
433 raise HTTPException(status_code=400, detail="Invalid request, parameter 'domain' missing")
435 # Tidy up domain name
436 domain = tidyup.domain(domain).encode("idna").decode("utf-8")
438 if not domain_helper.is_wanted(domain):
439 raise HTTPException(status_code=500, detail=f"domain='{domain}' is not wanted")
441 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/domain.json?domain={domain}")
443 if not response.ok or response.status_code > 200 or response.text.strip() == "":
444 raise HTTPException(status_code=response.status_code, detail=response.reason)
446 domain_data = response.json()
449 tformat = config.get("timestamp_format")
451 for key in domain_data.keys():
452 if key in ["last_nodeinfo", "last_blocked", "first_seen", "last_updated", "last_instance_fetch"] and isinstance(domain_data[key], float):
454 instance[key] = datetime.utcfromtimestamp(domain_data[key]).strftime(tformat)
457 instance[key] = domain_data[key]
459 return templates.TemplateResponse("views/infos.html", {
462 "instance": instance,
463 "theme" : config.get("theme"),
464 "slogan" : config.get("slogan"),
467 @router.get(config.get("base_url") + "/rss")
468 def rss(request: Request, domain: str = None):
469 if domain is not None:
470 domain = tidyup.domain(domain).encode("idna").decode("utf-8")
472 wildchar = "*." + ".".join(domain.split(".")[-domain.count("."):])
473 punycode = domain.encode("idna").decode("utf-8")
475 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
477 WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? \
478 ORDER BY first_seen DESC \
481 "*." + domain, wildchar,
482 utils.get_hash(domain),
485 config.get("rss_limit")
488 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
490 ORDER BY first_seen DESC \
491 LIMIT ?", [config.get("rss_limit")])
493 result = database.cursor.fetchall()
500 "block_level": row[2],
501 "reason" : "Provided reason: '" + row[3] + "'" if row[3] is not None and row[3] != "" else "No reason provided.",
502 "first_seen" : format_datetime(datetime.fromtimestamp(row[4])),
503 "last_seen" : format_datetime(datetime.fromtimestamp(row[5])),
506 return templates.TemplateResponse("views/rss.xml", {
508 "timestamp": format_datetime(datetime.now()),
510 "scheme" : config.get("scheme"),
511 "hostname" : config.get("hostname"),
514 "Content-Type": "routerlication/rss+xml"
517 @router.get(config.get("base_url") + "/robots.txt", response_class=PlainTextResponse)
518 def robots(request: Request):
519 return templates.TemplateResponse("views/robots.txt", {
521 "base_url": config.get("base_url")
524 @router.get(config.get("base_url") + "/")
525 def index(request: Request):
527 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/info.json")
530 raise HTTPException(status_code=response.status_code, detail=response.text)
532 return templates.TemplateResponse("views/index.html", {
534 "theme" : config.get("theme"),
535 "info" : response.json(),
536 "slogan" : config.get("slogan"),
539 if __name__ == "__main__":
542 host=config.get("host"),
543 port=config.get("port"),
544 log_level=config.get("log_level"),