2 # -*- coding: utf-8 -*-
4 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
5 # Copyright (C) 2023 Free Software Foundation
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published
9 # by the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <https://www.gnu.org/licenses/>.
22 from datetime import datetime
23 from email.utils import format_datetime
24 from pathlib import Path
27 from fastapi import Request, HTTPException, Query
28 from fastapi.responses import JSONResponse
29 from fastapi.responses import PlainTextResponse
30 from fastapi.staticfiles import StaticFiles
31 from fastapi.templating import Jinja2Templates
36 from fba import database
39 from fba.helpers import blacklist
40 from fba.helpers import config
41 from fba.helpers import domain as domain_helper
42 from fba.helpers import json as json_helper
43 from fba.helpers import tidyup
45 from fba.models import blocks
46 from fba.models import instances
48 router = fastapi.FastAPI(docs_url=config.get("base_url") + "/docs", redoc_url=config.get("base_url") + "/redoc")
51 StaticFiles(directory=Path(__file__).parent.absolute() / "static"),
55 templates = Jinja2Templates(directory="templates")
57 @router.get(config.get("base_url") + "/api/info.json", response_class=JSONResponse)
59 database.cursor.execute("SELECT (SELECT COUNT(domain) FROM instances), (SELECT COUNT(domain) FROM instances WHERE software IN ('pleroma', 'mastodon', 'lemmy', 'friendica', 'misskey', 'peertube', 'takahe', 'gotosocial', 'brighteon', 'wildebeest', 'bookwyrm', 'mitra', 'areionskey', 'mammuthus', 'neodb')), (SELECT COUNT(blocker) FROM blocks), (SELECT COUNT(domain) FROM instances WHERE last_error_details IS NOT NULL)")
60 row = database.cursor.fetchone()
62 return JSONResponse(status_code=200, content={
63 "known_instances" : row[0],
64 "supported_instances": row[1],
65 "blocks_recorded" : row[2],
66 "erroneous_instances": row[3],
69 @router.get(config.get("base_url") + "/api/scoreboard.json", response_class=JSONResponse)
70 def api_scoreboard(mode: str, amount: int):
71 if amount > config.get("api_limit"):
72 raise HTTPException(status_code=400, detail="Too many results")
75 database.cursor.execute("SELECT blocked, COUNT(blocked) AS score FROM blocks GROUP BY blocked ORDER BY score DESC LIMIT ?", [amount])
76 elif mode == "blocker":
77 database.cursor.execute("SELECT blocker, COUNT(blocker) AS score FROM blocks GROUP BY blocker ORDER BY score DESC LIMIT ?", [amount])
78 elif mode == "reference":
79 database.cursor.execute("SELECT origin, COUNT(domain) AS score FROM instances WHERE origin IS NOT NULL GROUP BY origin ORDER BY score DESC LIMIT ?", [amount])
80 elif mode == "software":
81 database.cursor.execute("SELECT software, COUNT(domain) AS score FROM instances WHERE software IS NOT NULL GROUP BY software ORDER BY score DESC, software ASC LIMIT ?", [amount])
82 elif mode == "command":
83 database.cursor.execute("SELECT command, COUNT(domain) AS score FROM instances WHERE command IS NOT NULL GROUP BY command ORDER BY score DESC, command ASC LIMIT ?", [amount])
84 elif mode == "error_code":
85 database.cursor.execute("SELECT last_status_code, COUNT(domain) AS score FROM instances WHERE last_status_code IS NOT NULL AND last_status_code != '200' GROUP BY last_status_code ORDER BY score DESC LIMIT ?", [amount])
86 elif mode == "detection_mode":
87 database.cursor.execute("SELECT detection_mode, COUNT(domain) AS cnt FROM instances GROUP BY detection_mode ORDER BY cnt DESC LIMIT ?", [amount])
88 elif mode == "avg_peers":
89 database.cursor.execute("SELECT software, AVG(total_peers) AS average FROM instances WHERE software IS NOT NULL AND total_peers IS NOT NULL GROUP BY software HAVING average > 0 ORDER BY average DESC LIMIT ?", [amount])
90 elif mode == "avg_blocks":
91 database.cursor.execute("SELECT software, AVG(total_blocks) AS average FROM instances WHERE software IS NOT NULL AND total_blocks IS NOT NULL GROUP BY software HAVING average > 0 ORDER BY average DESC LIMIT ?", [amount])
92 elif mode == "obfuscator":
93 database.cursor.execute("SELECT software, COUNT(domain) AS cnt FROM instances WHERE has_obfuscation = 1 GROUP BY software ORDER BY cnt DESC LIMIT ?", [amount])
94 elif mode == "obfuscation":
95 database.cursor.execute("SELECT has_obfuscation, COUNT(domain) AS cnt FROM instances WHERE software IN ('pleroma', 'lemmy', 'mastodon', 'friendica') GROUP BY has_obfuscation ORDER BY cnt DESC LIMIT ?", [amount])
96 elif mode == "block_level":
97 database.cursor.execute("SELECT block_level, COUNT(rowid) AS cnt FROM blocks GROUP BY block_level ORDER BY cnt DESC LIMIT ?", [amount])
99 raise HTTPException(status_code=400, detail="No filter specified")
103 for row in database.cursor.fetchall():
106 "score" : round(row[1]),
109 return JSONResponse(status_code=200, content=scores)
111 @router.get(config.get("base_url") + "/api/list.json", response_class=JSONResponse)
112 def api_list(request: Request, mode: str, value: str, amount: int):
113 if mode is None or value is None or amount is None:
114 raise HTTPException(status_code=500, detail="No filter specified")
115 elif amount > config.get("api_limit"):
116 raise HTTPException(status_code=500, detail=f"amount={amount} is to big")
118 if mode in ("detection_mode", "software", "command", "origin"):
119 database.cursor.execute(
124 LIMIT ?", [value, amount]
126 elif mode == "recently":
127 database.cursor.execute(
130 ORDER BY first_seen DESC \
134 raise HTTPException(status_code=500, detail=f"mode='{mode}' is unsupported")
136 domainlist = database.cursor.fetchall()
139 @router.get(config.get("base_url") + "/api/top.json", response_class=JSONResponse)
140 def api_index(request: Request, mode: str, value: str, amount: int):
141 if mode is None or value is None or amount is None:
142 raise HTTPException(status_code=500, detail="No filter specified")
143 elif amount > config.get("api_limit"):
144 raise HTTPException(status_code=500, detail=f"amount={amount} is to big")
146 domain = wildchar = punycode = reason = None
148 if mode == "block_level":
149 database.cursor.execute(
150 "SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks WHERE block_level = ? LIMIT ?", [value, amount]
152 elif mode in ["domain", "reverse"]:
153 domain = tidyup.domain(value)
154 if not domain_helper.is_wanted(domain):
155 raise HTTPException(status_code=500, detail=f"domain='{domain}' is not wanted")
157 wildchar = "*." + ".".join(domain.split(".")[-domain.count("."):])
158 punycode = domain.encode("idna").decode("utf-8")
159 elif mode == "reason":
160 reason = re.sub("(%|_)", "", tidyup.reason(value))
162 raise HTTPException(status_code=400, detail="Keyword is shorter than three characters")
165 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
167 WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? \
168 ORDER BY block_level ASC, first_seen ASC \
174 utils.get_hash(domain),
180 elif mode == "reverse":
181 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
183 WHERE blocker = ? OR blocker = ? OR blocker = ? OR blocker = ? OR blocker = ? OR blocker = ? \
184 ORDER BY first_seen ASC \
189 utils.get_hash(domain),
194 elif mode == "reason":
195 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
197 WHERE reason LIKE ? AND reason != '' \
198 ORDER BY first_seen ASC \
204 blocklist = database.cursor.fetchall()
207 for blocker, blocked, block_level, reason, first_seen, last_seen in blocklist:
208 if reason is not None and reason != "":
209 reason = reason.replace(",", " ").replace(" ", " ")
215 "first_seen": first_seen,
216 "last_seen" : last_seen
219 if block_level in result:
220 result[block_level].append(entry)
222 result[block_level] = [entry]
226 @router.get(config.get("base_url") + "/api/domain.json", response_class=JSONResponse)
227 def api_domain(domain: str):
229 raise HTTPException(status_code=400, detail="Invalid request, parameter 'domain' missing")
231 # Tidy up domain name
232 domain = tidyup.domain(domain).encode("idna").decode("utf-8")
234 if not domain_helper.is_wanted(domain):
235 raise HTTPException(status_code=500, detail=f"domain='{domain}' is not wanted")
238 database.cursor.execute("SELECT * FROM instances WHERE domain = ? LIMIT 1", [domain])
239 domain_data = database.cursor.fetchone()
241 if domain_data is None:
242 raise HTTPException(status_code=404, detail=f"domain='{domain}' not found")
244 return JSONResponse(status_code=200, content=dict(domain_data))
246 @router.get(config.get("base_url") + "/api/mutual.json", response_class=JSONResponse)
247 def api_mutual(domains: list[str] = Query()):
248 """Return 200 if federation is open between the two, 4xx otherwise"""
249 database.cursor.execute(
250 "SELECT block_level FROM blocks " \
251 "WHERE ((blocker = :a OR blocker = :b) AND (blocked = :b OR blocked = :a OR blocked = :aw OR blocked = :bw)) " \
252 "AND block_level = 'reject' " \
257 "aw": "*." + domains[0],
258 "bw": "*." + domains[1],
262 if database.cursor.fetchone() is not None:
264 return JSONResponse(status_code=418, content={})
267 return JSONResponse(status_code=200, content={})
269 @router.get(config.get("base_url") + "/.well-known/nodeinfo", response_class=JSONResponse)
270 def wellknown_nodeinfo(request: Request):
271 return JSONResponse(status_code=200, content={
273 "rel" : "http://nodeinfo.diaspora.software/ns/schema/1.0",
274 "href": f"{config.get('scheme')}://{config.get('hostname')}{config.get('base_url')}/nodeinfo/1.0"
278 @router.get(config.get("base_url") + "/nodeinfo/1.0", response_class=JSONResponse)
279 def nodeinfo_1_0(request: Request):
280 return JSONResponse(status_code=200, content={
301 "openRegistrations": False,
303 "nodeName": "Fedi Block API",
316 "explicitContent": False,
320 @router.get(config.get("base_url") + "/api/v1/instance/domain_blocks", response_class=JSONResponse)
321 def api_domain_blocks(request: Request):
322 blocked = blacklist.get_all()
325 for block in blocked:
328 "digest" : utils.get_hash(block),
329 "severity": "suspend",
330 "comment" : blocked[block],
333 return JSONResponse(status_code=200, content=blocking)
335 @router.get(config.get("base_url") + "/api/v1/instance/peers", response_class=JSONResponse)
336 def api_peers(request: Request):
337 database.cursor.execute("SELECT domain FROM instances WHERE nodeinfo_url IS NOT NULL")
340 for row in database.cursor.fetchall():
341 peers.append(row["domain"])
343 return JSONResponse(status_code=200, content=peers)
345 @router.get(config.get("base_url") + "/scoreboard")
346 def scoreboard(request: Request, mode: str, amount: int):
348 raise HTTPException(status_code=400, detail="No mode specified")
350 raise HTTPException(status_code=500, detail="Invalid amount specified")
352 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode={mode}&amount={amount}")
355 raise HTTPException(status_code=500, detail="Could not determine scores")
356 elif not response.ok:
357 raise HTTPException(status_code=response.status_code, detail=response.text)
359 return templates.TemplateResponse("views/scoreboard.html", {
360 "base_url" : utils.base_url(),
361 "slogan" : config.get("slogan"),
362 "theme" : config.get("theme"),
367 "scores" : json_helper.from_response(response)
370 @router.get(config.get("base_url") + "/list")
371 def list_domains(request: Request, mode: str, value: str, amount: int = config.get("api_limit")):
372 if mode == "detection_mode" and not instances.valid(value, "detection_mode"):
373 raise HTTPException(status_code=500, detail="Invalid detection mode provided")
375 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/list.json?mode={mode}&value={value}&amount={amount}")
378 if response is not None and response.ok:
379 domainlist = response.json()
380 tformat = config.get("timestamp_format")
381 for row in domainlist:
382 row["first_seen"] = datetime.utcfromtimestamp(row["first_seen"]).strftime(tformat)
383 row["last_updated"] = datetime.utcfromtimestamp(row["last_updated"]).strftime(tformat) if isinstance(row["last_updated"], float) else None
385 return templates.TemplateResponse("views/list.html", {
387 "mode" : mode if response is not None else None,
388 "value" : value if response is not None else None,
389 "amount" : amount if response is not None else None,
390 "found" : len(domainlist),
391 "domainlist": domainlist,
392 "slogan" : config.get("slogan"),
393 "theme" : config.get("theme"),
396 @router.get(config.get("base_url") + "/top")
397 def top(request: Request, mode: str, value: str, amount: int = config.get("api_limit")):
398 if mode == "block_level" and not blocks.valid(value, "block_level"):
399 raise HTTPException(status_code=500, detail="Invalid block level provided")
400 elif mode in ["domain", "reverse"] and not domain_helper.is_wanted(value):
401 raise HTTPException(status_code=500, detail="Invalid or blocked domain specified")
403 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/top.json?mode={mode}&value={value}&amount={amount}")
407 if response.ok and response.status_code == 200 and len(response.text) > 0:
408 blocklist = response.json()
410 tformat = config.get("timestamp_format")
411 for block_level in blocklist:
412 for row in blocklist[block_level]:
413 row["first_seen"] = datetime.utcfromtimestamp(row["first_seen"]).strftime(tformat)
414 row["last_seen"] = datetime.utcfromtimestamp(row["last_seen"]).strftime(tformat) if isinstance(row["last_seen"], float) else None
417 return templates.TemplateResponse("views/top.html", {
419 "mode" : mode if response is not None else None,
420 "value" : value if response is not None else None,
421 "amount" : amount if response is not None else None,
423 "blocklist": blocklist,
424 "slogan" : config.get("slogan"),
425 "theme" : config.get("theme"),
428 @router.get(config.get("base_url") + "/infos")
429 def infos(request: Request, domain: str):
431 raise HTTPException(status_code=400, detail="Invalid request, parameter 'domain' missing")
433 # Tidy up domain name
434 domain = tidyup.domain(domain).encode("idna").decode("utf-8")
436 if not domain_helper.is_wanted(domain):
437 raise HTTPException(status_code=500, detail=f"domain='{domain}' is not wanted")
439 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/domain.json?domain={domain}")
441 if not response.ok or response.status_code > 200 or response.text.strip() == "":
442 raise HTTPException(status_code=response.status_code, detail=response.reason)
444 domain_data = response.json()
447 tformat = config.get("timestamp_format")
449 for key in domain_data.keys():
450 if key in ["last_nodeinfo", "last_blocked", "first_seen", "last_updated", "last_instance_fetch", "last_response_time"] and isinstance(domain_data[key], float):
452 instance[key] = datetime.utcfromtimestamp(domain_data[key]).strftime(tformat)
455 instance[key] = domain_data[key]
457 return templates.TemplateResponse("views/infos.html", {
460 "instance": instance,
461 "theme" : config.get("theme"),
462 "slogan" : config.get("slogan"),
465 @router.get(config.get("base_url") + "/rss")
466 def rss(request: Request, domain: str = None):
467 if domain is not None:
468 domain = tidyup.domain(domain).encode("idna").decode("utf-8")
470 wildchar = "*." + ".".join(domain.split(".")[-domain.count("."):])
471 punycode = domain.encode("idna").decode("utf-8")
473 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? ORDER BY first_seen DESC LIMIT ?", [
475 "*." + domain, wildchar,
476 utils.get_hash(domain),
479 config.get("rss_limit")
482 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks ORDER BY first_seen DESC LIMIT ?", [config.get("rss_limit")])
484 result = database.cursor.fetchall()
491 "block_level": row[2],
492 "reason" : "Provided reason: '" + row[3] + "'" if row[3] is not None and row[3] != "" else "No reason provided.",
493 "first_seen" : format_datetime(datetime.fromtimestamp(row[4])),
494 "last_seen" : format_datetime(datetime.fromtimestamp(row[5])),
497 return templates.TemplateResponse("views/rss.xml", {
499 "timestamp": format_datetime(datetime.now()),
501 "scheme" : config.get("scheme"),
502 "hostname" : config.get("hostname"),
505 "Content-Type": "routerlication/rss+xml"
508 @router.get(config.get("base_url") + "/robots.txt", response_class=PlainTextResponse)
509 def robots(request: Request):
510 return templates.TemplateResponse("views/robots.txt", {
512 "base_url": config.get("base_url")
515 @router.get(config.get("base_url") + "/")
516 def index(request: Request):
518 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/info.json")
521 raise HTTPException(status_code=response.status_code, detail=response.text)
523 return templates.TemplateResponse("views/index.html", {
525 "theme" : config.get("theme"),
526 "info" : response.json(),
527 "slogan" : config.get("slogan"),
530 if __name__ == "__main__":
531 uvicorn.run("daemon:router", host=config.get("host"), port=config.get("port"), log_level=config.get("log_level"), proxy_headers=True)