2 # -*- coding: utf-8 -*-
4 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
5 # Copyright (C) 2023 Free Software Foundation
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published
9 # by the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <https://www.gnu.org/licenses/>.
22 from datetime import datetime
23 from email.utils import format_datetime
24 from pathlib import Path
25 from urllib.parse import urlparse
28 from fastapi import Request, HTTPException, Query
29 from fastapi.responses import JSONResponse
30 from fastapi.responses import PlainTextResponse
31 from fastapi.staticfiles import StaticFiles
32 from fastapi.templating import Jinja2Templates
37 from fba import database
40 from fba.helpers import blacklist
41 from fba.helpers import config
42 from fba.helpers import json as json_helper
43 from fba.helpers import tidyup
45 from fba.models import blocks
46 from fba.models import instances
48 router = fastapi.FastAPI(docs_url=config.get("base_url") + "/docs", redoc_url=config.get("base_url") + "/redoc")
51 StaticFiles(directory=Path(__file__).parent.absolute() / "static"),
55 templates = Jinja2Templates(directory="templates")
57 @router.get(config.get("base_url") + "/api/info.json", response_class=JSONResponse)
59 database.cursor.execute("SELECT (SELECT COUNT(domain) FROM instances), (SELECT COUNT(domain) FROM instances WHERE software IN ('pleroma', 'mastodon', 'lemmy', 'friendica', 'misskey', 'peertube', 'takahe', 'gotosocial', 'brighteon', 'wildebeest', 'bookwyrm')), (SELECT COUNT(blocker) FROM blocks), (SELECT COUNT(domain) FROM instances WHERE last_error_details IS NOT NULL)")
60 row = database.cursor.fetchone()
62 return JSONResponse(status_code=200, content={
63 "known_instances" : row[0],
64 "supported_instances": row[1],
65 "blocks_recorded" : row[2],
66 "erroneous_instances": row[3],
69 @router.get(config.get("base_url") + "/api/scoreboard.json", response_class=JSONResponse)
70 def api_scoreboard(mode: str, amount: int):
71 if amount > config.get("api_limit"):
72 raise HTTPException(status_code=400, detail="Too many results")
75 database.cursor.execute("SELECT blocked, COUNT(blocked) AS score FROM blocks GROUP BY blocked ORDER BY score DESC LIMIT ?", [amount])
76 elif mode == "blocker":
77 database.cursor.execute("SELECT blocker, COUNT(blocker) AS score FROM blocks GROUP BY blocker ORDER BY score DESC LIMIT ?", [amount])
78 elif mode == "reference":
79 database.cursor.execute("SELECT origin, COUNT(domain) AS score FROM instances WHERE origin IS NOT NULL GROUP BY origin ORDER BY score DESC LIMIT ?", [amount])
80 elif mode == "software":
81 database.cursor.execute("SELECT software, COUNT(domain) AS score FROM instances WHERE software IS NOT NULL GROUP BY software ORDER BY score DESC, software ASC LIMIT ?", [amount])
82 elif mode == "command":
83 database.cursor.execute("SELECT command, COUNT(domain) AS score FROM instances WHERE command IS NOT NULL GROUP BY command ORDER BY score DESC, command ASC LIMIT ?", [amount])
84 elif mode == "error_code":
85 database.cursor.execute("SELECT last_status_code, COUNT(domain) AS score FROM instances WHERE last_status_code IS NOT NULL AND last_status_code != '200' GROUP BY last_status_code ORDER BY score DESC LIMIT ?", [amount])
86 elif mode == "detection_mode":
87 database.cursor.execute("SELECT detection_mode, COUNT(domain) AS cnt FROM instances GROUP BY detection_mode ORDER BY cnt DESC LIMIT ?", [amount])
88 elif mode == "avg_peers":
89 database.cursor.execute("SELECT software, AVG(total_peers) AS average FROM instances WHERE software IS NOT NULL GROUP BY software HAVING average > 0 ORDER BY average DESC LIMIT ?", [amount])
90 elif mode == "obfuscator":
91 database.cursor.execute("SELECT software, COUNT(domain) AS cnt FROM instances WHERE has_obfuscation = 1 GROUP BY software ORDER BY cnt DESC LIMIT ?", [amount])
92 elif mode == "obfuscation":
93 database.cursor.execute("SELECT has_obfuscation, COUNT(domain) AS cnt FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica') GROUP BY has_obfuscation ORDER BY cnt DESC LIMIT ?", [amount])
94 elif mode == "block_level":
95 database.cursor.execute("SELECT block_level, COUNT(rowid) AS cnt FROM blocks GROUP BY block_level ORDER BY cnt DESC LIMIT ?", [amount])
97 raise HTTPException(status_code=400, detail="No filter specified")
101 for domain, score in database.cursor.fetchall():
104 "score" : round(score)
107 return JSONResponse(status_code=200, content=scores)
109 @router.get(config.get("base_url") + "/api/list.json", response_class=JSONResponse)
110 def api_list(request: Request, mode: str, value: str, amount: int):
111 if mode is None or value is None or amount is None:
112 raise HTTPException(status_code=500, detail="No filter specified")
113 elif amount > config.get("api_limit"):
114 raise HTTPException(status_code=500, detail=f"amount={amount} is to big")
116 if mode in ("detection_mode", "software", "command"):
117 database.cursor.execute(
118 f"SELECT domain, origin, software, detection_mode, command, total_peers, total_blocks, first_seen, last_updated \
122 LIMIT ?", [value, amount]
125 domainlist = database.cursor.fetchall()
127 return JSONResponse(status_code=200, content=dict(domainlist))
129 @router.get(config.get("base_url") + "/api/top.json", response_class=JSONResponse)
130 def api_index(request: Request, mode: str, value: str, amount: int):
131 if mode is None or value is None or amount is None:
132 raise HTTPException(status_code=500, detail="No filter specified")
133 elif amount > config.get("api_limit"):
134 raise HTTPException(status_code=500, detail=f"amount={amount} is to big")
136 domain = wildchar = punycode = reason = None
138 if mode == "block_level":
139 database.cursor.execute(
140 "SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks WHERE block_level = ? LIMIT ?", [value, amount]
142 elif mode in ["domain", "reverse"]:
143 domain = tidyup.domain(value)
144 if not utils.is_domain_wanted(domain):
145 raise HTTPException(status_code=500, detail=f"domain='{domain}' is not wanted")
147 wildchar = "*." + ".".join(domain.split(".")[-domain.count("."):])
148 punycode = domain.encode("idna").decode("utf-8")
149 elif mode == "reason":
150 reason = re.sub("(%|_)", "", tidyup.reason(value))
152 raise HTTPException(status_code=400, detail="Keyword is shorter than three characters")
155 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
157 WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? \
158 ORDER BY block_level ASC, first_seen ASC \
164 utils.get_hash(domain),
170 elif mode == "reverse":
171 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
173 WHERE blocker = ? OR blocker = ? OR blocker = ? OR blocker = ? OR blocker = ? OR blocker = ? \
174 ORDER BY first_seen ASC \
179 utils.get_hash(domain),
184 elif mode == "reason":
185 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
187 WHERE reason LIKE ? AND reason != '' \
188 ORDER BY first_seen ASC \
194 blocklist = database.cursor.fetchall()
197 for blocker, blocked, block_level, reason, first_seen, last_seen in blocklist:
198 if reason is not None and reason != "":
199 reason = reason.replace(",", " ").replace(" ", " ")
205 "first_seen": first_seen,
206 "last_seen" : last_seen
209 if block_level in result:
210 result[block_level].append(entry)
212 result[block_level] = [entry]
216 @router.get(config.get("base_url") + "/api/domain.json", response_class=JSONResponse)
217 def api_domain(domain: str):
219 raise HTTPException(status_code=400, detail="Invalid request, parameter 'domain' missing")
221 # Tidy up domain name
222 domain = tidyup.domain(domain).encode("idna").decode("utf-8")
224 if not utils.is_domain_wanted(domain):
225 raise HTTPException(status_code=500, detail=f"domain='{domain}' is not wanted")
228 database.cursor.execute("SELECT * FROM instances WHERE domain = ? LIMIT 1", [domain])
229 domain_data = database.cursor.fetchone()
231 if domain_data is None:
232 raise HTTPException(status_code=404, detail=f"domain='{domain}' not found")
234 return JSONResponse(status_code=200, content=dict(domain_data))
236 @router.get(config.get("base_url") + "/api/mutual.json", response_class=JSONResponse)
237 def api_mutual(domains: list[str] = Query()):
238 """Return 200 if federation is open between the two, 4xx otherwise"""
239 database.cursor.execute(
240 "SELECT block_level FROM blocks " \
241 "WHERE ((blocker = :a OR blocker = :b) AND (blocked = :b OR blocked = :a OR blocked = :aw OR blocked = :bw)) " \
242 "AND block_level = 'reject' " \
247 "aw": "*." + domains[0],
248 "bw": "*." + domains[1],
252 if database.cursor.fetchone() is not None:
254 return JSONResponse(status_code=418, content={})
257 return JSONResponse(status_code=200, content={})
259 @router.get(config.get("base_url") + "/.well-known/nodeinfo", response_class=JSONResponse)
260 def wellknown_nodeinfo(request: Request):
261 return JSONResponse(status_code=200, content={
263 "rel" : "http://nodeinfo.diaspora.software/ns/schema/1.0",
264 "href": f"{config.get('scheme')}://{config.get('hostname')}{config.get('base_url')}/nodeinfo/1.0"
268 @router.get(config.get("base_url") + "/nodeinfo/1.0", response_class=JSONResponse)
269 def nodeinfo_1_0(request: Request):
270 return JSONResponse(status_code=200, content={
291 "openRegistrations": False,
293 "nodeName": "Fedi Block API",
306 "explicitContent": False,
310 @router.get(config.get("base_url") + "/api/v1/instance/domain_blocks", response_class=JSONResponse)
311 def api_domain_blocks(request: Request):
312 blocked = blacklist.get_all()
315 for block in blocked:
318 "digest" : utils.get_hash(block),
319 "severity": "suspend",
320 "comment" : blocked[block],
323 return JSONResponse(status_code=200, content=blocking)
325 @router.get(config.get("base_url") + "/api/v1/instance/peers", response_class=JSONResponse)
326 def api_peers(request: Request):
327 database.cursor.execute("SELECT domain FROM instances WHERE nodeinfo_url IS NOT NULL")
330 for row in database.cursor.fetchall():
331 peers.append(row["domain"])
333 return JSONResponse(status_code=200, content=peers)
335 @router.get(config.get("base_url") + "/scoreboard")
336 def scoreboard(request: Request, mode: str, amount: int):
339 if mode == "blocker" and amount > 0:
340 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=blocker&amount={amount}")
341 elif mode == "blocked" and amount > 0:
342 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=blocked&amount={amount}")
343 elif mode == "reference" and amount > 0:
344 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=reference&amount={amount}")
345 elif mode == "software" and amount > 0:
346 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=software&amount={amount}")
347 elif mode == "command" and amount > 0:
348 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=command&amount={amount}")
349 elif mode == "error_code" and amount > 0:
350 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=error_code&amount={amount}")
351 elif mode == "detection_mode" and amount > 0:
352 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=detection_mode&amount={amount}")
353 elif mode == "avg_peers" and amount > 0:
354 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=avg_peers&amount={amount}")
355 elif mode == "obfuscator" and amount > 0:
356 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=obfuscator&amount={amount}")
357 elif mode == "obfuscation" and amount > 0:
358 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=obfuscation&amount={amount}")
359 elif mode == "block_level" and amount > 0:
360 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=block_level&amount={amount}")
362 raise HTTPException(status_code=400, detail="No filter specified")
365 raise HTTPException(status_code=500, detail="Could not determine scores")
366 elif not response.ok:
367 raise HTTPException(status_code=response.status_code, detail=response.text)
369 return templates.TemplateResponse("views/scoreboard.html", {
370 "base_url" : utils.base_url(),
371 "slogan" : config.get("slogan"),
372 "theme" : config.get("theme"),
377 "scores" : json_helper.from_response(response)
380 @router.get(config.get("base_url") + "/list")
381 def list_domains(request: Request, mode: str, value: str, amount: int = config.get("api_limit")):
382 if mode == "detection_mode" and not instances.valid(value, "detection_mode"):
383 raise HTTPException(status_code=500, detail="Invalid detection mode provided")
385 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/list.json?mode={mode}&value={value}&amount={amount}")
388 if response is not None and response.ok:
389 domainlist = response.json()
390 tformat = config.get("timestamp_format")
391 for row in domainlist:
392 row["first_seen"] = datetime.utcfromtimestamp(row["first_seen"]).strftime(tformat)
393 row["last_updated"] = datetime.utcfromtimestamp(row["last_updated"]).strftime(tformat) if isinstance(row["last_updated"], float) else None
395 return templates.TemplateResponse("views/list.html", {
397 "mode" : mode if response is not None else None,
398 "value" : value if response is not None else None,
399 "amount" : amount if response is not None else None,
400 "found" : len(domainlist),
401 "domainlist": domainlist,
402 "slogan" : config.get("slogan"),
403 "theme" : config.get("theme"),
406 @router.get(config.get("base_url") + "/top")
407 def top(request: Request, mode: str, value: str, amount: int = config.get("api_limit")):
408 if mode == "block_level" and not blocks.valid(value, "block_level"):
409 raise HTTPException(status_code=500, detail="Invalid block level provided")
410 elif mode in ["domain", "reverse"] and not utils.is_domain_wanted(value):
411 raise HTTPException(status_code=500, detail="Invalid or blocked domain specified")
413 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/top.json?mode={mode}&value={value}&amount={amount}")
417 if response.ok and response.status_code == 200 and len(response.text) > 0:
418 blocklist = response.json()
420 tformat = config.get("timestamp_format")
421 for block_level in blocklist:
422 for row in blocklist[block_level]:
423 row["first_seen"] = datetime.utcfromtimestamp(row["first_seen"]).strftime(tformat)
424 row["last_seen"] = datetime.utcfromtimestamp(row["last_seen"]).strftime(tformat) if isinstance(row["last_seen"], float) else None
427 return templates.TemplateResponse("views/top.html", {
429 "mode" : mode if response is not None else None,
430 "value" : value if response is not None else None,
431 "amount" : amount if response is not None else None,
433 "blocklist": blocklist,
434 "slogan" : config.get("slogan"),
435 "theme" : config.get("theme"),
438 @router.get(config.get("base_url") + "/infos")
439 def infos(request: Request, domain: str):
441 raise HTTPException(status_code=400, detail="Invalid request, parameter 'domain' missing")
443 # Tidy up domain name
444 domain = tidyup.domain(domain).encode("idna").decode("utf-8")
446 if not utils.is_domain_wanted(domain):
447 raise HTTPException(status_code=500, detail=f"domain='{domain}' is not wanted")
449 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/domain.json?domain={domain}")
451 if not response.ok or response.status_code >= 300 or response.text.strip() == "":
452 raise HTTPException(status_code=response.status_code, detail=response.reason)
454 domain_data = response.json()
457 tformat = config.get("timestamp_format")
459 for key in domain_data.keys():
460 if key in ["last_nodeinfo", "last_blocked", "first_seen", "last_updated", "last_instance_fetch"] and isinstance(domain_data[key], float):
462 instance[key] = datetime.utcfromtimestamp(domain_data[key]).strftime(tformat)
465 instance[key] = domain_data[key]
467 return templates.TemplateResponse("views/infos.html", {
470 "instance": instance,
471 "theme" : config.get("theme"),
472 "slogan" : config.get("slogan"),
475 @router.get(config.get("base_url") + "/rss")
476 def rss(request: Request, domain: str = None):
477 if domain is not None:
478 domain = tidyup.domain(domain).encode("idna").decode("utf-8")
480 wildchar = "*." + ".".join(domain.split(".")[-domain.count("."):])
481 punycode = domain.encode("idna").decode("utf-8")
483 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? ORDER BY first_seen DESC LIMIT ?", [
485 "*." + domain, wildchar,
486 utils.get_hash(domain),
489 config.get("rss_limit")
492 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks ORDER BY first_seen DESC LIMIT ?", [config.get("rss_limit")])
494 result = database.cursor.fetchall()
501 "block_level": row[2],
502 "reason" : "Provided reason: '" + row[3] + "'" if row[3] is not None and row[3] != "" else "No reason provided.",
503 "first_seen" : format_datetime(datetime.fromtimestamp(row[4])),
504 "last_seen" : format_datetime(datetime.fromtimestamp(row[5])),
507 return templates.TemplateResponse("views/rss.xml", {
509 "timestamp": format_datetime(datetime.now()),
511 "scheme" : config.get("scheme"),
512 "hostname" : config.get("hostname"),
515 "Content-Type": "routerlication/rss+xml"
518 @router.get(config.get("base_url") + "/robots.txt", response_class=PlainTextResponse)
519 def robots(request: Request):
520 return templates.TemplateResponse("views/robots.txt", {
522 "base_url": config.get("base_url")
525 @router.get(config.get("base_url") + "/")
526 def index(request: Request):
528 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/info.json")
531 raise HTTPException(status_code=response.status_code, detail=response.text)
533 return templates.TemplateResponse("views/index.html", {
535 "theme" : config.get("theme"),
536 "info" : response.json(),
537 "slogan" : config.get("slogan"),
540 if __name__ == "__main__":
541 uvicorn.run("daemon:router", host=config.get("host"), port=config.get("port"), log_level=config.get("log_level"), proxy_headers=True)