2 # -*- coding: utf-8 -*-
4 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
5 # Copyright (C) 2023 Free Software Foundation
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published
9 # by the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <https://www.gnu.org/licenses/>.
22 from datetime import datetime
23 from email.utils import format_datetime
24 from pathlib import Path
25 from urllib.parse import urlparse
28 from fastapi import Request, HTTPException, Query
29 from fastapi.responses import JSONResponse
30 from fastapi.responses import PlainTextResponse
31 from fastapi.staticfiles import StaticFiles
32 from fastapi.templating import Jinja2Templates
37 from fba import database
40 from fba.helpers import blacklist
41 from fba.helpers import config
42 from fba.helpers import json as json_helper
43 from fba.helpers import tidyup
45 from fba.models import blocks
46 from fba.models import instances
48 router = fastapi.FastAPI(docs_url=config.get("base_url") + "/docs", redoc_url=config.get("base_url") + "/redoc")
51 StaticFiles(directory=Path(__file__).parent.absolute() / "static"),
55 templates = Jinja2Templates(directory="templates")
57 @router.get(config.get("base_url") + "/api/info.json", response_class=JSONResponse)
59 database.cursor.execute("SELECT (SELECT COUNT(domain) FROM instances), (SELECT COUNT(domain) FROM instances WHERE software IN ('pleroma', 'mastodon', 'lemmy', 'friendica', 'misskey', 'peertube', 'takahe', 'gotosocial', 'brighteon', 'wildebeest', 'bookwyrm', 'mitra')), (SELECT COUNT(blocker) FROM blocks), (SELECT COUNT(domain) FROM instances WHERE last_error_details IS NOT NULL)")
60 row = database.cursor.fetchone()
62 return JSONResponse(status_code=200, content={
63 "known_instances" : row[0],
64 "supported_instances": row[1],
65 "blocks_recorded" : row[2],
66 "erroneous_instances": row[3],
69 @router.get(config.get("base_url") + "/api/scoreboard.json", response_class=JSONResponse)
70 def api_scoreboard(mode: str, amount: int):
71 if amount > config.get("api_limit"):
72 raise HTTPException(status_code=400, detail="Too many results")
75 database.cursor.execute("SELECT blocked, COUNT(blocked) AS score FROM blocks GROUP BY blocked ORDER BY score DESC LIMIT ?", [amount])
76 elif mode == "blocker":
77 database.cursor.execute("SELECT blocker, COUNT(blocker) AS score FROM blocks GROUP BY blocker ORDER BY score DESC LIMIT ?", [amount])
78 elif mode == "reference":
79 database.cursor.execute("SELECT origin, COUNT(domain) AS score FROM instances WHERE origin IS NOT NULL GROUP BY origin ORDER BY score DESC LIMIT ?", [amount])
80 elif mode == "software":
81 database.cursor.execute("SELECT software, COUNT(domain) AS score FROM instances WHERE software IS NOT NULL GROUP BY software ORDER BY score DESC, software ASC LIMIT ?", [amount])
82 elif mode == "command":
83 database.cursor.execute("SELECT command, COUNT(domain) AS score FROM instances WHERE command IS NOT NULL GROUP BY command ORDER BY score DESC, command ASC LIMIT ?", [amount])
84 elif mode == "error_code":
85 database.cursor.execute("SELECT last_status_code, COUNT(domain) AS score FROM instances WHERE last_status_code IS NOT NULL AND last_status_code != '200' GROUP BY last_status_code ORDER BY score DESC LIMIT ?", [amount])
86 elif mode == "detection_mode":
87 database.cursor.execute("SELECT detection_mode, COUNT(domain) AS cnt FROM instances GROUP BY detection_mode ORDER BY cnt DESC LIMIT ?", [amount])
88 elif mode == "avg_peers":
89 database.cursor.execute("SELECT software, AVG(total_peers) AS average FROM instances WHERE software IS NOT NULL GROUP BY software HAVING average > 0 ORDER BY average DESC LIMIT ?", [amount])
90 elif mode == "obfuscator":
91 database.cursor.execute("SELECT software, COUNT(domain) AS cnt FROM instances WHERE has_obfuscation = 1 GROUP BY software ORDER BY cnt DESC LIMIT ?", [amount])
92 elif mode == "obfuscation":
93 database.cursor.execute("SELECT has_obfuscation, COUNT(domain) AS cnt FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica') GROUP BY has_obfuscation ORDER BY cnt DESC LIMIT ?", [amount])
94 elif mode == "block_level":
95 database.cursor.execute("SELECT block_level, COUNT(rowid) AS cnt FROM blocks GROUP BY block_level ORDER BY cnt DESC LIMIT ?", [amount])
97 raise HTTPException(status_code=400, detail="No filter specified")
101 for domain, score in database.cursor.fetchall():
104 "score" : round(score)
107 return JSONResponse(status_code=200, content=scores)
109 @router.get(config.get("base_url") + "/api/list.json", response_class=JSONResponse)
110 def api_list(request: Request, mode: str, value: str, amount: int):
111 if mode is None or value is None or amount is None:
112 raise HTTPException(status_code=500, detail="No filter specified")
113 elif amount > config.get("api_limit"):
114 raise HTTPException(status_code=500, detail=f"amount={amount} is to big")
116 if mode in ("detection_mode", "software", "command"):
117 database.cursor.execute(
118 f"SELECT domain, origin, software, detection_mode, command, total_peers, total_blocks, first_seen, last_updated \
122 LIMIT ?", [value, amount]
125 domainlist = database.cursor.fetchall()
128 @router.get(config.get("base_url") + "/api/top.json", response_class=JSONResponse)
129 def api_index(request: Request, mode: str, value: str, amount: int):
130 if mode is None or value is None or amount is None:
131 raise HTTPException(status_code=500, detail="No filter specified")
132 elif amount > config.get("api_limit"):
133 raise HTTPException(status_code=500, detail=f"amount={amount} is to big")
135 domain = wildchar = punycode = reason = None
137 if mode == "block_level":
138 database.cursor.execute(
139 "SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks WHERE block_level = ? LIMIT ?", [value, amount]
141 elif mode in ["domain", "reverse"]:
142 domain = tidyup.domain(value)
143 if not utils.is_domain_wanted(domain):
144 raise HTTPException(status_code=500, detail=f"domain='{domain}' is not wanted")
146 wildchar = "*." + ".".join(domain.split(".")[-domain.count("."):])
147 punycode = domain.encode("idna").decode("utf-8")
148 elif mode == "reason":
149 reason = re.sub("(%|_)", "", tidyup.reason(value))
151 raise HTTPException(status_code=400, detail="Keyword is shorter than three characters")
154 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
156 WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? \
157 ORDER BY block_level ASC, first_seen ASC \
163 utils.get_hash(domain),
169 elif mode == "reverse":
170 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
172 WHERE blocker = ? OR blocker = ? OR blocker = ? OR blocker = ? OR blocker = ? OR blocker = ? \
173 ORDER BY first_seen ASC \
178 utils.get_hash(domain),
183 elif mode == "reason":
184 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
186 WHERE reason LIKE ? AND reason != '' \
187 ORDER BY first_seen ASC \
193 blocklist = database.cursor.fetchall()
196 for blocker, blocked, block_level, reason, first_seen, last_seen in blocklist:
197 if reason is not None and reason != "":
198 reason = reason.replace(",", " ").replace(" ", " ")
204 "first_seen": first_seen,
205 "last_seen" : last_seen
208 if block_level in result:
209 result[block_level].append(entry)
211 result[block_level] = [entry]
215 @router.get(config.get("base_url") + "/api/domain.json", response_class=JSONResponse)
216 def api_domain(domain: str):
218 raise HTTPException(status_code=400, detail="Invalid request, parameter 'domain' missing")
220 # Tidy up domain name
221 domain = tidyup.domain(domain).encode("idna").decode("utf-8")
223 if not utils.is_domain_wanted(domain):
224 raise HTTPException(status_code=500, detail=f"domain='{domain}' is not wanted")
227 database.cursor.execute("SELECT * FROM instances WHERE domain = ? LIMIT 1", [domain])
228 domain_data = database.cursor.fetchone()
230 if domain_data is None:
231 raise HTTPException(status_code=404, detail=f"domain='{domain}' not found")
233 return JSONResponse(status_code=200, content=dict(domain_data))
235 @router.get(config.get("base_url") + "/api/mutual.json", response_class=JSONResponse)
236 def api_mutual(domains: list[str] = Query()):
237 """Return 200 if federation is open between the two, 4xx otherwise"""
238 database.cursor.execute(
239 "SELECT block_level FROM blocks " \
240 "WHERE ((blocker = :a OR blocker = :b) AND (blocked = :b OR blocked = :a OR blocked = :aw OR blocked = :bw)) " \
241 "AND block_level = 'reject' " \
246 "aw": "*." + domains[0],
247 "bw": "*." + domains[1],
251 if database.cursor.fetchone() is not None:
253 return JSONResponse(status_code=418, content={})
256 return JSONResponse(status_code=200, content={})
258 @router.get(config.get("base_url") + "/.well-known/nodeinfo", response_class=JSONResponse)
259 def wellknown_nodeinfo(request: Request):
260 return JSONResponse(status_code=200, content={
262 "rel" : "http://nodeinfo.diaspora.software/ns/schema/1.0",
263 "href": f"{config.get('scheme')}://{config.get('hostname')}{config.get('base_url')}/nodeinfo/1.0"
267 @router.get(config.get("base_url") + "/nodeinfo/1.0", response_class=JSONResponse)
268 def nodeinfo_1_0(request: Request):
269 return JSONResponse(status_code=200, content={
290 "openRegistrations": False,
292 "nodeName": "Fedi Block API",
305 "explicitContent": False,
309 @router.get(config.get("base_url") + "/api/v1/instance/domain_blocks", response_class=JSONResponse)
310 def api_domain_blocks(request: Request):
311 blocked = blacklist.get_all()
314 for block in blocked:
317 "digest" : utils.get_hash(block),
318 "severity": "suspend",
319 "comment" : blocked[block],
322 return JSONResponse(status_code=200, content=blocking)
324 @router.get(config.get("base_url") + "/api/v1/instance/peers", response_class=JSONResponse)
325 def api_peers(request: Request):
326 database.cursor.execute("SELECT domain FROM instances WHERE nodeinfo_url IS NOT NULL")
329 for row in database.cursor.fetchall():
330 peers.append(row["domain"])
332 return JSONResponse(status_code=200, content=peers)
334 @router.get(config.get("base_url") + "/scoreboard")
335 def scoreboard(request: Request, mode: str, amount: int):
338 if mode == "blocker" and amount > 0:
339 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=blocker&amount={amount}")
340 elif mode == "blocked" and amount > 0:
341 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=blocked&amount={amount}")
342 elif mode == "reference" and amount > 0:
343 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=reference&amount={amount}")
344 elif mode == "software" and amount > 0:
345 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=software&amount={amount}")
346 elif mode == "command" and amount > 0:
347 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=command&amount={amount}")
348 elif mode == "error_code" and amount > 0:
349 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=error_code&amount={amount}")
350 elif mode == "detection_mode" and amount > 0:
351 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=detection_mode&amount={amount}")
352 elif mode == "avg_peers" and amount > 0:
353 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=avg_peers&amount={amount}")
354 elif mode == "obfuscator" and amount > 0:
355 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=obfuscator&amount={amount}")
356 elif mode == "obfuscation" and amount > 0:
357 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=obfuscation&amount={amount}")
358 elif mode == "block_level" and amount > 0:
359 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=block_level&amount={amount}")
361 raise HTTPException(status_code=400, detail="No filter specified")
364 raise HTTPException(status_code=500, detail="Could not determine scores")
365 elif not response.ok:
366 raise HTTPException(status_code=response.status_code, detail=response.text)
368 return templates.TemplateResponse("views/scoreboard.html", {
369 "base_url" : utils.base_url(),
370 "slogan" : config.get("slogan"),
371 "theme" : config.get("theme"),
376 "scores" : json_helper.from_response(response)
379 @router.get(config.get("base_url") + "/list")
380 def list_domains(request: Request, mode: str, value: str, amount: int = config.get("api_limit")):
381 if mode == "detection_mode" and not instances.valid(value, "detection_mode"):
382 raise HTTPException(status_code=500, detail="Invalid detection mode provided")
384 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/list.json?mode={mode}&value={value}&amount={amount}")
387 if response is not None and response.ok:
388 domainlist = response.json()
389 tformat = config.get("timestamp_format")
390 for row in domainlist:
391 row["first_seen"] = datetime.utcfromtimestamp(row["first_seen"]).strftime(tformat)
392 row["last_updated"] = datetime.utcfromtimestamp(row["last_updated"]).strftime(tformat) if isinstance(row["last_updated"], float) else None
394 return templates.TemplateResponse("views/list.html", {
396 "mode" : mode if response is not None else None,
397 "value" : value if response is not None else None,
398 "amount" : amount if response is not None else None,
399 "found" : len(domainlist),
400 "domainlist": domainlist,
401 "slogan" : config.get("slogan"),
402 "theme" : config.get("theme"),
405 @router.get(config.get("base_url") + "/top")
406 def top(request: Request, mode: str, value: str, amount: int = config.get("api_limit")):
407 if mode == "block_level" and not blocks.valid(value, "block_level"):
408 raise HTTPException(status_code=500, detail="Invalid block level provided")
409 elif mode in ["domain", "reverse"] and not utils.is_domain_wanted(value):
410 raise HTTPException(status_code=500, detail="Invalid or blocked domain specified")
412 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/top.json?mode={mode}&value={value}&amount={amount}")
416 if response.ok and response.status_code == 200 and len(response.text) > 0:
417 blocklist = response.json()
419 tformat = config.get("timestamp_format")
420 for block_level in blocklist:
421 for row in blocklist[block_level]:
422 row["first_seen"] = datetime.utcfromtimestamp(row["first_seen"]).strftime(tformat)
423 row["last_seen"] = datetime.utcfromtimestamp(row["last_seen"]).strftime(tformat) if isinstance(row["last_seen"], float) else None
426 return templates.TemplateResponse("views/top.html", {
428 "mode" : mode if response is not None else None,
429 "value" : value if response is not None else None,
430 "amount" : amount if response is not None else None,
432 "blocklist": blocklist,
433 "slogan" : config.get("slogan"),
434 "theme" : config.get("theme"),
437 @router.get(config.get("base_url") + "/infos")
438 def infos(request: Request, domain: str):
440 raise HTTPException(status_code=400, detail="Invalid request, parameter 'domain' missing")
442 # Tidy up domain name
443 domain = tidyup.domain(domain).encode("idna").decode("utf-8")
445 if not utils.is_domain_wanted(domain):
446 raise HTTPException(status_code=500, detail=f"domain='{domain}' is not wanted")
448 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/domain.json?domain={domain}")
450 if not response.ok or response.status_code >= 300 or response.text.strip() == "":
451 raise HTTPException(status_code=response.status_code, detail=response.reason)
453 domain_data = response.json()
456 tformat = config.get("timestamp_format")
458 for key in domain_data.keys():
459 if key in ["last_nodeinfo", "last_blocked", "first_seen", "last_updated", "last_instance_fetch"] and isinstance(domain_data[key], float):
461 instance[key] = datetime.utcfromtimestamp(domain_data[key]).strftime(tformat)
464 instance[key] = domain_data[key]
466 return templates.TemplateResponse("views/infos.html", {
469 "instance": instance,
470 "theme" : config.get("theme"),
471 "slogan" : config.get("slogan"),
474 @router.get(config.get("base_url") + "/rss")
475 def rss(request: Request, domain: str = None):
476 if domain is not None:
477 domain = tidyup.domain(domain).encode("idna").decode("utf-8")
479 wildchar = "*." + ".".join(domain.split(".")[-domain.count("."):])
480 punycode = domain.encode("idna").decode("utf-8")
482 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? ORDER BY first_seen DESC LIMIT ?", [
484 "*." + domain, wildchar,
485 utils.get_hash(domain),
488 config.get("rss_limit")
491 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks ORDER BY first_seen DESC LIMIT ?", [config.get("rss_limit")])
493 result = database.cursor.fetchall()
500 "block_level": row[2],
501 "reason" : "Provided reason: '" + row[3] + "'" if row[3] is not None and row[3] != "" else "No reason provided.",
502 "first_seen" : format_datetime(datetime.fromtimestamp(row[4])),
503 "last_seen" : format_datetime(datetime.fromtimestamp(row[5])),
506 return templates.TemplateResponse("views/rss.xml", {
508 "timestamp": format_datetime(datetime.now()),
510 "scheme" : config.get("scheme"),
511 "hostname" : config.get("hostname"),
514 "Content-Type": "routerlication/rss+xml"
517 @router.get(config.get("base_url") + "/robots.txt", response_class=PlainTextResponse)
518 def robots(request: Request):
519 return templates.TemplateResponse("views/robots.txt", {
521 "base_url": config.get("base_url")
524 @router.get(config.get("base_url") + "/")
525 def index(request: Request):
527 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/info.json")
530 raise HTTPException(status_code=response.status_code, detail=response.text)
532 return templates.TemplateResponse("views/index.html", {
534 "theme" : config.get("theme"),
535 "info" : response.json(),
536 "slogan" : config.get("slogan"),
539 if __name__ == "__main__":
540 uvicorn.run("daemon:router", host=config.get("host"), port=config.get("port"), log_level=config.get("log_level"), proxy_headers=True)