2 # -*- coding: utf-8 -*-
4 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
5 # Copyright (C) 2023 Free Software Foundation
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published
9 # by the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <https://www.gnu.org/licenses/>.
22 from datetime import datetime
23 from email.utils import format_datetime
24 from pathlib import Path
25 from urllib.parse import urlparse
28 from fastapi import Request, HTTPException, Query
29 from fastapi.responses import JSONResponse
30 from fastapi.responses import PlainTextResponse
31 from fastapi.staticfiles import StaticFiles
32 from fastapi.templating import Jinja2Templates
37 from fba import database
40 from fba.helpers import config
41 from fba.helpers import json as json_helper
42 from fba.helpers import tidyup
44 from fba.models import blocks
45 from fba.models import instances
47 router = fastapi.FastAPI(docs_url=config.get("base_url") + "/docs", redoc_url=config.get("base_url") + "/redoc")
50 StaticFiles(directory=Path(__file__).parent.absolute() / "static"),
54 templates = Jinja2Templates(directory="templates")
56 @router.get(config.get("base_url") + "/api/info.json", response_class=JSONResponse)
58 database.cursor.execute("SELECT (SELECT COUNT(domain) FROM instances), (SELECT COUNT(domain) FROM instances WHERE software IN ('pleroma', 'mastodon', 'lemmy', 'friendica', 'misskey', 'peertube', 'takahe', 'gotosocial', 'brighteon', 'wildebeest', 'bookwyrm')), (SELECT COUNT(blocker) FROM blocks), (SELECT COUNT(domain) FROM instances WHERE last_error_details IS NOT NULL)")
59 row = database.cursor.fetchone()
61 return JSONResponse(status_code=200, content={
62 "known_instances" : row[0],
63 "supported_instances": row[1],
64 "blocks_recorded" : row[2],
65 "erroneous_instances": row[3],
68 @router.get(config.get("base_url") + "/api/scoreboard.json", response_class=JSONResponse)
69 def api_scoreboard(mode: str, amount: int):
70 if amount > config.get("api_limit"):
71 raise HTTPException(status_code=400, detail="Too many results")
74 database.cursor.execute("SELECT blocked, COUNT(blocked) AS score FROM blocks GROUP BY blocked ORDER BY score DESC LIMIT ?", [amount])
75 elif mode == "blocker":
76 database.cursor.execute("SELECT blocker, COUNT(blocker) AS score FROM blocks GROUP BY blocker ORDER BY score DESC LIMIT ?", [amount])
77 elif mode == "reference":
78 database.cursor.execute("SELECT origin, COUNT(domain) AS score FROM instances WHERE origin IS NOT NULL GROUP BY origin ORDER BY score DESC LIMIT ?", [amount])
79 elif mode == "software":
80 database.cursor.execute("SELECT software, COUNT(domain) AS score FROM instances WHERE software IS NOT NULL GROUP BY software ORDER BY score DESC, software ASC LIMIT ?", [amount])
81 elif mode == "command":
82 database.cursor.execute("SELECT command, COUNT(domain) AS score FROM instances WHERE command IS NOT NULL GROUP BY command ORDER BY score DESC, command ASC LIMIT ?", [amount])
83 elif mode == "error_code":
84 database.cursor.execute("SELECT last_status_code, COUNT(domain) AS score FROM instances WHERE last_status_code IS NOT NULL AND last_status_code != '200' GROUP BY last_status_code ORDER BY score DESC LIMIT ?", [amount])
85 elif mode == "detection_mode":
86 database.cursor.execute("SELECT detection_mode, COUNT(domain) AS cnt FROM instances GROUP BY detection_mode ORDER BY cnt DESC LIMIT ?", [amount])
87 elif mode == "avg_peers":
88 database.cursor.execute("SELECT software, AVG(total_peers) AS average FROM instances WHERE software IS NOT NULL GROUP BY software HAVING average > 0 ORDER BY average DESC LIMIT ?", [amount])
89 elif mode == "obfuscator":
90 database.cursor.execute("SELECT software, COUNT(domain) AS cnt FROM instances WHERE has_obfuscation = 1 GROUP BY software ORDER BY cnt DESC LIMIT ?", [amount])
91 elif mode == "obfuscation":
92 database.cursor.execute("SELECT has_obfuscation, COUNT(domain) AS cnt FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica') GROUP BY has_obfuscation ORDER BY cnt DESC LIMIT ?", [amount])
93 elif mode == "block_level":
94 database.cursor.execute("SELECT block_level, COUNT(rowid) AS cnt FROM blocks GROUP BY block_level ORDER BY cnt DESC LIMIT ?", [amount])
96 raise HTTPException(status_code=400, detail="No filter specified")
100 for domain, score in database.cursor.fetchall():
103 "score" : round(score)
106 return JSONResponse(status_code=200, content=scores)
108 @router.get(config.get("base_url") + "/api/list.json", response_class=JSONResponse)
109 def api_list(request: Request, mode: str, value: str, amount: int):
110 if mode is None or value is None or amount is None:
111 raise HTTPException(status_code=500, detail="No filter specified")
112 elif amount > config.get("api_limit"):
113 raise HTTPException(status_code=500, detail=f"amount={amount} is to big")
115 if mode in ("detection_mode", "software", "command"):
116 database.cursor.execute(
117 f"SELECT domain, origin, software, detection_mode, command, total_peers, total_blocks, first_seen, last_updated \
121 LIMIT ?", [value, amount]
124 domainlist = database.cursor.fetchall()
126 return JSONResponse(status_code=200, content=dict(domainlist))
128 @router.get(config.get("base_url") + "/api/top.json", response_class=JSONResponse)
129 def api_index(request: Request, mode: str, value: str, amount: int):
130 if mode is None or value is None or amount is None:
131 raise HTTPException(status_code=500, detail="No filter specified")
132 elif amount > config.get("api_limit"):
133 raise HTTPException(status_code=500, detail=f"amount={amount} is to big")
135 domain = wildchar = punycode = reason = None
137 if mode == "block_level":
138 database.cursor.execute(
139 "SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks WHERE block_level = ? LIMIT ?", [value, amount]
141 elif mode in ["domain", "reverse"]:
142 domain = tidyup.domain(value)
143 if not utils.is_domain_wanted(domain):
144 raise HTTPException(status_code=500, detail=f"domain='{domain}' is not wanted")
146 wildchar = "*." + ".".join(domain.split(".")[-domain.count("."):])
147 punycode = domain.encode("idna").decode("utf-8")
148 elif mode == "reason":
149 reason = re.sub("(%|_)", "", tidyup.reason(value))
151 raise HTTPException(status_code=400, detail="Keyword is shorter than three characters")
154 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
156 WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? \
157 ORDER BY block_level ASC, first_seen ASC \
163 utils.get_hash(domain),
169 elif mode == "reverse":
170 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
172 WHERE blocker = ? OR blocker = ? OR blocker = ? OR blocker = ? OR blocker = ? OR blocker = ? \
173 ORDER BY first_seen ASC \
178 utils.get_hash(domain),
183 elif mode == "reason":
184 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
186 WHERE reason LIKE ? AND reason != '' \
187 ORDER BY first_seen ASC \
193 blocklist = database.cursor.fetchall()
196 for blocker, blocked, block_level, reason, first_seen, last_seen in blocklist:
197 if reason is not None and reason != "":
198 reason = reason.replace(",", " ").replace(" ", " ")
204 "first_seen": first_seen,
205 "last_seen" : last_seen
208 if block_level in result:
209 result[block_level].append(entry)
211 result[block_level] = [entry]
215 @router.get(config.get("base_url") + "/api/domain.json", response_class=JSONResponse)
216 def api_domain(domain: str):
218 raise HTTPException(status_code=400, detail="Invalid request, parameter 'domain' missing")
220 # Tidy up domain name
221 domain = tidyup.domain(domain).encode("idna").decode("utf-8")
223 if not utils.is_domain_wanted(domain):
224 raise HTTPException(status_code=500, detail=f"domain='{domain}' is not wanted")
227 database.cursor.execute("SELECT * FROM instances WHERE domain = ? LIMIT 1", [domain])
228 domain_data = database.cursor.fetchone()
230 if domain_data is None:
231 raise HTTPException(status_code=404, detail=f"domain='{domain}' not found")
233 return JSONResponse(status_code=200, content=dict(domain_data))
235 @router.get(config.get("base_url") + "/api/mutual.json", response_class=JSONResponse)
236 def api_mutual(domains: list[str] = Query()):
237 """Return 200 if federation is open between the two, 4xx otherwise"""
238 database.cursor.execute(
239 "SELECT block_level FROM blocks " \
240 "WHERE ((blocker = :a OR blocker = :b) AND (blocked = :b OR blocked = :a OR blocked = :aw OR blocked = :bw)) " \
241 "AND block_level = 'reject' " \
246 "aw": "*." + domains[0],
247 "bw": "*." + domains[1],
251 if database.cursor.fetchone() is not None:
253 return JSONResponse(status_code=418, content={})
256 return JSONResponse(status_code=200, content={})
258 @router.get(config.get("base_url") + "/.well-known/nodeinfo", response_class=JSONResponse)
259 def wellknown_nodeinfo(request: Request):
260 components = urlparse(str(request.url))
262 return JSONResponse(status_code=200, content={
264 "rel" : "http://nodeinfo.diaspora.software/ns/schema/1.0",
265 "href": f"{components.scheme}://{config.get('hostname')}{config.get('base_url')}/nodeinfo/1.0"
269 @router.get(config.get("base_url") + "/nodeinfo/1.0", response_class=JSONResponse)
270 def nodeinfo_1_0(request: Request):
271 return JSONResponse(status_code=200, content={
292 "openRegistrations": False,
294 "nodeName": "Fedi Block API",
307 "explicitContent": False,
311 @router.get(config.get("base_url") + "/api/v1/instance/peers", response_class=JSONResponse)
312 def api_peers(request: Request):
313 database.cursor.execute("SELECT domain FROM instances WHERE nodeinfo_url IS NOT NULL")
316 for row in database.cursor.fetchall():
317 peers.append(row["domain"])
319 return JSONResponse(status_code=200, content=peers)
321 @router.get(config.get("base_url") + "/scoreboard")
322 def scoreboard(request: Request, mode: str, amount: int):
325 if mode == "blocker" and amount > 0:
326 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=blocker&amount={amount}")
327 elif mode == "blocked" and amount > 0:
328 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=blocked&amount={amount}")
329 elif mode == "reference" and amount > 0:
330 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=reference&amount={amount}")
331 elif mode == "software" and amount > 0:
332 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=software&amount={amount}")
333 elif mode == "command" and amount > 0:
334 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=command&amount={amount}")
335 elif mode == "error_code" and amount > 0:
336 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=error_code&amount={amount}")
337 elif mode == "detection_mode" and amount > 0:
338 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=detection_mode&amount={amount}")
339 elif mode == "avg_peers" and amount > 0:
340 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=avg_peers&amount={amount}")
341 elif mode == "obfuscator" and amount > 0:
342 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=obfuscator&amount={amount}")
343 elif mode == "obfuscation" and amount > 0:
344 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=obfuscation&amount={amount}")
345 elif mode == "block_level" and amount > 0:
346 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=block_level&amount={amount}")
348 raise HTTPException(status_code=400, detail="No filter specified")
351 raise HTTPException(status_code=500, detail="Could not determine scores")
352 elif not response.ok:
353 raise HTTPException(status_code=response.status_code, detail=response.text)
355 return templates.TemplateResponse("views/scoreboard.html", {
356 "base_url" : config.get("base_url"),
357 "slogan" : config.get("slogan"),
358 "theme" : config.get("theme"),
363 "scores" : json_helper.from_response(response)
366 @router.get(config.get("base_url") + "/list")
367 def list_domains(request: Request, mode: str, value: str, amount: int = config.get("api_limit")):
368 if mode == "detection_mode" and not instances.valid(value, "detection_mode"):
369 raise HTTPException(status_code=500, detail="Invalid detection mode provided")
371 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/list.json?mode={mode}&value={value}&amount={amount}")
374 if response is not None and response.ok:
375 domainlist = response.json()
376 tformat = config.get("timestamp_format")
377 for row in domainlist:
378 row["first_seen"] = datetime.utcfromtimestamp(row["first_seen"]).strftime(tformat)
379 row["last_updated"] = datetime.utcfromtimestamp(row["last_updated"]).strftime(tformat) if isinstance(row["last_updated"], float) else None
381 return templates.TemplateResponse("views/list.html", {
383 "mode" : mode if response is not None else None,
384 "value" : value if response is not None else None,
385 "amount" : amount if response is not None else None,
386 "found" : len(domainlist),
387 "domainlist": domainlist,
388 "slogan" : config.get("slogan"),
389 "theme" : config.get("theme"),
392 @router.get(config.get("base_url") + "/top")
393 def top(request: Request, mode: str, value: str, amount: int = config.get("api_limit")):
394 if mode == "block_level" and not blocks.valid(value, "block_level"):
395 raise HTTPException(status_code=500, detail="Invalid block level provided")
396 elif mode in ["domain", "reverse"] and not utils.is_domain_wanted(value):
397 raise HTTPException(status_code=500, detail="Invalid or blocked domain specified")
399 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/top.json?mode={mode}&value={value}&amount={amount}")
403 if response.ok and response.status_code == 200 and len(response.text) > 0:
404 blocklist = response.json()
406 tformat = config.get("timestamp_format")
407 for block_level in blocklist:
408 for row in blocklist[block_level]:
409 row["first_seen"] = datetime.utcfromtimestamp(row["first_seen"]).strftime(tformat)
410 row["last_seen"] = datetime.utcfromtimestamp(row["last_seen"]).strftime(tformat) if isinstance(row["last_seen"], float) else None
413 return templates.TemplateResponse("views/top.html", {
415 "mode" : mode if response is not None else None,
416 "value" : value if response is not None else None,
417 "amount" : amount if response is not None else None,
419 "blocklist": blocklist,
420 "slogan" : config.get("slogan"),
421 "theme" : config.get("theme"),
424 @router.get(config.get("base_url") + "/infos")
425 def infos(request: Request, domain: str):
427 raise HTTPException(status_code=400, detail="Invalid request, parameter 'domain' missing")
429 # Tidy up domain name
430 domain = tidyup.domain(domain).encode("idna").decode("utf-8")
432 if not utils.is_domain_wanted(domain):
433 raise HTTPException(status_code=500, detail=f"domain='{domain}' is not wanted")
435 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/domain.json?domain={domain}")
437 if not response.ok or response.status_code >= 300 or response.text.strip() == "":
438 raise HTTPException(status_code=response.status_code, detail=response.reason)
440 domain_data = response.json()
443 tformat = config.get("timestamp_format")
445 for key in domain_data.keys():
446 if key in ["last_nodeinfo", "last_blocked", "first_seen", "last_updated", "last_instance_fetch"] and isinstance(domain_data[key], float):
448 instance[key] = datetime.utcfromtimestamp(domain_data[key]).strftime(tformat)
451 instance[key] = domain_data[key]
453 return templates.TemplateResponse("views/infos.html", {
456 "instance": instance,
457 "theme" : config.get("theme"),
458 "slogan" : config.get("slogan"),
461 @router.get(config.get("base_url") + "/rss")
462 def rss(request: Request, domain: str = None):
463 if domain is not None:
464 domain = tidyup.domain(domain).encode("idna").decode("utf-8")
466 wildchar = "*." + ".".join(domain.split(".")[-domain.count("."):])
467 punycode = domain.encode("idna").decode("utf-8")
469 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? ORDER BY first_seen DESC LIMIT ?", [
471 "*." + domain, wildchar,
472 utils.get_hash(domain),
475 config.get("rss_limit")
478 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks ORDER BY first_seen DESC LIMIT ?", [config.get("rss_limit")])
480 result = database.cursor.fetchall()
487 "block_level": row[2],
488 "reason" : "Provided reason: '" + row[3] + "'" if row[3] is not None and row[3] != "" else "No reason provided.",
489 "first_seen" : format_datetime(datetime.fromtimestamp(row[4])),
490 "last_seen" : format_datetime(datetime.fromtimestamp(row[5])),
493 return templates.TemplateResponse("views/rss.xml", {
495 "timestamp": format_datetime(datetime.now()),
497 "hostname" : config.get("hostname"),
500 "Content-Type": "routerlication/rss+xml"
503 @router.get(config.get("base_url") + "/robots.txt", response_class=PlainTextResponse)
504 def robots(request: Request):
505 return templates.TemplateResponse("views/robots.txt", {
507 "base_url": config.get("base_url")
510 @router.get(config.get("base_url") + "/")
511 def index(request: Request):
513 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/info.json")
516 raise HTTPException(status_code=response.status_code, detail=response.text)
518 return templates.TemplateResponse("views/index.html", {
520 "theme" : config.get("theme"),
521 "info" : response.json(),
522 "slogan" : config.get("slogan"),
525 if __name__ == "__main__":
526 uvicorn.run("daemon:router", host=config.get("host"), port=config.get("port"), log_level=config.get("log_level"))