2 # -*- coding: utf-8 -*-
4 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
5 # Copyright (C) 2023 Free Software Foundation
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published
9 # by the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <https://www.gnu.org/licenses/>.
22 from datetime import datetime
23 from email.utils import format_datetime
24 from pathlib import Path
27 from fastapi import Request, HTTPException, Query
28 from fastapi.responses import JSONResponse
29 from fastapi.responses import PlainTextResponse
30 from fastapi.staticfiles import StaticFiles
31 from fastapi.templating import Jinja2Templates
36 from fba import database
39 from fba.helpers import config
40 from fba.helpers import json as json_helper
41 from fba.helpers import tidyup
43 from fba.models import blocks
44 from fba.models import instances
46 router = fastapi.FastAPI(docs_url=config.get("base_url") + "/docs", redoc_url=config.get("base_url") + "/redoc")
49 StaticFiles(directory=Path(__file__).parent.absolute() / "static"),
53 templates = Jinja2Templates(directory="templates")
55 @router.get(config.get("base_url") + "/api/info.json", response_class=JSONResponse)
57 database.cursor.execute("SELECT (SELECT COUNT(domain) FROM instances), (SELECT COUNT(domain) FROM instances WHERE software IN ('pleroma', 'mastodon', 'lemmy', 'friendica', 'misskey', 'peertube', 'takahe', 'gotosocial', 'brighteon', 'wildebeest', 'bookwyrm')), (SELECT COUNT(blocker) FROM blocks), (SELECT COUNT(domain) FROM instances WHERE last_error_details IS NOT NULL)")
58 row = database.cursor.fetchone()
61 "known_instances" : row[0],
62 "supported_instances": row[1],
63 "blocks_recorded" : row[2],
64 "erroneous_instances": row[3],
67 @router.get(config.get("base_url") + "/api/scoreboard.json", response_class=JSONResponse)
68 def api_scoreboard(mode: str, amount: int):
69 if amount > config.get("api_limit"):
70 raise HTTPException(status_code=400, detail="Too many results")
73 database.cursor.execute("SELECT blocked, COUNT(blocked) AS score FROM blocks GROUP BY blocked ORDER BY score DESC LIMIT ?", [amount])
74 elif mode == "blocker":
75 database.cursor.execute("SELECT blocker, COUNT(blocker) AS score FROM blocks GROUP BY blocker ORDER BY score DESC LIMIT ?", [amount])
76 elif mode == "reference":
77 database.cursor.execute("SELECT origin, COUNT(domain) AS score FROM instances WHERE origin IS NOT NULL GROUP BY origin ORDER BY score DESC LIMIT ?", [amount])
78 elif mode == "software":
79 database.cursor.execute("SELECT software, COUNT(domain) AS score FROM instances WHERE software IS NOT NULL GROUP BY software ORDER BY score DESC, software ASC LIMIT ?", [amount])
80 elif mode == "command":
81 database.cursor.execute("SELECT command, COUNT(domain) AS score FROM instances WHERE command IS NOT NULL GROUP BY command ORDER BY score DESC, command ASC LIMIT ?", [amount])
82 elif mode == "error_code":
83 database.cursor.execute("SELECT last_status_code, COUNT(domain) AS score FROM instances WHERE last_status_code IS NOT NULL AND last_status_code != '200' GROUP BY last_status_code ORDER BY score DESC LIMIT ?", [amount])
84 elif mode == "detection_mode":
85 database.cursor.execute("SELECT detection_mode, COUNT(domain) AS cnt FROM instances GROUP BY detection_mode ORDER BY cnt DESC LIMIT ?", [amount])
86 elif mode == "avg_peers":
87 database.cursor.execute("SELECT software, AVG(total_peers) AS average FROM instances WHERE software IS NOT NULL GROUP BY software HAVING average > 0 ORDER BY average DESC LIMIT ?", [amount])
88 elif mode == "obfuscator":
89 database.cursor.execute("SELECT software, COUNT(domain) AS cnt FROM instances WHERE has_obfuscation = 1 GROUP BY software ORDER BY cnt DESC LIMIT ?", [amount])
90 elif mode == "obfuscation":
91 database.cursor.execute("SELECT has_obfuscation, COUNT(domain) AS cnt FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica') GROUP BY has_obfuscation ORDER BY cnt DESC LIMIT ?", [amount])
92 elif mode == "block_level":
93 database.cursor.execute("SELECT block_level, COUNT(rowid) AS cnt FROM blocks GROUP BY block_level ORDER BY cnt DESC LIMIT ?", [amount])
95 raise HTTPException(status_code=400, detail="No filter specified")
99 for domain, score in database.cursor.fetchall():
102 "score" : round(score)
107 @router.get(config.get("base_url") + "/api/list.json", response_class=JSONResponse)
108 def api_list(request: Request, mode: str, value: str, amount: int):
109 if mode is None or value is None or amount is None:
110 raise HTTPException(status_code=500, detail="No filter specified")
111 elif amount > config.get("api_limit"):
112 raise HTTPException(status_code=500, detail=f"amount={amount} is to big")
114 if mode in ("detection_mode", "software", "command"):
115 database.cursor.execute(
116 f"SELECT domain, origin, software, detection_mode, command, total_peers, total_blocks, first_seen, last_updated \
120 LIMIT ?", [value, amount]
123 domainlist = database.cursor.fetchall()
127 @router.get(config.get("base_url") + "/api/top.json", response_class=JSONResponse)
128 def api_index(request: Request, mode: str, value: str, amount: int):
129 if mode is None or value is None or amount is None:
130 raise HTTPException(status_code=500, detail="No filter specified")
131 elif amount > config.get("api_limit"):
132 raise HTTPException(status_code=500, detail=f"amount={amount} is to big")
134 domain = wildchar = punycode = reason = None
136 if mode == "block_level":
137 database.cursor.execute(
138 "SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks WHERE block_level = ? LIMIT ?", [value, amount]
140 elif mode in ["domain", "reverse"]:
141 domain = tidyup.domain(value)
142 if not utils.is_domain_wanted(domain):
143 raise HTTPException(status_code=500, detail=f"domain='{domain}' is not wanted")
145 wildchar = "*." + ".".join(domain.split(".")[-domain.count("."):])
146 punycode = domain.encode("idna").decode("utf-8")
147 elif mode == "reason":
148 reason = re.sub("(%|_)", "", tidyup.reason(value))
150 raise HTTPException(status_code=400, detail="Keyword is shorter than three characters")
153 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
155 WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? \
156 ORDER BY block_level ASC, first_seen ASC \
162 utils.get_hash(domain),
168 elif mode == "reverse":
169 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
171 WHERE blocker = ? OR blocker = ? OR blocker = ? OR blocker = ? OR blocker = ? OR blocker = ? \
172 ORDER BY first_seen ASC \
177 utils.get_hash(domain),
182 elif mode == "reason":
183 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
185 WHERE reason LIKE ? AND reason != '' \
186 ORDER BY first_seen ASC \
192 blocklist = database.cursor.fetchall()
195 for blocker, blocked, block_level, reason, first_seen, last_seen in blocklist:
196 if reason is not None and reason != "":
197 reason = reason.replace(",", " ").replace(" ", " ")
203 "first_seen": first_seen,
204 "last_seen" : last_seen
207 if block_level in result:
208 result[block_level].append(entry)
210 result[block_level] = [entry]
214 @router.get(config.get("base_url") + "/api/domain.json", response_class=JSONResponse)
215 def api_domain(domain: str):
217 raise HTTPException(status_code=400, detail="Invalid request, parameter 'domain' missing")
219 # Tidy up domain name
220 domain = tidyup.domain(domain).encode("idna").decode("utf-8")
222 if not utils.is_domain_wanted(domain):
223 raise HTTPException(status_code=500, detail=f"domain='{domain}' is not wanted")
226 database.cursor.execute("SELECT * FROM instances WHERE domain = ? LIMIT 1", [domain])
227 domain_data = database.cursor.fetchone()
229 if domain_data is None:
230 raise HTTPException(status_code=404, detail=f"domain='{domain}' not found")
234 @router.get(config.get("base_url") + "/api/mutual.json", response_class=JSONResponse)
235 def api_mutual(domains: list[str] = Query()):
236 """Return 200 if federation is open between the two, 4xx otherwise"""
237 database.cursor.execute(
238 "SELECT block_level FROM blocks " \
239 "WHERE ((blocker = :a OR blocker = :b) AND (blocked = :b OR blocked = :a OR blocked = :aw OR blocked = :bw)) " \
240 "AND block_level = 'reject' " \
245 "aw": "*." + domains[0],
246 "bw": "*." + domains[1],
250 if database.cursor.fetchone() is not None:
252 return JSONResponse(status_code=418, content={})
255 return JSONResponse(status_code=200, content={})
257 @router.get(config.get("base_url") + "/scoreboard")
258 def scoreboard(request: Request, mode: str, amount: int):
261 if mode == "blocker" and amount > 0:
262 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=blocker&amount={amount}")
263 elif mode == "blocked" and amount > 0:
264 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=blocked&amount={amount}")
265 elif mode == "reference" and amount > 0:
266 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=reference&amount={amount}")
267 elif mode == "software" and amount > 0:
268 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=software&amount={amount}")
269 elif mode == "command" and amount > 0:
270 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=command&amount={amount}")
271 elif mode == "error_code" and amount > 0:
272 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=error_code&amount={amount}")
273 elif mode == "detection_mode" and amount > 0:
274 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=detection_mode&amount={amount}")
275 elif mode == "avg_peers" and amount > 0:
276 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=avg_peers&amount={amount}")
277 elif mode == "obfuscator" and amount > 0:
278 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=obfuscator&amount={amount}")
279 elif mode == "obfuscation" and amount > 0:
280 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=obfuscation&amount={amount}")
281 elif mode == "block_level" and amount > 0:
282 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=block_level&amount={amount}")
284 raise HTTPException(status_code=400, detail="No filter specified")
287 raise HTTPException(status_code=500, detail="Could not determine scores")
288 elif not response.ok:
289 raise HTTPException(status_code=response.status_code, detail=response.text)
291 return templates.TemplateResponse("views/scoreboard.html", {
292 "base_url" : config.get("base_url"),
293 "slogan" : config.get("slogan"),
294 "theme" : config.get("theme"),
299 "scores" : json_helper.from_response(response)
302 @router.get(config.get("base_url") + "/list")
303 def list_domains(request: Request, mode: str, value: str, amount: int = config.get("api_limit")):
304 if mode == "detection_mode" and not instances.valid(value, "detection_mode"):
305 raise HTTPException(status_code=500, detail="Invalid detection mode provided")
307 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/list.json?mode={mode}&value={value}&amount={amount}")
310 if response is not None and response.ok:
311 domainlist = response.json()
312 tformat = config.get("timestamp_format")
313 for row in domainlist:
314 row["first_seen"] = datetime.utcfromtimestamp(row["first_seen"]).strftime(tformat)
315 row["last_updated"] = datetime.utcfromtimestamp(row["last_updated"]).strftime(tformat) if isinstance(row["last_updated"], float) else None
317 return templates.TemplateResponse("views/list.html", {
319 "mode" : mode if response is not None else None,
320 "value" : value if response is not None else None,
321 "amount" : amount if response is not None else None,
322 "found" : len(domainlist),
323 "domainlist": domainlist,
324 "slogan" : config.get("slogan"),
325 "theme" : config.get("theme"),
328 @router.get(config.get("base_url") + "/top")
329 def top(request: Request, mode: str, value: str, amount: int = config.get("api_limit")):
330 if mode == "block_level" and not blocks.valid(value, "block_level"):
331 raise HTTPException(status_code=500, detail="Invalid block level provided")
332 elif mode in ["domain", "reverse"] and not utils.is_domain_wanted(value):
333 raise HTTPException(status_code=500, detail="Invalid or blocked domain specified")
335 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/top.json?mode={mode}&value={value}&amount={amount}")
339 if response.ok and response.status_code == 200 and len(response.text) > 0:
340 blocklist = response.json()
342 tformat = config.get("timestamp_format")
343 for block_level in blocklist:
344 for row in blocklist[block_level]:
345 row["first_seen"] = datetime.utcfromtimestamp(row["first_seen"]).strftime(tformat)
346 row["last_seen"] = datetime.utcfromtimestamp(row["last_seen"]).strftime(tformat) if isinstance(row["last_seen"], float) else None
349 return templates.TemplateResponse("views/top.html", {
351 "mode" : mode if response is not None else None,
352 "value" : value if response is not None else None,
353 "amount" : amount if response is not None else None,
355 "blocklist": blocklist,
356 "slogan" : config.get("slogan"),
357 "theme" : config.get("theme"),
360 @router.get(config.get("base_url") + "/infos")
361 def infos(request: Request, domain: str):
363 raise HTTPException(status_code=400, detail="Invalid request, parameter 'domain' missing")
365 # Tidy up domain name
366 domain = tidyup.domain(domain).encode("idna").decode("utf-8")
368 if not utils.is_domain_wanted(domain):
369 raise HTTPException(status_code=500, detail=f"domain='{domain}' is not wanted")
371 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/domain.json?domain={domain}")
373 if not response.ok or response.status_code >= 300 or response.text.strip() == "":
374 raise HTTPException(status_code=response.status_code, detail=response.reason)
376 domain_data = response.json()
379 tformat = config.get("timestamp_format")
381 for key in domain_data.keys():
382 if key in ["last_nodeinfo", "last_blocked", "first_seen", "last_updated", "last_instance_fetch"] and isinstance(domain_data[key], float):
384 instance[key] = datetime.utcfromtimestamp(domain_data[key]).strftime(tformat)
387 instance[key] = domain_data[key]
389 return templates.TemplateResponse("views/infos.html", {
392 "instance": instance,
393 "theme" : config.get("theme"),
394 "slogan" : config.get("slogan"),
397 @router.get(config.get("base_url") + "/rss")
398 def rss(request: Request, domain: str = None):
399 if domain is not None:
400 domain = tidyup.domain(domain).encode("idna").decode("utf-8")
402 wildchar = "*." + ".".join(domain.split(".")[-domain.count("."):])
403 punycode = domain.encode("idna").decode("utf-8")
405 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? ORDER BY first_seen DESC LIMIT ?", [
407 "*." + domain, wildchar,
408 utils.get_hash(domain),
411 config.get("rss_limit")
414 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks ORDER BY first_seen DESC LIMIT ?", [config.get("rss_limit")])
416 result = database.cursor.fetchall()
423 "block_level": row[2],
424 "reason" : "Provided reason: '" + row[3] + "'" if row[3] is not None and row[3] != "" else "No reason provided.",
425 "first_seen" : format_datetime(datetime.fromtimestamp(row[4])),
426 "last_seen" : format_datetime(datetime.fromtimestamp(row[5])),
429 return templates.TemplateResponse("views/rss.xml", {
431 "timestamp": format_datetime(datetime.now()),
433 "hostname" : config.get("hostname"),
436 "Content-Type": "routerlication/rss+xml"
439 @router.get(config.get("base_url") + "/robots.txt", response_class=PlainTextResponse)
440 def robots(request: Request):
441 return templates.TemplateResponse("views/robots.txt", {
443 "base_url": config.get("base_url")
446 @router.get(config.get("base_url") + "/")
447 def index(request: Request):
449 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/info.json")
452 raise HTTPException(status_code=response.status_code, detail=response.text)
454 return templates.TemplateResponse("views/index.html", {
456 "theme" : config.get("theme"),
457 "info" : response.json(),
458 "slogan" : config.get("slogan"),
461 if __name__ == "__main__":
462 uvicorn.run("daemon:router", host=config.get("host"), port=config.get("port"), log_level=config.get("log_level"))