2 # -*- coding: utf-8 -*-
4 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
5 # Copyright (C) 2023 Free Software Foundation
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published
9 # by the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <https://www.gnu.org/licenses/>.
22 from datetime import datetime
23 from email.utils import format_datetime
24 from pathlib import Path
27 from fastapi import Request, HTTPException, Query
28 from fastapi.responses import JSONResponse
29 from fastapi.responses import PlainTextResponse
30 from fastapi.staticfiles import StaticFiles
31 from fastapi.templating import Jinja2Templates
36 from fba import database
39 from fba.helpers import config
40 from fba.helpers import json as json_helper
41 from fba.helpers import tidyup
43 from fba.models import blocks
45 router = fastapi.FastAPI(docs_url=config.get("base_url") + "/docs", redoc_url=config.get("base_url") + "/redoc")
48 StaticFiles(directory=Path(__file__).parent.absolute() / "static"),
52 templates = Jinja2Templates(directory="templates")
54 @router.get(config.get("base_url") + "/api/info.json", response_class=JSONResponse)
56 database.cursor.execute("SELECT (SELECT COUNT(domain) FROM instances), (SELECT COUNT(domain) FROM instances WHERE software IN ('pleroma', 'mastodon', 'lemmy', 'friendica', 'misskey', 'peertube', 'takahe')), (SELECT COUNT(blocker) FROM blocks), (SELECT COUNT(domain) FROM instances WHERE last_error_details IS NOT NULL)")
57 row = database.cursor.fetchone()
60 "known_instances" : row[0],
61 "supported_instances": row[1],
62 "blocks_recorded" : row[2],
63 "erroneous_instances": row[3],
66 @router.get(config.get("base_url") + "/api/scoreboard.json", response_class=JSONResponse)
67 def api_scoreboard(mode: str, amount: int):
68 if amount > config.get("api_limit"):
69 raise HTTPException(status_code=400, detail="Too many results")
72 database.cursor.execute("SELECT blocked, COUNT(blocked) AS score FROM blocks GROUP BY blocked ORDER BY score DESC LIMIT ?", [amount])
73 elif mode == "blocker":
74 database.cursor.execute("SELECT blocker, COUNT(blocker) AS score FROM blocks GROUP BY blocker ORDER BY score DESC LIMIT ?", [amount])
75 elif mode == "reference":
76 database.cursor.execute("SELECT origin, COUNT(domain) AS score FROM instances WHERE origin IS NOT NULL GROUP BY origin ORDER BY score DESC LIMIT ?", [amount])
77 elif mode == "software":
78 database.cursor.execute("SELECT software, COUNT(domain) AS score FROM instances WHERE software IS NOT NULL GROUP BY software ORDER BY score DESC, software ASC LIMIT ?", [amount])
79 elif mode == "command":
80 database.cursor.execute("SELECT command, COUNT(domain) AS score FROM instances WHERE command IS NOT NULL GROUP BY command ORDER BY score DESC, command ASC LIMIT ?", [amount])
81 elif mode == "error_code":
82 database.cursor.execute("SELECT last_status_code, COUNT(domain) AS score FROM instances WHERE last_status_code IS NOT NULL AND last_status_code != '200' GROUP BY last_status_code ORDER BY score DESC LIMIT ?", [amount])
83 elif mode == "detection_mode":
84 database.cursor.execute("SELECT detection_mode, COUNT(domain) AS cnt FROM instances GROUP BY detection_mode ORDER BY cnt DESC LIMIT ?", [amount])
85 elif mode == "avg_peers":
86 database.cursor.execute("SELECT software, AVG(total_peers) AS average FROM instances WHERE software IS NOT NULL GROUP BY software HAVING average>0 ORDER BY average DESC LIMIT ?", [amount])
87 elif mode == "obfuscator":
88 database.cursor.execute("SELECT software, COUNT(domain) AS cnt FROM instances WHERE has_obfuscation = 1 GROUP BY software ORDER BY cnt DESC LIMIT ?", [amount])
89 elif mode == "obfuscation":
90 database.cursor.execute("SELECT has_obfuscation, COUNT(domain) AS cnt FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica') GROUP BY has_obfuscation ORDER BY cnt DESC LIMIT ?", [amount])
91 elif mode == "block_level":
92 database.cursor.execute("SELECT block_level, COUNT(rowid) AS cnt FROM blocks GROUP BY block_level ORDER BY cnt DESC LIMIT ?", [amount])
94 raise HTTPException(status_code=400, detail="No filter specified")
98 for domain, score in database.cursor.fetchall():
101 "score" : round(score)
106 @router.get(config.get("base_url") + "/api/top.json", response_class=JSONResponse)
107 def api_index(request: Request, mode: str, value: str, amount: int):
108 if mode is None or value is None or amount is None:
109 raise HTTPException(status_code=500, detail="No filter specified")
110 elif amount > config.get("api_limit"):
111 raise HTTPException(status_code=500, detail=f"amount={amount} is to big")
113 domain = wildchar = punycode = reason = None
115 if mode == "block_level":
116 database.cursor.execute(
117 "SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks WHERE block_level = ? LIMIT ?", [value, amount]
119 elif mode in ["domain", "reverse"]:
120 domain = tidyup.domain(value)
121 if not utils.is_domain_wanted(domain):
122 raise HTTPException(status_code=500, detail=f"domain='{domain}' is not wanted")
124 wildchar = "*." + ".".join(domain.split(".")[-domain.count("."):])
125 punycode = domain.encode("idna").decode("utf-8")
126 elif mode == "reason":
127 reason = re.sub("(%|_)", "", tidyup.reason(value))
129 raise HTTPException(status_code=400, detail="Keyword is shorter than three characters")
132 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
134 WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? \
135 ORDER BY block_level ASC, first_seen ASC \
141 utils.get_hash(domain),
147 elif mode == "reverse":
148 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
150 WHERE blocker = ? OR blocker = ? OR blocker = ? OR blocker = ? OR blocker = ? OR blocker = ? \
151 ORDER BY first_seen ASC \
156 utils.get_hash(domain),
161 elif mode == "reason":
162 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
164 WHERE reason LIKE ? AND reason != '' \
165 ORDER BY first_seen ASC \
171 blocklist = database.cursor.fetchall()
174 for blocker, blocked, block_level, reason, first_seen, last_seen in blocklist:
175 if reason is not None and reason != "":
176 reason = reason.replace(",", " ").replace(" ", " ")
182 "first_seen": first_seen,
183 "last_seen" : last_seen
186 if block_level in result:
187 result[block_level].append(entry)
189 result[block_level] = [entry]
193 @router.get(config.get("base_url") + "/api/domain.json", response_class=JSONResponse)
194 def api_domain(domain: str):
195 # Tidy up domain name
196 domain = tidyup.domain(domain)
198 if not utils.is_domain_wanted(domain):
199 raise HTTPException(status_code=500, detail=f"domain='{domain}' is not wanted")
202 database.cursor.execute("SELECT * FROM instances WHERE domain = ? LIMIT 1", [domain])
203 domain_data = database.cursor.fetchone()
205 if domain_data is None:
206 raise HTTPException(status_code=404, detail=f"domain='{domain}' not found")
210 @router.get(config.get("base_url") + "/api/mutual.json", response_class=JSONResponse)
211 def api_mutual(domains: list[str] = Query()):
212 """Return 200 if federation is open between the two, 4xx otherwise"""
213 database.cursor.execute(
214 "SELECT block_level FROM blocks " \
215 "WHERE ((blocker = :a OR blocker = :b) AND (blocked = :b OR blocked = :a OR blocked = :aw OR blocked = :bw)) " \
216 "AND block_level = 'reject' " \
221 "aw": "*." + domains[0],
222 "bw": "*." + domains[1],
226 if database.cursor.fetchone() is not None:
228 return JSONResponse(status_code=418, content={})
231 return JSONResponse(status_code=200, content={})
233 @router.get(config.get("base_url") + "/scoreboard")
234 def scoreboard(request: Request, mode: str, amount: int):
237 if mode == "blocker" and amount > 0:
238 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=blocker&amount={amount}")
239 elif mode == "blocked" and amount > 0:
240 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=blocked&amount={amount}")
241 elif mode == "reference" and amount > 0:
242 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=reference&amount={amount}")
243 elif mode == "software" and amount > 0:
244 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=software&amount={amount}")
245 elif mode == "command" and amount > 0:
246 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=command&amount={amount}")
247 elif mode == "error_code" and amount > 0:
248 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=error_code&amount={amount}")
249 elif mode == "detection_mode" and amount > 0:
250 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=detection_mode&amount={amount}")
251 elif mode == "avg_peers" and amount > 0:
252 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=avg_peers&amount={amount}")
253 elif mode == "obfuscator" and amount > 0:
254 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=obfuscator&amount={amount}")
255 elif mode == "obfuscation" and amount > 0:
256 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=obfuscation&amount={amount}")
257 elif mode == "block_level" and amount > 0:
258 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=block_level&amount={amount}")
260 raise HTTPException(status_code=400, detail="No filter specified")
263 raise HTTPException(status_code=500, detail="Could not determine scores")
264 elif not response.ok:
265 raise HTTPException(status_code=response.status_code, detail=response.text)
267 return templates.TemplateResponse("views/scoreboard.html", {
268 "base_url" : config.get("base_url"),
269 "slogan" : config.get("slogan"),
270 "theme" : config.get("theme"),
275 "scores" : json_helper.from_response(response)
278 @router.get(config.get("base_url") + "/")
279 def index(request: Request):
281 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/info.json")
284 raise HTTPException(status_code=response.status_code, detail=response.text)
286 return templates.TemplateResponse("views/index.html", {
288 "theme" : config.get("theme"),
289 "info" : response.json(),
290 "slogan" : config.get("slogan"),
293 @router.get(config.get("base_url") + "/top")
294 def top(request: Request, mode: str, value: str, amount: int = config.get("api_limit")):
295 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/info.json")
298 raise HTTPException(status_code=response.status_code, detail=response.text)
299 elif mode == "" or value == "" or amount == 0:
300 raise HTTPException(status_code=500, detail="Parameter mode, value and amount must always be set")
301 elif amount > config.get("api_limit"):
302 raise HTTPException(status_code=500, detail=f"amount='{amount}' is to big")
304 info = response.json()
307 if mode == "block_level" and not blocks.is_valid_level(value):
308 raise HTTPException(status_code=500, detail="Invalid block level provided")
309 elif mode in ["domain", "reverse"] and not utils.is_domain_wanted(value):
310 raise HTTPException(status_code=500, detail="Invalid or blocked domain specified")
312 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/top.json?mode={mode}&value={value}&amount={amount}")
314 if response is not None and response.ok:
315 blocklist = response.json()
318 for block_level in blocklist:
319 for block in blocklist[block_level]:
320 block["first_seen"] = datetime.utcfromtimestamp(block["first_seen"]).strftime(config.get("timestamp_format"))
321 block["last_seen"] = datetime.utcfromtimestamp(block["last_seen"]).strftime(config.get("timestamp_format"))
324 return templates.TemplateResponse("views/top.html", {
326 "mode" : mode if response is not None else None,
327 "value" : value if response is not None else None,
328 "amount" : amount if response is not None else None,
330 "blocklist": blocklist,
332 "theme" : config.get("theme"),
335 @router.get(config.get("base_url") + "/infos")
336 def rss(request: Request, domain: str):
337 # Tidy up domain name
338 domain = tidyup.domain(domain)
340 if not utils.is_domain_wanted(domain):
341 raise HTTPException(status_code=500, detail=f"domain='{domain}' is not wanted")
343 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/domain.json?domain={domain}")
345 if not response.ok or response.status_code > 200 or response.text.strip() == "":
346 raise HTTPException(status_code=response.status_code, detail=response.reason)
348 domain_data = response.json()
351 format = config.get("timestamp_format")
353 for key in domain_data.keys():
354 if key in ["last_nodeinfo", "last_blocked", "first_seen", "last_updated", "last_instance_fetch"]:
356 instance[key] = datetime.utcfromtimestamp(domain_data[key]).strftime(format) if isinstance(domain_data[key], float) else "-"
359 instance[key] = domain_data[key]
361 return templates.TemplateResponse("views/infos.html", {
364 "instance": instance,
365 "theme" : config.get("theme"),
366 "slogan" : config.get("slogan"),
369 @router.get(config.get("base_url") + "/rss")
370 def rss(request: Request, domain: str = None):
371 if domain is not None:
372 domain = tidyup.domain(domain)
374 wildchar = "*." + ".".join(domain.split(".")[-domain.count("."):])
375 punycode = domain.encode("idna").decode("utf-8")
377 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? ORDER BY first_seen DESC LIMIT ?", [
379 "*." + domain, wildchar,
380 utils.get_hash(domain),
383 config.get("rss_limit")
386 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks ORDER BY first_seen DESC LIMIT ?", [config.get("rss_limit")])
388 result = database.cursor.fetchall()
395 "block_level": row[2],
396 "reason" : "Provided reason: '" + row[3] + "'" if row[3] is not None and row[3] != "" else "No reason provided.",
397 "first_seen" : format_datetime(datetime.fromtimestamp(row[4])),
398 "last_seen" : format_datetime(datetime.fromtimestamp(row[5])),
401 return templates.TemplateResponse("views/rss.xml", {
403 "timestamp": format_datetime(datetime.now()),
405 "hostname" : config.get("hostname"),
408 "Content-Type": "routerlication/rss+xml"
411 @router.get(config.get("base_url") + "/robots.txt", response_class=PlainTextResponse)
412 def robots(request: Request):
413 return templates.TemplateResponse("views/robots.txt", {
415 "base_url": config.get("base_url")
418 if __name__ == "__main__":
419 uvicorn.run("daemon:router", host=config.get("host"), port=config.get("port"), log_level=config.get("log_level"))