2 # -*- coding: utf-8 -*-
4 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
5 # Copyright (C) 2023 Free Software Foundation
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published
9 # by the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <https://www.gnu.org/licenses/>.
22 from datetime import datetime
23 from email.utils import format_datetime
24 from pathlib import Path
27 from fastapi import Request, HTTPException, Query
28 from fastapi.responses import JSONResponse
29 from fastapi.responses import PlainTextResponse
30 from fastapi.staticfiles import StaticFiles
31 from fastapi.templating import Jinja2Templates
36 from fba import database
39 from fba.helpers import config
40 from fba.helpers import json as json_helper
41 from fba.helpers import tidyup
43 from fba.models import blocks
44 from fba.models import instances
46 router = fastapi.FastAPI(docs_url=config.get("base_url") + "/docs", redoc_url=config.get("base_url") + "/redoc")
49 StaticFiles(directory=Path(__file__).parent.absolute() / "static"),
53 templates = Jinja2Templates(directory="templates")
55 @router.get(config.get("base_url") + "/api/info.json", response_class=JSONResponse)
57 database.cursor.execute("SELECT (SELECT COUNT(domain) FROM instances), (SELECT COUNT(domain) FROM instances WHERE software IN ('pleroma', 'mastodon', 'lemmy', 'friendica', 'misskey', 'peertube', 'takahe')), (SELECT COUNT(blocker) FROM blocks), (SELECT COUNT(domain) FROM instances WHERE last_error_details IS NOT NULL)")
58 row = database.cursor.fetchone()
61 "known_instances" : row[0],
62 "supported_instances": row[1],
63 "blocks_recorded" : row[2],
64 "erroneous_instances": row[3],
67 @router.get(config.get("base_url") + "/api/scoreboard.json", response_class=JSONResponse)
68 def api_scoreboard(mode: str, amount: int):
69 if amount > config.get("api_limit"):
70 raise HTTPException(status_code=400, detail="Too many results")
73 database.cursor.execute("SELECT blocked, COUNT(blocked) AS score FROM blocks GROUP BY blocked ORDER BY score DESC LIMIT ?", [amount])
74 elif mode == "blocker":
75 database.cursor.execute("SELECT blocker, COUNT(blocker) AS score FROM blocks GROUP BY blocker ORDER BY score DESC LIMIT ?", [amount])
76 elif mode == "reference":
77 database.cursor.execute("SELECT origin, COUNT(domain) AS score FROM instances WHERE origin IS NOT NULL GROUP BY origin ORDER BY score DESC LIMIT ?", [amount])
78 elif mode == "software":
79 database.cursor.execute("SELECT software, COUNT(domain) AS score FROM instances WHERE software IS NOT NULL GROUP BY software ORDER BY score DESC, software ASC LIMIT ?", [amount])
80 elif mode == "command":
81 database.cursor.execute("SELECT command, COUNT(domain) AS score FROM instances WHERE command IS NOT NULL GROUP BY command ORDER BY score DESC, command ASC LIMIT ?", [amount])
82 elif mode == "error_code":
83 database.cursor.execute("SELECT last_status_code, COUNT(domain) AS score FROM instances WHERE last_status_code IS NOT NULL AND last_status_code != '200' GROUP BY last_status_code ORDER BY score DESC LIMIT ?", [amount])
84 elif mode == "detection_mode":
85 database.cursor.execute("SELECT detection_mode, COUNT(domain) AS cnt FROM instances GROUP BY detection_mode ORDER BY cnt DESC LIMIT ?", [amount])
86 elif mode == "avg_peers":
87 database.cursor.execute("SELECT software, AVG(total_peers) AS average FROM instances WHERE software IS NOT NULL GROUP BY software HAVING average>0 ORDER BY average DESC LIMIT ?", [amount])
88 elif mode == "obfuscator":
89 database.cursor.execute("SELECT software, COUNT(domain) AS cnt FROM instances WHERE has_obfuscation = 1 GROUP BY software ORDER BY cnt DESC LIMIT ?", [amount])
90 elif mode == "obfuscation":
91 database.cursor.execute("SELECT has_obfuscation, COUNT(domain) AS cnt FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica') GROUP BY has_obfuscation ORDER BY cnt DESC LIMIT ?", [amount])
92 elif mode == "block_level":
93 database.cursor.execute("SELECT block_level, COUNT(rowid) AS cnt FROM blocks GROUP BY block_level ORDER BY cnt DESC LIMIT ?", [amount])
95 raise HTTPException(status_code=400, detail="No filter specified")
99 for domain, score in database.cursor.fetchall():
102 "score" : round(score)
107 @router.get(config.get("base_url") + "/api/list.json", response_class=JSONResponse)
108 def api_index(request: Request, mode: str, value: str, amount: int):
109 if mode is None or value is None or amount is None:
110 raise HTTPException(status_code=500, detail="No filter specified")
111 elif amount > config.get("api_limit"):
112 raise HTTPException(status_code=500, detail=f"amount={amount} is to big")
114 domain = wildchar = punycode = reason = None
116 if mode == "detection_mode":
117 database.cursor.execute(
118 f"SELECT domain, origin, software, command, total_peers, total_blocks, first_seen, last_updated FROM instances WHERE {mode} = ? LIMIT ?", [value, amount]
121 domainlist = database.cursor.fetchall()
125 @router.get(config.get("base_url") + "/api/top.json", response_class=JSONResponse)
126 def api_index(request: Request, mode: str, value: str, amount: int):
127 if mode is None or value is None or amount is None:
128 raise HTTPException(status_code=500, detail="No filter specified")
129 elif amount > config.get("api_limit"):
130 raise HTTPException(status_code=500, detail=f"amount={amount} is to big")
132 domain = wildchar = punycode = reason = None
134 if mode == "block_level":
135 database.cursor.execute(
136 "SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks WHERE block_level = ? LIMIT ?", [value, amount]
138 elif mode in ["domain", "reverse"]:
139 domain = tidyup.domain(value)
140 if not utils.is_domain_wanted(domain):
141 raise HTTPException(status_code=500, detail=f"domain='{domain}' is not wanted")
143 wildchar = "*." + ".".join(domain.split(".")[-domain.count("."):])
144 punycode = domain.encode("idna").decode("utf-8")
145 elif mode == "reason":
146 reason = re.sub("(%|_)", "", tidyup.reason(value))
148 raise HTTPException(status_code=400, detail="Keyword is shorter than three characters")
151 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
153 WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? \
154 ORDER BY block_level ASC, first_seen ASC \
160 utils.get_hash(domain),
166 elif mode == "reverse":
167 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
169 WHERE blocker = ? OR blocker = ? OR blocker = ? OR blocker = ? OR blocker = ? OR blocker = ? \
170 ORDER BY first_seen ASC \
175 utils.get_hash(domain),
180 elif mode == "reason":
181 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
183 WHERE reason LIKE ? AND reason != '' \
184 ORDER BY first_seen ASC \
190 blocklist = database.cursor.fetchall()
193 for blocker, blocked, block_level, reason, first_seen, last_seen in blocklist:
194 if reason is not None and reason != "":
195 reason = reason.replace(",", " ").replace(" ", " ")
201 "first_seen": first_seen,
202 "last_seen" : last_seen
205 if block_level in result:
206 result[block_level].append(entry)
208 result[block_level] = [entry]
212 @router.get(config.get("base_url") + "/api/domain.json", response_class=JSONResponse)
213 def api_domain(domain: str):
214 # Tidy up domain name
215 domain = tidyup.domain(domain)
217 if not utils.is_domain_wanted(domain):
218 raise HTTPException(status_code=500, detail=f"domain='{domain}' is not wanted")
221 database.cursor.execute("SELECT * FROM instances WHERE domain = ? LIMIT 1", [domain])
222 domain_data = database.cursor.fetchone()
224 if domain_data is None:
225 raise HTTPException(status_code=404, detail=f"domain='{domain}' not found")
229 @router.get(config.get("base_url") + "/api/mutual.json", response_class=JSONResponse)
230 def api_mutual(domains: list[str] = Query()):
231 """Return 200 if federation is open between the two, 4xx otherwise"""
232 database.cursor.execute(
233 "SELECT block_level FROM blocks " \
234 "WHERE ((blocker = :a OR blocker = :b) AND (blocked = :b OR blocked = :a OR blocked = :aw OR blocked = :bw)) " \
235 "AND block_level = 'reject' " \
240 "aw": "*." + domains[0],
241 "bw": "*." + domains[1],
245 if database.cursor.fetchone() is not None:
247 return JSONResponse(status_code=418, content={})
250 return JSONResponse(status_code=200, content={})
252 @router.get(config.get("base_url") + "/scoreboard")
253 def scoreboard(request: Request, mode: str, amount: int):
256 if mode == "blocker" and amount > 0:
257 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=blocker&amount={amount}")
258 elif mode == "blocked" and amount > 0:
259 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=blocked&amount={amount}")
260 elif mode == "reference" and amount > 0:
261 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=reference&amount={amount}")
262 elif mode == "software" and amount > 0:
263 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=software&amount={amount}")
264 elif mode == "command" and amount > 0:
265 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=command&amount={amount}")
266 elif mode == "error_code" and amount > 0:
267 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=error_code&amount={amount}")
268 elif mode == "detection_mode" and amount > 0:
269 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=detection_mode&amount={amount}")
270 elif mode == "avg_peers" and amount > 0:
271 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=avg_peers&amount={amount}")
272 elif mode == "obfuscator" and amount > 0:
273 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=obfuscator&amount={amount}")
274 elif mode == "obfuscation" and amount > 0:
275 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=obfuscation&amount={amount}")
276 elif mode == "block_level" and amount > 0:
277 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=block_level&amount={amount}")
279 raise HTTPException(status_code=400, detail="No filter specified")
282 raise HTTPException(status_code=500, detail="Could not determine scores")
283 elif not response.ok:
284 raise HTTPException(status_code=response.status_code, detail=response.text)
286 return templates.TemplateResponse("views/scoreboard.html", {
287 "base_url" : config.get("base_url"),
288 "slogan" : config.get("slogan"),
289 "theme" : config.get("theme"),
294 "scores" : json_helper.from_response(response)
297 @router.get(config.get("base_url") + "/")
298 def index(request: Request):
300 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/info.json")
303 raise HTTPException(status_code=response.status_code, detail=response.text)
305 return templates.TemplateResponse("views/index.html", {
307 "theme" : config.get("theme"),
308 "info" : response.json(),
309 "slogan" : config.get("slogan"),
312 @router.get(config.get("base_url") + "/list")
313 def list_domains(request: Request, mode: str, value: str, amount: int = config.get("api_limit")):
314 if mode == "detection_mode" and not instances.valid(value, "detection_mode"):
315 raise HTTPException(status_code=500, detail="Invalid detection mode provided")
317 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/list.json?mode={mode}&value={value}&amount={amount}")
320 if response is not None and response.ok:
321 domainlist = response.json()
322 format = config.get("timestamp_format")
323 for row in domainlist:
324 row["first_seen"] = datetime.utcfromtimestamp(row["first_seen"]).strftime(format)
325 row["last_updated"] = datetime.utcfromtimestamp(row["last_updated"]).strftime(format)
327 return templates.TemplateResponse("views/list.html", {
329 "mode" : mode if response is not None else None,
330 "value" : value if response is not None else None,
331 "amount" : amount if response is not None else None,
332 "found" : len(domainlist),
333 "domainlist": domainlist,
334 "slogan" : config.get("slogan"),
335 "theme" : config.get("theme"),
338 @router.get(config.get("base_url") + "/top")
339 def top(request: Request, mode: str, value: str, amount: int = config.get("api_limit")):
340 if mode == "block_level" and not blocks.valid(value, "block_level"):
341 raise HTTPException(status_code=500, detail="Invalid block level provided")
342 elif mode in ["domain", "reverse"] and not utils.is_domain_wanted(value):
343 raise HTTPException(status_code=500, detail="Invalid or blocked domain specified")
345 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/top.json?mode={mode}&value={value}&amount={amount}")
349 if response is not None and response.ok:
350 blocklist = response.json()
352 format = config.get("timestamp_format")
353 for block_level in blocklist:
354 for block in blocklist[block_level]:
355 block["first_seen"] = datetime.utcfromtimestamp(block["first_seen"]).strftime(format)
356 block["last_seen"] = datetime.utcfromtimestamp(block["last_seen"]).strftime(format)
359 return templates.TemplateResponse("views/top.html", {
361 "mode" : mode if response is not None else None,
362 "value" : value if response is not None else None,
363 "amount" : amount if response is not None else None,
365 "blocklist": blocklist,
366 "slogan" : config.get("slogan"),
367 "theme" : config.get("theme"),
370 @router.get(config.get("base_url") + "/infos")
371 def rss(request: Request, domain: str):
372 # Tidy up domain name
373 domain = tidyup.domain(domain)
375 if not utils.is_domain_wanted(domain):
376 raise HTTPException(status_code=500, detail=f"domain='{domain}' is not wanted")
378 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/domain.json?domain={domain}")
380 if not response.ok or response.status_code > 200 or response.text.strip() == "":
381 raise HTTPException(status_code=response.status_code, detail=response.reason)
383 domain_data = response.json()
386 format = config.get("timestamp_format")
388 for key in domain_data.keys():
389 if key in ["last_nodeinfo", "last_blocked", "first_seen", "last_updated", "last_instance_fetch"]:
391 instance[key] = datetime.utcfromtimestamp(domain_data[key]).strftime(format) if isinstance(domain_data[key], float) else "-"
394 instance[key] = domain_data[key]
396 return templates.TemplateResponse("views/infos.html", {
399 "instance": instance,
400 "theme" : config.get("theme"),
401 "slogan" : config.get("slogan"),
404 @router.get(config.get("base_url") + "/rss")
405 def rss(request: Request, domain: str = None):
406 if domain is not None:
407 domain = tidyup.domain(domain)
409 wildchar = "*." + ".".join(domain.split(".")[-domain.count("."):])
410 punycode = domain.encode("idna").decode("utf-8")
412 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? ORDER BY first_seen DESC LIMIT ?", [
414 "*." + domain, wildchar,
415 utils.get_hash(domain),
418 config.get("rss_limit")
421 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks ORDER BY first_seen DESC LIMIT ?", [config.get("rss_limit")])
423 result = database.cursor.fetchall()
430 "block_level": row[2],
431 "reason" : "Provided reason: '" + row[3] + "'" if row[3] is not None and row[3] != "" else "No reason provided.",
432 "first_seen" : format_datetime(datetime.fromtimestamp(row[4])),
433 "last_seen" : format_datetime(datetime.fromtimestamp(row[5])),
436 return templates.TemplateResponse("views/rss.xml", {
438 "timestamp": format_datetime(datetime.now()),
440 "hostname" : config.get("hostname"),
443 "Content-Type": "routerlication/rss+xml"
446 @router.get(config.get("base_url") + "/robots.txt", response_class=PlainTextResponse)
447 def robots(request: Request):
448 return templates.TemplateResponse("views/robots.txt", {
450 "base_url": config.get("base_url")
453 if __name__ == "__main__":
454 uvicorn.run("daemon:router", host=config.get("host"), port=config.get("port"), log_level=config.get("log_level"))