2 # -*- coding: utf-8 -*-
4 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
5 # Copyright (C) 2023 Free Software Foundation
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published
9 # by the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <https://www.gnu.org/licenses/>.
22 from datetime import datetime
23 from email.utils import format_datetime
24 from pathlib import Path
27 from fastapi import Request, HTTPException, Query
28 from fastapi.responses import JSONResponse
29 from fastapi.responses import PlainTextResponse
30 from fastapi.staticfiles import StaticFiles
31 from fastapi.templating import Jinja2Templates
36 from fba import database
39 from fba.helpers import config
40 from fba.helpers import tidyup
42 from fba.http import network
44 from fba.models import blocks
46 router = fastapi.FastAPI(docs_url=config.get("base_url") + "/docs", redoc_url=config.get("base_url") + "/redoc")
49 StaticFiles(directory=Path(__file__).parent.absolute() / "static"),
53 templates = Jinja2Templates(directory="templates")
55 @router.get(config.get("base_url") + "/api/info.json", response_class=JSONResponse)
57 database.cursor.execute("SELECT (SELECT COUNT(domain) FROM instances), (SELECT COUNT(domain) FROM instances WHERE software IN ('pleroma', 'mastodon', 'lemmy', 'friendica', 'misskey', 'peertube')), (SELECT COUNT(blocker) FROM blocks), (SELECT COUNT(domain) FROM instances WHERE last_error_details IS NOT NULL)")
58 row = database.cursor.fetchone()
61 "known_instances" : row[0],
62 "supported_instances": row[1],
63 "blocks_recorded" : row[2],
64 "erroneous_instances": row[3],
67 @router.get(config.get("base_url") + "/api/scoreboard.json", response_class=JSONResponse)
68 def api_scoreboard(mode: str, amount: int):
69 if amount > config.get("api_limit"):
70 raise HTTPException(status_code=400, detail="Too many results")
73 database.cursor.execute("SELECT blocked, COUNT(blocked) AS score FROM blocks GROUP BY blocked ORDER BY score DESC LIMIT ?", [amount])
74 elif mode == "blocker":
75 database.cursor.execute("SELECT blocker, COUNT(blocker) AS score FROM blocks GROUP BY blocker ORDER BY score DESC LIMIT ?", [amount])
76 elif mode == "reference":
77 database.cursor.execute("SELECT origin, COUNT(domain) AS score FROM instances WHERE origin IS NOT NULL GROUP BY origin ORDER BY score DESC LIMIT ?", [amount])
78 elif mode == "software":
79 database.cursor.execute("SELECT software, COUNT(domain) AS score FROM instances WHERE software IS NOT NULL GROUP BY software ORDER BY score DESC, software ASC LIMIT ?", [amount])
80 elif mode == "command":
81 database.cursor.execute("SELECT command, COUNT(domain) AS score FROM instances WHERE command IS NOT NULL GROUP BY command ORDER BY score DESC, command ASC LIMIT ?", [amount])
82 elif mode == "error_code":
83 database.cursor.execute("SELECT last_status_code, COUNT(domain) AS score FROM instances WHERE last_status_code IS NOT NULL AND last_status_code != '200' GROUP BY last_status_code ORDER BY score DESC LIMIT ?", [amount])
84 elif mode == "detection_mode":
85 database.cursor.execute("SELECT detection_mode, COUNT(domain) AS cnt FROM instances GROUP BY detection_mode ORDER BY cnt DESC LIMIT ?", [amount])
86 elif mode == "avg_peers":
87 database.cursor.execute("SELECT software, AVG(total_peers) AS average FROM instances WHERE software IS NOT NULL GROUP BY software HAVING average>0 ORDER BY average DESC LIMIT ?", [amount])
88 elif mode == "obfuscator":
89 database.cursor.execute("SELECT software, COUNT(domain) AS cnt FROM instances WHERE has_obfuscation = 1 GROUP BY software ORDER BY cnt DESC LIMIT ?", [amount])
90 elif mode == "obfuscation":
91 database.cursor.execute("SELECT has_obfuscation, COUNT(domain) AS cnt FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica') GROUP BY has_obfuscation ORDER BY cnt DESC LIMIT ?", [amount])
92 elif mode == "block_level":
93 database.cursor.execute("SELECT block_level, COUNT(rowid) AS cnt FROM blocks GROUP BY block_level ORDER BY cnt DESC LIMIT ?", [amount])
95 raise HTTPException(status_code=400, detail="No filter specified")
99 for domain, score in database.cursor.fetchall():
102 "score" : round(score)
107 @router.get(config.get("base_url") + "/api/index.json", response_class=JSONResponse)
108 def api_index(request: Request, mode: str, value: str, amount: int):
109 if mode is None or value is None or amount is None:
110 raise HTTPException(status_code=500, detail="No filter specified")
111 elif amount > config.get("api_limit"):
112 raise HTTPException(status_code=500, detail=f"amount={amount} is to big")
114 domain = wildchar = punycode = reason = None
116 if mode == "block_level":
117 database.cursor.execute(
118 "SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks WHERE block_level = ? LIMIT ?", [value, amount]
120 elif mode in ["domain", "reverse"]:
121 domain = tidyup.domain(value)
122 if not utils.is_domain_wanted(domain):
123 raise HTTPException(status_code=500, detail=f"domain='{domain}' is not wanted")
125 wildchar = "*." + ".".join(domain.split(".")[-domain.count("."):])
126 punycode = domain.encode('idna').decode('utf-8')
127 elif mode == "reason":
128 reason = re.sub("(%|_)", "", tidyup.reason(value))
130 raise HTTPException(status_code=400, detail="Keyword is shorter than three characters")
133 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
135 WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? ORDER BY first_seen ASC LIMIT ?",
140 utils.get_hash(domain),
146 elif mode == "reverse":
147 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
149 WHERE blocker = ? OR blocker = ? OR blocker = ? OR blocker = ? OR blocker = ? OR blocker = ? \
150 ORDER BY first_seen ASC \
155 utils.get_hash(domain),
160 elif mode == "reason":
161 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
163 WHERE reason LIKE ? AND reason != '' \
164 ORDER BY first_seen ASC \
170 blocklist = database.cursor.fetchall()
173 for blocker, blocked, block_level, reason, first_seen, last_seen in blocklist:
174 if reason is not None and reason != "":
175 reason = reason.replace(",", " ").replace(" ", " ")
181 "first_seen": first_seen,
182 "last_seen" : last_seen
185 if block_level in result:
186 result[block_level].append(entry)
188 result[block_level] = [entry]
192 @router.get(config.get("base_url") + "/api/mutual.json", response_class=JSONResponse)
193 def api_mutual(domains: list[str] = Query()):
194 """Return 200 if federation is open between the two, 4xx otherwise"""
195 database.cursor.execute(
196 "SELECT block_level FROM blocks " \
197 "WHERE ((blocker = :a OR blocker = :b) AND (blocked = :b OR blocked = :a OR blocked = :aw OR blocked = :bw)) " \
198 "AND block_level = 'reject' " \
203 "aw": "*." + domains[0],
204 "bw": "*." + domains[1],
207 response = database.cursor.fetchone()
209 if response is not None:
211 return JSONResponse(status_code=418, content={})
214 return JSONResponse(status_code=200, content={})
216 @router.get(config.get("base_url") + "/scoreboard")
217 def scoreboard(request: Request, mode: str, amount: int):
220 if mode == "blocker" and amount > 0:
221 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=blocker&amount={amount}")
222 elif mode == "blocked" and amount > 0:
223 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=blocked&amount={amount}")
224 elif mode == "reference" and amount > 0:
225 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=reference&amount={amount}")
226 elif mode == "software" and amount > 0:
227 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=software&amount={amount}")
228 elif mode == "command" and amount > 0:
229 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=command&amount={amount}")
230 elif mode == "error_code" and amount > 0:
231 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=error_code&amount={amount}")
232 elif mode == "detection_mode" and amount > 0:
233 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=detection_mode&amount={amount}")
234 elif mode == "avg_peers" and amount > 0:
235 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=avg_peers&amount={amount}")
236 elif mode == "obfuscator" and amount > 0:
237 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=obfuscator&amount={amount}")
238 elif mode == "obfuscation" and amount > 0:
239 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=obfuscation&amount={amount}")
240 elif mode == "block_level" and amount > 0:
241 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=block_level&amount={amount}")
243 raise HTTPException(status_code=400, detail="No filter specified")
246 raise HTTPException(status_code=500, detail="Could not determine scores")
247 elif not response.ok:
248 raise HTTPException(status_code=response.status_code, detail=response.text)
250 return templates.TemplateResponse("views/scoreboard.html", {
251 "base_url" : config.get("base_url"),
252 "slogan" : config.get("slogan"),
253 "theme" : config.get("theme"),
258 "scores" : network.json_from_response(response)
261 @router.get(config.get("base_url") + "/")
262 def index(request: Request):
264 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/info.json")
267 raise HTTPException(status_code=response.status_code, detail=response.text)
269 return templates.TemplateResponse("views/index.html", {
271 "theme" : config.get("theme"),
272 "info" : response.json(),
273 "slogan" : config.get("slogan"),
276 @router.get(config.get("base_url") + "/top")
277 def top(request: Request, mode: str, value: str, amount: int = config.get("api_limit")):
278 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/info.json")
281 raise HTTPException(status_code=response.status_code, detail=response.text)
282 elif mode == "" or value == "" or amount == 0:
283 raise HTTPException(status_code=500, detail="Parameter mode, value and amount must always be set")
284 elif amount > config.get("api_limit"):
285 raise HTTPException(status_code=500, detail=f"amount='{amount}' is to big")
287 info = response.json()
291 if mode == "block_level" and not blocks.is_valid_level(value):
292 raise HTTPException(status_code=500, detail="Invalid block level provided")
293 elif mode in ["domain", "reverse"] and not utils.is_domain_wanted(value):
294 raise HTTPException(status_code=500, detail="Invalid or blocked domain specified")
296 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/index.json?mode={mode}&value={value}&amount={amount}")
298 if response is not None:
299 blocklist = response.json()
302 for block_level in blocklist:
303 for block in blocklist[block_level]:
304 block["first_seen"] = datetime.utcfromtimestamp(block["first_seen"]).strftime(config.get("timestamp_format"))
305 block["last_seen"] = datetime.utcfromtimestamp(block["last_seen"]).strftime(config.get("timestamp_format"))
308 return templates.TemplateResponse("views/top.html", {
310 "mode" : mode if response is not None else None,
311 "value" : value if response is not None else None,
312 "amount" : amount if response is not None else None,
314 "blocklist": blocklist,
316 "theme" : config.get("theme"),
319 @router.get(config.get("base_url") + "/infos")
320 def rss(request: Request, domain: str):
321 # Tidy up domain name
322 domain = tidyup.domain(domain)
324 if not utils.is_domain_wanted(domain):
325 raise HTTPException(status_code=500, detail=f"domain='{domain}' is not wanted")
328 database.cursor.execute("SELECT * FROM instances WHERE domain = ? LIMIT 1", [domain])
329 domain_data = database.cursor.fetchone()
332 format = config.get("timestamp_format")
334 for key in domain_data.keys():
335 if key in ["last_nodeinfo", "last_blocked", "first_seen", "last_updated", "last_instance_fetch"]:
337 instance[key] = datetime.utcfromtimestamp(domain_data[key]).strftime(format) if isinstance(domain_data[key], float) else "-"
340 instance[key] = domain_data[key]
342 print(domain_data.keys())
343 return templates.TemplateResponse("views/infos.html", {
346 "instance": instance,
347 "theme" : config.get("theme"),
348 "slogan" : config.get("slogan"),
351 @router.get(config.get("base_url") + "/rss")
352 def rss(request: Request, domain: str = None):
353 if domain is not None:
354 domain = tidyup.domain(domain)
356 wildchar = "*." + ".".join(domain.split(".")[-domain.count("."):])
357 punycode = domain.encode("idna").decode("utf-8")
359 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? ORDER BY first_seen DESC LIMIT ?", [
361 "*." + domain, wildchar,
362 utils.get_hash(domain),
365 config.get("rss_limit")
368 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks ORDER BY first_seen DESC LIMIT ?", [config.get("rss_limit")])
370 result = database.cursor.fetchall()
377 "block_level": row[2],
378 "reason" : "Provided reason: '" + row[3] + "'" if row[3] is not None and row[3] != "" else "No reason provided.",
379 "first_seen" : format_datetime(datetime.fromtimestamp(row[4])),
380 "last_seen" : format_datetime(datetime.fromtimestamp(row[5])),
383 return templates.TemplateResponse("views/rss.xml", {
385 "timestamp": format_datetime(datetime.now()),
387 "hostname" : config.get("hostname"),
390 "Content-Type": "routerlication/rss+xml"
393 @router.get(config.get("base_url") + "/robots.txt", response_class=PlainTextResponse)
394 def robots(request: Request):
395 return templates.TemplateResponse("views/robots.txt", {
397 "base_url": config.get("base_url")
400 if __name__ == "__main__":
401 uvicorn.run("daemon:router", host=config.get("host"), port=config.get("port"), log_level=config.get("log_level"))