1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
2 # Copyright (C) 2023 Free Software Foundation
4 # This program is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU Affero General Public License as published
6 # by the Free Software Foundation, either version 3 of the License, or
7 # (at your option) any later version.
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU Affero General Public License for more details.
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <https://www.gnu.org/licenses/>.
19 from datetime import datetime
20 from email.utils import format_datetime
21 from fastapi import Request, HTTPException, Query
22 from fastapi.responses import JSONResponse
23 from fastapi.responses import PlainTextResponse
24 from fastapi.templating import Jinja2Templates
31 from fba import database
34 from fba.helpers import config
35 from fba.helpers import tidyup
37 from fba.http import network
39 from fba.models import blocks
41 router = fastapi.FastAPI(docs_url=config.get("base_url") + "/docs", redoc_url=config.get("base_url") + "/redoc")
42 templates = Jinja2Templates(directory="templates")
44 @router.get(config.get("base_url") + "/api/info.json", response_class=JSONResponse)
46 database.cursor.execute("SELECT (SELECT COUNT(domain) FROM instances), (SELECT COUNT(domain) FROM instances WHERE software IN ('pleroma', 'mastodon', 'lemmy', 'friendica', 'misskey', 'peertube')), (SELECT COUNT(blocker) FROM blocks), (SELECT COUNT(domain) FROM instances WHERE last_error_details IS NOT NULL)")
47 row = database.cursor.fetchone()
50 "known_instances" : row[0],
51 "supported_instances": row[1],
52 "blocks_recorded" : row[2],
53 "erroneous_instances": row[3],
54 "slogan" : config.get("slogan"),
57 @router.get(config.get("base_url") + "/api/scoreboard.json", response_class=JSONResponse)
58 def api_scoreboard(mode: str, amount: int):
59 if amount > config.get("api_limit"):
60 raise HTTPException(status_code=400, detail="Too many results")
63 database.cursor.execute("SELECT blocked, COUNT(blocked) AS score FROM blocks GROUP BY blocked ORDER BY score DESC LIMIT ?", [amount])
64 elif mode == "blocker":
65 database.cursor.execute("SELECT blocker, COUNT(blocker) AS score FROM blocks GROUP BY blocker ORDER BY score DESC LIMIT ?", [amount])
66 elif mode == "reference":
67 database.cursor.execute("SELECT origin, COUNT(domain) AS score FROM instances WHERE software IS NOT NULL GROUP BY origin ORDER BY score DESC LIMIT ?", [amount])
68 elif mode == "software":
69 database.cursor.execute("SELECT software, COUNT(domain) AS score FROM instances WHERE software IS NOT NULL GROUP BY software ORDER BY score DESC, software ASC LIMIT ?", [amount])
70 elif mode == "command":
71 database.cursor.execute("SELECT command, COUNT(domain) AS score FROM instances WHERE command IS NOT NULL GROUP BY command ORDER BY score DESC, command ASC LIMIT ?", [amount])
72 elif mode == "error_code":
73 database.cursor.execute("SELECT last_status_code, COUNT(domain) AS score FROM instances WHERE last_status_code IS NOT NULL AND last_status_code != '200' GROUP BY last_status_code ORDER BY score DESC LIMIT ?", [amount])
74 elif mode == "detection_mode":
75 database.cursor.execute("SELECT detection_mode, COUNT(domain) AS cnt FROM instances GROUP BY detection_mode ORDER BY cnt DESC LIMIT ?", [amount])
76 elif mode == "avg_peers":
77 database.cursor.execute("SELECT software, AVG(total_peers) AS average FROM instances WHERE software IS NOT NULL GROUP BY software HAVING average>0 ORDER BY average DESC LIMIT ?", [amount])
78 elif mode == "obfuscator":
79 database.cursor.execute("SELECT software, COUNT(domain) AS cnt FROM instances WHERE has_obfuscation = 1 GROUP BY software ORDER BY cnt DESC LIMIT ?", [amount])
80 elif mode == "obfuscation":
81 database.cursor.execute("SELECT has_obfuscation, COUNT(domain) AS cnt FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica') GROUP BY has_obfuscation ORDER BY cnt DESC LIMIT ?", [amount])
82 elif mode == "block_level":
83 database.cursor.execute("SELECT block_level, COUNT(rowid) AS cnt FROM blocks GROUP BY block_level ORDER BY cnt DESC LIMIT ?", [amount])
85 raise HTTPException(status_code=400, detail="No filter specified")
89 for domain, score in database.cursor.fetchall():
92 "score" : round(score)
97 @router.get(config.get("base_url") + "/api/index.json", response_class=JSONResponse)
98 def api_index(request: Request, mode: str, value: str, amount: int):
99 if mode is None or value is None or amount is None:
100 raise HTTPException(status_code=500, detail="No filter specified")
101 elif amount > config.get("api_limit"):
102 raise HTTPException(status_code=500, detail=f"amount={amount} is to big")
104 domain = whildchar = punycode = reason = None
106 if mode == "block_level":
107 database.cursor.execute(
108 "SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks WHERE block_level = ? LIMIT ?", [value, amount]
110 elif mode in ["domain", "reverse"]:
111 domain = tidyup.domain(value)
112 if not utils.is_domain_wanted(domain):
113 raise HTTPException(status_code=500, detail=f"domain='{domain}' is not wanted")
115 wildchar = "*." + ".".join(domain.split(".")[-domain.count("."):])
116 punycode = domain.encode('idna').decode('utf-8')
117 elif mode == "reason":
118 reason = re.sub("(%|_)", "", tidyup.reason(value))
120 raise HTTPException(status_code=400, detail="Keyword is shorter than three characters")
123 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
125 WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? ORDER BY first_seen ASC LIMIT ?",
130 utils.get_hash(domain),
136 elif mode == "reverse":
137 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
139 WHERE blocker = ? OR blocker = ? OR blocker = ? OR blocker = ? OR blocker = ? OR blocker = ? \
140 ORDER BY first_seen ASC \
145 utils.get_hash(domain),
150 elif mode == "reason":
151 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen \
153 WHERE reason LIKE ? AND reason != '' \
154 ORDER BY first_seen ASC \
160 blocklist = database.cursor.fetchall()
163 for blocker, blocked, block_level, reason, first_seen, last_seen in blocklist:
164 if reason is not None and reason != "":
165 reason = reason.replace(",", " ").replace(" ", " ")
171 "first_seen": first_seen,
172 "last_seen" : last_seen
175 if block_level in result:
176 result[block_level].append(entry)
178 result[block_level] = [entry]
182 @router.get(config.get("base_url") + "/api/mutual.json", response_class=JSONResponse)
183 def api_mutual(domains: list[str] = Query()):
184 """Return 200 if federation is open between the two, 4xx otherwise"""
185 database.cursor.execute(
186 "SELECT block_level FROM blocks " \
187 "WHERE ((blocker = :a OR blocker = :b) AND (blocked = :b OR blocked = :a OR blocked = :aw OR blocked = :bw)) " \
188 "AND block_level = 'reject' " \
193 "aw": "*." + domains[0],
194 "bw": "*." + domains[1],
197 response = database.cursor.fetchone()
199 if response is not None:
201 return JSONResponse(status_code=418, content={})
204 return JSONResponse(status_code=200, content={})
206 @router.get(config.get("base_url") + "/scoreboard")
207 def scoreboard(request: Request, mode: str, amount: int):
210 if mode == "blocker" and amount > 0:
211 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=blocker&amount={amount}")
212 elif mode == "blocked" and amount > 0:
213 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=blocked&amount={amount}")
214 elif mode == "reference" and amount > 0:
215 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=reference&amount={amount}")
216 elif mode == "software" and amount > 0:
217 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=software&amount={amount}")
218 elif mode == "command" and amount > 0:
219 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=command&amount={amount}")
220 elif mode == "error_code" and amount > 0:
221 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=error_code&amount={amount}")
222 elif mode == "detection_mode" and amount > 0:
223 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=detection_mode&amount={amount}")
224 elif mode == "avg_peers" and amount > 0:
225 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=avg_peers&amount={amount}")
226 elif mode == "obfuscator" and amount > 0:
227 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=obfuscator&amount={amount}")
228 elif mode == "obfuscation" and amount > 0:
229 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=obfuscation&amount={amount}")
230 elif mode == "block_level" and amount > 0:
231 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/scoreboard.json?mode=block_level&amount={amount}")
233 raise HTTPException(status_code=400, detail="No filter specified")
236 raise HTTPException(status_code=500, detail="Could not determine scores")
237 elif not response.ok:
238 raise HTTPException(status_code=response.status_code, detail=response.text)
240 return templates.TemplateResponse("views/scoreboard.html", {
241 "base_url" : config.get("base_url"),
242 "slogan" : config.get("slogan"),
247 "scores" : network.json_from_response(response)
250 @router.get(config.get("base_url") + "/")
251 def index(request: Request):
253 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/info.json")
256 raise HTTPException(status_code=response.status_code, detail=response.text)
258 return templates.TemplateResponse("views/index.html", {
260 "info" : response.json()
263 @router.get(config.get("base_url") + "/top")
264 def top(request: Request, mode: str, value: str, amount: int = config.get("api_limit")):
265 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/info.json")
268 raise HTTPException(status_code=response.status_code, detail=response.text)
269 elif mode == "" or value == "" or amount == 0:
270 raise HTTPException(status_code=500, detail="Parameter mode, value and amount must always be set")
271 elif amount > config.get("api_limit"):
272 raise HTTPException(status_code=500, detail=f"amount='{amount}' is to big")
274 info = response.json()
278 if mode == "block_level" and not blocks.is_valid_level(value):
279 raise HTTPException(status_code=500, detail="Invalid block level provided")
280 elif mode in ["domain", "reverse"] and not utils.is_domain_wanted(value):
281 raise HTTPException(status_code=500, detail="Invalid or blocked domain specified")
283 response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/index.json?mode={mode}&value={value}&amount={amount}")
285 if response is not None:
286 blocklist = response.json()
289 for block_level in blocklist:
290 for block in blocklist[block_level]:
291 block["first_seen"] = datetime.utcfromtimestamp(block["first_seen"]).strftime(config.get("timestamp_format"))
292 block["last_seen"] = datetime.utcfromtimestamp(block["last_seen"]).strftime(config.get("timestamp_format"))
295 return templates.TemplateResponse("views/top.html", {
297 "mode" : mode if response is not None else None,
298 "value" : value if response is not None else None,
299 "amount" : amount if response is not None else None,
301 "blocklist": blocklist,
305 @router.get(config.get("base_url") + "/rss")
306 def rss(request: Request, domain: str = None):
307 if domain is not None:
308 domain = tidyup.domain(domain)
310 wildchar = "*." + ".".join(domain.split(".")[-domain.count("."):])
311 punycode = domain.encode("idna").decode("utf-8")
313 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? ORDER BY first_seen DESC LIMIT ?", [
315 "*." + domain, wildchar,
316 utils.get_hash(domain),
319 config.get("rss_limit")
322 database.cursor.execute("SELECT blocker, blocked, block_level, reason, first_seen, last_seen FROM blocks ORDER BY first_seen DESC LIMIT ?", [config.get("rss_limit")])
324 result = database.cursor.fetchall()
331 "block_level": row[2],
332 "reason" : "Provided reason: '" + row[3] + "'" if row[3] is not None and row[3] != "" else "No reason provided.",
333 "first_seen" : format_datetime(datetime.fromtimestamp(row[4])),
334 "last_seen" : format_datetime(datetime.fromtimestamp(row[5])),
337 return templates.TemplateResponse("views/rss.xml", {
339 "timestamp": format_datetime(datetime.now()),
341 "hostname" : config.get("hostname"),
344 "Content-Type": "routerlication/rss+xml"
347 @router.get(config.get("base_url") + "/robots.txt", response_class=PlainTextResponse)
348 def robots(request: Request):
349 return templates.TemplateResponse("views/robots.txt", {
351 "base_url": config.get("base_url")
354 if __name__ == "__main__":
355 uvicorn.run("api:router", host=config.get("host"), port=config.get("port"), log_level=config.get("log_level"))