1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
2 # Copyright (C) 2023 Free Software Foundation
4 # This program is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU Affero General Public License as published
6 # by the Free Software Foundation, either version 3 of the License, or
7 # (at your option) any later version.
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU Affero General Public License for more details.
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <https://www.gnu.org/licenses/>.
19 from fba import blacklist
20 from fba import config
22 from fba import network
24 from fba.helpers import dicts
25 from fba.helpers import tidyup
27 from fba.models import instances
29 def fetch_peers(domain: str) -> list:
30 # DEBUG: print(f"DEBUG: domain({len(domain)})={domain} - CALLED!")
31 if not isinstance(domain, str):
32 raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
34 raise ValueError("Parameter 'domain' is empty")
36 # DEBUG: print(f"DEBUG: domain='{domain}' is misskey, sending API POST request ...")
39 step = config.get("misskey_limit")
41 # No CSRF by default, you don't have to add network.api_headers by yourself here
45 # DEBUG: print(f"DEBUG: Checking CSRF for domain='{domain}'")
46 headers = csrf.determine(domain, dict())
47 except network.exceptions as exception:
48 print(f"WARNING: Exception '{type(exception)}' during checking CSRF (fetch_peers,{__name__}) - EXIT!")
51 # iterating through all "suspended" (follow-only in its terminology)
52 # instances page-by-page, since that troonware doesn't support
53 # sending them all at once
55 # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...")
57 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
63 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
71 # DEBUG: print(f"DEBUG: fetched[]='{type(fetched)}'")
72 if "error_message" in fetched:
73 print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}")
74 instances.update_last_error(domain, fetched)
76 elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
77 print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}")
78 instances.update_last_error(domain, fetched["json"]["error"]["message"])
81 rows = fetched["json"]
83 # DEBUG: print(f"DEBUG: rows()={len(rows)}")
85 # DEBUG: print(f"DEBUG: Returned zero bytes, exiting loop, domain='{domain}'")
87 elif len(rows) != config.get("misskey_limit"):
88 # DEBUG: print(f"DEBUG: Fetched '{len(rows)}' row(s) but expected: '{config.get('misskey_limit')}'")
89 offset = offset + (config.get("misskey_limit") - len(rows))
91 # DEBUG: print(f"DEBUG: Raising offset by step={step}")
92 offset = offset + step
95 # DEBUG: print(f"DEBUG: rows({len(rows)})[]='{type(rows)}'")
97 # DEBUG: print(f"DEBUG: row()={len(row)}")
99 print(f"WARNING: row()={len(row)} does not contain key 'host': {row},domain='{domain}'")
101 elif not isinstance(row["host"], str):
102 print(f"WARNING: row[host][]='{type(row['host'])}' is not 'str'")
104 elif blacklist.is_blacklisted(row["host"]):
105 # DEBUG: print(f"DEBUG: row[host]='{row['host']}' is blacklisted. domain='{domain}'")
107 elif row["host"] in peers:
108 # DEBUG: print(f"DEBUG: Not adding row[host]='{row['host']}', already found.")
109 already = already + 1
112 # DEBUG: print(f"DEBUG: Adding peer: '{row['host']}'")
113 peers.append(row["host"])
115 if already == len(rows):
116 # DEBUG: print(f"DEBUG: Host returned same set of '{already}' instances, aborting loop!")
119 # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'")
120 instances.set_data("total_peers", domain, len(peers))
122 # DEBUG: print(f"DEBUG: Returning peers[]='{type(peers)}'")
125 def fetch_blocks(domain: str) -> dict:
126 # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
127 if not isinstance(domain, str):
128 raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
130 raise ValueError("Parameter 'domain' is empty")
132 # DEBUG: print(f"DEBUG: Fetching misskey blocks from domain={domain}")
139 step = config.get("misskey_limit")
141 # No CSRF by default, you don't have to add network.api_headers by yourself here
145 # DEBUG: print(f"DEBUG: Checking CSRF for domain='{domain}'")
146 headers = csrf.determine(domain, dict())
147 except network.exceptions as exception:
148 print(f"WARNING: Exception '{type(exception)}' during checking CSRF (fetch_blocks,{__name__}) - EXIT!")
151 # iterating through all "suspended" (follow-only in its terminology)
152 # instances page-by-page since it doesn't support sending them all at once
155 # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...")
157 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
158 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
165 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
166 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
171 "offset" : offset - 1
174 # DEBUG: print(f"DEBUG: fetched[]='{type(fetched)}'")
175 if "error_message" in fetched:
176 print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}")
177 instances.update_last_error(domain, fetched)
179 elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
180 print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}")
181 instances.update_last_error(domain, fetched["json"]["error"]["message"])
184 rows = fetched["json"]
186 # DEBUG: print(f"DEBUG: rows({len(rows)})={rows} - suspend")
188 # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
190 elif len(rows) != config.get("misskey_limit"):
191 # DEBUG: print(f"DEBUG: Fetched '{len(rows)}' row(s) but expected: '{config.get('misskey_limit')}'")
192 offset = offset + (config.get("misskey_limit") - len(rows))
194 # DEBUG: print("DEBUG: Raising offset by step:", step)
195 offset = offset + step
198 for instance in rows:
200 # DEBUG: print(f"DEBUG: instance[{type(instance)}]='{instance}' - suspend")
201 if "isSuspended" in instance and instance["isSuspended"] and not dicts.has_key(blocklist["suspended"], "domain", instance["host"]):
203 blocklist["suspended"].append({
204 "domain": tidyup.domain(instance["host"]),
205 # no reason field, nothing
209 # DEBUG: print(f"DEBUG: count={count}")
211 # DEBUG: print("DEBUG: API is no more returning new instances, aborting loop!")
214 except network.exceptions as exception:
215 print(f"WARNING: Caught error, exiting loop: domain='{domain}',exception[{type(exception)}]='{str(exception)}'")
216 instances.update_last_error(domain, exception)
221 # Fetch blocked (full suspended) instances
224 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
225 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
232 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
233 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
238 "offset" : offset - 1
241 # DEBUG: print(f"DEBUG: fetched[]='{type(fetched)}'")
242 if "error_message" in fetched:
243 print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}")
244 instances.update_last_error(domain, fetched)
246 elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
247 print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}")
248 instances.update_last_error(domain, fetched["json"]["error"]["message"])
251 rows = fetched["json"]
253 # DEBUG: print(f"DEBUG: rows({len(rows)})={rows} - blocked")
255 # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
257 elif len(rows) != config.get("misskey_limit"):
258 # DEBUG: print(f"DEBUG: Fetched '{len(rows)}' row(s) but expected: '{config.get('misskey_limit')}'")
259 offset = offset + (config.get("misskey_limit") - len(rows))
261 # DEBUG: print("DEBUG: Raising offset by step:", step)
262 offset = offset + step
265 for instance in rows:
267 # DEBUG: print(f"DEBUG: instance[{type(instance)}]='{instance}' - blocked")
268 if "isBlocked" in instance and instance["isBlocked"] and not dicts.has_key(blocklist["blocked"], "domain", instance["host"]):
270 blocklist["blocked"].append({
271 "domain": tidyup.domain(instance["host"]),
275 # DEBUG: print(f"DEBUG: count={count}")
277 # DEBUG: print("DEBUG: API is no more returning new instances, aborting loop!")
280 except network.exceptions as exception:
281 print(f"WARNING: Caught error, exiting loop: domain='{domain}',exception[{type(exception)}]='{str(exception)}'")
282 instances.update_last_error(domain, exception)
286 # DEBUG: print(f"DEBUG: Returning for domain='{domain}',blocked()={len(blocklist['blocked'])},suspended()={len(blocklist['suspended'])}")
288 "reject" : blocklist["blocked"],
289 "followers_only": blocklist["suspended"]