1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
2 # Copyright (C) 2023 Free Software Foundation
4 # This program is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU Affero General Public License as published
6 # by the Free Software Foundation, either version 3 of the License, or
7 # (at your option) any later version.
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU Affero General Public License for more details.
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <https://www.gnu.org/licenses/>.
19 from fba import blacklist
20 from fba import config
22 from fba import instances
23 from fba import network
25 from fba.helpers import dicts
26 from fba.helpers import tidyup
28 def fetch_peers(domain: str) -> list:
29 # DEBUG: print(f"DEBUG: domain({len(domain)})={domain} - CALLED!")
30 if not isinstance(domain, str):
31 raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
33 raise ValueError("Parameter 'domain' is empty")
35 # DEBUG: print(f"DEBUG: domain='{domain}' is misskey, sending API POST request ...")
38 step = config.get("misskey_limit")
40 # No CSRF by default, you don't have to add network.api_headers by yourself here
44 # DEBUG: print(f"DEBUG: Checking CSRF for domain='{domain}'")
45 headers = csrf.determine(domain, dict())
46 except network.exceptions as exception:
47 print(f"WARNING: Exception '{type(exception)}' during checking CSRF (fetch_peers,{__name__}) - EXIT!")
50 # iterating through all "suspended" (follow-only in its terminology)
51 # instances page-by-page, since that troonware doesn't support
52 # sending them all at once
54 # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...")
56 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
62 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
70 # DEBUG: print(f"DEBUG: fetched[]='{type(fetched)}'")
71 if "error_message" in fetched:
72 print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}")
73 instances.update_last_error(domain, fetched)
75 elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
76 print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}")
77 instances.update_last_error(domain, fetched["json"]["error"]["message"])
80 rows = fetched["json"]
82 # DEBUG: print(f"DEBUG: rows()={len(rows)}")
84 # DEBUG: print(f"DEBUG: Returned zero bytes, exiting loop, domain='{domain}'")
86 elif len(rows) != config.get("misskey_limit"):
87 # DEBUG: print(f"DEBUG: Fetched '{len(rows)}' row(s) but expected: '{config.get('misskey_limit')}'")
88 offset = offset + (config.get("misskey_limit") - len(rows))
90 # DEBUG: print(f"DEBUG: Raising offset by step={step}")
91 offset = offset + step
94 # DEBUG: print(f"DEBUG: rows({len(rows)})[]='{type(rows)}'")
96 # DEBUG: print(f"DEBUG: row()={len(row)}")
98 print(f"WARNING: row()={len(row)} does not contain key 'host': {row},domain='{domain}'")
100 elif not isinstance(row["host"], str):
101 print(f"WARNING: row[host][]='{type(row['host'])}' is not 'str'")
103 elif blacklist.is_blacklisted(row["host"]):
104 # DEBUG: print(f"DEBUG: row[host]='{row['host']}' is blacklisted. domain='{domain}'")
106 elif row["host"] in peers:
107 # DEBUG: print(f"DEBUG: Not adding row[host]='{row['host']}', already found.")
108 already = already + 1
111 # DEBUG: print(f"DEBUG: Adding peer: '{row['host']}'")
112 peers.append(row["host"])
114 if already == len(rows):
115 # DEBUG: print(f"DEBUG: Host returned same set of '{already}' instances, aborting loop!")
118 # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'")
119 instances.set_data("total_peers", domain, len(peers))
121 # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...")
122 instances.update_last_instance_fetch(domain)
124 # DEBUG: print(f"DEBUG: Returning peers[]='{type(peers)}'")
127 def fetch_blocks(domain: str) -> dict:
128 # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
129 if not isinstance(domain, str):
130 raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
132 raise ValueError("Parameter 'domain' is empty")
134 # DEBUG: print(f"DEBUG: Fetching misskey blocks from domain={domain}")
141 step = config.get("misskey_limit")
143 # No CSRF by default, you don't have to add network.api_headers by yourself here
147 # DEBUG: print(f"DEBUG: Checking CSRF for domain='{domain}'")
148 headers = csrf.determine(domain, dict())
149 except network.exceptions as exception:
150 print(f"WARNING: Exception '{type(exception)}' during checking CSRF (fetch_blocks,{__name__}) - EXIT!")
153 # iterating through all "suspended" (follow-only in its terminology)
154 # instances page-by-page since it doesn't support sending them all at once
157 # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...")
159 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
160 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
167 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
168 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
173 "offset" : offset - 1
176 # DEBUG: print(f"DEBUG: fetched[]='{type(fetched)}'")
177 if "error_message" in fetched:
178 print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}")
179 instances.update_last_error(domain, fetched)
181 elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
182 print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}")
183 instances.update_last_error(domain, fetched["json"]["error"]["message"])
186 rows = fetched["json"]
188 # DEBUG: print(f"DEBUG: rows({len(rows)})={rows} - suspend")
190 # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
192 elif len(rows) != config.get("misskey_limit"):
193 # DEBUG: print(f"DEBUG: Fetched '{len(rows)}' row(s) but expected: '{config.get('misskey_limit')}'")
194 offset = offset + (config.get("misskey_limit") - len(rows))
196 # DEBUG: print("DEBUG: Raising offset by step:", step)
197 offset = offset + step
200 for instance in rows:
202 # DEBUG: print(f"DEBUG: instance[{type(instance)}]='{instance}' - suspend")
203 if "isSuspended" in instance and instance["isSuspended"] and not dicts.has_key(blocklist["suspended"], "domain", instance["host"]):
205 blocklist["suspended"].append(
207 "domain": tidyup.domain(instance["host"]),
208 # no reason field, nothing
213 # DEBUG: print(f"DEBUG: count={count}")
215 # DEBUG: print("DEBUG: API is no more returning new instances, aborting loop!")
218 except network.exceptions as exception:
219 print(f"WARNING: Caught error, exiting loop: domain='{domain}',exception[{type(exception)}]='{str(exception)}'")
220 instances.update_last_error(domain, exception)
225 # Fetch blocked (full suspended) instances
228 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
229 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
236 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
237 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
242 "offset" : offset - 1
245 # DEBUG: print(f"DEBUG: fetched[]='{type(fetched)}'")
246 if "error_message" in fetched:
247 print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}")
248 instances.update_last_error(domain, fetched)
250 elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
251 print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}")
252 instances.update_last_error(domain, fetched["json"]["error"]["message"])
255 rows = fetched["json"]
257 # DEBUG: print(f"DEBUG: rows({len(rows)})={rows} - blocked")
259 # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
261 elif len(rows) != config.get("misskey_limit"):
262 # DEBUG: print(f"DEBUG: Fetched '{len(rows)}' row(s) but expected: '{config.get('misskey_limit')}'")
263 offset = offset + (config.get("misskey_limit") - len(rows))
265 # DEBUG: print("DEBUG: Raising offset by step:", step)
266 offset = offset + step
269 for instance in rows:
271 # DEBUG: print(f"DEBUG: instance[{type(instance)}]='{instance}' - blocked")
272 if "isBlocked" in instance and instance["isBlocked"] and not dicts.has_key(blocklist["blocked"], "domain", instance["host"]):
274 blocklist["blocked"].append({
275 "domain": tidyup.domain(instance["host"]),
279 # DEBUG: print(f"DEBUG: count={count}")
281 # DEBUG: print("DEBUG: API is no more returning new instances, aborting loop!")
284 except network.exceptions as exception:
285 print(f"WARNING: Caught error, exiting loop: domain='{domain}',exception[{type(exception)}]='{str(exception)}'")
286 instances.update_last_error(domain, exception)
290 # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...")
291 instances.update_last_instance_fetch(domain)
293 # DEBUG: print(f"DEBUG: Returning for domain='{domain}',blocked()={len(blocklist['blocked'])},suspended()={len(blocklist['suspended'])}")
295 "reject" : blocklist["blocked"],
296 "followers_only": blocklist["suspended"]