1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
2 # Copyright (C) 2023 Free Software Foundation
4 # This program is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU Affero General Public License as published
6 # by the Free Software Foundation, either version 3 of the License, or
7 # (at your option) any later version.
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU Affero General Public License for more details.
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <https://www.gnu.org/licenses/>.
19 from fba import blacklist
20 from fba import config
22 from fba import instances
23 from fba import network
25 def fetch_peers(domain: str) -> list:
26 # DEBUG: print(f"DEBUG: domain({len(domain)})={domain} - CALLED!")
27 if not isinstance(domain, str):
28 raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
30 raise ValueError("Parameter 'domain' is empty")
32 # DEBUG: print(f"DEBUG: domain='{domain}' is misskey, sending API POST request ...")
35 step = config.get("misskey_limit")
37 # iterating through all "suspended" (follow-only in its terminology)
38 # instances page-by-page, since that troonware doesn't support
39 # sending them all at once
41 # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...")
43 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
51 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
60 # DEBUG: print(f"DEBUG: fetched()={len(fetched)}")
62 # DEBUG: print(f"DEBUG: Returned zero bytes, exiting loop, domain='{domain}'")
64 elif len(fetched) != config.get("misskey_limit"):
65 # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'")
66 offset = offset + (config.get("misskey_limit") - len(fetched))
68 # DEBUG: print(f"DEBUG: Raising offset by step={step}")
69 offset = offset + step
72 # DEBUG: print(f"DEBUG: fetched({len(fetched)})[]={type(fetched)}")
73 if isinstance(fetched, dict) and "error" in fetched and "message" in fetched["error"]:
74 print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}")
75 instances.update_last_error(domain, fetched["error"]["message"])
80 # DEBUG: print(f"DEBUG: row()={len(row)}")
82 print(f"WARNING: row()={len(row)} does not contain key 'host': {row},domain='{domain}'")
84 elif not isinstance(row["host"], str):
85 print(f"WARNING: row[host][]={type(row['host'])} is not 'str'")
87 elif blacklist.is_blacklisted(row["host"]):
88 # DEBUG: print(f"DEBUG: row[host]='{row['host']}' is blacklisted. domain='{domain}'")
90 elif row["host"] in peers:
91 # DEBUG: print(f"DEBUG: Not adding row[host]='{row['host']}', already found.")
95 # DEBUG: print(f"DEBUG: Adding peer: '{row['host']}'")
96 peers.append(row["host"])
98 if already == len(fetched):
99 print(f"WARNING: Host returned same set of '{already}' instances, aborting loop!")
102 # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'")
103 instances.set_data("total_peers", domain, len(peers))
105 # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...")
106 instances.update_last_instance_fetch(domain)
108 # DEBUG: print(f"DEBUG: Returning peers[]='{type(peers)}'")
111 def fetch_blocks(domain: str) -> dict:
112 # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
113 if not isinstance(domain, str):
114 raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
116 raise ValueError("Parameter 'domain' is empty")
118 # DEBUG: print("DEBUG: Fetching misskey blocks from domain:", domain)
125 step = config.get("misskey_limit")
127 # iterating through all "suspended" (follow-only in its terminology)
128 # instances page-by-page, since that troonware doesn't support
129 # sending them all at once
131 # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...")
133 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
134 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
143 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
144 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
149 "offset" : offset - 1
154 # DEBUG: print("DEBUG: fetched():", len(fetched))
155 if len(fetched) == 0:
156 # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
158 elif len(fetched) != config.get("misskey_limit"):
159 # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'")
160 offset = offset + (config.get("misskey_limit") - len(fetched))
162 # DEBUG: print("DEBUG: Raising offset by step:", step)
163 offset = offset + step
166 for instance in fetched:
168 if instance["isSuspended"] and not fba.has_key(blocklist["suspended"], "domain", instance):
170 blocklist["suspended"].append(
172 "domain": fba.tidyup_domain(instance["host"]),
173 # no reason field, nothing
178 # DEBUG: print(f"DEBUG: count={count}")
180 # DEBUG: print("DEBUG: API is no more returning new instances, aborting loop!")
183 except BaseException as exc:
184 print("WARNING: Caught error, exiting loop:", domain, exc)
185 instances.update_last_error(domain, exc)
190 # same shit, different asshole ("blocked" aka full suspend)
193 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
194 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
203 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
204 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
209 "offset" : offset - 1
214 # DEBUG: print("DEBUG: fetched():", len(fetched))
215 if len(fetched) == 0:
216 # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
218 elif len(fetched) != config.get("misskey_limit"):
219 # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'")
220 offset = offset + (config.get("misskey_limit") - len(fetched))
222 # DEBUG: print("DEBUG: Raising offset by step:", step)
223 offset = offset + step
226 for instance in fetched:
228 if instance["isBlocked"] and not fba.has_key(blocklist["blocked"], "domain", instance):
230 blocklist["blocked"].append({
231 "domain": fba.tidyup_domain(instance["host"]),
235 # DEBUG: print(f"DEBUG: count={count}")
237 # DEBUG: print("DEBUG: API is no more returning new instances, aborting loop!")
240 except BaseException as exc:
241 print("ERROR: Exception during POST:", domain, exc)
242 instances.update_last_error(domain, exc)
246 # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...")
247 instances.update_last_instance_fetch(domain)
249 # DEBUG: print(f"DEBUG: Returning for domain='{domain}',blocked()={len(blocklist['blocked'])},suspended()={len(blocklist['suspended'])}")
251 "reject" : blocklist["blocked"],
252 "followers_only": blocklist["suspended"]