1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
2 # Copyright (C) 2023 Free Software Foundation
4 # This program is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU Affero General Public License as published
6 # by the Free Software Foundation, either version 3 of the License, or
7 # (at your option) any later version.
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU Affero General Public License for more details.
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <https://www.gnu.org/licenses/>.
20 from fba.helpers import blacklist
21 from fba.helpers import config
22 from fba.helpers import dicts as dict_helper
23 from fba.helpers import domain as domain_helper
24 from fba.helpers import tidyup
26 from fba.http import csrf
27 from fba.http import network
29 from fba.models import instances
31 logging.basicConfig(level=logging.INFO)
32 logger = logging.getLogger(__name__)
34 def fetch_peers(domain: str) -> list:
35 logger.debug("domain='%s' - CALLED!", domain)
36 domain_helper.raise_on(domain)
38 if blacklist.is_blacklisted(domain):
39 raise Exception(f"domain='{domain}' is blacklisted but function is invoked.")
40 elif not instances.is_registered(domain):
41 raise Exception(f"domain='{domain}' is not registered but function is invoked.")
43 logger.debug("domain='%s' is misskey, sending API POST request ...", domain)
46 step = config.get("misskey_limit")
48 # No CSRF by default, you don't have to add network.api_headers by yourself here
52 logger.debug("Checking CSRF for domain='%s'", domain)
53 headers = csrf.determine(domain, dict())
54 except network.exceptions as exception:
55 logger.warning("Exception '%s' during checking CSRF (fetch_peers,%s)", type(exception), __name__)
56 instances.set_last_error(domain, exception)
58 logger.debug("Returning empty list ... - EXIT!")
61 # iterating through all "suspended" (follow-only in its terminology)
62 # instances page-by-page, since that troonware doesn't support
63 # sending them all at once
65 logger.debug("Fetching offset=%d from domain='%s' ...", offset, domain)
67 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
73 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
81 logger.debug("fetched[]='%s'", type(fetched))
82 if "error_message" in fetched:
83 logger.warning("post_json_api() for domain='%s' returned error message: '%s'", domain, fetched['error_message'])
84 instances.set_last_error(domain, fetched)
86 elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
87 logger.warning("post_json_api() returned error: '%s'", fetched["json"]["error"]["message"])
88 instances.set_last_error(domain, fetched["json"]["error"]["message"])
91 rows = fetched["json"]
93 logger.debug("rows(%d)[]='%s',step=%d", len(rows), type(rows), step)
95 logger.debug("Returned zero bytes, domain='%s' - BREAK!", domain)
97 elif len(rows) != config.get("misskey_limit"):
98 logger.debug("Fetched %d row(s) but expected: %d", len(rows), config.get('misskey_limit'))
99 offset = offset + (config.get("misskey_limit") - len(rows))
101 logger.debug("Raising offset by step=%d", step)
102 offset = offset + step
105 logger.debug("rows(%d))[]='%s'", len(rows), type(rows))
107 logger.debug("row()=%d", len(row))
108 if "host" not in row:
109 logger.warning("row()=%d does not contain key 'host': row='%s',domain='%s' - SKIPPED!", len(row), row, domain)
111 elif not isinstance(row["host"], str):
112 logger.warning("row[host][]='%s' is not of type 'str' - SKIPPED!", type(row['host']))
114 elif row["host"] in peers:
115 logger.debug("Not adding row[host]='%s', already found - SKIPPED!", row['host'])
117 elif not domain_helper.is_wanted(row["host"]):
118 logger.debug("row[host]='%s' is not wanted - SKIPPED!", row["host"])
121 logger.debug("Adding peer: row[host]='%s'", row['host'])
123 peers.append(row["host"])
125 logger.debug("added=%d,rows()=%d", added, len(rows))
127 logger.debug("Host returned already added (%d) peers - BREAK!", len(rows))
130 logger.debug("peers()=%d - EXIT!", len(peers))
133 def fetch_blocks(domain: str) -> list:
134 logger.debug("domain='%s' - CALLED!", domain)
135 domain_helper.raise_on(domain)
137 if blacklist.is_blacklisted(domain):
138 raise Exception(f"domain='{domain}' is blacklisted but function is invoked.")
139 elif not instances.is_registered(domain):
140 raise Exception(f"domain='{domain}' is not registered but function is invoked.")
142 # No CSRF by default, you don't have to add network.api_headers by yourself here
146 logger.debug("Checking CSRF for domain='%s' ...", domain)
147 headers = csrf.determine(domain, dict())
148 except network.exceptions as exception:
149 logger.warning("Exception '%s' during checking CSRF (fetch_blocks,%s)", type(exception), __name__)
150 instances.set_last_error(domain, exception)
152 logger.debug("Returning empty list ... - EXIT!")
157 step = config.get("misskey_limit")
159 # iterating through all "suspended" (follow-only in its terminology)
160 # instances page-by-page since it doesn't support sending them all at once
161 logger.debug("Fetching misskey blocks from domain='%s'", domain)
163 logger.debug("offset=%d", offset)
165 logger.debug("Fetching offset=%d from domain='%s' ...", offset, domain)
167 logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d", domain, step, offset)
168 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
175 logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d", domain, step, offset)
176 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
181 "offset" : offset - 1
184 logger.debug("fetched[]='%s'", type(fetched))
185 if "error_message" in fetched:
186 logger.warning("post_json_api() for domain='%s' returned error message: '%s'", domain, fetched['error_message'])
187 instances.set_last_error(domain, fetched)
189 elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
190 logger.warning("post_json_api() returned error: '%s'", fetched["json"]["error"]["message"])
191 instances.set_last_error(domain, fetched["json"]["error"]["message"])
194 rows = fetched["json"]
196 logger.debug("rows(%d)[]='%s'", len(rows), type(rows))
198 logger.debug("Returned zero bytes, domain='%s' - BREAK!", domain)
200 elif len(rows) != config.get("misskey_limit"):
201 logger.debug("Fetched %d row(s) but expected: %d", len(rows), config.get('misskey_limit'))
202 offset = offset + (config.get("misskey_limit") - len(rows))
204 logger.debug("Raising offset by step=%d", step)
205 offset = offset + step
208 logger.debug("Checking %d row(s) of instances ...", len(rows))
209 for instance in rows:
211 logger.debug("instance[]='%s'", type(instance))
212 if "host" not in instance:
213 logger.warning("instance(%d)='%s' has no key 'host' - SKIPPED!", len(instance), instance)
215 elif instance["host"] in [None, ""]:
216 logger.debug("instance[host]='%s' is None or empty - SKIPPED!", instance["host"])
219 logger.debug("instance[host]='%s' - BEFORE!", instance["host"])
220 blocked = tidyup.domain(instance["host"])
221 logger.debug("blocked[%s]='%s' - AFTER!", type(blocked), blocked)
223 if blocked in [None, ""]:
224 logger.warning("instance[host]='%s' is None or empty after tidyup.domain() - SKIPPED!", instance["host"])
226 elif not domain_helper.is_wanted(blocked):
227 logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
229 elif "isSuspended" in instance and instance["isSuspended"] and not dict_helper.has_key(blocklist, "blocked", blocked):
231 logger.debug("Appending blocker='%s',blocked='%s',block_level='suspended'", domain, blocked)
236 "block_level": "suspended",
239 logger.debug("count=%d", count)
241 logger.debug("API is no more returning new instances, aborting loop! domain='%s'", domain)
244 except network.exceptions as exception:
245 logger.warning("Caught error, exiting loop: domain='%s',exception[%s]='%s'", domain, type(exception), str(exception))
246 instances.set_last_error(domain, exception)
251 # Fetch blocked (full suspended) instances
252 logger.debug("offset=%d", offset)
255 logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d", domain, step, offset)
256 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
263 logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d", domain, step, offset)
264 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
269 "offset" : offset - 1
272 logger.debug("fetched[]='%s'", type(fetched))
273 if "error_message" in fetched:
274 logger.warning("post_json_api() for domain='%s' returned error message: '%s'", domain, fetched['error_message'])
275 instances.set_last_error(domain, fetched)
277 elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
278 logger.warning("post_json_api() returned error: '%s'", fetched["json"]["error"]["message"])
279 instances.set_last_error(domain, fetched["json"]["error"]["message"])
282 rows = fetched["json"]
284 logger.debug("rows(%d)[]='%s'", len(rows), type(rows))
286 logger.debug("Returned zero bytes, domain='%s' - BREAK!", domain)
288 elif len(rows) != config.get("misskey_limit"):
289 logger.debug("Fetched %d row(s) but expected: %d'", len(rows), config.get('misskey_limit'))
290 offset = offset + (config.get("misskey_limit") - len(rows))
292 logger.debug("Raising offset by step=%d", step)
293 offset = offset + step
296 logger.debug("Checking %d row(s) of instances ...", len(rows))
297 for instance in rows:
299 logger.debug("instance[]='%s'", type(instance))
300 blocked = tidyup.domain(instance["host"]) if instance["host"] != "" else None
301 logger.debug("blocked='%s' - AFTER!", blocked)
303 if blocked in [None, ""]:
304 logger.warning("instance[host]='%s' is None or empty after tidyup.domain() - SKIPPED!", instance["host"])
306 elif not domain_helper.is_wanted(blocked):
307 logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
309 elif "isBlocked" in instance and instance["isBlocked"] and not dict_helper.has_key(blocklist, "blocked", blocked):
311 logger.debug("Appending blocker='%s',blocked='%s',block_level='reject'", domain, blocked)
316 "block_level": "reject",
319 logger.debug("count=%d", count)
321 logger.debug("API is no more returning new instances, aborting loop!")
324 except network.exceptions as exception:
325 logger.warning("Caught error, exiting loop: domain='%s',exception[%s]='%s'", domain, type(exception), str(exception))
326 instances.set_last_error(domain, exception)
330 logger.debug("blocklist()=%d - EXIT!", len(blocklist))