1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
2 # Copyright (C) 2023 Free Software Foundation
4 # This program is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU Affero General Public License as published
6 # by the Free Software Foundation, either version 3 of the License, or
7 # (at your option) any later version.
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU Affero General Public License for more details.
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <https://www.gnu.org/licenses/>.
23 from fba.helpers import config
24 from fba.helpers import domain as domain_helper
25 from fba.helpers import tidyup
27 from fba.http import network
29 from fba.models import instances
31 logging.basicConfig(level=logging.INFO)
32 logger = logging.getLogger(__name__)
34 def fetch_peers(domain: str) -> list:
35 logger.debug("domain='%s' - CALLED!", domain)
36 domain_helper.raise_on(domain)
38 logger.debug("domain='%s' is misskey, sending API POST request ...", domain)
41 step = config.get("misskey_limit")
43 # No CSRF by default, you don't have to add network.api_headers by yourself here
47 logger.debug("Checking CSRF for domain='%s'", domain)
48 headers = csrf.determine(domain, dict())
49 except network.exceptions as exception:
50 logger.warning("Exception '%s' during checking CSRF (fetch_peers,%s) - EXIT!", type(exception), __name__)
51 instances.set_last_error(domain, exception)
54 # iterating through all "suspended" (follow-only in its terminology)
55 # instances page-by-page, since that troonware doesn't support
56 # sending them all at once
58 logger.debug("Fetching offset='%d' from domain='%s' ...", offset, domain)
60 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
66 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
74 logger.debug("fetched[]='%s'", type(fetched))
75 if "error_message" in fetched:
76 logger.warning("post_json_api() for domain='%s' returned error message: '%s'", domain, fetched['error_message'])
77 instances.set_last_error(domain, fetched)
79 elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
80 logger.warning("post_json_api() returned error: '%s'", fetched['error']['message'])
81 instances.set_last_error(domain, fetched["json"]["error"]["message"])
84 rows = fetched["json"]
86 logger.debug("rows(%d)[]='%s'", len(rows), type(rows))
88 logger.debug("Returned zero bytes, domain='%s' - BREAK!", domain)
90 elif len(rows) != config.get("misskey_limit"):
91 logger.debug("Fetched %d row(s) but expected: %d", len(rows), config.get('misskey_limit'))
92 offset = offset + (config.get("misskey_limit") - len(rows))
94 logger.debug("Raising offset by step='%d'", step)
95 offset = offset + step
98 logger.debug("rows(%d))[]='%s'", len(rows), type(rows))
100 logger.debug(f"row()={len(row)}")
101 if "host" not in row:
102 logger.warning("row()=%d does not contain key 'host': row='%s',domain='%s' - SKIPPED!", len(row), row, domain)
104 elif not isinstance(row["host"], str):
105 logger.warning("row[host][]='%s' is not 'str' - SKIPPED!", type(row['host']))
107 elif not utils.is_domain_wanted(row["host"]):
108 logger.debug("row[host]='%s' is not wanted, domain='{domain}' - SKIPPED!", row['host'])
110 elif row["host"] in peers:
111 logger.debug("Not adding row[host]='%s', already found - SKIPPED!", row['host'])
112 already = already + 1
115 logger.debug("Adding peer: '%s'", row['host'])
116 peers.append(row["host"])
118 if already == len(rows):
119 logger.debug("Host returned same set of %d instance(s) - BREAK!", already)
122 logger.debug("Adding %d for domain='%s'", len(peers), domain)
123 instances.set_total_peers(domain, peers)
125 logger.debug("peers()=%d - EXIT!", len(peers))
128 def fetch_blocks(domain: str) -> list:
129 logger.debug("domain='%s' - CALLED!", domain)
130 domain_helper.raise_on(domain)
132 logger.debug("Fetching misskey blocks from domain='%s'", domain)
136 step = config.get("misskey_limit")
138 # No CSRF by default, you don't have to add network.api_headers by yourself here
142 logger.debug("Checking CSRF for domain='%s'", domain)
143 headers = csrf.determine(domain, dict())
144 except network.exceptions as exception:
145 logger.warning("Exception '%s' during checking CSRF (fetch_blocks,%s) - EXIT!", type(exception), __name__)
146 instances.set_last_error(domain, exception)
149 # iterating through all "suspended" (follow-only in its terminology)
150 # instances page-by-page since it doesn't support sending them all at once
153 logger.debug("Fetching offset='%d' from domain='%s' ...", offset, domain)
155 logger.debug("Sending JSON API request to domain='%s',step='%d',offset='%d'", domain, step, offset)
156 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
163 logger.debug("Sending JSON API request to domain='%s',step='%d',offset='%d'", domain, step, offset)
164 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
169 "offset" : offset - 1
172 logger.debug("fetched[]='%s'", type(fetched))
173 if "error_message" in fetched:
174 logger.warning("post_json_api() for domain='%s' returned error message: '%s'", domain, fetched['error_message'])
175 instances.set_last_error(domain, fetched)
177 elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
178 logger.warning("post_json_api() returned error: '%s'", fetched['error']['message'])
179 instances.set_last_error(domain, fetched["json"]["error"]["message"])
182 rows = fetched["json"]
184 logger.debug("rows(%d)[]='%s'", len(rows), type(rows))
186 logger.debug("Returned zero bytes, domain='%s' - BREAK!", domain)
188 elif len(rows) != config.get("misskey_limit"):
189 logger.debug("Fetched %d row(s) but expected: %d", len(rows), config.get('misskey_limit'))
190 offset = offset + (config.get("misskey_limit") - len(rows))
192 logger.debug("Raising offset by step='%d'", step)
193 offset = offset + step
196 for instance in rows:
198 logger.debug("instance[%s]='%s'", type(instance), instance)
199 if "isSuspended" in instance and instance["isSuspended"]:
203 "blocked" : tidyup.domain(instance["host"]),
205 "block_level": "suspend",
208 logger.debug("count=%d", count)
210 logger.debug("API is no more returning new instances, aborting loop! domain='%s'", domain)
213 except network.exceptions as exception:
214 logger.warning("Caught error, exiting loop: domain='%s',exception[%s]='%s'", domain, type(exception), str(exception))
215 instances.set_last_error(domain, exception)
220 # Fetch blocked (full suspended) instances
223 logger.debug("Sending JSON API request to domain='%s',step='%d',offset='%d'", domain, step, offset)
224 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
231 logger.debug("Sending JSON API request to domain='%s',step='%d',offset='%d'", domain, step, offset)
232 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
237 "offset" : offset - 1
240 logger.debug("fetched[]='%s'", type(fetched))
241 if "error_message" in fetched:
242 logger.warning("post_json_api() for domain='%s' returned error message: '%s'", domain, fetched['error_message'])
243 instances.set_last_error(domain, fetched)
245 elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
246 logger.warning("post_json_api() returned error: '%s'", fetched['error']['message'])
247 instances.set_last_error(domain, fetched["json"]["error"]["message"])
250 rows = fetched["json"]
252 logger.debug("rows(%d)[]='%s'", len(rows), type(rows))
254 logger.debug("Returned zero bytes, domain='%s' - BREAK!", domain)
256 elif len(rows) != config.get("misskey_limit"):
257 logger.debug("Fetched %d row(s) but expected: '{config.get('misskey_limit')}'", len(rows))
258 offset = offset + (config.get("misskey_limit") - len(rows))
260 logger.debug("Raising offset by step='%d'", step)
261 offset = offset + step
264 for instance in rows:
266 logger.debug("instance[%s]='%s'", type(instance), instance)
267 if "isBlocked" in instance and instance["isBlocked"]:
269 blocked = tidyup.domain(instance["host"])
270 logger.debug("Appending blocker='%s',blocked='%s',block_level='reject'", domain, blocked)
275 "block_level": "reject",
278 logger.debug("count=%d", count)
280 logger.debug("API is no more returning new instances, aborting loop!")
283 except network.exceptions as exception:
284 logger.warning("Caught error, exiting loop: domain='%s',exception[%s]='%s'", domain, type(exception), str(exception))
285 instances.set_last_error(domain, exception)
289 logger.debug("blocklist()=%d - EXIT!", len(blocklist))