1 # Copyright (C) 2023 Free Software Foundation
3 # This program is free software: you can redistribute it and/or modify
4 # it under the terms of the GNU Affero General Public License as published
5 # by the Free Software Foundation, either version 3 of the License, or
6 # (at your option) any later version.
8 # This program is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # GNU Affero General Public License for more details.
13 # You should have received a copy of the GNU Affero General Public License
14 # along with this program. If not, see <https://www.gnu.org/licenses/>.
18 from urllib.parse import urlparse
24 from fba.helpers import blacklist
25 from fba.helpers import config
26 from fba.helpers import cookies
27 from fba.helpers import domain as domain_helper
28 from fba.helpers import software as software_helper
29 from fba.helpers import tidyup
30 from fba.helpers import version
32 from fba.http import csrf
33 from fba.http import network
34 from fba.http import nodeinfo
36 from fba.models import blocks
37 from fba.models import instances
39 from fba.networks import lemmy
40 from fba.networks import misskey
41 from fba.networks import peertube
43 # Depth counter, being raised and lowered
48 "/api/v1/instance/peers",
52 logging.basicConfig(level=logging.INFO)
53 logger = logging.getLogger(__name__)
55 def fetch_instances(domain: str, origin: str, software: str, command: str, path: str = None):
57 logger.debug("domain='%s',origin='%s',software='%s',command='%s',path='%s',_DEPTH=%d - CALLED!", domain, origin, software, command, path, _DEPTH)
58 domain_helper.raise_on(domain)
60 if blacklist.is_blacklisted(domain):
61 raise Exception(f"domain='{domain}' is blacklisted but function was invoked")
62 elif not isinstance(origin, str) and origin is not None:
63 raise ValueError(f"Parameter origin[]='{type(origin)}' is not of type 'str'")
64 elif not isinstance(command, str):
65 raise ValueError(f"Parameter command[]='{type(command)}' is not of type 'str'")
67 raise ValueError("Parameter 'command' is empty")
68 elif command in ["fetch_blocks", "fetch_cs", "fetch_bkali", "fetch_relays", "fetch_fedipact", "fetch_joinmobilizon", "fetch_joinmisskey", "fetch_joinfediverse", "fetch_relaylist"] and origin is None:
69 raise ValueError(f"Parameter command='{command}' but origin is None, please fix invoking this function.")
70 elif not isinstance(path, str) and path is not None:
71 raise ValueError(f"Parameter path[]='{type(path)}' is not of type 'str'")
72 elif path is not None and not path.startswith("/"):
73 raise ValueError(f"path='{path}' does not start with a slash")
74 elif _DEPTH > 0 and instances.is_recent(domain, "last_instance_fetch"):
75 raise ValueError(f"domain='{domain}' has recently been fetched but function was invoked")
76 elif software is None and not instances.is_recent(domain, "last_nodeinfo"):
78 logger.debug("Software for domain='%s',path='%s' is not set, determining ...", domain, path)
79 software = determine_software(domain, path)
80 except network.exceptions as exception:
81 logger.warning("Exception '%s' during determining software type", type(exception))
82 instances.set_last_error(domain, exception)
84 logger.debug("Determined software='%s' for domain='%s'", software, domain)
85 elif software is None:
86 logger.debug("domain='%s' has unknown software or nodeinfo has recently being fetched", domain)
87 elif not isinstance(software, str):
88 raise ValueError(f"Parameter software[]='{type(software)}' is not of type 'str'")
93 logger.debug("Checking if domain='%s' is registered ...", domain)
94 if not instances.is_registered(domain):
95 logger.debug("Adding new domain='%s',origin='%s',command='%s',path='%s',software='%s'", domain, origin, command, path, software)
96 instances.add(domain, origin, command, path, software)
98 logger.debug("software='%s'", software)
99 if software is not None and software_helper.is_relay(software):
100 logger.debug("software='%s' is a relay software - EXIT!", software)
104 logger.debug("Updating last_instance_fetch for domain='%s' ...", domain)
105 instances.set_last_instance_fetch(domain)
108 logger.debug("software='%s'", software)
109 if software is not None:
111 logger.debug("Fetching instances for domain='%s',software='%s',origin='%s'", domain, software, origin)
112 peerlist = fetch_peers(domain, software, origin)
113 except network.exceptions as exception:
117 logger.debug("peerlist[]='%s'", type(peerlist))
118 if isinstance(peerlist, list):
119 logger.debug("Invoking instances.set_total_peerlist(%s,%d) ...", domain, len(peerlist))
120 instances.set_total_peers(domain, peerlist)
122 logger.debug("Invoking cookies.clear(%s) ...", domain)
123 cookies.clear(domain)
125 logger.debug("peerlist[]='%s'", type(peerlist))
127 logger.warning("Cannot fetch peers: domain='%s',software='%s'", domain, software)
128 if instances.has_pending(domain):
129 logger.debug("Flushing updates for domain='%s' ...", domain)
130 instances.update(domain)
133 logger.debug("EXIT!")
135 elif len(peerlist) == 0:
136 logger.info("domain='%s' returned an empty peer list.", domain)
137 if instances.has_pending(domain):
138 logger.debug("Flushing updates for domain='%s' ...", domain)
139 instances.update(domain)
142 logger.debug("domain='%s',software='%s' has an empty peer list returned - EXIT!", domain, software)
145 logger.info("Checking %d instance(s) from domain='%s',software='%s',depth=%d ...", len(peerlist), domain, software, _DEPTH)
146 for instance in peerlist:
147 logger.debug("instance[%s]='%s'", type(instance), instance)
148 if instance in [None, ""]:
149 logger.debug("instance[%s]='%s' is either None or empty - SKIPPED!", type(instance), instance)
152 logger.debug("instance='%s' - BEFORE!", instance)
153 instance = tidyup.domain(instance) if isinstance(instance, str) and instance != "" else None
154 logger.debug("instance='%s' - AFTER!", instance)
156 if instance in [None, ""]:
157 logger.warning("instance='%s' is empty after tidyup.domain(), domain='%s'", instance, domain)
159 elif ".." in instance:
160 logger.warning("instance='%s' contains double-dot, removing ...", instance)
161 instance = instance.replace("..", ".")
163 logger.debug("instance='%s' - BEFORE!", instance)
164 instance = instance.encode("idna").decode("utf-8")
165 logger.debug("instance='%s' - AFTER!", instance)
167 if not domain_helper.is_wanted(instance):
168 logger.debug("instance='%s' is not wanted - SKIPPED!", instance)
170 elif instance.find("/profile/") > 0 or instance.find("/users/") > 0 or (instances.is_registered(instance.split("/")[0]) and instance.find("/c/") > 0):
171 logger.debug("instance='%s' is a link to a single user profile - SKIPPED!", instance)
173 elif instance.find("/tag/") > 0:
174 logger.debug("instance='%s' is a link to a tag - SKIPPED!", instance)
176 elif not instances.is_registered(instance):
177 logger.debug("Checking if domain='%s' has pending updates ...", domain)
178 if instances.has_pending(domain):
179 logger.debug("Flushing updates for domain='%s' ...", domain)
180 instances.update(domain)
182 logger.debug("instance='%s',origin='%s',_DEPTH=%d reached!", instance, origin, _DEPTH)
183 if _DEPTH <= config.get("max_crawl_depth") and len(peerlist) >= config.get("min_peers_length"):
184 logger.debug("Fetching instance='%s',origin='%s',command='%s',path='%s',_DEPTH=%d ...", instance, domain, command, path, _DEPTH)
185 fetch_instances(instance, domain, None, command, path)
187 logger.debug("Adding instance='%s',domain='%s',command='%s',_DEPTH=%d ...", instance, domain, command, _DEPTH)
188 instances.add(instance, domain, command)
190 logger.debug("Checking if domain='%s' has pending updates ...", domain)
191 if instances.has_pending(domain):
192 logger.debug("Flushing updates for domain='%s' ...", domain)
193 instances.update(domain)
196 logger.debug("EXIT!")
198 def fetch_peers(domain: str, software: str, origin: str) -> list:
199 logger.debug("domain='%s',software='%s',origin='%s' - CALLED!", domain, software, origin)
200 domain_helper.raise_on(domain)
202 if blacklist.is_blacklisted(domain):
203 raise Exception(f"domain='{domain}' is blacklisted but function was invoked")
204 elif not isinstance(software, str) and software is not None:
205 raise ValueError(f"Parameter software[]='{type(software)}' is not of type 'str'")
206 elif isinstance(software, str) and software == "":
207 raise ValueError("Parameter 'software' is empty")
208 elif software is not None and software_helper.is_relay(software):
209 raise ValueError(f"domain='{domain}' is of software='{software}' and isn't supported here.")
210 elif not isinstance(origin, str) and origin is not None:
211 raise ValueError(f"Parameter origin[]='{type(origin)}' is not of type 'str'")
212 elif isinstance(origin, str) and origin == "":
213 raise ValueError("Parameter 'origin' is empty")
215 if software == "misskey":
216 logger.debug("Invoking misskey.fetch_peers(%s) ...", domain)
217 return misskey.fetch_peers(domain)
218 elif software == "lemmy":
219 logger.debug("Invoking lemmy.fetch_peers(%s,%s) ...", domain, origin)
220 return lemmy.fetch_peers(domain, origin)
221 elif software == "peertube":
222 logger.debug("Invoking peertube.fetch_peers(%s) ...", domain)
223 return peertube.fetch_peers(domain)
225 # No CSRF by default, you don't have to add network.api_headers by yourself here
229 logger.debug("Checking CSRF for domain='%s'", domain)
230 headers = csrf.determine(domain, dict())
231 except network.exceptions as exception:
232 logger.warning("Exception '%s' during checking CSRF (fetch_peers,%s)", type(exception), __name__)
233 instances.set_last_error(domain, exception)
235 logger.debug("Returning empty list ... - EXIT!")
238 # Init peers variable
241 logger.debug("Checking %d API paths ...", len(_api_paths))
242 for path in _api_paths:
243 logger.debug("Fetching path='%s' from domain='%s',software='%s' ...", path, domain, software)
244 data = network.get_json_api(
248 timeout=(config.get("connection_timeout"), config.get("read_timeout"))
251 logger.debug("data(%d)[]='%s'", len(data), type(data))
252 if "error_message" in data:
253 logger.debug("Was not able to fetch peers from path='%s',domain='%s' ...", path, domain)
254 instances.set_last_error(domain, data)
255 elif "json" in data and len(data["json"]) > 0:
256 logger.debug("Querying API path='%s' was successful: domain='%s',data[json][%s]()=%d", path, domain, type(data['json']), len(data['json']))
259 logger.debug("Marking domain='%s' as successfully handled ...", domain)
260 instances.set_success(domain)
263 if not isinstance(peers, list):
264 logger.warning("peers[]='%s' is not of type 'list', maybe bad API response?", type(peers))
267 logger.debug("Invoking instances.set_total_peers(%s,%d) ...", domain, len(peers))
268 instances.set_total_peers(domain, peers)
270 logger.debug("peers()=%d - EXIT!", len(peers))
273 def fetch_generator_from_path(domain: str, path: str = "/") -> str:
274 logger.debug("domain='%s',path='%s' - CALLED!", domain, path)
275 domain_helper.raise_on(domain)
277 if blacklist.is_blacklisted(domain):
278 raise Exception(f"domain='{domain}' is blacklisted but function was invoked")
279 elif not isinstance(path, str):
280 raise ValueError(f"path[]='{type(path)}' is not of type 'str'")
282 raise ValueError("Parameter 'path' is empty")
283 elif not path.startswith("/"):
284 raise ValueError(f"path='{path}' does not start with / but should")
288 logger.debug("Fetching path='%s' from domain='%s' ...", path, domain)
289 response = network.fetch_response(
292 headers=network.web_headers,
293 timeout=(config.get("connection_timeout"), config.get("read_timeout")),
297 logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text))
298 if ((response.ok and response.status_code == 200) or response.status_code == 410) and response.text.find("<html") > 0 and domain_helper.is_in_url(domain, response.url):
299 logger.debug("Parsing response.text()=%d Bytes ...", len(response.text))
300 doc = bs4.BeautifulSoup(response.text, "html.parser")
302 logger.debug("doc[]='%s'", type(doc))
303 platform = doc.find("meta", {"property": "og:platform"})
304 generator = doc.find("meta", {"name" : "generator"})
305 site_name = doc.find("meta", {"property": "og:site_name"})
306 app_name = doc.find("meta", {"name" : "application-name"})
308 logger.debug("generator[]='%s',site_name[]='%s',platform[]='%s',app_name[]='%s'", type(generator), type(site_name), type(platform), type(app_name))
309 if isinstance(platform, bs4.element.Tag) and isinstance(platform.get("content"), str) and platform.get("content") != "":
310 logger.debug("Found property=og:platform, domain='%s'", domain)
311 software = tidyup.domain(platform.get("content"))
312 logger.debug("software[%s]='%s' after tidyup.domain() ...", type(software), software)
314 if software is not None and software != "":
315 logger.debug("domain='%s' has og:platform='%s' - Setting detection_mode=PLATFORM ...", domain, software)
316 instances.set_detection_mode(domain, "PLATFORM")
317 elif isinstance(generator, bs4.element.Tag) and isinstance(generator.get("content"), str) and generator.get("content") != "":
318 logger.debug("Found generator meta tag: domain='%s'", domain)
319 software = tidyup.domain(generator.get("content"))
321 logger.debug("software[%s]='%s'", type(software), software)
322 if software is not None and software != "":
323 logger.info("domain='%s' is generated by software='%s' - Setting detection_mode=GENERATOR ...", domain, software)
324 instances.set_detection_mode(domain, "GENERATOR")
325 elif isinstance(app_name, bs4.element.Tag) and isinstance(app_name.get("content"), str) and app_name.get("content") != "":
326 logger.debug("Found property=og:app_name, domain='%s'", domain)
327 software = tidyup.domain(app_name.get("content"))
329 logger.debug("software[%s]='%s'", type(software), software)
330 if software is not None and software != "":
331 logger.debug("domain='%s' has application-name='%s' - Setting detection_mode=app_name ...", domain, software)
332 instances.set_detection_mode(domain, "APP_NAME")
333 elif isinstance(site_name, bs4.element.Tag) and isinstance(site_name.get("content"), str) and site_name.get("content") != "":
334 logger.debug("Found property=og:site_name, domain='%s'", domain)
335 software = tidyup.domain(site_name.get("content"))
337 logger.debug("software[%s]='%s'", type(software), software)
338 if software is not None and software != "":
339 logger.debug("domain='%s' has og:site_name='%s' - Setting detection_mode=SITE_NAME ...", domain, software)
340 instances.set_detection_mode(domain, "SITE_NAME")
341 elif not domain_helper.is_in_url(domain, response.url):
342 logger.warning("domain='%s' doesn't match response.url='%s', maybe redirect to other domain?", domain, response.url)
344 components = urlparse(response.url)
345 domain2 = components.netloc.lower().split(":")[0]
347 logger.debug("domain2='%s'", domain2)
348 if not domain_helper.is_wanted(domain2):
349 logger.debug("domain2='%s' is not wanted - EXIT!", domain2)
351 elif not instances.is_registered(domain2):
352 logger.info("components.netloc='%s' is not registered, adding ...", components.netloc)
353 instances.add(domain2, domain, "redirect_target")
355 message = f"Redirect from domain='{domain}' to response.url='{response.url}'"
356 instances.set_last_error(domain, message)
357 instances.set_software(domain, None)
358 instances.set_detection_mode(domain, None)
359 instances.set_nodeinfo_url(domain, None)
361 raise requests.exceptions.TooManyRedirects(message)
363 logger.debug("software[]='%s'", type(software))
364 if isinstance(software, str) and software == "":
365 logger.debug("Corrected empty string to None for software of domain='%s'", domain)
367 elif isinstance(software, str) and ("." in software or " " in software):
368 logger.debug("software='%s' may contain a version number, domain='%s', removing it ...", software, domain)
369 software = version.remove(software)
371 logger.debug("software[%s]='%s'", type(software), software)
372 if isinstance(software, str) and "powered by " in software:
373 logger.debug("software='%s' has 'powered by' in it", software)
374 software = version.remove(software_helper.strip_powered_by(software))
375 elif isinstance(software, str) and " hosted on " in software:
376 logger.debug("software='%s' has 'hosted on' in it", software)
377 software = version.remove(software_helper.strip_hosted_on(software))
378 elif isinstance(software, str) and " by " in software:
379 logger.debug("software='%s' has ' by ' in it", software)
380 software = software_helper.strip_until(software, " by ")
381 elif isinstance(software, str) and " see " in software:
382 logger.debug("software='%s' has ' see ' in it", software)
383 software = software_helper.strip_until(software, " see ")
385 logger.debug("software[%s]='%s' - EXIT!", type(software), software)
388 def determine_software(domain: str, path: str = None) -> str:
389 logger.debug("domain='%s',path='%s' - CALLED!", domain, path)
390 domain_helper.raise_on(domain)
392 if blacklist.is_blacklisted(domain):
393 raise Exception(f"domain='{domain}' is blacklisted but function was invoked")
394 elif not isinstance(path, str) and path is not None:
395 raise ValueError(f"Parameter path[]='{type(path)}' is not of type 'str'")
396 elif path is not None and not path.startswith("/"):
397 raise ValueError(f"path='{path}' does not start with a slash")
399 logger.debug("Fetching nodeinfo from domain='%s',path='%s' ...", domain, path)
400 data = nodeinfo.fetch(domain, path)
403 logger.debug("data[%s]='%s'", type(data), data)
404 if "exception" in data:
405 # Continue raising it
406 logger.debug("data()=%d contains exception='%s' - raising ...", len(data), type(data["exception"]))
407 raise data["exception"]
408 elif "error_message" in data:
409 logger.debug("Returned error_message during fetching nodeinfo: '%s',status_code=%d", data['error_message'], data['status_code'])
410 software = fetch_generator_from_path(domain)
411 logger.debug("Generator for domain='%s' is: '%s'", domain, software)
413 logger.debug("domain='%s',path='%s',data[json] found ...", domain, path)
416 logger.debug("Auto-detection for domain='%s' was failing, fetching / ...", domain)
417 software = fetch_generator_from_path(domain)
418 logger.debug("Generator for domain='%s' is: '%s'", domain, software)
420 if "status" in data and data["status"] == "error" and "message" in data:
421 logger.warning("JSON response is an error: '%s' - Resetting detection_mode,nodeinfo_url ...", data["message"])
422 instances.set_last_error(domain, data["message"])
423 instances.set_detection_mode(domain, None)
424 instances.set_nodeinfo_url(domain, None)
425 software = fetch_generator_from_path(domain)
426 logger.debug("Generator for domain='%s' is: '%s'", domain, software)
427 elif "software" in data and "name" in data["software"]:
428 logger.debug("Found data[json][software][name] in JSON response")
429 software = data["software"]["name"]
430 logger.debug("software[%s]='%s' - FOUND!", type(software), software)
431 elif "message" in data:
432 logger.warning("JSON response contains only a message: '%s' - Resetting detection_mode,nodeinfo_url ...", data["message"])
433 instances.set_last_error(domain, data["message"])
434 instances.set_detection_mode(domain, None)
435 instances.set_nodeinfo_url(domain, None)
437 logger.debug("Invoking fetch_generator_from_path(%s) ...", domain)
438 software = fetch_generator_from_path(domain)
439 logger.debug("Generator for domain='%s' is: '%s'", domain, software)
440 elif "server" in data and "software" in data["server"]:
441 logger.debug("Found data[server][software]='%s' for domain='%s'", data["server"]["software"].lower(), domain)
442 software = data["server"]["software"].lower()
443 logger.debug("Detected software for domain='%s' is: '%s'", domain, software)
444 elif "software" not in data or "name" not in data["software"]:
445 logger.debug("JSON response from domain='%s' does not include [software][name] - Resetting detection_mode,nodeinfo_url ...", domain)
446 instances.set_detection_mode(domain, None)
447 instances.set_nodeinfo_url(domain, None)
449 logger.debug("Invoking fetch_generator_from_path(%s) ...", domain)
450 software = fetch_generator_from_path(domain)
451 logger.debug("Generator for domain='%s' is: '%s'", domain, software)
453 logger.debug("software[%s]='%s'", type(software), software)
454 if software in [None, ""]:
455 logger.debug("Returning None - EXIT!")
458 logger.debug("Setting original software='%s' for domain='%s' ...", software, domain)
459 instances.set_original_software(domain, software)
461 logger.debug("software='%s'- BEFORE!", software)
462 software = software_helper.alias(software)
463 logger.debug("software['%s']='%s' - AFTER!", type(software), software)
465 if software in [None, ""]:
466 logger.debug("software for domain='%s' was not detected, trying generator ...", domain)
467 software = fetch_generator_from_path(domain)
468 elif len(str(software)) > 0 and ("." in software or " " in software):
469 logger.debug("software='%s' may contain a version number, domain='%s', removing it ...", software, domain)
470 software = version.remove(software)
472 logger.debug("software[]='%s'", type(software))
473 if isinstance(software, str) and "powered by" in software:
474 logger.debug("software='%s' has 'powered by' in it", software)
475 software = version.remove(software_helper.strip_powered_by(software))
477 software = software.strip()
479 logger.debug("software[%s]='%s' - EXIT!", type(software), software)
482 def find_domains(tag: bs4.element.Tag) -> list:
483 logger.debug("tag[]='%s' - CALLED!", type(tag))
485 if not isinstance(tag, bs4.element.Tag):
486 raise ValueError(f"Parameter tag[]='{type(tag)}' is not type of bs4.element.Tag")
487 elif len(tag.select("tr")) == 0:
488 raise KeyError("No table rows found in table!")
491 for element in tag.select("tr"):
492 logger.debug("element[]='%s'", type(element))
493 if not element.find("td"):
494 logger.debug("Skipping element, no <td> found")
497 domain = tidyup.domain(element.find("td").text)
498 reason = tidyup.reason(element.findAll("td")[1].text)
500 logger.debug("domain='%s',reason='%s'", domain, reason)
502 if not domain_helper.is_wanted(domain):
503 logger.debug("domain='%s' is blacklisted - SKIPPED!", domain)
505 elif domain == "gab.com/.ai, develop.gab.com":
506 logger.debug("Multiple gab.com domains detected in one row")
516 "domain": "develop.gab.com",
520 elif not validators.domain(domain.split("/")[0]):
521 logger.warning("domain='%s' is not a valid domain - SKIPPED!", domain)
524 logger.debug("Adding domain='%s',reason='%s' ...", domain, reason)
530 logger.debug("domains()=%d - EXIT!", len(domains))
533 def add_peers(rows: dict) -> list:
534 logger.debug("rows[]='%s' - CALLED!", type(rows))
536 if not isinstance(rows, dict):
537 raise ValueError(f"Parameter rows[]='{type(rows)}' is not of type 'dict'")
539 raise ValueError("Parameter 'rows' is empty")
542 for key in ["linked", "allowed", "blocked"]:
543 logger.debug("key='%s'", key)
544 if key not in rows or rows[key] is None:
545 logger.debug("Cannot find key='%s' or it is NoneType - SKIPPED!", key)
548 logger.debug("Adding %d peer(s) to peers list ...", len(rows[key]))
549 for peer in rows[key]:
550 logger.debug("peer[%s]='%s' - BEFORE!", type(peer), peer)
551 if peer in [None, ""]:
552 logger.debug("peer is empty - SKIPPED")
554 elif isinstance(peer, dict) and "domain" in peer:
555 logger.debug("peer[domain]='%s'", peer["domain"])
556 peer = tidyup.domain(peer["domain"])
557 elif isinstance(peer, str):
558 logger.debug("peer='%s'", peer)
559 peer = tidyup.domain(peer)
561 raise ValueError(f"peer[]='{type(peer)}' is not supported,key='{key}'")
563 logger.debug("peer[%s]='%s' - AFTER!", type(peer), peer)
564 if not domain_helper.is_wanted(peer):
565 logger.debug("peer='%s' is not wanted - SKIPPED!", peer)
568 logger.debug("Appending peer='%s' ...", peer)
571 logger.debug("peers()=%d - EXIT!", len(peers))
574 def fetch_blocks(domain: str) -> list:
575 logger.debug("domain='%s' - CALLED!", domain)
576 domain_helper.raise_on(domain)
578 if blacklist.is_blacklisted(domain):
579 raise Exception(f"domain='{domain}' is blacklisted but function was invoked")
580 elif not instances.is_registered(domain):
581 raise Exception(f"domain='{domain}' is not registered but function is invoked.")
586 # No CSRF by default, you don't have to add network.api_headers by yourself here
590 logger.debug("Checking CSRF for domain='%s'", domain)
591 headers = csrf.determine(domain, dict())
592 except network.exceptions as exception:
593 logger.warning("Exception '%s' during checking CSRF (fetch_blocks,%s)", type(exception), __name__)
594 instances.set_last_error(domain, exception)
596 logger.debug("Returning empty list ... - EXIT!")
600 # json endpoint for newer mastodongs
601 logger.info("Fetching domain_blocks from domain='%s' ...", domain)
602 data = network.get_json_api(
604 "/api/v1/instance/domain_blocks",
606 timeout=(config.get("connection_timeout"), config.get("read_timeout"))
610 logger.debug("data(%d)[]='%s'", len(data), type(data))
611 if "error_message" in data:
612 logger.debug("Was not able to fetch domain_blocks from domain='%s': status_code=%d,error_message='%s'", domain, data['status_code'], data['error_message'])
613 instances.set_last_error(domain, data)
615 logger.debug("blocklist()=%d - EXIT!", len(blocklist))
617 elif "json" in data and "error" in data["json"]:
618 logger.warning("JSON API returned error message: '%s'", data["json"]["error"])
619 instances.set_last_error(domain, data)
621 logger.debug("blocklist()=%d - EXIT!", len(blocklist))
627 logger.debug("Marking domain='%s' as successfully handled ...", domain)
628 instances.set_success(domain)
630 logger.debug("rows(%d)[]='%s'", len(rows), type(rows))
632 logger.debug("Checking %d entries from domain='%s' ...", len(rows), domain)
635 logger.debug("block[]='%s'", type(block))
636 if not isinstance(block, dict):
637 logger.debug("block[]='%s' is of type 'dict' - SKIPPED!", type(block))
639 elif "domain" not in block:
640 logger.warning("block()=%d does not contain element 'domain' - SKIPPED!", len(block))
642 elif "severity" not in block:
643 logger.warning("block()=%d does not contain element 'severity' - SKIPPED!", len(block))
645 elif block["severity"] in ["accept", "accepted"]:
646 logger.debug("block[domain]='%s' has unwanted severity level '%s' - SKIPPED!", block["domain"], block["severity"])
648 elif "digest" in block and not validators.hashes.sha256(block["digest"]):
649 logger.warning("block[domain]='%s' has invalid block[digest]='%s' - SKIPPED!", block["domain"], block["digest"])
652 reason = tidyup.reason(block["comment"]) if "comment" in block and block["comment"] is not None and block["comment"] != "" else None
654 logger.debug("Appending blocker='%s',blocked='%s',reason='%s',block_level='%s' ...", domain, block["domain"], reason, block["severity"])
657 "blocked" : block["domain"],
658 "digest" : block["digest"] if "digest" in block else None,
660 "block_level": blocks.alias_block_level(block["severity"]),
663 logger.debug("domain='%s' has no block list", domain)
665 except network.exceptions as exception:
666 logger.warning("domain='%s',exception[%s]='%s'", domain, type(exception), str(exception))
667 instances.set_last_error(domain, exception)
669 logger.debug("blocklist()=%d - EXIT!", len(blocklist))