1 # Copyright (C) 2023 Free Software Foundation
3 # This program is free software: you can redistribute it and/or modify
4 # it under the terms of the GNU Affero General Public License as published
5 # by the Free Software Foundation, either version 3 of the License, or
6 # (at your option) any later version.
8 # This program is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # GNU Affero General Public License for more details.
13 # You should have received a copy of the GNU Affero General Public License
14 # along with this program. If not, see <https://www.gnu.org/licenses/>.
18 from urllib.parse import urlparse
24 from fba.helpers import config
25 from fba.helpers import cookies
26 from fba.helpers import domain as domain_helper
27 from fba.helpers import software as software_helper
28 from fba.helpers import tidyup
29 from fba.helpers import version
31 from fba.http import csrf
32 from fba.http import network
33 from fba.http import nodeinfo
35 from fba.models import blocks
36 from fba.models import instances
38 from fba.networks import lemmy
39 from fba.networks import misskey
40 from fba.networks import peertube
42 # Depth counter, being raised and lowered
45 logging.basicConfig(level=logging.INFO)
46 logger = logging.getLogger(__name__)
48 def fetch_instances(domain: str, origin: str, software: str, command: str, path: str = None):
50 logger.debug("domain='%s',origin='%s',software='%s',command='%s',path='%s',_DEPTH=%d - CALLED!", domain, origin, software, command, path, _DEPTH)
51 domain_helper.raise_on(domain)
53 if not isinstance(origin, str) and origin is not None:
54 raise ValueError(f"Parameter origin[]='{type(origin)}' is not of type 'str'")
55 elif not isinstance(command, str):
56 raise ValueError(f"Parameter command[]='{type(command)}' is not of type 'str'")
58 raise ValueError("Parameter 'command' is empty")
59 elif command in ["fetch_blocks", "fetch_cs", "fetch_bkali", "fetch_relays", "fetch_fedipact", "fetch_joinmobilizon", "fetch_joinmisskey", "fetch_joinfediverse"] and origin is None:
60 raise ValueError(f"Parameter command='{command}' but origin is None, please fix invoking this function.")
61 elif not isinstance(path, str) and path is not None:
62 raise ValueError(f"Parameter path[]='{type(path)}' is not of type 'str'")
63 elif _DEPTH > 0 and instances.is_recent(domain, "last_instance_fetch"):
64 raise ValueError(f"domain='{domain}' has recently been fetched but function was invoked")
65 elif software is None and not instances.is_recent(domain, "last_nodeinfo"):
67 logger.debug("Software for domain='%s' is not set, determining ...", domain)
68 software = determine_software(domain, path)
69 except network.exceptions as exception:
70 logger.warning("Exception '%s' during determining software type", type(exception))
71 instances.set_last_error(domain, exception)
73 logger.debug("Determined software='%s' for domain='%s'", software, domain)
74 elif software is None:
75 logger.debug("domain='%s' has unknown software or nodeinfo has recently being fetched", domain)
76 elif not isinstance(software, str):
77 raise ValueError(f"Parameter software[]='{type(software)}' is not of type 'str'")
82 logger.debug("Checking if domain='%s' is registered ...", domain)
83 if not instances.is_registered(domain):
84 logger.debug("Adding new domain='%s',origin='%s',command='%s',path='%s',software='%s'", domain, origin, command, path, software)
85 instances.add(domain, origin, command, path, software)
87 logger.debug("Updating last_instance_fetch for domain='%s' ...", domain)
88 instances.set_last_instance_fetch(domain)
91 logger.debug("software='%s'", software)
92 if software is not None:
94 logger.debug("Fetching instances for domain='%s',software='%s',origin='%s'", domain, software, origin)
95 peerlist = fetch_peers(domain, software, origin)
96 except network.exceptions as exception:
97 logger.warning("Cannot fetch peers from domain='%s',software='%s': '%s'", domain, software, type(exception))
99 logger.debug("peerlist[]='%s'", type(peerlist))
100 if isinstance(peerlist, list):
101 logger.debug("Invoking instances.set_total_peerlist(%s,%d) ...", domain, len(peerlist))
102 instances.set_total_peers(domain, peerlist)
104 logger.debug("peerlist[]='%s'", type(peerlist))
105 if peerlist is None or len(peerlist) == 0:
106 logger.warning("Cannot fetch peers: domain='%s',software='%s'", domain, software)
108 if instances.has_pending(domain):
109 logger.debug("Flushing updates for domain='%s' ...", domain)
110 instances.update(domain)
112 logger.debug("Invoking cookies.clear(%s) ...", domain)
113 cookies.clear(domain)
116 logger.debug("EXIT!")
119 logger.info("Checking %d instance(s) from domain='%s',software='%s',depth=%d ...", len(peerlist), domain, software, _DEPTH)
120 for instance in peerlist:
121 logger.debug("instance='%s'", instance)
122 if instance is None or instance == "":
123 logger.debug("instance[%s]='%s' is either None or empty - SKIPPED!", type(instance), instance)
126 logger.debug("instance='%s' - BEFORE!", instance)
127 instance = tidyup.domain(instance)
128 logger.debug("instance='%s' - AFTER!", instance)
131 logger.warning("Empty instance after tidyup.domain(), domain='%s'", domain)
133 elif ".." in instance:
134 logger.warning("instance='%s' contains double-dot, removing ...", instance)
135 instance = instance.replace("..", ".")
137 logger.debug("instance='%s' - BEFORE!", instance)
138 instance = instance.encode("idna").decode("utf-8")
139 logger.debug("instance='%s' - AFTER!", instance)
141 if not domain_helper.is_wanted(instance):
142 logger.debug("instance='%s' is not wanted - SKIPPED!", instance)
144 elif instance.find("/profile/") > 0 or instance.find("/users/") > 0 or (instances.is_registered(instance.split("/")[0]) and instance.find("/c/") > 0):
145 logger.debug("instance='%s' is a link to a single user profile - SKIPPED!", instance)
147 elif instance.find("/tag/") > 0:
148 logger.debug("instance='%s' is a link to a tag - SKIPPED!", instance)
150 elif not instances.is_registered(instance):
151 logger.debug("Checking if domain='%s' has pending updates ...", domain)
152 if instances.has_pending(domain):
153 logger.debug("Flushing updates for domain='%s' ...", domain)
154 instances.update(domain)
156 logger.debug("instance='%s',origin='%s',_DEPTH=%d reached!", instance, origin, _DEPTH)
157 if _DEPTH <= config.get("max_crawl_depth") and len(peerlist) >= config.get("min_peers_length"):
158 logger.debug("Fetching instance='%s',origin='%s',command='%s',path='%s',_DEPTH=%d ...", instance, domain, command, path, _DEPTH)
159 fetch_instances(instance, domain, None, command, path)
161 logger.debug("Adding instance='%s',domain='%s',command='%s',_DEPTH=%d ...", instance, domain, command, _DEPTH)
162 instances.add(instance, domain, command)
164 logger.debug("Invoking cookies.clear(%s) ...", domain)
165 cookies.clear(domain)
167 logger.debug("Checking if domain='%s' has pending updates ...", domain)
168 if instances.has_pending(domain):
169 logger.debug("Flushing updates for domain='%s' ...", domain)
170 instances.update(domain)
173 logger.debug("EXIT!")
175 def fetch_peers(domain: str, software: str, origin: str) -> list:
176 logger.debug("domain='%s',software='%s',origin='%s' - CALLED!", domain, software, origin)
177 domain_helper.raise_on(domain)
179 if not isinstance(software, str) and software is not None:
180 raise ValueError(f"Parameter software[]='{type(software)}' is not of type 'str'")
181 elif software_helper.is_relay(software):
182 raise ValueError(f"domain='{domain}' is of software='{software}' and isn't supported here.")
183 elif not isinstance(origin, str) and origin is not None:
184 raise ValueError(f"Parameter origin[]='{type(origin)}' is not of type 'str'")
185 elif isinstance(origin, str) and origin == "":
186 raise ValueError("Parameter 'origin' is empty")
188 if software == "misskey":
189 logger.debug("Invoking misskey.fetch_peers(%s) ...", domain)
190 return misskey.fetch_peers(domain)
191 elif software == "lemmy":
192 logger.debug("Invoking lemmy.fetch_peers(%s,%s) ...", domain, origin)
193 return lemmy.fetch_peers(domain, origin)
194 elif software == "peertube":
195 logger.debug("Invoking peertube.fetch_peers(%s) ...", domain)
196 return peertube.fetch_peers(domain)
198 # No CSRF by default, you don't have to add network.api_headers by yourself here
202 logger.debug("Checking CSRF for domain='%s'", domain)
203 headers = csrf.determine(domain, dict())
204 except network.exceptions as exception:
205 logger.warning("Exception '%s' during checking CSRF (fetch_peers,%s)", type(exception), __name__)
206 instances.set_last_error(domain, exception)
208 logger.debug("Returning empty list ... - EXIT!")
212 "/api/v1/instance/peers",
216 # Init peers variable
219 logger.debug("Checking %d paths ...", len(paths))
221 logger.debug("Fetching path='%s' from domain='%s',software='%s' ...", path, domain, software)
222 data = network.get_json_api(
226 (config.get("connection_timeout"), config.get("read_timeout"))
229 logger.debug("data[]='%s'", type(data))
230 if "error_message" in data:
231 logger.debug("Was not able to fetch peers from path='%s',domain='%s' ...", path, domain)
232 instances.set_last_error(domain, data)
233 elif "json" in data and len(data["json"]) > 0:
234 logger.debug("Querying API path='%s' was successful: domain='%s',data[json][%s]()=%d", path, domain, type(data['json']), len(data['json']))
237 logger.debug("Marking domain='%s' as successfully handled ...", domain)
238 instances.set_success(domain)
241 if not isinstance(peers, list):
242 logger.warning("peers[]='%s' is not of type 'list', maybe bad API response?", type(peers))
245 logger.debug("Invoking instances.set_total_peers(%s,%d) ...", domain, len(peers))
246 instances.set_total_peers(domain, peers)
248 logger.debug("peers()=%d - EXIT!", len(peers))
251 def fetch_generator_from_path(domain: str, path: str = "/") -> str:
252 logger.debug("domain='%s',path='%s' - CALLED!", domain, path)
253 domain_helper.raise_on(domain)
255 if not isinstance(path, str):
256 raise ValueError(f"path[]='{type(path)}' is not of type 'str'")
258 raise ValueError("Parameter 'path' is empty")
260 logger.debug("domain='%s',path='%s' - CALLED!", domain, path)
263 logger.debug("Fetching path='%s' from domain='%s' ...", path, domain)
264 response = network.fetch_response(
268 (config.get("connection_timeout"), config.get("read_timeout")),
272 logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text))
273 if ((response.ok and response.status_code == 200) or response.status_code == 410) and response.text.find("<html") > 0 and domain_helper.is_in_url(domain, response.url):
274 logger.debug("Parsing response.text()=%d Bytes ...", len(response.text))
275 doc = bs4.BeautifulSoup(response.text, "html.parser")
277 logger.debug("doc[]='%s'", type(doc))
278 platform = doc.find("meta", {"property": "og:platform"})
279 generator = doc.find("meta", {"name" : "generator"})
280 site_name = doc.find("meta", {"property": "og:site_name"})
281 app_name = doc.find("meta", {"name" : "application-name"})
283 logger.debug("generator[]='%s',site_name[]='%s',platform[]='%s',app_name[]='%s'", type(generator), type(site_name), type(platform), type(app_name))
284 if isinstance(platform, bs4.element.Tag) and isinstance(platform.get("content"), str):
285 logger.debug("Found property=og:platform, domain='%s'", domain)
286 software = tidyup.domain(platform.get("content"))
288 logger.debug("software[%s]='%s'", type(software), software)
289 if software is not None and software != "":
290 logger.debug("domain='%s' has og:platform='%s' - Setting detection_mode=PLATFORM ...", domain, software)
291 instances.set_detection_mode(domain, "PLATFORM")
292 elif isinstance(generator, bs4.element.Tag) and isinstance(generator.get("content"), str):
293 logger.debug("Found generator meta tag: domain='%s'", domain)
294 software = tidyup.domain(generator.get("content"))
296 logger.debug("software[%s]='%s'", type(software), software)
297 if software is not None and software != "":
298 logger.info("domain='%s' is generated by software='%s' - Setting detection_mode=GENERATOR ...", domain, software)
299 instances.set_detection_mode(domain, "GENERATOR")
300 elif isinstance(app_name, bs4.element.Tag) and isinstance(app_name.get("content"), str):
301 logger.debug("Found property=og:app_name, domain='%s'", domain)
302 software = tidyup.domain(app_name.get("content"))
304 logger.debug("software[%s]='%s'", type(software), software)
305 if software is not None and software != "":
306 logger.debug("domain='%s' has application-name='%s' - Setting detection_mode=app_name ...", domain, software)
307 instances.set_detection_mode(domain, "APP_NAME")
308 elif isinstance(site_name, bs4.element.Tag) and isinstance(site_name.get("content"), str):
309 logger.debug("Found property=og:site_name, domain='%s'", domain)
310 software = tidyup.domain(site_name.get("content"))
312 logger.debug("software[%s]='%s'", type(software), software)
313 if software is not None and software != "":
314 logger.debug("domain='%s' has og:site_name='%s' - Setting detection_mode=SITE_NAME ...", domain, software)
315 instances.set_detection_mode(domain, "SITE_NAME")
316 elif not domain_helper.is_in_url(domain, response.url):
317 logger.warning("domain='%s' doesn't match response.url='%s', maybe redirect to other domain?", domain, response.url)
319 components = urlparse(response.url)
320 domain2 = components.netloc.lower().split(":")[0]
322 logger.debug("domain2='%s'", domain2)
323 if not domain_helper.is_wanted(domain2):
324 logger.debug("domain2='%s' is not wanted - EXIT!", domain2)
326 elif not instances.is_registered(domain2):
327 logger.info("components.netloc='%s' is not registered, adding ...", components.netloc)
328 instances.add(domain2, domain, "redirect_target")
330 message = f"Redirect from domain='{domain}' to response.url='{response.url}'"
331 instances.set_last_error(domain, message)
332 instances.set_software(domain, None)
333 instances.set_detection_mode(domain, None)
334 instances.set_nodeinfo_url(domain, None)
336 raise requests.exceptions.TooManyRedirects(message)
338 logger.debug("software[]='%s'", type(software))
339 if isinstance(software, str) and software == "":
340 logger.debug("Corrected empty string to None for software of domain='%s'", domain)
342 elif isinstance(software, str) and ("." in software or " " in software):
343 logger.debug("software='%s' may contain a version number, domain='%s', removing it ...", software, domain)
344 software = version.remove(software)
346 logger.debug("software[]='%s'", type(software))
347 if isinstance(software, str) and "powered by " in software:
348 logger.debug("software='%s' has 'powered by' in it", software)
349 software = version.remove(software_helper.strip_powered_by(software))
350 elif isinstance(software, str) and " hosted on " in software:
351 logger.debug("software='%s' has 'hosted on' in it", software)
352 software = version.remove(software_helper.strip_hosted_on(software))
353 elif isinstance(software, str) and " by " in software:
354 logger.debug("software='%s' has ' by ' in it", software)
355 software = software_helper.strip_until(software, " by ")
356 elif isinstance(software, str) and " see " in software:
357 logger.debug("software='%s' has ' see ' in it", software)
358 software = software_helper.strip_until(software, " see ")
360 logger.debug("software='%s' - EXIT!", software)
363 def determine_software(domain: str, path: str = None) -> str:
364 logger.debug("domain='%s',path='%s' - CALLED!", domain, path)
365 domain_helper.raise_on(domain)
367 if not isinstance(path, str) and path is not None:
368 raise ValueError(f"Parameter path[]='{type(path)}' is not of type 'str'")
370 logger.debug("Fetching nodeinfo from domain='%s',path='%s' ...", domain, path)
371 data = nodeinfo.fetch(domain, path)
374 logger.debug("data[%s]='%s'", type(data), data)
375 if "exception" in data:
376 # Continue raising it
377 logger.debug("data()=%d contains exception='%s' - raising ...", len(data), type(data["exception"]))
378 raise data["exception"]
379 elif "error_message" in data:
380 logger.debug("Returned error_message during fetching nodeinfo: '%s',status_code=%d", data['error_message'], data['status_code'])
381 software = fetch_generator_from_path(domain)
382 logger.debug("Generator for domain='%s' is: '%s'", domain, software)
384 logger.debug("domain='%s',path='%s',data[json] found ...", domain, path)
387 logger.debug("Auto-detection for domain='%s' was failing, fetching / ...", domain)
388 software = fetch_generator_from_path(domain)
389 logger.debug("Generator for domain='%s' is: '%s'", domain, software)
391 if "status" in data and data["status"] == "error" and "message" in data:
392 logger.warning("JSON response is an error: '%s' - Resetting detection_mode,nodeinfo_url ...", data["message"])
393 instances.set_last_error(domain, data["message"])
394 instances.set_detection_mode(domain, None)
395 instances.set_nodeinfo_url(domain, None)
396 software = fetch_generator_from_path(domain)
397 logger.debug("Generator for domain='%s' is: '%s'", domain, software)
398 elif "software" in data and "name" in data["software"]:
399 logger.debug("Found data[json][software][name] in JSON response")
400 software = data["software"]["name"]
401 logger.debug("software[%s]='%s' - FOUND!", type(software), software)
402 elif "message" in data:
403 logger.warning("JSON response contains only a message: '%s' - Resetting detection_mode,nodeinfo_url ...", data["message"])
404 instances.set_last_error(domain, data["message"])
405 instances.set_detection_mode(domain, None)
406 instances.set_nodeinfo_url(domain, None)
408 logger.debug("Invoking fetch_generator_from_path(%s) ...", domain)
409 software = fetch_generator_from_path(domain)
410 logger.debug("Generator for domain='%s' is: '%s'", domain, software)
411 elif "server" in data and "software" in data["server"]:
412 logger.debug("Found data[server][software]='%s' for domain='%s'", data["server"]["software"].lower(), domain)
413 software = data["server"]["software"].lower()
414 logger.debug("Detected software for domain='%s' is: '%s'", domain, software)
415 elif "software" not in data or "name" not in data["software"]:
416 logger.debug("JSON response from domain='%s' does not include [software][name] - Resetting detection_mode,nodeinfo_url ...", domain)
417 instances.set_detection_mode(domain, None)
418 instances.set_nodeinfo_url(domain, None)
420 logger.debug("Invoking fetch_generator_from_path(%s) ...", domain)
421 software = fetch_generator_from_path(domain)
422 logger.debug("Generator for domain='%s' is: '%s'", domain, software)
424 logger.debug("software[%s]='%s'", type(software), software)
426 logger.debug("Returning None - EXIT!")
429 logger.debug("software='%s'- BEFORE!", software)
430 software = software_helper.alias(software)
431 logger.debug("software['%s']='%s' - AFTER!", type(software), software)
433 if str(software) == "":
434 logger.debug("software for domain='%s' was not detected, trying generator ...", domain)
435 software = fetch_generator_from_path(domain)
436 elif len(str(software)) > 0 and ("." in software or " " in software):
437 logger.debug("software='%s' may contain a version number, domain='%s', removing it ...", software, domain)
438 software = version.remove(software)
440 logger.debug("software[]='%s'", type(software))
441 if isinstance(software, str) and "powered by" in software:
442 logger.debug("software='%s' has 'powered by' in it", software)
443 software = version.remove(software_helper.strip_powered_by(software))
445 software = software.strip()
447 logger.debug("software='%s' - EXIT!", software)
450 def find_domains(tag: bs4.element.Tag) -> list:
451 logger.debug("tag[]='%s' - CALLED!", type(tag))
452 if not isinstance(tag, bs4.element.Tag):
453 raise ValueError(f"Parameter tag[]='{type(tag)}' is not type of bs4.element.Tag")
454 elif len(tag.select("tr")) == 0:
455 raise KeyError("No table rows found in table!")
458 for element in tag.select("tr"):
459 logger.debug("element[]='%s'", type(element))
460 if not element.find("td"):
461 logger.debug("Skipping element, no <td> found")
464 domain = tidyup.domain(element.find("td").text)
465 reason = tidyup.reason(element.findAll("td")[1].text)
467 logger.debug("domain='%s',reason='%s'", domain, reason)
469 if not domain_helper.is_wanted(domain):
470 logger.debug("domain='%s' is blacklisted - SKIPPED!", domain)
472 elif domain == "gab.com/.ai, develop.gab.com":
473 logger.debug("Multiple domains detected in one row")
483 "domain": "develop.gab.com",
487 elif not validators.domain(domain.split("/")[0]):
488 logger.warning("domain='%s' is not a valid domain - SKIPPED!", domain)
491 logger.debug("Adding domain='%s',reason='%s' ...", domain, reason)
497 logger.debug("domains()=%d - EXIT!", len(domains))
500 def add_peers(rows: dict) -> list:
501 logger.debug("rows[]='%s' - CALLED!", type(rows))
502 if not isinstance(rows, dict):
503 raise ValueError(f"Parameter rows[]='{type(rows)}' is not of type 'dict'")
506 for key in ["linked", "allowed", "blocked"]:
507 logger.debug("Checking key='%s'", key)
508 if key not in rows or rows[key] is None:
509 logger.debug("Cannot find key='%s' or it is NoneType - SKIPPED!", key)
512 logger.debug("Adding %d peer(s) to peers list ...", len(rows[key]))
513 for peer in rows[key]:
514 logger.debug("peer[%s]='%s' - BEFORE!", type(peer), peer)
515 if peer is None or peer == "":
516 logger.debug("peer is empty - SKIPPED")
518 elif isinstance(peer, dict) and "domain" in peer:
519 logger.debug("peer[domain]='%s'", peer["domain"])
520 peer = tidyup.domain(peer["domain"])
521 elif isinstance(peer, str):
522 logger.debug("peer='%s'", peer)
523 peer = tidyup.domain(peer)
525 raise ValueError(f"peer[]='{type(peer)}' is not supported,key='{key}'")
527 logger.debug("peer[%s]='%s' - AFTER!", type(peer), peer)
528 if not domain_helper.is_wanted(peer):
529 logger.debug("peer='%s' is not wanted - SKIPPED!", peer)
532 logger.debug("Appending peer='%s' ...", peer)
535 logger.debug("peers()=%d - EXIT!", len(peers))
538 def fetch_blocks(domain: str) -> list:
539 logger.debug("domain='%s' - CALLED!", domain)
540 domain_helper.raise_on(domain)
542 if not instances.is_registered(domain):
543 raise Exception(f"domain='{domain}' is not registered but function is invoked.")
548 # No CSRF by default, you don't have to add network.api_headers by yourself here
552 logger.debug("Checking CSRF for domain='%s'", domain)
553 headers = csrf.determine(domain, dict())
554 except network.exceptions as exception:
555 logger.warning("Exception '%s' during checking CSRF (fetch_blocks,%s)", type(exception), __name__)
556 instances.set_last_error(domain, exception)
558 logger.debug("Returning empty list ... - EXIT!")
562 # json endpoint for newer mastodongs
563 logger.info("Fetching domain_blocks from domain='%s' ...", domain)
564 data = network.get_json_api(
566 "/api/v1/instance/domain_blocks",
568 (config.get("connection_timeout"), config.get("read_timeout"))
572 logger.debug("data[]='%s'", type(data))
573 if "error_message" in data:
574 logger.debug("Was not able to fetch domain_blocks from domain='%s': status_code=%d,error_message='%s'", domain, data['status_code'], data['error_message'])
575 instances.set_last_error(domain, data)
577 elif "json" in data and "error" in data["json"]:
578 logger.warning("JSON API returned error message: '%s'", data["json"]["error"])
579 instances.set_last_error(domain, data)
585 logger.debug("Marking domain='%s' as successfully handled ...", domain)
586 instances.set_success(domain)
588 logger.debug("rows[%s]()=%d", type(rows), len(rows))
590 logger.debug("Checking %d entries from domain='%s' ...", len(rows), domain)
593 logger.debug("block[]='%s'", type(block))
594 if not isinstance(block, dict):
595 logger.debug("block[]='%s' is of type 'dict' - SKIPPED!", type(block))
597 elif "domain" not in block:
598 logger.warning("block()=%d does not contain element 'domain' - SKIPPED!", len(block))
600 elif "severity" not in block:
601 logger.warning("block()=%d does not contain element 'severity' - SKIPPED!", len(block))
603 elif block["severity"] in ["accept", "accepted"]:
604 logger.debug("block[domain]='%s' has unwanted severity level '%s' - SKIPPED!", block["domain"], block["severity"])
606 elif "digest" in block and not validators.hashes.sha256(block["digest"]):
607 logger.warning("block[domain]='%s' has invalid block[digest]='%s' - SKIPPED!", block["domain"], block["digest"])
610 reason = tidyup.reason(block["comment"]) if "comment" in block and block["comment"] is not None and block["comment"] != "" else None
612 logger.debug("Appending blocker='%s',blocked='%s',reason='%s',block_level='%s'", domain, block["domain"], reason, block["severity"])
615 "blocked" : block["domain"],
616 "hash" : block["digest"] if "digest" in block else None,
618 "block_level": blocks.alias_block_level(block["severity"]),
621 logger.debug("domain='%s' has no block list", domain)
623 except network.exceptions as exception:
624 logger.warning("domain='%s',exception[%s]='%s'", domain, type(exception), str(exception))
625 instances.set_last_error(domain, exception)
627 logger.debug("blocklist()=%d - EXIT!", len(blocklist))