1 # Copyright (C) 2023 Free Software Foundation
3 # This program is free software: you can redistribute it and/or modify
4 # it under the terms of the GNU Affero General Public License as published
5 # by the Free Software Foundation, either version 3 of the License, or
6 # (at your option) any later version.
8 # This program is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # GNU Affero General Public License for more details.
13 # You should have received a copy of the GNU Affero General Public License
14 # along with this program. If not, see <https://www.gnu.org/licenses/>.
18 from urllib.parse import urlparse
24 from fba.helpers import config
25 from fba.helpers import cookies
26 from fba.helpers import domain as domain_helper
27 from fba.helpers import software as software_helper
28 from fba.helpers import tidyup
29 from fba.helpers import version
31 from fba.http import csrf
32 from fba.http import network
33 from fba.http import nodeinfo
35 from fba.models import blocks
36 from fba.models import instances
38 from fba.networks import lemmy
39 from fba.networks import misskey
40 from fba.networks import peertube
42 # Depth counter, being raised and lowered
45 logging.basicConfig(level=logging.INFO)
46 logger = logging.getLogger(__name__)
48 def fetch_instances(domain: str, origin: str, software: str, command: str, path: str = None):
50 logger.debug("domain='%s',origin='%s',software='%s',command='%s',path='%s',_DEPTH=%d - CALLED!", domain, origin, software, command, path, _DEPTH)
51 domain_helper.raise_on(domain)
53 if not isinstance(origin, str) and origin is not None:
54 raise ValueError(f"Parameter origin[]='{type(origin)}' is not of type 'str'")
55 elif not isinstance(command, str):
56 raise ValueError(f"Parameter command[]='{type(command)}' is not of type 'str'")
58 raise ValueError("Parameter 'command' is empty")
59 elif command in ["fetch_blocks", "fetch_cs", "fetch_bkali", "fetch_relays", "fetch_fedipact", "fetch_joinmobilizon", "fetch_joinmisskey", "fetch_joinfediverse"] and origin is None:
60 raise ValueError(f"Parameter command='{command}' but origin is None, please fix invoking this function.")
61 elif not isinstance(path, str) and path is not None:
62 raise ValueError(f"Parameter path[]='{type(path)}' is not of type 'str'")
63 elif _DEPTH > 0 and instances.is_recent(domain, "last_instance_fetch"):
64 raise ValueError(f"domain='{domain}' has recently been fetched but function was invoked")
65 elif software is None and not instances.is_recent(domain, "last_nodeinfo"):
67 logger.debug("Software for domain='%s' is not set, determining ...", domain)
68 software = determine_software(domain, path)
69 except network.exceptions as exception:
70 logger.warning("Exception '%s' during determining software type", type(exception))
71 instances.set_last_error(domain, exception)
73 logger.debug("Determined software='%s' for domain='%s'", software, domain)
74 elif software is None:
75 logger.debug("domain='%s' has unknown software or nodeinfo has recently being fetched", domain)
76 elif not isinstance(software, str):
77 raise ValueError(f"Parameter software[]='{type(software)}' is not of type 'str'")
82 logger.debug("Checking if domain='%s' is registered ...", domain)
83 if not instances.is_registered(domain):
84 logger.debug("Adding new domain='%s',origin='%s',command='%s',path='%s',software='%s'", domain, origin, command, path, software)
85 instances.add(domain, origin, command, path, software)
87 logger.debug("Updating last_instance_fetch for domain='%s' ...", domain)
88 instances.set_last_instance_fetch(domain)
91 logger.debug("software='%s'", software)
92 if software is not None:
94 logger.debug("Fetching instances for domain='%s',software='%s',origin='%s'", domain, software, origin)
95 peerlist = fetch_peers(domain, software, origin)
96 except network.exceptions as exception:
97 logger.warning("Cannot fetch peers from domain='%s',software='%s': '%s'", domain, software, type(exception))
99 logger.debug("peerlist[]='%s'", type(peerlist))
100 if isinstance(peerlist, list):
101 logger.debug("Invoking instances.set_total_peerlist(%s,%d) ...", domain, len(peerlist))
102 instances.set_total_peers(domain, peerlist)
104 logger.debug("peerlist[]='%s'", type(peerlist))
105 if peerlist is None or len(peerlist) == 0:
106 logger.warning("Cannot fetch peers: domain='%s',software='%s'", domain, software)
108 if instances.has_pending(domain):
109 logger.debug("Flushing updates for domain='%s' ...", domain)
110 instances.update(domain)
112 logger.debug("Invoking cookies.clear(%s) ...", domain)
113 cookies.clear(domain)
116 logger.debug("EXIT!")
119 logger.info("Checking %d instance(s) from domain='%s',software='%s',depth=%d ...", len(peerlist), domain, software, _DEPTH)
120 for instance in peerlist:
121 logger.debug("instance='%s'", instance)
122 if instance is None or instance == "":
123 logger.debug("instance[%s]='%s' is either None or empty - SKIPPED!", type(instance), instance)
126 logger.debug("instance='%s' - BEFORE!", instance)
127 instance = tidyup.domain(instance)
128 logger.debug("instance='%s' - AFTER!", instance)
131 logger.warning("Empty instance after tidyup.domain(), domain='%s'", domain)
133 elif ".." in instance:
134 logger.warning("instance='%s' contains double-dot, removing ...", instance)
135 instance = instance.replace("..", ".")
137 logger.debug("instance='%s' - BEFORE!", instance)
138 instance = instance.encode("idna").decode("utf-8")
139 logger.debug("instance='%s' - AFTER!", instance)
141 if not domain_helper.is_wanted(instance):
142 logger.debug("instance='%s' is not wanted - SKIPPED!", instance)
144 elif instance.find("/profile/") > 0 or instance.find("/users/") > 0 or (instances.is_registered(instance.split("/")[0]) and instance.find("/c/") > 0):
145 logger.debug("instance='%s' is a link to a single user profile - SKIPPED!", instance)
147 elif instance.find("/tag/") > 0:
148 logger.debug("instance='%s' is a link to a tag - SKIPPED!", instance)
150 elif not instances.is_registered(instance):
151 logger.debug("Checking if domain='%s' has pending updates ...", domain)
152 if instances.has_pending(domain):
153 logger.debug("Flushing updates for domain='%s' ...", domain)
154 instances.update(domain)
156 logger.debug("instance='%s',origin='%s',_DEPTH=%d reached!", instance, origin, _DEPTH)
157 if _DEPTH <= config.get("max_crawl_depth") and len(peerlist) >= config.get("min_peers_length"):
158 logger.debug("Fetching instance='%s',origin='%s',command='%s',path='%s',_DEPTH=%d ...", instance, domain, command, path, _DEPTH)
159 fetch_instances(instance, domain, None, command, path)
161 logger.debug("Adding instance='%s',domain='%s',command='%s',_DEPTH=%d ...", instance, domain, command, _DEPTH)
162 instances.add(instance, domain, command)
164 logger.debug("Invoking cookies.clear(%s) ...", domain)
165 cookies.clear(domain)
167 logger.debug("Checking if domain='%s' has pending updates ...", domain)
168 if instances.has_pending(domain):
169 logger.debug("Flushing updates for domain='%s' ...", domain)
170 instances.update(domain)
173 logger.debug("EXIT!")
175 def fetch_peers(domain: str, software: str, origin: str) -> list:
176 logger.debug("domain='%s',software='%s',origin='%s' - CALLED!", domain, software, origin)
177 domain_helper.raise_on(domain)
179 if not isinstance(software, str) and software is not None:
180 raise ValueError(f"Parameter software[]='{type(software)}' is not of type 'str'")
181 elif software_helper.is_relay(software):
182 raise ValueError(f"domain='{domain}' is of software='{software}' and isn't supported here.")
183 elif not isinstance(origin, str) and origin is not None:
184 raise ValueError(f"Parameter origin[]='{type(origin)}' is not of type 'str'")
185 elif isinstance(origin, str) and origin == "":
186 raise ValueError("Parameter 'origin' is empty")
188 if software == "misskey":
189 logger.debug("Invoking misskey.fetch_peers(%s) ...", domain)
190 return misskey.fetch_peers(domain)
191 elif software == "lemmy":
192 logger.debug("Invoking lemmy.fetch_peers(%s,%s) ...", domain, origin)
193 return lemmy.fetch_peers(domain, origin)
194 elif software == "peertube":
195 logger.debug("Invoking peertube.fetch_peers(%s) ...", domain)
196 return peertube.fetch_peers(domain)
198 # No CSRF by default, you don't have to add network.api_headers by yourself here
202 logger.debug("Checking CSRF for domain='%s'", domain)
203 headers = csrf.determine(domain, dict())
204 except network.exceptions as exception:
205 logger.warning("Exception '%s' during checking CSRF (fetch_peers,%s)", type(exception), __name__)
206 instances.set_last_error(domain, exception)
208 logger.debug("Returning empty list ... - EXIT!")
212 "/api/v1/instance/peers",
216 # Init peers variable
219 logger.debug("Checking %d paths ...", len(paths))
221 logger.debug("Fetching path='%s' from domain='%s',software='%s' ...", path, domain, software)
222 data = network.get_json_api(
226 (config.get("connection_timeout"), config.get("read_timeout"))
229 logger.debug("data[]='%s'", type(data))
230 if "error_message" in data:
231 logger.debug("Was not able to fetch peers from path='%s',domain='%s' ...", path, domain)
232 instances.set_last_error(domain, data)
233 elif "json" in data and len(data["json"]) > 0:
234 logger.debug("Querying API path='%s' was successful: domain='%s',data[json][%s]()=%d", path, domain, type(data['json']), len(data['json']))
237 logger.debug("Marking domain='%s' as successfully handled ...", domain)
238 instances.set_success(domain)
241 if not isinstance(peers, list):
242 logger.warning("peers[]='%s' is not of type 'list', maybe bad API response?", type(peers))
245 logger.debug("Invoking instances.set_total_peers(%s,%d) ...", domain, len(peers))
246 instances.set_total_peers(domain, peers)
248 logger.debug("peers()=%d - EXIT!", len(peers))
251 def fetch_generator_from_path(domain: str, path: str = "/") -> str:
252 logger.debug("domain='%s',path='%s' - CALLED!", domain, path)
253 domain_helper.raise_on(domain)
255 if not isinstance(path, str):
256 raise ValueError(f"path[]='{type(path)}' is not of type 'str'")
258 raise ValueError("Parameter 'path' is empty")
260 logger.debug("domain='%s',path='%s' - CALLED!", domain, path)
263 logger.debug("Fetching path='%s' from domain='%s' ...", path, domain)
264 response = network.fetch_response(
268 (config.get("connection_timeout"), config.get("read_timeout")),
272 logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text))
273 if ((response.ok and response.status_code == 200) or response.status_code == 410) and response.text.find("<html") > 0 and domain_helper.is_in_url(domain, response.url):
274 logger.debug("Parsing response.text()=%d Bytes ...", len(response.text))
275 doc = bs4.BeautifulSoup(response.text, "html.parser")
277 logger.debug("doc[]='%s'", type(doc))
278 platform = doc.find("meta", {"property": "og:platform"})
279 generator = doc.find("meta", {"name" : "generator"})
280 site_name = doc.find("meta", {"property": "og:site_name"})
281 app_name = doc.find("meta", {"name" : "application-name"})
283 logger.debug("generator[]='%s',site_name[]='%s',platform[]='%s',app_name[]='%s'", type(generator), type(site_name), type(platform), type(app_name))
284 if isinstance(platform, bs4.element.Tag) and isinstance(platform.get("content"), str):
285 logger.debug("Found property=og:platform, domain='%s'", domain)
286 software = tidyup.domain(platform.get("content"))
288 logger.debug("software[%s]='%s'", type(software), software)
289 if software is not None and software != "":
290 logger.debug("domain='%s' has og:platform='%s' - Setting detection_mode=PLATFORM ...", domain, software)
291 instances.set_detection_mode(domain, "PLATFORM")
292 elif isinstance(generator, bs4.element.Tag) and isinstance(generator.get("content"), str):
293 logger.debug("Found generator meta tag: domain='%s'", domain)
294 software = tidyup.domain(generator.get("content"))
296 logger.debug("software[%s]='%s'", type(software), software)
297 if software is not None and software != "":
298 logger.info("domain='%s' is generated by software='%s' - Setting detection_mode=GENERATOR ...", domain, software)
299 instances.set_detection_mode(domain, "GENERATOR")
300 elif isinstance(app_name, bs4.element.Tag) and isinstance(app_name.get("content"), str):
301 logger.debug("Found property=og:app_name, domain='%s'", domain)
302 software = tidyup.domain(app_name.get("content"))
304 logger.debug("software[%s]='%s'", type(software), software)
305 if software is not None and software != "":
306 logger.debug("domain='%s' has application-name='%s' - Setting detection_mode=app_name ...", domain, software)
307 instances.set_detection_mode(domain, "APP_NAME")
308 elif isinstance(site_name, bs4.element.Tag) and isinstance(site_name.get("content"), str):
309 logger.debug("Found property=og:site_name, domain='%s'", domain)
310 software = tidyup.domain(site_name.get("content"))
312 logger.debug("software[%s]='%s'", type(software), software)
313 if software is not None and software != "":
314 logger.debug("domain='%s' has og:site_name='%s' - Setting detection_mode=SITE_NAME ...", domain, software)
315 instances.set_detection_mode(domain, "SITE_NAME")
316 elif not domain_helper.is_in_url(domain, response.url):
317 logger.warning("domain='%s' doesn't match response.url='%s', maybe redirect to other domain?", domain, response.url)
319 components = urlparse(response.url)
320 domain2 = components.netloc.lower().split(":")[0]
322 logger.debug("domain2='%s'", domain2)
323 if not instances.is_registered(domain2):
324 logger.info("components.netloc='%s' is not registered, adding ...", components.netloc)
325 fetch_instances(domain2, domain, None, "redirect_target")
327 message = f"Redirect from domain='{domain}' to response.url='{response.url}'"
328 instances.set_last_error(domain, message)
329 instances.set_software(domain, None)
330 instances.set_detection_mode(domain, None)
331 instances.set_nodeinfo_url(domain, None)
333 raise requests.exceptions.TooManyRedirects(message)
335 logger.debug("software[]='%s'", type(software))
336 if isinstance(software, str) and software == "":
337 logger.debug("Corrected empty string to None for software of domain='%s'", domain)
339 elif isinstance(software, str) and ("." in software or " " in software):
340 logger.debug("software='%s' may contain a version number, domain='%s', removing it ...", software, domain)
341 software = version.remove(software)
343 logger.debug("software[]='%s'", type(software))
344 if isinstance(software, str) and "powered by " in software:
345 logger.debug("software='%s' has 'powered by' in it", software)
346 software = version.remove(software_helper.strip_powered_by(software))
347 elif isinstance(software, str) and " hosted on " in software:
348 logger.debug("software='%s' has 'hosted on' in it", software)
349 software = version.remove(software_helper.strip_hosted_on(software))
350 elif isinstance(software, str) and " by " in software:
351 logger.debug("software='%s' has ' by ' in it", software)
352 software = software_helper.strip_until(software, " by ")
353 elif isinstance(software, str) and " see " in software:
354 logger.debug("software='%s' has ' see ' in it", software)
355 software = software_helper.strip_until(software, " see ")
357 logger.debug("software='%s' - EXIT!", software)
360 def determine_software(domain: str, path: str = None) -> str:
361 logger.debug("domain='%s',path='%s' - CALLED!", domain, path)
362 domain_helper.raise_on(domain)
364 if not isinstance(path, str) and path is not None:
365 raise ValueError(f"Parameter path[]='{type(path)}' is not of type 'str'")
367 logger.debug("Fetching nodeinfo from domain='%s',path='%s' ...", domain, path)
368 data = nodeinfo.fetch(domain, path)
371 logger.debug("data[%s]='%s'", type(data), data)
372 if "exception" in data:
373 # Continue raising it
374 logger.debug("data()=%d contains exception='%s' - raising ...", len(data), type(data["exception"]))
375 raise data["exception"]
376 elif "error_message" in data:
377 logger.debug("Returned error_message during fetching nodeinfo: '%s',status_code=%d", data['error_message'], data['status_code'])
378 software = fetch_generator_from_path(domain)
379 logger.debug("Generator for domain='%s' is: '%s'", domain, software)
381 logger.debug("domain='%s',path='%s',data[json] found ...", domain, path)
384 logger.debug("Auto-detection for domain='%s' was failing, fetching / ...", domain)
385 software = fetch_generator_from_path(domain)
386 logger.debug("Generator for domain='%s' is: '%s'", domain, software)
388 if "status" in data and data["status"] == "error" and "message" in data:
389 logger.warning("JSON response is an error: '%s' - Resetting detection_mode,nodeinfo_url ...", data["message"])
390 instances.set_last_error(domain, data["message"])
391 instances.set_detection_mode(domain, None)
392 instances.set_nodeinfo_url(domain, None)
393 software = fetch_generator_from_path(domain)
394 logger.debug("Generator for domain='%s' is: '%s'", domain, software)
395 elif "software" in data and "name" in data["software"]:
396 logger.debug("Found data[json][software][name] in JSON response")
397 software = data["software"]["name"]
398 logger.debug("software[%s]='%s' - FOUND!", type(software), software)
399 elif "message" in data:
400 logger.warning("JSON response contains only a message: '%s' - Resetting detection_mode,nodeinfo_url ...", data["message"])
401 instances.set_last_error(domain, data["message"])
402 instances.set_detection_mode(domain, None)
403 instances.set_nodeinfo_url(domain, None)
405 logger.debug("Invoking fetch_generator_from_path(%s) ...", domain)
406 software = fetch_generator_from_path(domain)
407 logger.debug("Generator for domain='%s' is: '%s'", domain, software)
408 elif "server" in data and "software" in data["server"]:
409 logger.debug("Found data[server][software]='%s' for domain='%s'", data["server"]["software"].lower(), domain)
410 software = data["server"]["software"].lower()
411 logger.debug("Detected software for domain='%s' is: '%s'", domain, software)
412 elif "software" not in data or "name" not in data["software"]:
413 logger.debug("JSON response from domain='%s' does not include [software][name] - Resetting detection_mode,nodeinfo_url ...", domain)
414 instances.set_detection_mode(domain, None)
415 instances.set_nodeinfo_url(domain, None)
417 logger.debug("Invoking fetch_generator_from_path(%s) ...", domain)
418 software = fetch_generator_from_path(domain)
419 logger.debug("Generator for domain='%s' is: '%s'", domain, software)
421 logger.debug("software[%s]='%s'", type(software), software)
423 logger.debug("Returning None - EXIT!")
426 logger.debug("software='%s'- BEFORE!", software)
427 software = software_helper.alias(software)
428 logger.debug("software['%s']='%s' - AFTER!", type(software), software)
430 if str(software) == "":
431 logger.debug("software for domain='%s' was not detected, trying generator ...", domain)
432 software = fetch_generator_from_path(domain)
433 elif len(str(software)) > 0 and ("." in software or " " in software):
434 logger.debug("software='%s' may contain a version number, domain='%s', removing it ...", software, domain)
435 software = version.remove(software)
437 logger.debug("software[]='%s'", type(software))
438 if isinstance(software, str) and "powered by" in software:
439 logger.debug("software='%s' has 'powered by' in it", software)
440 software = version.remove(software_helper.strip_powered_by(software))
442 software = software.strip()
444 logger.debug("software='%s' - EXIT!", software)
447 def find_domains(tag: bs4.element.Tag) -> list:
448 logger.debug("tag[]='%s' - CALLED!", type(tag))
449 if not isinstance(tag, bs4.element.Tag):
450 raise ValueError(f"Parameter tag[]='{type(tag)}' is not type of bs4.element.Tag")
451 elif len(tag.select("tr")) == 0:
452 raise KeyError("No table rows found in table!")
455 for element in tag.select("tr"):
456 logger.debug("element[]='%s'", type(element))
457 if not element.find("td"):
458 logger.debug("Skipping element, no <td> found")
461 domain = tidyup.domain(element.find("td").text)
462 reason = tidyup.reason(element.findAll("td")[1].text)
464 logger.debug("domain='%s',reason='%s'", domain, reason)
466 if not domain_helper.is_wanted(domain):
467 logger.debug("domain='%s' is blacklisted - SKIPPED!", domain)
469 elif domain == "gab.com/.ai, develop.gab.com":
470 logger.debug("Multiple domains detected in one row")
480 "domain": "develop.gab.com",
484 elif not validators.domain(domain.split("/")[0]):
485 logger.warning("domain='%s' is not a valid domain - SKIPPED!", domain)
488 logger.debug("Adding domain='%s',reason='%s' ...", domain, reason)
494 logger.debug("domains()=%d - EXIT!", len(domains))
497 def add_peers(rows: dict) -> list:
498 logger.debug("rows[]='%s' - CALLED!", type(rows))
499 if not isinstance(rows, dict):
500 raise ValueError(f"Parameter rows[]='{type(rows)}' is not of type 'dict'")
503 for key in ["linked", "allowed", "blocked"]:
504 logger.debug("Checking key='%s'", key)
505 if key not in rows or rows[key] is None:
506 logger.debug("Cannot find key='%s' or it is NoneType - SKIPPED!", key)
509 logger.debug("Adding %d peer(s) to peers list ...", len(rows[key]))
510 for peer in rows[key]:
511 logger.debug("peer[%s]='%s' - BEFORE!", type(peer), peer)
512 if peer is None or peer == "":
513 logger.debug("peer is empty - SKIPPED")
515 elif isinstance(peer, dict) and "domain" in peer:
516 logger.debug("peer[domain]='%s'", peer["domain"])
517 peer = tidyup.domain(peer["domain"])
518 elif isinstance(peer, str):
519 logger.debug("peer='%s'", peer)
520 peer = tidyup.domain(peer)
522 raise ValueError(f"peer[]='{type(peer)}' is not supported,key='{key}'")
524 logger.debug("peer[%s]='%s' - AFTER!", type(peer), peer)
525 if not domain_helper.is_wanted(peer):
526 logger.debug("peer='%s' is not wanted - SKIPPED!", peer)
529 logger.debug("Appending peer='%s' ...", peer)
532 logger.debug("peers()=%d - EXIT!", len(peers))
535 def fetch_blocks(domain: str) -> list:
536 logger.debug("domain='%s' - CALLED!", domain)
537 domain_helper.raise_on(domain)
539 if not instances.is_registered(domain):
540 raise Exception(f"domain='{domain}' is not registered but function is invoked.")
545 # No CSRF by default, you don't have to add network.api_headers by yourself here
549 logger.debug("Checking CSRF for domain='%s'", domain)
550 headers = csrf.determine(domain, dict())
551 except network.exceptions as exception:
552 logger.warning("Exception '%s' during checking CSRF (fetch_blocks,%s)", type(exception), __name__)
553 instances.set_last_error(domain, exception)
555 logger.debug("Returning empty list ... - EXIT!")
559 # json endpoint for newer mastodongs
560 logger.info("Fetching domain_blocks from domain='%s' ...", domain)
561 data = network.get_json_api(
563 "/api/v1/instance/domain_blocks",
565 (config.get("connection_timeout"), config.get("read_timeout"))
569 logger.debug("data[]='%s'", type(data))
570 if "error_message" in data:
571 logger.debug("Was not able to fetch domain_blocks from domain='%s': status_code=%d,error_message='%s'", domain, data['status_code'], data['error_message'])
572 instances.set_last_error(domain, data)
574 elif "json" in data and "error" in data["json"]:
575 logger.warning("JSON API returned error message: '%s'", data["json"]["error"])
576 instances.set_last_error(domain, data)
582 logger.debug("Marking domain='%s' as successfully handled ...", domain)
583 instances.set_success(domain)
585 logger.debug("rows[%s]()=%d", type(rows), len(rows))
587 logger.debug("Checking %d entries from domain='%s' ...", len(rows), domain)
590 logger.debug("block[]='%s'", type(block))
591 if not isinstance(block, dict):
592 logger.debug("block[]='%s' is of type 'dict' - SKIPPED!", type(block))
594 elif "domain" not in block:
595 logger.warning("block()=%d does not contain element 'domain' - SKIPPED!", len(block))
597 elif "severity" not in block:
598 logger.warning("block()=%d does not contain element 'severity' - SKIPPED!", len(block))
600 elif block["severity"] in ["accept", "accepted"]:
601 logger.debug("block[domain]='%s' has unwanted severity level '%s' - SKIPPED!", block["domain"], block["severity"])
603 elif "digest" in block and not validators.hashes.sha256(block["digest"]):
604 logger.warning("block[domain]='%s' has invalid block[digest]='%s' - SKIPPED!", block["domain"], block["digest"])
607 reason = tidyup.reason(block["comment"]) if "comment" in block and block["comment"] is not None and block["comment"] != "" else None
609 logger.debug("Appending blocker='%s',blocked='%s',reason='%s',block_level='%s'", domain, block["domain"], reason, block["severity"])
612 "blocked" : block["domain"],
613 "hash" : block["digest"] if "digest" in block else None,
615 "block_level": blocks.alias_block_level(block["severity"]),
618 logger.debug("domain='%s' has no block list", domain)
620 except network.exceptions as exception:
621 logger.warning("domain='%s',exception[%s]='%s'", domain, type(exception), str(exception))
622 instances.set_last_error(domain, exception)
624 logger.debug("blocklist()=%d - EXIT!", len(blocklist))