1 # Copyright (C) 2023 Free Software Foundation
3 # This program is free software: you can redistribute it and/or modify
4 # it under the terms of the GNU Affero General Public License as published
5 # by the Free Software Foundation, either version 3 of the License, or
6 # (at your option) any later version.
8 # This program is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # GNU Affero General Public License for more details.
13 # You should have received a copy of the GNU Affero General Public License
14 # along with this program. If not, see <https://www.gnu.org/licenses/>.
18 from urllib.parse import urlparse
24 from fba.helpers import config
25 from fba.helpers import cookies
26 from fba.helpers import domain as domain_helper
27 from fba.helpers import software as software_helper
28 from fba.helpers import tidyup
29 from fba.helpers import version
31 from fba.http import csrf
32 from fba.http import network
33 from fba.http import nodeinfo
35 from fba.models import blocks
36 from fba.models import instances
38 from fba.networks import lemmy
39 from fba.networks import misskey
40 from fba.networks import peertube
42 # Depth counter, being raised and lowered
45 logging.basicConfig(level=logging.INFO)
46 logger = logging.getLogger(__name__)
48 def fetch_instances(domain: str, origin: str, software: str, command: str, path: str = None):
50 logger.debug("domain='%s',origin='%s',software='%s',command='%s',path='%s',_DEPTH=%d - CALLED!", domain, origin, software, command, path, _DEPTH)
51 domain_helper.raise_on(domain)
53 if not isinstance(origin, str) and origin is not None:
54 raise ValueError(f"Parameter origin[]='{type(origin)}' is not of type 'str'")
55 elif not isinstance(command, str):
56 raise ValueError(f"Parameter command[]='{type(command)}' is not of type 'str'")
58 raise ValueError("Parameter 'command' is empty")
59 elif command in ["fetch_blocks", "fetch_cs", "fetch_bkali", "fetch_relays", "fetch_fedipact", "fetch_joinmobilizon", "fetch_joinmisskey", "fetch_joinfediverse"] and origin is None:
60 raise ValueError(f"Parameter command='{command}' but origin is None, please fix invoking this function.")
61 elif not isinstance(path, str) and path is not None:
62 raise ValueError(f"Parameter path[]='{type(path)}' is not of type 'str'")
63 elif _DEPTH > 0 and instances.is_recent(domain, "last_instance_fetch"):
64 raise ValueError(f"domain='{domain}' has recently been fetched but function was invoked")
65 elif software is None and not instances.is_recent(domain, "last_nodeinfo"):
67 logger.debug("Software for domain='%s' is not set, determining ...", domain)
68 software = determine_software(domain, path)
69 except network.exceptions as exception:
70 logger.warning("Exception '%s' during determining software type", type(exception))
71 instances.set_last_error(domain, exception)
73 logger.debug("Determined software='%s' for domain='%s'", software, domain)
74 elif software is None:
75 logger.debug("domain='%s' has unknown software or nodeinfo has recently being fetched", domain)
76 elif not isinstance(software, str):
77 raise ValueError(f"Parameter software[]='{type(software)}' is not of type 'str'")
82 logger.debug("Checking if domain='%s' is registered ...", domain)
83 if not instances.is_registered(domain):
84 logger.debug("Adding new domain='%s',origin='%s',command='%s',path='%s',software='%s'", domain, origin, command, path, software)
85 instances.add(domain, origin, command, path, software)
87 logger.debug("software='%s'", software)
88 if software_helper.is_relay(software):
89 logger.debug("software='%s' is a relay software - EXIT!", software)
92 logger.debug("Updating last_instance_fetch for domain='%s' ...", domain)
93 instances.set_last_instance_fetch(domain)
96 logger.debug("software='%s'", software)
97 if software is not None:
99 logger.debug("Fetching instances for domain='%s',software='%s',origin='%s'", domain, software, origin)
100 peerlist = fetch_peers(domain, software, origin)
101 except network.exceptions as exception:
102 logger.warning("Cannot fetch peers from domain='%s',software='%s': '%s'", domain, software, type(exception))
104 logger.debug("peerlist[]='%s'", type(peerlist))
105 if isinstance(peerlist, list):
106 logger.debug("Invoking instances.set_total_peerlist(%s,%d) ...", domain, len(peerlist))
107 instances.set_total_peers(domain, peerlist)
109 logger.debug("peerlist[]='%s'", type(peerlist))
110 if peerlist is None or len(peerlist) == 0:
111 logger.warning("Cannot fetch peers: domain='%s',software='%s'", domain, software)
113 if instances.has_pending(domain):
114 logger.debug("Flushing updates for domain='%s' ...", domain)
115 instances.update(domain)
117 logger.debug("Invoking cookies.clear(%s) ...", domain)
118 cookies.clear(domain)
121 logger.debug("EXIT!")
124 logger.info("Checking %d instance(s) from domain='%s',software='%s',depth=%d ...", len(peerlist), domain, software, _DEPTH)
125 for instance in peerlist:
126 logger.debug("instance='%s'", instance)
127 if instance is None or instance == "":
128 logger.debug("instance[%s]='%s' is either None or empty - SKIPPED!", type(instance), instance)
131 logger.debug("instance='%s' - BEFORE!", instance)
132 instance = tidyup.domain(instance)
133 logger.debug("instance='%s' - AFTER!", instance)
136 logger.warning("Empty instance after tidyup.domain(), domain='%s'", domain)
138 elif ".." in instance:
139 logger.warning("instance='%s' contains double-dot, removing ...", instance)
140 instance = instance.replace("..", ".")
142 logger.debug("instance='%s' - BEFORE!", instance)
143 instance = instance.encode("idna").decode("utf-8")
144 logger.debug("instance='%s' - AFTER!", instance)
146 if not domain_helper.is_wanted(instance):
147 logger.debug("instance='%s' is not wanted - SKIPPED!", instance)
149 elif instance.find("/profile/") > 0 or instance.find("/users/") > 0 or (instances.is_registered(instance.split("/")[0]) and instance.find("/c/") > 0):
150 logger.debug("instance='%s' is a link to a single user profile - SKIPPED!", instance)
152 elif instance.find("/tag/") > 0:
153 logger.debug("instance='%s' is a link to a tag - SKIPPED!", instance)
155 elif not instances.is_registered(instance):
156 logger.debug("Checking if domain='%s' has pending updates ...", domain)
157 if instances.has_pending(domain):
158 logger.debug("Flushing updates for domain='%s' ...", domain)
159 instances.update(domain)
161 logger.debug("instance='%s',origin='%s',_DEPTH=%d reached!", instance, origin, _DEPTH)
162 if _DEPTH <= config.get("max_crawl_depth") and len(peerlist) >= config.get("min_peers_length"):
163 logger.debug("Fetching instance='%s',origin='%s',command='%s',path='%s',_DEPTH=%d ...", instance, domain, command, path, _DEPTH)
164 fetch_instances(instance, domain, None, command, path)
166 logger.debug("Adding instance='%s',domain='%s',command='%s',_DEPTH=%d ...", instance, domain, command, _DEPTH)
167 instances.add(instance, domain, command)
169 logger.debug("Invoking cookies.clear(%s) ...", domain)
170 cookies.clear(domain)
172 logger.debug("Checking if domain='%s' has pending updates ...", domain)
173 if instances.has_pending(domain):
174 logger.debug("Flushing updates for domain='%s' ...", domain)
175 instances.update(domain)
178 logger.debug("EXIT!")
180 def fetch_peers(domain: str, software: str, origin: str) -> list:
181 logger.debug("domain='%s',software='%s',origin='%s' - CALLED!", domain, software, origin)
182 domain_helper.raise_on(domain)
184 if not isinstance(software, str) and software is not None:
185 raise ValueError(f"Parameter software[]='{type(software)}' is not of type 'str'")
186 elif software_helper.is_relay(software):
187 raise ValueError(f"domain='{domain}' is of software='{software}' and isn't supported here.")
188 elif not isinstance(origin, str) and origin is not None:
189 raise ValueError(f"Parameter origin[]='{type(origin)}' is not of type 'str'")
190 elif isinstance(origin, str) and origin == "":
191 raise ValueError("Parameter 'origin' is empty")
193 if software == "misskey":
194 logger.debug("Invoking misskey.fetch_peers(%s) ...", domain)
195 return misskey.fetch_peers(domain)
196 elif software == "lemmy":
197 logger.debug("Invoking lemmy.fetch_peers(%s,%s) ...", domain, origin)
198 return lemmy.fetch_peers(domain, origin)
199 elif software == "peertube":
200 logger.debug("Invoking peertube.fetch_peers(%s) ...", domain)
201 return peertube.fetch_peers(domain)
203 # No CSRF by default, you don't have to add network.api_headers by yourself here
207 logger.debug("Checking CSRF for domain='%s'", domain)
208 headers = csrf.determine(domain, dict())
209 except network.exceptions as exception:
210 logger.warning("Exception '%s' during checking CSRF (fetch_peers,%s)", type(exception), __name__)
211 instances.set_last_error(domain, exception)
213 logger.debug("Returning empty list ... - EXIT!")
217 "/api/v1/instance/peers",
221 # Init peers variable
224 logger.debug("Checking %d paths ...", len(paths))
226 logger.debug("Fetching path='%s' from domain='%s',software='%s' ...", path, domain, software)
227 data = network.get_json_api(
231 (config.get("connection_timeout"), config.get("read_timeout"))
234 logger.debug("data[]='%s'", type(data))
235 if "error_message" in data:
236 logger.debug("Was not able to fetch peers from path='%s',domain='%s' ...", path, domain)
237 instances.set_last_error(domain, data)
238 elif "json" in data and len(data["json"]) > 0:
239 logger.debug("Querying API path='%s' was successful: domain='%s',data[json][%s]()=%d", path, domain, type(data['json']), len(data['json']))
242 logger.debug("Marking domain='%s' as successfully handled ...", domain)
243 instances.set_success(domain)
246 if not isinstance(peers, list):
247 logger.warning("peers[]='%s' is not of type 'list', maybe bad API response?", type(peers))
250 logger.debug("Invoking instances.set_total_peers(%s,%d) ...", domain, len(peers))
251 instances.set_total_peers(domain, peers)
253 logger.debug("peers()=%d - EXIT!", len(peers))
256 def fetch_generator_from_path(domain: str, path: str = "/") -> str:
257 logger.debug("domain='%s',path='%s' - CALLED!", domain, path)
258 domain_helper.raise_on(domain)
260 if not isinstance(path, str):
261 raise ValueError(f"path[]='{type(path)}' is not of type 'str'")
263 raise ValueError("Parameter 'path' is empty")
265 logger.debug("domain='%s',path='%s' - CALLED!", domain, path)
268 logger.debug("Fetching path='%s' from domain='%s' ...", path, domain)
269 response = network.fetch_response(
273 (config.get("connection_timeout"), config.get("read_timeout")),
277 logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text))
278 if ((response.ok and response.status_code == 200) or response.status_code == 410) and response.text.find("<html") > 0 and domain_helper.is_in_url(domain, response.url):
279 logger.debug("Parsing response.text()=%d Bytes ...", len(response.text))
280 doc = bs4.BeautifulSoup(response.text, "html.parser")
282 logger.debug("doc[]='%s'", type(doc))
283 platform = doc.find("meta", {"property": "og:platform"})
284 generator = doc.find("meta", {"name" : "generator"})
285 site_name = doc.find("meta", {"property": "og:site_name"})
286 app_name = doc.find("meta", {"name" : "application-name"})
288 logger.debug("generator[]='%s',site_name[]='%s',platform[]='%s',app_name[]='%s'", type(generator), type(site_name), type(platform), type(app_name))
289 if isinstance(platform, bs4.element.Tag) and isinstance(platform.get("content"), str):
290 logger.debug("Found property=og:platform, domain='%s'", domain)
291 software = tidyup.domain(platform.get("content"))
293 logger.debug("software[%s]='%s'", type(software), software)
294 if software is not None and software != "":
295 logger.debug("domain='%s' has og:platform='%s' - Setting detection_mode=PLATFORM ...", domain, software)
296 instances.set_detection_mode(domain, "PLATFORM")
297 elif isinstance(generator, bs4.element.Tag) and isinstance(generator.get("content"), str):
298 logger.debug("Found generator meta tag: domain='%s'", domain)
299 software = tidyup.domain(generator.get("content"))
301 logger.debug("software[%s]='%s'", type(software), software)
302 if software is not None and software != "":
303 logger.info("domain='%s' is generated by software='%s' - Setting detection_mode=GENERATOR ...", domain, software)
304 instances.set_detection_mode(domain, "GENERATOR")
305 elif isinstance(app_name, bs4.element.Tag) and isinstance(app_name.get("content"), str):
306 logger.debug("Found property=og:app_name, domain='%s'", domain)
307 software = tidyup.domain(app_name.get("content"))
309 logger.debug("software[%s]='%s'", type(software), software)
310 if software is not None and software != "":
311 logger.debug("domain='%s' has application-name='%s' - Setting detection_mode=app_name ...", domain, software)
312 instances.set_detection_mode(domain, "APP_NAME")
313 elif isinstance(site_name, bs4.element.Tag) and isinstance(site_name.get("content"), str):
314 logger.debug("Found property=og:site_name, domain='%s'", domain)
315 software = tidyup.domain(site_name.get("content"))
317 logger.debug("software[%s]='%s'", type(software), software)
318 if software is not None and software != "":
319 logger.debug("domain='%s' has og:site_name='%s' - Setting detection_mode=SITE_NAME ...", domain, software)
320 instances.set_detection_mode(domain, "SITE_NAME")
321 elif not domain_helper.is_in_url(domain, response.url):
322 logger.warning("domain='%s' doesn't match response.url='%s', maybe redirect to other domain?", domain, response.url)
324 components = urlparse(response.url)
325 domain2 = components.netloc.lower().split(":")[0]
327 logger.debug("domain2='%s'", domain2)
328 if not domain_helper.is_wanted(domain2):
329 logger.debug("domain2='%s' is not wanted - EXIT!", domain2)
331 elif not instances.is_registered(domain2):
332 logger.info("components.netloc='%s' is not registered, adding ...", components.netloc)
333 instances.add(domain2, domain, "redirect_target")
335 message = f"Redirect from domain='{domain}' to response.url='{response.url}'"
336 instances.set_last_error(domain, message)
337 instances.set_software(domain, None)
338 instances.set_detection_mode(domain, None)
339 instances.set_nodeinfo_url(domain, None)
341 raise requests.exceptions.TooManyRedirects(message)
343 logger.debug("software[]='%s'", type(software))
344 if isinstance(software, str) and software == "":
345 logger.debug("Corrected empty string to None for software of domain='%s'", domain)
347 elif isinstance(software, str) and ("." in software or " " in software):
348 logger.debug("software='%s' may contain a version number, domain='%s', removing it ...", software, domain)
349 software = version.remove(software)
351 logger.debug("software[]='%s'", type(software))
352 if isinstance(software, str) and "powered by " in software:
353 logger.debug("software='%s' has 'powered by' in it", software)
354 software = version.remove(software_helper.strip_powered_by(software))
355 elif isinstance(software, str) and " hosted on " in software:
356 logger.debug("software='%s' has 'hosted on' in it", software)
357 software = version.remove(software_helper.strip_hosted_on(software))
358 elif isinstance(software, str) and " by " in software:
359 logger.debug("software='%s' has ' by ' in it", software)
360 software = software_helper.strip_until(software, " by ")
361 elif isinstance(software, str) and " see " in software:
362 logger.debug("software='%s' has ' see ' in it", software)
363 software = software_helper.strip_until(software, " see ")
365 logger.debug("software='%s' - EXIT!", software)
368 def determine_software(domain: str, path: str = None) -> str:
369 logger.debug("domain='%s',path='%s' - CALLED!", domain, path)
370 domain_helper.raise_on(domain)
372 if not isinstance(path, str) and path is not None:
373 raise ValueError(f"Parameter path[]='{type(path)}' is not of type 'str'")
375 logger.debug("Fetching nodeinfo from domain='%s',path='%s' ...", domain, path)
376 data = nodeinfo.fetch(domain, path)
379 logger.debug("data[%s]='%s'", type(data), data)
380 if "exception" in data:
381 # Continue raising it
382 logger.debug("data()=%d contains exception='%s' - raising ...", len(data), type(data["exception"]))
383 raise data["exception"]
384 elif "error_message" in data:
385 logger.debug("Returned error_message during fetching nodeinfo: '%s',status_code=%d", data['error_message'], data['status_code'])
386 software = fetch_generator_from_path(domain)
387 logger.debug("Generator for domain='%s' is: '%s'", domain, software)
389 logger.debug("domain='%s',path='%s',data[json] found ...", domain, path)
392 logger.debug("Auto-detection for domain='%s' was failing, fetching / ...", domain)
393 software = fetch_generator_from_path(domain)
394 logger.debug("Generator for domain='%s' is: '%s'", domain, software)
396 if "status" in data and data["status"] == "error" and "message" in data:
397 logger.warning("JSON response is an error: '%s' - Resetting detection_mode,nodeinfo_url ...", data["message"])
398 instances.set_last_error(domain, data["message"])
399 instances.set_detection_mode(domain, None)
400 instances.set_nodeinfo_url(domain, None)
401 software = fetch_generator_from_path(domain)
402 logger.debug("Generator for domain='%s' is: '%s'", domain, software)
403 elif "software" in data and "name" in data["software"]:
404 logger.debug("Found data[json][software][name] in JSON response")
405 software = data["software"]["name"]
406 logger.debug("software[%s]='%s' - FOUND!", type(software), software)
407 elif "message" in data:
408 logger.warning("JSON response contains only a message: '%s' - Resetting detection_mode,nodeinfo_url ...", data["message"])
409 instances.set_last_error(domain, data["message"])
410 instances.set_detection_mode(domain, None)
411 instances.set_nodeinfo_url(domain, None)
413 logger.debug("Invoking fetch_generator_from_path(%s) ...", domain)
414 software = fetch_generator_from_path(domain)
415 logger.debug("Generator for domain='%s' is: '%s'", domain, software)
416 elif "server" in data and "software" in data["server"]:
417 logger.debug("Found data[server][software]='%s' for domain='%s'", data["server"]["software"].lower(), domain)
418 software = data["server"]["software"].lower()
419 logger.debug("Detected software for domain='%s' is: '%s'", domain, software)
420 elif "software" not in data or "name" not in data["software"]:
421 logger.debug("JSON response from domain='%s' does not include [software][name] - Resetting detection_mode,nodeinfo_url ...", domain)
422 instances.set_detection_mode(domain, None)
423 instances.set_nodeinfo_url(domain, None)
425 logger.debug("Invoking fetch_generator_from_path(%s) ...", domain)
426 software = fetch_generator_from_path(domain)
427 logger.debug("Generator for domain='%s' is: '%s'", domain, software)
429 logger.debug("software[%s]='%s'", type(software), software)
431 logger.debug("Returning None - EXIT!")
434 logger.debug("software='%s'- BEFORE!", software)
435 software = software_helper.alias(software)
436 logger.debug("software['%s']='%s' - AFTER!", type(software), software)
438 if str(software) == "":
439 logger.debug("software for domain='%s' was not detected, trying generator ...", domain)
440 software = fetch_generator_from_path(domain)
441 elif len(str(software)) > 0 and ("." in software or " " in software):
442 logger.debug("software='%s' may contain a version number, domain='%s', removing it ...", software, domain)
443 software = version.remove(software)
445 logger.debug("software[]='%s'", type(software))
446 if isinstance(software, str) and "powered by" in software:
447 logger.debug("software='%s' has 'powered by' in it", software)
448 software = version.remove(software_helper.strip_powered_by(software))
450 software = software.strip()
452 logger.debug("software='%s' - EXIT!", software)
455 def find_domains(tag: bs4.element.Tag) -> list:
456 logger.debug("tag[]='%s' - CALLED!", type(tag))
457 if not isinstance(tag, bs4.element.Tag):
458 raise ValueError(f"Parameter tag[]='{type(tag)}' is not type of bs4.element.Tag")
459 elif len(tag.select("tr")) == 0:
460 raise KeyError("No table rows found in table!")
463 for element in tag.select("tr"):
464 logger.debug("element[]='%s'", type(element))
465 if not element.find("td"):
466 logger.debug("Skipping element, no <td> found")
469 domain = tidyup.domain(element.find("td").text)
470 reason = tidyup.reason(element.findAll("td")[1].text)
472 logger.debug("domain='%s',reason='%s'", domain, reason)
474 if not domain_helper.is_wanted(domain):
475 logger.debug("domain='%s' is blacklisted - SKIPPED!", domain)
477 elif domain == "gab.com/.ai, develop.gab.com":
478 logger.debug("Multiple domains detected in one row")
488 "domain": "develop.gab.com",
492 elif not validators.domain(domain.split("/")[0]):
493 logger.warning("domain='%s' is not a valid domain - SKIPPED!", domain)
496 logger.debug("Adding domain='%s',reason='%s' ...", domain, reason)
502 logger.debug("domains()=%d - EXIT!", len(domains))
505 def add_peers(rows: dict) -> list:
506 logger.debug("rows[]='%s' - CALLED!", type(rows))
507 if not isinstance(rows, dict):
508 raise ValueError(f"Parameter rows[]='{type(rows)}' is not of type 'dict'")
511 for key in ["linked", "allowed", "blocked"]:
512 logger.debug("Checking key='%s'", key)
513 if key not in rows or rows[key] is None:
514 logger.debug("Cannot find key='%s' or it is NoneType - SKIPPED!", key)
517 logger.debug("Adding %d peer(s) to peers list ...", len(rows[key]))
518 for peer in rows[key]:
519 logger.debug("peer[%s]='%s' - BEFORE!", type(peer), peer)
520 if peer is None or peer == "":
521 logger.debug("peer is empty - SKIPPED")
523 elif isinstance(peer, dict) and "domain" in peer:
524 logger.debug("peer[domain]='%s'", peer["domain"])
525 peer = tidyup.domain(peer["domain"])
526 elif isinstance(peer, str):
527 logger.debug("peer='%s'", peer)
528 peer = tidyup.domain(peer)
530 raise ValueError(f"peer[]='{type(peer)}' is not supported,key='{key}'")
532 logger.debug("peer[%s]='%s' - AFTER!", type(peer), peer)
533 if not domain_helper.is_wanted(peer):
534 logger.debug("peer='%s' is not wanted - SKIPPED!", peer)
537 logger.debug("Appending peer='%s' ...", peer)
540 logger.debug("peers()=%d - EXIT!", len(peers))
543 def fetch_blocks(domain: str) -> list:
544 logger.debug("domain='%s' - CALLED!", domain)
545 domain_helper.raise_on(domain)
547 if not instances.is_registered(domain):
548 raise Exception(f"domain='{domain}' is not registered but function is invoked.")
553 # No CSRF by default, you don't have to add network.api_headers by yourself here
557 logger.debug("Checking CSRF for domain='%s'", domain)
558 headers = csrf.determine(domain, dict())
559 except network.exceptions as exception:
560 logger.warning("Exception '%s' during checking CSRF (fetch_blocks,%s)", type(exception), __name__)
561 instances.set_last_error(domain, exception)
563 logger.debug("Returning empty list ... - EXIT!")
567 # json endpoint for newer mastodongs
568 logger.info("Fetching domain_blocks from domain='%s' ...", domain)
569 data = network.get_json_api(
571 "/api/v1/instance/domain_blocks",
573 (config.get("connection_timeout"), config.get("read_timeout"))
577 logger.debug("data[]='%s'", type(data))
578 if "error_message" in data:
579 logger.debug("Was not able to fetch domain_blocks from domain='%s': status_code=%d,error_message='%s'", domain, data['status_code'], data['error_message'])
580 instances.set_last_error(domain, data)
582 elif "json" in data and "error" in data["json"]:
583 logger.warning("JSON API returned error message: '%s'", data["json"]["error"])
584 instances.set_last_error(domain, data)
590 logger.debug("Marking domain='%s' as successfully handled ...", domain)
591 instances.set_success(domain)
593 logger.debug("rows[%s]()=%d", type(rows), len(rows))
595 logger.debug("Checking %d entries from domain='%s' ...", len(rows), domain)
598 logger.debug("block[]='%s'", type(block))
599 if not isinstance(block, dict):
600 logger.debug("block[]='%s' is of type 'dict' - SKIPPED!", type(block))
602 elif "domain" not in block:
603 logger.warning("block()=%d does not contain element 'domain' - SKIPPED!", len(block))
605 elif "severity" not in block:
606 logger.warning("block()=%d does not contain element 'severity' - SKIPPED!", len(block))
608 elif block["severity"] in ["accept", "accepted"]:
609 logger.debug("block[domain]='%s' has unwanted severity level '%s' - SKIPPED!", block["domain"], block["severity"])
611 elif "digest" in block and not validators.hashes.sha256(block["digest"]):
612 logger.warning("block[domain]='%s' has invalid block[digest]='%s' - SKIPPED!", block["domain"], block["digest"])
615 reason = tidyup.reason(block["comment"]) if "comment" in block and block["comment"] is not None and block["comment"] != "" else None
617 logger.debug("Appending blocker='%s',blocked='%s',reason='%s',block_level='%s'", domain, block["domain"], reason, block["severity"])
620 "blocked" : block["domain"],
621 "digest" : block["digest"] if "digest" in block else None,
623 "block_level": blocks.alias_block_level(block["severity"]),
626 logger.debug("domain='%s' has no block list", domain)
628 except network.exceptions as exception:
629 logger.warning("domain='%s',exception[%s]='%s'", domain, type(exception), str(exception))
630 instances.set_last_error(domain, exception)
632 logger.debug("blocklist()=%d - EXIT!", len(blocklist))