1 # Copyright (C) 2023 Free Software Foundation
3 # This program is free software: you can redistribute it and/or modify
4 # it under the terms of the GNU Affero General Public License as published
5 # by the Free Software Foundation, either version 3 of the License, or
6 # (at your option) any later version.
8 # This program is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # GNU Affero General Public License for more details.
13 # You should have received a copy of the GNU Affero General Public License
14 # along with this program. If not, see <https://www.gnu.org/licenses/>.
18 from urllib.parse import urlparse
24 from fba.helpers import config
25 from fba.helpers import cookies
26 from fba.helpers import domain as domain_helper
27 from fba.helpers import software as software_helper
28 from fba.helpers import tidyup
29 from fba.helpers import version
31 from fba.http import csrf
32 from fba.http import network
33 from fba.http import nodeinfo
35 from fba.models import blocks
36 from fba.models import instances
38 from fba.networks import lemmy
39 from fba.networks import misskey
40 from fba.networks import peertube
42 # Depth counter, being raised and lowered
45 logging.basicConfig(level=logging.INFO)
46 logger = logging.getLogger(__name__)
48 def fetch_instances(domain: str, origin: str, software: str, command: str, path: str = None):
50 logger.debug("domain='%s',origin='%s',software='%s',command='%s',path='%s',_DEPTH=%d - CALLED!", domain, origin, software, command, path, _DEPTH)
51 domain_helper.raise_on(domain)
53 if not isinstance(origin, str) and origin is not None:
54 raise ValueError(f"Parameter origin[]='{type(origin)}' is not of type 'str'")
55 elif not isinstance(command, str):
56 raise ValueError(f"Parameter command[]='{type(command)}' is not of type 'str'")
58 raise ValueError("Parameter 'command' is empty")
59 elif command in ["fetch_blocks", "fetch_cs", "fetch_bkali", "fetch_relays", "fetch_fedipact", "fetch_joinmobilizon", "fetch_joinmisskey", "fetch_joinfediverse", "fetch_relaylist"] and origin is None:
60 raise ValueError(f"Parameter command='{command}' but origin is None, please fix invoking this function.")
61 elif not isinstance(path, str) and path is not None:
62 raise ValueError(f"Parameter path[]='{type(path)}' is not of type 'str'")
63 elif _DEPTH > 0 and instances.is_recent(domain, "last_instance_fetch"):
64 raise ValueError(f"domain='{domain}' has recently been fetched but function was invoked")
65 elif software is None and not instances.is_recent(domain, "last_nodeinfo"):
67 logger.debug("Software for domain='%s' is not set, determining ...", domain)
68 software = determine_software(domain, path)
69 except network.exceptions as exception:
70 logger.warning("Exception '%s' during determining software type", type(exception))
71 instances.set_last_error(domain, exception)
73 logger.debug("Determined software='%s' for domain='%s'", software, domain)
74 elif software is None:
75 logger.debug("domain='%s' has unknown software or nodeinfo has recently being fetched", domain)
76 elif not isinstance(software, str):
77 raise ValueError(f"Parameter software[]='{type(software)}' is not of type 'str'")
82 logger.debug("Checking if domain='%s' is registered ...", domain)
83 if not instances.is_registered(domain):
84 logger.debug("Adding new domain='%s',origin='%s',command='%s',path='%s',software='%s'", domain, origin, command, path, software)
85 instances.add(domain, origin, command, path, software)
87 logger.debug("software='%s'", software)
88 if software_helper.is_relay(software):
89 logger.debug("software='%s' is a relay software - EXIT!", software)
92 logger.debug("Updating last_instance_fetch for domain='%s' ...", domain)
93 instances.set_last_instance_fetch(domain)
96 logger.debug("software='%s'", software)
97 if software is not None:
99 logger.debug("Fetching instances for domain='%s',software='%s',origin='%s'", domain, software, origin)
100 peerlist = fetch_peers(domain, software, origin)
101 except network.exceptions as exception:
102 logger.warning("Cannot fetch peers from domain='%s',software='%s': '%s'", domain, software, type(exception))
104 logger.debug("peerlist[]='%s'", type(peerlist))
105 if isinstance(peerlist, list):
106 logger.debug("Invoking instances.set_total_peerlist(%s,%d) ...", domain, len(peerlist))
107 instances.set_total_peers(domain, peerlist)
109 logger.debug("peerlist[]='%s'", type(peerlist))
111 logger.warning("Cannot fetch peers: domain='%s',software='%s'", domain, software)
113 if instances.has_pending(domain):
114 logger.debug("Flushing updates for domain='%s' ...", domain)
115 instances.update(domain)
117 logger.debug("Invoking cookies.clear(%s) ...", domain)
118 cookies.clear(domain)
121 logger.debug("EXIT!")
123 elif len(peerlist) == 0:
124 logger.debug("domain='%s',software='%s' has an empty peer list returned - EXIT!", domain, software)
127 logger.info("Checking %d instance(s) from domain='%s',software='%s',depth=%d ...", len(peerlist), domain, software, _DEPTH)
128 for instance in peerlist:
129 logger.debug("instance='%s'", instance)
130 if instance is None or instance == "":
131 logger.debug("instance[%s]='%s' is either None or empty - SKIPPED!", type(instance), instance)
134 logger.debug("instance='%s' - BEFORE!", instance)
135 instance = tidyup.domain(instance) if isinstance(instance, str) and instance != "" else None
136 logger.debug("instance='%s' - AFTER!", instance)
138 if instance is None or instance == "":
139 logger.warning("instance='%s' is empty after tidyup.domain(), domain='%s'", instance, domain)
141 elif ".." in instance:
142 logger.warning("instance='%s' contains double-dot, removing ...", instance)
143 instance = instance.replace("..", ".")
145 logger.debug("instance='%s' - BEFORE!", instance)
146 instance = instance.encode("idna").decode("utf-8")
147 logger.debug("instance='%s' - AFTER!", instance)
149 if not domain_helper.is_wanted(instance):
150 logger.debug("instance='%s' is not wanted - SKIPPED!", instance)
152 elif instance.find("/profile/") > 0 or instance.find("/users/") > 0 or (instances.is_registered(instance.split("/")[0]) and instance.find("/c/") > 0):
153 logger.debug("instance='%s' is a link to a single user profile - SKIPPED!", instance)
155 elif instance.find("/tag/") > 0:
156 logger.debug("instance='%s' is a link to a tag - SKIPPED!", instance)
158 elif not instances.is_registered(instance):
159 logger.debug("Checking if domain='%s' has pending updates ...", domain)
160 if instances.has_pending(domain):
161 logger.debug("Flushing updates for domain='%s' ...", domain)
162 instances.update(domain)
164 logger.debug("instance='%s',origin='%s',_DEPTH=%d reached!", instance, origin, _DEPTH)
165 if _DEPTH <= config.get("max_crawl_depth") and len(peerlist) >= config.get("min_peers_length"):
166 logger.debug("Fetching instance='%s',origin='%s',command='%s',path='%s',_DEPTH=%d ...", instance, domain, command, path, _DEPTH)
167 fetch_instances(instance, domain, None, command, path)
169 logger.debug("Adding instance='%s',domain='%s',command='%s',_DEPTH=%d ...", instance, domain, command, _DEPTH)
170 instances.add(instance, domain, command)
172 logger.debug("Invoking cookies.clear(%s) ...", domain)
173 cookies.clear(domain)
175 logger.debug("Checking if domain='%s' has pending updates ...", domain)
176 if instances.has_pending(domain):
177 logger.debug("Flushing updates for domain='%s' ...", domain)
178 instances.update(domain)
181 logger.debug("EXIT!")
183 def fetch_peers(domain: str, software: str, origin: str) -> list:
184 logger.debug("domain='%s',software='%s',origin='%s' - CALLED!", domain, software, origin)
185 domain_helper.raise_on(domain)
187 if not isinstance(software, str) and software is not None:
188 raise ValueError(f"Parameter software[]='{type(software)}' is not of type 'str'")
189 elif isinstance(software, str) and software == "":
190 raise ValueError("Parameter 'software' is empty")
191 elif software_helper.is_relay(software):
192 raise ValueError(f"domain='{domain}' is of software='{software}' and isn't supported here.")
193 elif not isinstance(origin, str) and origin is not None:
194 raise ValueError(f"Parameter origin[]='{type(origin)}' is not of type 'str'")
195 elif isinstance(origin, str) and origin == "":
196 raise ValueError("Parameter 'origin' is empty")
198 if software == "misskey":
199 logger.debug("Invoking misskey.fetch_peers(%s) ...", domain)
200 return misskey.fetch_peers(domain)
201 elif software == "lemmy":
202 logger.debug("Invoking lemmy.fetch_peers(%s,%s) ...", domain, origin)
203 return lemmy.fetch_peers(domain, origin)
204 elif software == "peertube":
205 logger.debug("Invoking peertube.fetch_peers(%s) ...", domain)
206 return peertube.fetch_peers(domain)
208 # No CSRF by default, you don't have to add network.api_headers by yourself here
212 logger.debug("Checking CSRF for domain='%s'", domain)
213 headers = csrf.determine(domain, dict())
214 except network.exceptions as exception:
215 logger.warning("Exception '%s' during checking CSRF (fetch_peers,%s)", type(exception), __name__)
216 instances.set_last_error(domain, exception)
218 logger.debug("Returning empty list ... - EXIT!")
222 "/api/v1/instance/peers",
226 # Init peers variable
229 logger.debug("Checking %d paths ...", len(paths))
231 logger.debug("Fetching path='%s' from domain='%s',software='%s' ...", path, domain, software)
232 data = network.get_json_api(
236 timeout=(config.get("connection_timeout"), config.get("read_timeout"))
239 logger.debug("data[]='%s'", type(data))
240 if "error_message" in data:
241 logger.debug("Was not able to fetch peers from path='%s',domain='%s' ...", path, domain)
242 instances.set_last_error(domain, data)
243 elif "json" in data and len(data["json"]) > 0:
244 logger.debug("Querying API path='%s' was successful: domain='%s',data[json][%s]()=%d", path, domain, type(data['json']), len(data['json']))
247 logger.debug("Marking domain='%s' as successfully handled ...", domain)
248 instances.set_success(domain)
251 if not isinstance(peers, list):
252 logger.warning("peers[]='%s' is not of type 'list', maybe bad API response?", type(peers))
255 logger.debug("Invoking instances.set_total_peers(%s,%d) ...", domain, len(peers))
256 instances.set_total_peers(domain, peers)
258 logger.debug("peers()=%d - EXIT!", len(peers))
261 def fetch_generator_from_path(domain: str, path: str = "/") -> str:
262 logger.debug("domain='%s',path='%s' - CALLED!", domain, path)
263 domain_helper.raise_on(domain)
265 if not isinstance(path, str):
266 raise ValueError(f"path[]='{type(path)}' is not of type 'str'")
268 raise ValueError("Parameter 'path' is empty")
272 logger.debug("Fetching path='%s' from domain='%s' ...", path, domain)
273 response = network.fetch_response(
276 headers=network.web_headers,
277 timeout=(config.get("connection_timeout"), config.get("read_timeout")),
281 logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text))
282 if ((response.ok and response.status_code == 200) or response.status_code == 410) and response.text.find("<html") > 0 and domain_helper.is_in_url(domain, response.url):
283 logger.debug("Parsing response.text()=%d Bytes ...", len(response.text))
284 doc = bs4.BeautifulSoup(response.text, "html.parser")
286 logger.debug("doc[]='%s'", type(doc))
287 platform = doc.find("meta", {"property": "og:platform"})
288 generator = doc.find("meta", {"name" : "generator"})
289 site_name = doc.find("meta", {"property": "og:site_name"})
290 app_name = doc.find("meta", {"name" : "application-name"})
292 logger.debug("generator[]='%s',site_name[]='%s',platform[]='%s',app_name[]='%s'", type(generator), type(site_name), type(platform), type(app_name))
293 if isinstance(platform, bs4.element.Tag) and isinstance(platform.get("content"), str) and platform.get("content") != "":
294 logger.debug("Found property=og:platform, domain='%s'", domain)
295 software = tidyup.domain(platform.get("content"))
297 logger.debug("software[%s]='%s'", type(software), software)
298 if software is not None and software != "":
299 logger.debug("domain='%s' has og:platform='%s' - Setting detection_mode=PLATFORM ...", domain, software)
300 instances.set_detection_mode(domain, "PLATFORM")
301 elif isinstance(generator, bs4.element.Tag) and isinstance(generator.get("content"), str) and generator.get("content") != "":
302 logger.debug("Found generator meta tag: domain='%s'", domain)
303 software = tidyup.domain(generator.get("content"))
305 logger.debug("software[%s]='%s'", type(software), software)
306 if software is not None and software != "":
307 logger.info("domain='%s' is generated by software='%s' - Setting detection_mode=GENERATOR ...", domain, software)
308 instances.set_detection_mode(domain, "GENERATOR")
309 elif isinstance(app_name, bs4.element.Tag) and isinstance(app_name.get("content"), str) and app_name.get("content") != "":
310 logger.debug("Found property=og:app_name, domain='%s'", domain)
311 software = tidyup.domain(app_name.get("content"))
313 logger.debug("software[%s]='%s'", type(software), software)
314 if software is not None and software != "":
315 logger.debug("domain='%s' has application-name='%s' - Setting detection_mode=app_name ...", domain, software)
316 instances.set_detection_mode(domain, "APP_NAME")
317 elif isinstance(site_name, bs4.element.Tag) and isinstance(site_name.get("content"), str) and site_name.get("content") != "":
318 logger.debug("Found property=og:site_name, domain='%s'", domain)
319 software = tidyup.domain(site_name.get("content"))
321 logger.debug("software[%s]='%s'", type(software), software)
322 if software is not None and software != "":
323 logger.debug("domain='%s' has og:site_name='%s' - Setting detection_mode=SITE_NAME ...", domain, software)
324 instances.set_detection_mode(domain, "SITE_NAME")
325 elif not domain_helper.is_in_url(domain, response.url):
326 logger.warning("domain='%s' doesn't match response.url='%s', maybe redirect to other domain?", domain, response.url)
328 components = urlparse(response.url)
329 domain2 = components.netloc.lower().split(":")[0]
331 logger.debug("domain2='%s'", domain2)
332 if not domain_helper.is_wanted(domain2):
333 logger.debug("domain2='%s' is not wanted - EXIT!", domain2)
335 elif not instances.is_registered(domain2):
336 logger.info("components.netloc='%s' is not registered, adding ...", components.netloc)
337 instances.add(domain2, domain, "redirect_target")
339 message = f"Redirect from domain='{domain}' to response.url='{response.url}'"
340 instances.set_last_error(domain, message)
341 instances.set_software(domain, None)
342 instances.set_detection_mode(domain, None)
343 instances.set_nodeinfo_url(domain, None)
345 raise requests.exceptions.TooManyRedirects(message)
347 logger.debug("software[]='%s'", type(software))
348 if isinstance(software, str) and software == "":
349 logger.debug("Corrected empty string to None for software of domain='%s'", domain)
351 elif isinstance(software, str) and ("." in software or " " in software):
352 logger.debug("software='%s' may contain a version number, domain='%s', removing it ...", software, domain)
353 software = version.remove(software)
355 logger.debug("software[]='%s'", type(software))
356 if isinstance(software, str) and "powered by " in software:
357 logger.debug("software='%s' has 'powered by' in it", software)
358 software = version.remove(software_helper.strip_powered_by(software))
359 elif isinstance(software, str) and " hosted on " in software:
360 logger.debug("software='%s' has 'hosted on' in it", software)
361 software = version.remove(software_helper.strip_hosted_on(software))
362 elif isinstance(software, str) and " by " in software:
363 logger.debug("software='%s' has ' by ' in it", software)
364 software = software_helper.strip_until(software, " by ")
365 elif isinstance(software, str) and " see " in software:
366 logger.debug("software='%s' has ' see ' in it", software)
367 software = software_helper.strip_until(software, " see ")
369 logger.debug("software='%s' - EXIT!", software)
372 def determine_software(domain: str, path: str = None) -> str:
373 logger.debug("domain='%s',path='%s' - CALLED!", domain, path)
374 domain_helper.raise_on(domain)
376 if not isinstance(path, str) and path is not None:
377 raise ValueError(f"Parameter path[]='{type(path)}' is not of type 'str'")
379 logger.debug("Fetching nodeinfo from domain='%s',path='%s' ...", domain, path)
380 data = nodeinfo.fetch(domain, path)
383 logger.debug("data[%s]='%s'", type(data), data)
384 if "exception" in data:
385 # Continue raising it
386 logger.debug("data()=%d contains exception='%s' - raising ...", len(data), type(data["exception"]))
387 raise data["exception"]
388 elif "error_message" in data:
389 logger.debug("Returned error_message during fetching nodeinfo: '%s',status_code=%d", data['error_message'], data['status_code'])
390 software = fetch_generator_from_path(domain)
391 logger.debug("Generator for domain='%s' is: '%s'", domain, software)
393 logger.debug("domain='%s',path='%s',data[json] found ...", domain, path)
396 logger.debug("Auto-detection for domain='%s' was failing, fetching / ...", domain)
397 software = fetch_generator_from_path(domain)
398 logger.debug("Generator for domain='%s' is: '%s'", domain, software)
400 if "status" in data and data["status"] == "error" and "message" in data:
401 logger.warning("JSON response is an error: '%s' - Resetting detection_mode,nodeinfo_url ...", data["message"])
402 instances.set_last_error(domain, data["message"])
403 instances.set_detection_mode(domain, None)
404 instances.set_nodeinfo_url(domain, None)
405 software = fetch_generator_from_path(domain)
406 logger.debug("Generator for domain='%s' is: '%s'", domain, software)
407 elif "software" in data and "name" in data["software"]:
408 logger.debug("Found data[json][software][name] in JSON response")
409 software = data["software"]["name"]
410 logger.debug("software[%s]='%s' - FOUND!", type(software), software)
411 elif "message" in data:
412 logger.warning("JSON response contains only a message: '%s' - Resetting detection_mode,nodeinfo_url ...", data["message"])
413 instances.set_last_error(domain, data["message"])
414 instances.set_detection_mode(domain, None)
415 instances.set_nodeinfo_url(domain, None)
417 logger.debug("Invoking fetch_generator_from_path(%s) ...", domain)
418 software = fetch_generator_from_path(domain)
419 logger.debug("Generator for domain='%s' is: '%s'", domain, software)
420 elif "server" in data and "software" in data["server"]:
421 logger.debug("Found data[server][software]='%s' for domain='%s'", data["server"]["software"].lower(), domain)
422 software = data["server"]["software"].lower()
423 logger.debug("Detected software for domain='%s' is: '%s'", domain, software)
424 elif "software" not in data or "name" not in data["software"]:
425 logger.debug("JSON response from domain='%s' does not include [software][name] - Resetting detection_mode,nodeinfo_url ...", domain)
426 instances.set_detection_mode(domain, None)
427 instances.set_nodeinfo_url(domain, None)
429 logger.debug("Invoking fetch_generator_from_path(%s) ...", domain)
430 software = fetch_generator_from_path(domain)
431 logger.debug("Generator for domain='%s' is: '%s'", domain, software)
433 logger.debug("software[%s]='%s'", type(software), software)
434 if software is None or software == "":
435 logger.debug("Returning None - EXIT!")
438 logger.debug("software='%s'- BEFORE!", software)
439 software = software_helper.alias(software)
440 logger.debug("software['%s']='%s' - AFTER!", type(software), software)
442 if str(software) == "":
443 logger.debug("software for domain='%s' was not detected, trying generator ...", domain)
444 software = fetch_generator_from_path(domain)
445 elif len(str(software)) > 0 and ("." in software or " " in software):
446 logger.debug("software='%s' may contain a version number, domain='%s', removing it ...", software, domain)
447 software = version.remove(software)
449 logger.debug("software[]='%s'", type(software))
450 if isinstance(software, str) and "powered by" in software:
451 logger.debug("software='%s' has 'powered by' in it", software)
452 software = version.remove(software_helper.strip_powered_by(software))
454 software = software.strip()
456 logger.debug("software='%s' - EXIT!", software)
459 def find_domains(tag: bs4.element.Tag) -> list:
460 logger.debug("tag[]='%s' - CALLED!", type(tag))
461 if not isinstance(tag, bs4.element.Tag):
462 raise ValueError(f"Parameter tag[]='{type(tag)}' is not type of bs4.element.Tag")
463 elif len(tag.select("tr")) == 0:
464 raise KeyError("No table rows found in table!")
467 for element in tag.select("tr"):
468 logger.debug("element[]='%s'", type(element))
469 if not element.find("td"):
470 logger.debug("Skipping element, no <td> found")
473 domain = tidyup.domain(element.find("td").text)
474 reason = tidyup.reason(element.findAll("td")[1].text)
476 logger.debug("domain='%s',reason='%s'", domain, reason)
478 if not domain_helper.is_wanted(domain):
479 logger.debug("domain='%s' is blacklisted - SKIPPED!", domain)
481 elif domain == "gab.com/.ai, develop.gab.com":
482 logger.debug("Multiple domains detected in one row")
492 "domain": "develop.gab.com",
496 elif not validators.domain(domain.split("/")[0]):
497 logger.warning("domain='%s' is not a valid domain - SKIPPED!", domain)
500 logger.debug("Adding domain='%s',reason='%s' ...", domain, reason)
506 logger.debug("domains()=%d - EXIT!", len(domains))
509 def add_peers(rows: dict) -> list:
510 logger.debug("rows[]='%s' - CALLED!", type(rows))
511 if not isinstance(rows, dict):
512 raise ValueError(f"Parameter rows[]='{type(rows)}' is not of type 'dict'")
515 for key in ["linked", "allowed", "blocked"]:
516 logger.debug("Checking key='%s'", key)
517 if key not in rows or rows[key] is None:
518 logger.debug("Cannot find key='%s' or it is NoneType - SKIPPED!", key)
521 logger.debug("Adding %d peer(s) to peers list ...", len(rows[key]))
522 for peer in rows[key]:
523 logger.debug("peer[%s]='%s' - BEFORE!", type(peer), peer)
524 if peer is None or peer == "":
525 logger.debug("peer is empty - SKIPPED")
527 elif isinstance(peer, dict) and "domain" in peer:
528 logger.debug("peer[domain]='%s'", peer["domain"])
529 peer = tidyup.domain(peer["domain"])
530 elif isinstance(peer, str):
531 logger.debug("peer='%s'", peer)
532 peer = tidyup.domain(peer)
534 raise ValueError(f"peer[]='{type(peer)}' is not supported,key='{key}'")
536 logger.debug("peer[%s]='%s' - AFTER!", type(peer), peer)
537 if not domain_helper.is_wanted(peer):
538 logger.debug("peer='%s' is not wanted - SKIPPED!", peer)
541 logger.debug("Appending peer='%s' ...", peer)
544 logger.debug("peers()=%d - EXIT!", len(peers))
547 def fetch_blocks(domain: str) -> list:
548 logger.debug("domain='%s' - CALLED!", domain)
549 domain_helper.raise_on(domain)
551 if not instances.is_registered(domain):
552 raise Exception(f"domain='{domain}' is not registered but function is invoked.")
557 # No CSRF by default, you don't have to add network.api_headers by yourself here
561 logger.debug("Checking CSRF for domain='%s'", domain)
562 headers = csrf.determine(domain, dict())
563 except network.exceptions as exception:
564 logger.warning("Exception '%s' during checking CSRF (fetch_blocks,%s)", type(exception), __name__)
565 instances.set_last_error(domain, exception)
567 logger.debug("Returning empty list ... - EXIT!")
571 # json endpoint for newer mastodongs
572 logger.info("Fetching domain_blocks from domain='%s' ...", domain)
573 data = network.get_json_api(
575 "/api/v1/instance/domain_blocks",
577 timeout=(config.get("connection_timeout"), config.get("read_timeout"))
581 logger.debug("data[]='%s'", type(data))
582 if "error_message" in data:
583 logger.debug("Was not able to fetch domain_blocks from domain='%s': status_code=%d,error_message='%s'", domain, data['status_code'], data['error_message'])
584 instances.set_last_error(domain, data)
586 elif "json" in data and "error" in data["json"]:
587 logger.warning("JSON API returned error message: '%s'", data["json"]["error"])
588 instances.set_last_error(domain, data)
594 logger.debug("Marking domain='%s' as successfully handled ...", domain)
595 instances.set_success(domain)
597 logger.debug("rows[%s]()=%d", type(rows), len(rows))
599 logger.debug("Checking %d entries from domain='%s' ...", len(rows), domain)
602 logger.debug("block[]='%s'", type(block))
603 if not isinstance(block, dict):
604 logger.debug("block[]='%s' is of type 'dict' - SKIPPED!", type(block))
606 elif "domain" not in block:
607 logger.warning("block()=%d does not contain element 'domain' - SKIPPED!", len(block))
609 elif "severity" not in block:
610 logger.warning("block()=%d does not contain element 'severity' - SKIPPED!", len(block))
612 elif block["severity"] in ["accept", "accepted"]:
613 logger.debug("block[domain]='%s' has unwanted severity level '%s' - SKIPPED!", block["domain"], block["severity"])
615 elif "digest" in block and not validators.hashes.sha256(block["digest"]):
616 logger.warning("block[domain]='%s' has invalid block[digest]='%s' - SKIPPED!", block["domain"], block["digest"])
619 reason = tidyup.reason(block["comment"]) if "comment" in block and block["comment"] is not None and block["comment"] != "" else None
621 logger.debug("Appending blocker='%s',blocked='%s',reason='%s',block_level='%s'", domain, block["domain"], reason, block["severity"])
624 "blocked" : block["domain"],
625 "digest" : block["digest"] if "digest" in block else None,
627 "block_level": blocks.alias_block_level(block["severity"]),
630 logger.debug("domain='%s' has no block list", domain)
632 except network.exceptions as exception:
633 logger.warning("domain='%s',exception[%s]='%s'", domain, type(exception), str(exception))
634 instances.set_last_error(domain, exception)
636 logger.debug("blocklist()=%d - EXIT!", len(blocklist))