1 # Copyright (C) 2023 Free Software Foundation
3 # This program is free software: you can redistribute it and/or modify
4 # it under the terms of the GNU Affero General Public License as published
5 # by the Free Software Foundation, either version 3 of the License, or
6 # (at your option) any later version.
8 # This program is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # GNU Affero General Public License for more details.
13 # You should have received a copy of the GNU Affero General Public License
14 # along with this program. If not, see <https://www.gnu.org/licenses/>.
18 from urllib.parse import urlparse
24 from fba.helpers import blacklist
25 from fba.helpers import config
26 from fba.helpers import cookies
27 from fba.helpers import domain as domain_helper
28 from fba.helpers import software as software_helper
29 from fba.helpers import tidyup
30 from fba.helpers import version
32 from fba.http import csrf
33 from fba.http import network
34 from fba.http import nodeinfo
36 from fba.models import blocks
37 from fba.models import instances
39 from fba.networks import lemmy
40 from fba.networks import misskey
41 from fba.networks import peertube
43 # Depth counter, being raised and lowered
48 "/api/v1/instance/peers",
52 logging.basicConfig(level=logging.INFO)
53 logger = logging.getLogger(__name__)
55 def fetch_instances(domain: str, origin: str, software: str, command: str, path: str = None):
57 logger.debug("domain='%s',origin='%s',software='%s',command='%s',path='%s',_DEPTH=%d - CALLED!", domain, origin, software, command, path, _DEPTH)
58 domain_helper.raise_on(domain)
60 if blacklist.is_blacklisted(domain):
61 raise Exception(f"domain='{domain}' is blacklisted but function was invoked")
62 elif not isinstance(origin, str) and origin is not None:
63 raise ValueError(f"Parameter origin[]='{type(origin)}' is not of type 'str'")
64 elif not isinstance(command, str):
65 raise ValueError(f"Parameter command[]='{type(command)}' is not of type 'str'")
67 raise ValueError("Parameter 'command' is empty")
68 elif command in ["fetch_blocks", "fetch_cs", "fetch_bkali", "fetch_relays", "fetch_fedipact", "fetch_joinmobilizon", "fetch_joinmisskey", "fetch_joinfediverse", "fetch_relaylist"] and origin is None:
69 raise ValueError(f"Parameter command='{command}' but origin is None, please fix invoking this function.")
70 elif not isinstance(path, str) and path is not None:
71 raise ValueError(f"Parameter path[]='{type(path)}' is not of type 'str'")
72 elif _DEPTH > 0 and instances.is_recent(domain, "last_instance_fetch"):
73 raise ValueError(f"domain='{domain}' has recently been fetched but function was invoked")
74 elif software is None and not instances.is_recent(domain, "last_nodeinfo"):
76 logger.debug("Software for domain='%s' is not set, determining ...", domain)
77 software = determine_software(domain, path)
78 except network.exceptions as exception:
79 logger.warning("Exception '%s' during determining software type", type(exception))
80 instances.set_last_error(domain, exception)
82 logger.debug("Determined software='%s' for domain='%s'", software, domain)
83 elif software is None:
84 logger.debug("domain='%s' has unknown software or nodeinfo has recently being fetched", domain)
85 elif not isinstance(software, str):
86 raise ValueError(f"Parameter software[]='{type(software)}' is not of type 'str'")
91 logger.debug("Checking if domain='%s' is registered ...", domain)
92 if not instances.is_registered(domain):
93 logger.debug("Adding new domain='%s',origin='%s',command='%s',path='%s',software='%s'", domain, origin, command, path, software)
94 instances.add(domain, origin, command, path, software)
96 logger.debug("software='%s'", software)
97 if software_helper.is_relay(software):
98 logger.debug("software='%s' is a relay software - EXIT!", software)
102 logger.debug("Updating last_instance_fetch for domain='%s' ...", domain)
103 instances.set_last_instance_fetch(domain)
106 logger.debug("software='%s'", software)
107 if software is not None:
108 logger.debug("Fetching instances for domain='%s',software='%s',origin='%s'", domain, software, origin)
109 peerlist = fetch_peers(domain, software, origin)
111 logger.debug("peerlist[]='%s'", type(peerlist))
112 if isinstance(peerlist, list):
113 logger.debug("Invoking instances.set_total_peerlist(%s,%d) ...", domain, len(peerlist))
114 instances.set_total_peers(domain, peerlist)
116 logger.debug("Invoking cookies.clear(%s) ...", domain)
117 cookies.clear(domain)
119 logger.debug("peerlist[]='%s'", type(peerlist))
121 logger.warning("Cannot fetch peers: domain='%s',software='%s'", domain, software)
122 if instances.has_pending(domain):
123 logger.debug("Flushing updates for domain='%s' ...", domain)
124 instances.update(domain)
127 logger.debug("EXIT!")
129 elif len(peerlist) == 0:
130 logger.info("domain='%s' returned an empty peer list.", domain)
131 if instances.has_pending(domain):
132 logger.debug("Flushing updates for domain='%s' ...", domain)
133 instances.update(domain)
136 logger.debug("domain='%s',software='%s' has an empty peer list returned - EXIT!", domain, software)
139 logger.info("Checking %d instance(s) from domain='%s',software='%s',depth=%d ...", len(peerlist), domain, software, _DEPTH)
140 for instance in peerlist:
141 logger.debug("instance[%s]='%s'", type(instance), instance)
142 if instance is None or instance == "":
143 logger.debug("instance[%s]='%s' is either None or empty - SKIPPED!", type(instance), instance)
146 logger.debug("instance='%s' - BEFORE!", instance)
147 instance = tidyup.domain(instance) if isinstance(instance, str) and instance != "" else None
148 logger.debug("instance='%s' - AFTER!", instance)
150 if instance is None or instance == "":
151 logger.warning("instance='%s' is empty after tidyup.domain(), domain='%s'", instance, domain)
153 elif ".." in instance:
154 logger.warning("instance='%s' contains double-dot, removing ...", instance)
155 instance = instance.replace("..", ".")
157 logger.debug("instance='%s' - BEFORE!", instance)
158 instance = instance.encode("idna").decode("utf-8")
159 logger.debug("instance='%s' - AFTER!", instance)
161 if not domain_helper.is_wanted(instance):
162 logger.debug("instance='%s' is not wanted - SKIPPED!", instance)
164 elif instance.find("/profile/") > 0 or instance.find("/users/") > 0 or (instances.is_registered(instance.split("/")[0]) and instance.find("/c/") > 0):
165 logger.debug("instance='%s' is a link to a single user profile - SKIPPED!", instance)
167 elif instance.find("/tag/") > 0:
168 logger.debug("instance='%s' is a link to a tag - SKIPPED!", instance)
170 elif not instances.is_registered(instance):
171 logger.debug("Checking if domain='%s' has pending updates ...", domain)
172 if instances.has_pending(domain):
173 logger.debug("Flushing updates for domain='%s' ...", domain)
174 instances.update(domain)
176 logger.debug("instance='%s',origin='%s',_DEPTH=%d reached!", instance, origin, _DEPTH)
177 if _DEPTH <= config.get("max_crawl_depth") and len(peerlist) >= config.get("min_peers_length"):
178 logger.debug("Fetching instance='%s',origin='%s',command='%s',path='%s',_DEPTH=%d ...", instance, domain, command, path, _DEPTH)
179 fetch_instances(instance, domain, None, command, path)
181 logger.debug("Adding instance='%s',domain='%s',command='%s',_DEPTH=%d ...", instance, domain, command, _DEPTH)
182 instances.add(instance, domain, command)
184 logger.debug("Checking if domain='%s' has pending updates ...", domain)
185 if instances.has_pending(domain):
186 logger.debug("Flushing updates for domain='%s' ...", domain)
187 instances.update(domain)
190 logger.debug("EXIT!")
192 def fetch_peers(domain: str, software: str, origin: str) -> list:
193 logger.debug("domain='%s',software='%s',origin='%s' - CALLED!", domain, software, origin)
194 domain_helper.raise_on(domain)
196 if blacklist.is_blacklisted(domain):
197 raise Exception(f"domain='{domain}' is blacklisted but function was invoked")
198 elif not isinstance(software, str) and software is not None:
199 raise ValueError(f"Parameter software[]='{type(software)}' is not of type 'str'")
200 elif isinstance(software, str) and software == "":
201 raise ValueError("Parameter 'software' is empty")
202 elif software_helper.is_relay(software):
203 raise ValueError(f"domain='{domain}' is of software='{software}' and isn't supported here.")
204 elif not isinstance(origin, str) and origin is not None:
205 raise ValueError(f"Parameter origin[]='{type(origin)}' is not of type 'str'")
206 elif isinstance(origin, str) and origin == "":
207 raise ValueError("Parameter 'origin' is empty")
209 if software == "misskey":
210 logger.debug("Invoking misskey.fetch_peers(%s) ...", domain)
211 return misskey.fetch_peers(domain)
212 elif software == "lemmy":
213 logger.debug("Invoking lemmy.fetch_peers(%s,%s) ...", domain, origin)
214 return lemmy.fetch_peers(domain, origin)
215 elif software == "peertube":
216 logger.debug("Invoking peertube.fetch_peers(%s) ...", domain)
217 return peertube.fetch_peers(domain)
219 # No CSRF by default, you don't have to add network.api_headers by yourself here
223 logger.debug("Checking CSRF for domain='%s'", domain)
224 headers = csrf.determine(domain, dict())
225 except network.exceptions as exception:
226 logger.warning("Exception '%s' during checking CSRF (fetch_peers,%s)", type(exception), __name__)
227 instances.set_last_error(domain, exception)
229 logger.debug("Returning empty list ... - EXIT!")
232 # Init peers variable
235 logger.debug("Checking %d API paths ...", len(_api_paths))
236 for path in _api_paths:
237 logger.debug("Fetching path='%s' from domain='%s',software='%s' ...", path, domain, software)
238 data = network.get_json_api(
242 timeout=(config.get("connection_timeout"), config.get("read_timeout"))
245 logger.debug("data(%d)[]='%s'", len(data), type(data))
246 if "error_message" in data:
247 logger.debug("Was not able to fetch peers from path='%s',domain='%s' ...", path, domain)
248 instances.set_last_error(domain, data)
249 elif "json" in data and len(data["json"]) > 0:
250 logger.debug("Querying API path='%s' was successful: domain='%s',data[json][%s]()=%d", path, domain, type(data['json']), len(data['json']))
253 logger.debug("Marking domain='%s' as successfully handled ...", domain)
254 instances.set_success(domain)
257 if not isinstance(peers, list):
258 logger.warning("peers[]='%s' is not of type 'list', maybe bad API response?", type(peers))
261 logger.debug("Invoking instances.set_total_peers(%s,%d) ...", domain, len(peers))
262 instances.set_total_peers(domain, peers)
264 logger.debug("peers()=%d - EXIT!", len(peers))
267 def fetch_generator_from_path(domain: str, path: str = "/") -> str:
268 logger.debug("domain='%s',path='%s' - CALLED!", domain, path)
269 domain_helper.raise_on(domain)
271 if blacklist.is_blacklisted(domain):
272 raise Exception(f"domain='{domain}' is blacklisted but function was invoked")
273 elif not isinstance(path, str):
274 raise ValueError(f"path[]='{type(path)}' is not of type 'str'")
276 raise ValueError("Parameter 'path' is empty")
277 elif not path.startswith("/"):
278 raise ValueError(f"path='{path}' does not start with / but should")
282 logger.debug("Fetching path='%s' from domain='%s' ...", path, domain)
283 response = network.fetch_response(
286 headers=network.web_headers,
287 timeout=(config.get("connection_timeout"), config.get("read_timeout")),
291 logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text))
292 if ((response.ok and response.status_code == 200) or response.status_code == 410) and response.text.find("<html") > 0 and domain_helper.is_in_url(domain, response.url):
293 logger.debug("Parsing response.text()=%d Bytes ...", len(response.text))
294 doc = bs4.BeautifulSoup(response.text, "html.parser")
296 logger.debug("doc[]='%s'", type(doc))
297 platform = doc.find("meta", {"property": "og:platform"})
298 generator = doc.find("meta", {"name" : "generator"})
299 site_name = doc.find("meta", {"property": "og:site_name"})
300 app_name = doc.find("meta", {"name" : "application-name"})
302 logger.debug("generator[]='%s',site_name[]='%s',platform[]='%s',app_name[]='%s'", type(generator), type(site_name), type(platform), type(app_name))
303 if isinstance(platform, bs4.element.Tag) and isinstance(platform.get("content"), str) and platform.get("content") != "":
304 logger.debug("Found property=og:platform, domain='%s'", domain)
305 software = tidyup.domain(platform.get("content"))
306 logger.debug("software[%s]='%s' after tidyup.domain() ...", type(software), software)
308 if software is not None and software != "":
309 logger.debug("domain='%s' has og:platform='%s' - Setting detection_mode=PLATFORM ...", domain, software)
310 instances.set_detection_mode(domain, "PLATFORM")
311 elif isinstance(generator, bs4.element.Tag) and isinstance(generator.get("content"), str) and generator.get("content") != "":
312 logger.debug("Found generator meta tag: domain='%s'", domain)
313 software = tidyup.domain(generator.get("content"))
315 logger.debug("software[%s]='%s'", type(software), software)
316 if software is not None and software != "":
317 logger.info("domain='%s' is generated by software='%s' - Setting detection_mode=GENERATOR ...", domain, software)
318 instances.set_detection_mode(domain, "GENERATOR")
319 elif isinstance(app_name, bs4.element.Tag) and isinstance(app_name.get("content"), str) and app_name.get("content") != "":
320 logger.debug("Found property=og:app_name, domain='%s'", domain)
321 software = tidyup.domain(app_name.get("content"))
323 logger.debug("software[%s]='%s'", type(software), software)
324 if software is not None and software != "":
325 logger.debug("domain='%s' has application-name='%s' - Setting detection_mode=app_name ...", domain, software)
326 instances.set_detection_mode(domain, "APP_NAME")
327 elif isinstance(site_name, bs4.element.Tag) and isinstance(site_name.get("content"), str) and site_name.get("content") != "":
328 logger.debug("Found property=og:site_name, domain='%s'", domain)
329 software = tidyup.domain(site_name.get("content"))
331 logger.debug("software[%s]='%s'", type(software), software)
332 if software is not None and software != "":
333 logger.debug("domain='%s' has og:site_name='%s' - Setting detection_mode=SITE_NAME ...", domain, software)
334 instances.set_detection_mode(domain, "SITE_NAME")
335 elif not domain_helper.is_in_url(domain, response.url):
336 logger.warning("domain='%s' doesn't match response.url='%s', maybe redirect to other domain?", domain, response.url)
338 components = urlparse(response.url)
339 domain2 = components.netloc.lower().split(":")[0]
341 logger.debug("domain2='%s'", domain2)
342 if not domain_helper.is_wanted(domain2):
343 logger.debug("domain2='%s' is not wanted - EXIT!", domain2)
345 elif not instances.is_registered(domain2):
346 logger.info("components.netloc='%s' is not registered, adding ...", components.netloc)
347 instances.add(domain2, domain, "redirect_target")
349 message = f"Redirect from domain='{domain}' to response.url='{response.url}'"
350 instances.set_last_error(domain, message)
351 instances.set_software(domain, None)
352 instances.set_detection_mode(domain, None)
353 instances.set_nodeinfo_url(domain, None)
355 raise requests.exceptions.TooManyRedirects(message)
357 logger.debug("software[]='%s'", type(software))
358 if isinstance(software, str) and software == "":
359 logger.debug("Corrected empty string to None for software of domain='%s'", domain)
361 elif isinstance(software, str) and ("." in software or " " in software):
362 logger.debug("software='%s' may contain a version number, domain='%s', removing it ...", software, domain)
363 software = version.remove(software)
365 logger.debug("software[%s]='%s'", type(software), software)
366 if isinstance(software, str) and "powered by " in software:
367 logger.debug("software='%s' has 'powered by' in it", software)
368 software = version.remove(software_helper.strip_powered_by(software))
369 elif isinstance(software, str) and " hosted on " in software:
370 logger.debug("software='%s' has 'hosted on' in it", software)
371 software = version.remove(software_helper.strip_hosted_on(software))
372 elif isinstance(software, str) and " by " in software:
373 logger.debug("software='%s' has ' by ' in it", software)
374 software = software_helper.strip_until(software, " by ")
375 elif isinstance(software, str) and " see " in software:
376 logger.debug("software='%s' has ' see ' in it", software)
377 software = software_helper.strip_until(software, " see ")
379 logger.debug("software[%s]='%s' - EXIT!", type(software), software)
382 def determine_software(domain: str, path: str = None) -> str:
383 logger.debug("domain='%s',path='%s' - CALLED!", domain, path)
384 domain_helper.raise_on(domain)
386 if blacklist.is_blacklisted(domain):
387 raise Exception(f"domain='{domain}' is blacklisted but function was invoked")
388 elif not isinstance(path, str) and path is not None:
389 raise ValueError(f"Parameter path[]='{type(path)}' is not of type 'str'")
391 logger.debug("Fetching nodeinfo from domain='%s',path='%s' ...", domain, path)
392 data = nodeinfo.fetch(domain, path)
395 logger.debug("data[%s]='%s'", type(data), data)
396 if "exception" in data:
397 # Continue raising it
398 logger.debug("data()=%d contains exception='%s' - raising ...", len(data), type(data["exception"]))
399 raise data["exception"]
400 elif "error_message" in data:
401 logger.debug("Returned error_message during fetching nodeinfo: '%s',status_code=%d", data['error_message'], data['status_code'])
402 software = fetch_generator_from_path(domain)
403 logger.debug("Generator for domain='%s' is: '%s'", domain, software)
405 logger.debug("domain='%s',path='%s',data[json] found ...", domain, path)
408 logger.debug("Auto-detection for domain='%s' was failing, fetching / ...", domain)
409 software = fetch_generator_from_path(domain)
410 logger.debug("Generator for domain='%s' is: '%s'", domain, software)
412 if "status" in data and data["status"] == "error" and "message" in data:
413 logger.warning("JSON response is an error: '%s' - Resetting detection_mode,nodeinfo_url ...", data["message"])
414 instances.set_last_error(domain, data["message"])
415 instances.set_detection_mode(domain, None)
416 instances.set_nodeinfo_url(domain, None)
417 software = fetch_generator_from_path(domain)
418 logger.debug("Generator for domain='%s' is: '%s'", domain, software)
419 elif "software" in data and "name" in data["software"]:
420 logger.debug("Found data[json][software][name] in JSON response")
421 software = data["software"]["name"]
422 logger.debug("software[%s]='%s' - FOUND!", type(software), software)
423 elif "message" in data:
424 logger.warning("JSON response contains only a message: '%s' - Resetting detection_mode,nodeinfo_url ...", data["message"])
425 instances.set_last_error(domain, data["message"])
426 instances.set_detection_mode(domain, None)
427 instances.set_nodeinfo_url(domain, None)
429 logger.debug("Invoking fetch_generator_from_path(%s) ...", domain)
430 software = fetch_generator_from_path(domain)
431 logger.debug("Generator for domain='%s' is: '%s'", domain, software)
432 elif "server" in data and "software" in data["server"]:
433 logger.debug("Found data[server][software]='%s' for domain='%s'", data["server"]["software"].lower(), domain)
434 software = data["server"]["software"].lower()
435 logger.debug("Detected software for domain='%s' is: '%s'", domain, software)
436 elif "software" not in data or "name" not in data["software"]:
437 logger.debug("JSON response from domain='%s' does not include [software][name] - Resetting detection_mode,nodeinfo_url ...", domain)
438 instances.set_detection_mode(domain, None)
439 instances.set_nodeinfo_url(domain, None)
441 logger.debug("Invoking fetch_generator_from_path(%s) ...", domain)
442 software = fetch_generator_from_path(domain)
443 logger.debug("Generator for domain='%s' is: '%s'", domain, software)
445 logger.debug("software[%s]='%s'", type(software), software)
446 if software is None or software == "":
447 logger.debug("Returning None - EXIT!")
450 logger.debug("software='%s'- BEFORE!", software)
451 software = software_helper.alias(software)
452 logger.debug("software['%s']='%s' - AFTER!", type(software), software)
454 if str(software) == "":
455 logger.debug("software for domain='%s' was not detected, trying generator ...", domain)
456 software = fetch_generator_from_path(domain)
457 elif len(str(software)) > 0 and ("." in software or " " in software):
458 logger.debug("software='%s' may contain a version number, domain='%s', removing it ...", software, domain)
459 software = version.remove(software)
461 logger.debug("software[]='%s'", type(software))
462 if isinstance(software, str) and "powered by" in software:
463 logger.debug("software='%s' has 'powered by' in it", software)
464 software = version.remove(software_helper.strip_powered_by(software))
466 software = software.strip()
468 logger.debug("software[%s]='%s' - EXIT!", type(software), software)
471 def find_domains(tag: bs4.element.Tag) -> list:
472 logger.debug("tag[]='%s' - CALLED!", type(tag))
473 if not isinstance(tag, bs4.element.Tag):
474 raise ValueError(f"Parameter tag[]='{type(tag)}' is not type of bs4.element.Tag")
475 elif len(tag.select("tr")) == 0:
476 raise KeyError("No table rows found in table!")
479 for element in tag.select("tr"):
480 logger.debug("element[]='%s'", type(element))
481 if not element.find("td"):
482 logger.debug("Skipping element, no <td> found")
485 domain = tidyup.domain(element.find("td").text)
486 reason = tidyup.reason(element.findAll("td")[1].text)
488 logger.debug("domain='%s',reason='%s'", domain, reason)
490 if not domain_helper.is_wanted(domain):
491 logger.debug("domain='%s' is blacklisted - SKIPPED!", domain)
493 elif domain == "gab.com/.ai, develop.gab.com":
494 logger.debug("Multiple gab.com domains detected in one row")
504 "domain": "develop.gab.com",
508 elif not validators.domain(domain.split("/")[0]):
509 logger.warning("domain='%s' is not a valid domain - SKIPPED!", domain)
512 logger.debug("Adding domain='%s',reason='%s' ...", domain, reason)
518 logger.debug("domains()=%d - EXIT!", len(domains))
521 def add_peers(rows: dict) -> list:
522 logger.debug("rows[]='%s' - CALLED!", type(rows))
523 if not isinstance(rows, dict):
524 raise ValueError(f"Parameter rows[]='{type(rows)}' is not of type 'dict'")
527 for key in ["linked", "allowed", "blocked"]:
528 logger.debug("key='%s'", key)
529 if key not in rows or rows[key] is None:
530 logger.debug("Cannot find key='%s' or it is NoneType - SKIPPED!", key)
533 logger.debug("Adding %d peer(s) to peers list ...", len(rows[key]))
534 for peer in rows[key]:
535 logger.debug("peer[%s]='%s' - BEFORE!", type(peer), peer)
536 if peer is None or peer == "":
537 logger.debug("peer is empty - SKIPPED")
539 elif isinstance(peer, dict) and "domain" in peer:
540 logger.debug("peer[domain]='%s'", peer["domain"])
541 peer = tidyup.domain(peer["domain"])
542 elif isinstance(peer, str):
543 logger.debug("peer='%s'", peer)
544 peer = tidyup.domain(peer)
546 raise ValueError(f"peer[]='{type(peer)}' is not supported,key='{key}'")
548 logger.debug("peer[%s]='%s' - AFTER!", type(peer), peer)
549 if not domain_helper.is_wanted(peer):
550 logger.debug("peer='%s' is not wanted - SKIPPED!", peer)
553 logger.debug("Appending peer='%s' ...", peer)
556 logger.debug("peers()=%d - EXIT!", len(peers))
559 def fetch_blocks(domain: str) -> list:
560 logger.debug("domain='%s' - CALLED!", domain)
561 domain_helper.raise_on(domain)
563 if not instances.is_registered(domain):
564 raise Exception(f"domain='{domain}' is not registered but function is invoked.")
565 elif blacklist.is_blacklisted(domain):
566 raise Exception(f"domain='{domain}' is blacklisted but function was invoked")
571 # No CSRF by default, you don't have to add network.api_headers by yourself here
575 logger.debug("Checking CSRF for domain='%s'", domain)
576 headers = csrf.determine(domain, dict())
577 except network.exceptions as exception:
578 logger.warning("Exception '%s' during checking CSRF (fetch_blocks,%s)", type(exception), __name__)
579 instances.set_last_error(domain, exception)
581 logger.debug("Returning empty list ... - EXIT!")
585 # json endpoint for newer mastodongs
586 logger.info("Fetching domain_blocks from domain='%s' ...", domain)
587 data = network.get_json_api(
589 "/api/v1/instance/domain_blocks",
591 timeout=(config.get("connection_timeout"), config.get("read_timeout"))
595 logger.debug("data(%d)[]='%s'", len(data), type(data))
596 if "error_message" in data:
597 logger.debug("Was not able to fetch domain_blocks from domain='%s': status_code=%d,error_message='%s'", domain, data['status_code'], data['error_message'])
598 instances.set_last_error(domain, data)
600 logger.debug("blocklist()=%d - EXIT!", len(blocklist))
602 elif "json" in data and "error" in data["json"]:
603 logger.warning("JSON API returned error message: '%s'", data["json"]["error"])
604 instances.set_last_error(domain, data)
606 logger.debug("blocklist()=%d - EXIT!", len(blocklist))
612 logger.debug("Marking domain='%s' as successfully handled ...", domain)
613 instances.set_success(domain)
615 logger.debug("rows(%d)[]='%s'", len(rows), type(rows))
617 logger.debug("Checking %d entries from domain='%s' ...", len(rows), domain)
620 logger.debug("block[]='%s'", type(block))
621 if not isinstance(block, dict):
622 logger.debug("block[]='%s' is of type 'dict' - SKIPPED!", type(block))
624 elif "domain" not in block:
625 logger.warning("block()=%d does not contain element 'domain' - SKIPPED!", len(block))
627 elif "severity" not in block:
628 logger.warning("block()=%d does not contain element 'severity' - SKIPPED!", len(block))
630 elif block["severity"] in ["accept", "accepted"]:
631 logger.debug("block[domain]='%s' has unwanted severity level '%s' - SKIPPED!", block["domain"], block["severity"])
633 elif "digest" in block and not validators.hashes.sha256(block["digest"]):
634 logger.warning("block[domain]='%s' has invalid block[digest]='%s' - SKIPPED!", block["domain"], block["digest"])
637 reason = tidyup.reason(block["comment"]) if "comment" in block and block["comment"] is not None and block["comment"] != "" else None
639 logger.debug("Appending blocker='%s',blocked='%s',reason='%s',block_level='%s' ...", domain, block["domain"], reason, block["severity"])
642 "blocked" : block["domain"],
643 "digest" : block["digest"] if "digest" in block else None,
645 "block_level": blocks.alias_block_level(block["severity"]),
648 logger.debug("domain='%s' has no block list", domain)
650 except network.exceptions as exception:
651 logger.warning("domain='%s',exception[%s]='%s'", domain, type(exception), str(exception))
652 instances.set_last_error(domain, exception)
654 logger.debug("blocklist()=%d - EXIT!", len(blocklist))