logger.warning("header_text='%s' not found in language mapping table", header_text)
logger.debug("header_text='%s' - AFTER!", header_text)
- if header_text in blocklist:
- # replaced find_next_siblings with find_all_next to account for instances that for example hide lists in dropdown menu
- for line in header.find_all_next("table")[0].find_all("tr")[1:]:
- domain = line.find("span").text
- reason = line.find_all("td")[1].text
-
- logger.debug("domain='%s',reason='%s' - BEFORE!", domain, reason)
- domain = tidyup.domain(domain) if domain != "" else None
- reason = tidyup.reason(reason) if reason != "" else None
- logger.debug("domain='%s',reason='%s' - AFTER!", domain, reason)
-
- digest = line.find("span")["title"][9:]
-
- logger.debug("digest='%s' - BEFORE!", digest)
- if not validators.hashes.sha256(digest):
- logger.warning("domain='%s' has invalid digest='%s' - Setting to None ...", domain, digest)
- digest = None
- logger.debug("digest='%s' - AFTER!", digest)
-
- logger.debug("domain[]='%s'", type(domain))
- if domain in [None, ""]:
- logger.warning("domain='%s' is empty,line='%s' - SKIPPED!", domain, line)
- continue
-
- logger.debug("Appending domain='%s',digest='%s',reason='%s' to blocklist header_text='%s' ...", domain, digest, reason, header_text)
- blocklist[header_text].append({
- "domain": domain,
- "digest": digest,
- "reason": reason,
- })
- else:
- logger.warning("header_text='%s' not found in blocklist()=%d", header_text, len(blocklist))
+ if header_text not in blocklist:
+ logger.warning("header_text='%s' not found in blocklist()=%d - SKIPPED!", header_text, len(blocklist))
+ continue
+
+ # replaced find_next_siblings with find_all_next to account for instances that for example hide lists in dropdown menu
+ for line in header.find_all_next("table")[0].find_all("tr")[1:]:
+ domain = line.find("span").text
+ reason = line.find_all("td")[1].text
+
+ logger.debug("domain='%s',reason='%s' - BEFORE!", domain, reason)
+ domain = tidyup.domain(domain) if domain != "" else None
+ reason = tidyup.reason(reason) if reason != "" else None
+ logger.debug("domain='%s',reason='%s' - AFTER!", domain, reason)
+
+ digest = line.find("span")["title"][9:]
+
+ logger.debug("digest='%s' - BEFORE!", digest)
+ if not validators.hashes.sha256(digest):
+ logger.warning("domain='%s' has invalid digest='%s' - Setting to None ...", domain, digest)
+ digest = None
+ logger.debug("digest='%s' - AFTER!", digest)
+
+ logger.debug("domain[]='%s'", type(domain))
+ if domain in [None, ""]:
+ logger.warning("domain='%s' is empty,line='%s' - SKIPPED!", domain, line)
+ continue
+
+ logger.debug("Appending domain='%s',digest='%s',reason='%s' to blocklist header_text='%s' ...", domain, digest, reason, header_text)
+ blocklist[header_text].append({
+ "domain": domain,
+ "digest": digest,
+ "reason": reason,
+ })
logger.debug("Returning blocklist for domain='%s' - EXIT!", domain)
return {