From b3df9b4e215297e2103720cf1db1f46eff75858d Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Sat, 15 Jun 2024 15:11:40 +0200 Subject: [PATCH] Continued: - if the found digest isn't valid, then set it to None - ignore (not skip) any invalid digests --- fba/networks/mastodon.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/fba/networks/mastodon.py b/fba/networks/mastodon.py index 704d4f1..b379ae0 100644 --- a/fba/networks/mastodon.py +++ b/fba/networks/mastodon.py @@ -121,14 +121,22 @@ def fetch_blocks_from_about(domain: str) -> dict: # replaced find_next_siblings with find_all_next to account for instances that e.g. hide lists in dropdown menu for line in header.find_all_next("table")[0].find_all("tr")[1:]: domain = line.find("span").text - digest = line.find("span")["title"][9:] reason = line.find_all("td")[1].text logger.debug("domain='%s',reason='%s' - BEFORE!", domain, reason) domain = tidyup.domain(domain) if domain != "" else None reason = tidyup.reason(reason) if reason != "" else None - logger.debug("domain='%s',reason='%s' - AFTER!", domain, reason) + + digest = line.find("span")["title"][9:] + + logger.debug("digest='%s' - BEFORE!", digest) + if not validators.hashes.sha256(digest): + logger.warning("domain='%s' has invalid digest='%s' - Setting to None ...", domain, digest) + digest = None + logger.debug("digest='%s' - AFTER!", digest) + + logger.debug("domain[]='%s'", type(domain)) if domain in [None, ""]: logger.warning("domain='%s' is empty,line='%s' - SKIPPED!", domain, line) continue @@ -182,7 +190,7 @@ def fetch_blocks(domain: str) -> list: elif block_level in ["accept", "accepted"]: logger.debug("block[domain]='%s' has unwanted severity level '%s' - SKIPPED!", block["domain"], block_level) continue - elif "digest" in block and not validators.hashes.sha256(block["digest"]): + elif "digest" in block and not block["digest"] is None and not validators.hashes.sha256(block["digest"]): logger.warning("block[domain]='%s' has invalid block[digest]='%s' - SKIPPED!", block["domain"], block["digest"]) continue -- 2.39.5