From 88c47338c39987cc13590f6a1f221169f26a4cd1 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Wed, 21 Jun 2023 15:02:23 +0200 Subject: [PATCH] Continued: - some lemmy APIs return NULL (NoneType) in 'federated_instances' field - fixed some logging messages --- fba/commands.py | 16 ++++++++-------- fba/http/federation.py | 5 ++++- fba/networks/friendica.py | 2 +- fba/networks/lemmy.py | 4 ++-- fba/networks/mastodon.py | 2 +- fba/networks/pleroma.py | 4 ++-- 6 files changed, 18 insertions(+), 15 deletions(-) diff --git a/fba/commands.py b/fba/commands.py index 6596578..a55e8a3 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -357,7 +357,7 @@ def fetch_observer(args: argparse.Namespace): logger.debug(f"raw[{type(raw)}]()={len(raw)}") doc = bs4.BeautifulSoup(raw, features='html.parser') - logger.debug("doc[]='%'", type(doc)) + logger.debug("doc[]='%s'", type(doc)) except network.exceptions as exception: logger.warning(f"Cannot fetch software='{software}' from fediverse.observer: '{type(exception)}'") continue @@ -376,7 +376,7 @@ def fetch_observer(args: argparse.Namespace): logger.debug("domain='%s' is blacklisted - SKIPPED!", domain) continue elif instances.is_registered(domain): - logger.debug(f"domain='{domain}' is already registered - SKIPPED!") + logger.debug("domain='%s' is already registered - SKIPPED!", domain) continue logger.info(f"Fetching instances for domain='{domain}',software='{software}'") @@ -479,10 +479,10 @@ def fetch_fba_rss(args: argparse.Namespace): logger.debug("domain='%s' is blacklisted - SKIPPED!", domain) continue elif domain in domains: - logger.debug(f"domain='{domain}' is already added - SKIPPED!") + logger.debug("domain='%s' is already added - SKIPPED!", domain) continue elif instances.is_registered(domain): - logger.debug(f"domain='{domain}' is already registered - SKIPPED!") + logger.debug("domain='%s' is already registered - SKIPPED!", domain) continue logger.debug(f"Adding domain='{domain}'") @@ -524,10 +524,10 @@ def fetch_fbabot_atom(args: argparse.Namespace): for entry in atom.entries: logger.debug(f"entry[]='{type(entry)}'") doc = bs4.BeautifulSoup(entry.content.value, "html.parser") - logger.debug("doc[]='%'", type(doc)) + logger.debug("doc[]='%s'", type(doc)) for element in doc.findAll("a"): for href in element["href"].split(","): - logger.debug(f"href[{type(href)}]={href}") + logger.debug("href[%s]='%s", type(href), href) domain = tidyup.domain(href) logger.debug("domain='%s'", domain) @@ -535,10 +535,10 @@ def fetch_fbabot_atom(args: argparse.Namespace): logger.debug("domain='%s' is blacklisted - SKIPPED!", domain) continue elif domain in domains: - logger.debug(f"domain='{domain}' is already added - SKIPPED!") + logger.debug("domain='%s' is already added - SKIPPED!", domain) continue elif instances.is_registered(domain): - logger.debug(f"domain='{domain}' is already registered - SKIPPED!") + logger.debug("domain='%s' is already registered - SKIPPED!", domain) continue logger.debug(f"Adding domain='{domain}',domains()={len(domains)}") diff --git a/fba/http/federation.py b/fba/http/federation.py index 65fa1ef..a7b2db4 100644 --- a/fba/http/federation.py +++ b/fba/http/federation.py @@ -637,7 +637,10 @@ def find_domains(tag: bs4.element.Tag) -> list: return domains def add_peers(rows: dict) -> list: - logger.debug(f"rows()={len(rows)} - CALLED!") + logger.debug(f"rows[]={type(rows)} - CALLED!") + if not isinstance(rows, dict): + raise ValueError(f"Parameter rows[]='{type(rows)}' is not 'dict'") + peers = list() for key in ["linked", "allowed", "blocked"]: logger.debug(f"Checking key='{key}'") diff --git a/fba/networks/friendica.py b/fba/networks/friendica.py index ddeb608..b3f6bc4 100644 --- a/fba/networks/friendica.py +++ b/fba/networks/friendica.py @@ -59,7 +59,7 @@ def fetch_blocks(domain: str) -> dict: ).text, "html.parser", ) - logger.debug("doc[]='%'", type(doc)) + logger.debug("doc[]='%s'", type(doc)) block_tag = doc.find(id="about_blocklist") except network.exceptions as exception: diff --git a/fba/networks/lemmy.py b/fba/networks/lemmy.py index 172e1f8..408d863 100644 --- a/fba/networks/lemmy.py +++ b/fba/networks/lemmy.py @@ -77,12 +77,12 @@ def fetch_peers(domain: str) -> list: if "error_message" in data: logger.warning("Could not reach any JSON API:", domain) instances.set_last_error(domain, data) - elif "federated_instances" in data["json"]: + elif "federated_instances" in data["json"] and isinstance(data["json"]["federated_instances"], dict): logger.debug(f"Found federated_instances for domain='{domain}'") peers = peers + federation.add_peers(data["json"]["federated_instances"]) logger.debug("Added instance(s) to peers") else: - logger.warning("JSON response does not contain 'federated_instances':", domain) + logger.warning("JSON response does not contain 'federated_instances', domain='%s'", domain) instances.set_last_error(domain, data) except network.exceptions as exception: diff --git a/fba/networks/mastodon.py b/fba/networks/mastodon.py index 3f90a9b..21e8a73 100644 --- a/fba/networks/mastodon.py +++ b/fba/networks/mastodon.py @@ -108,7 +108,7 @@ def fetch_blocks_from_about(domain: str) -> dict: "Silenced servers" : [], } - logger.debug("doc[]='%'", type(doc)) + logger.debug("doc[]='%s'", type(doc)) if doc is None: logger.warning(f"Cannot fetch any /about pages for domain='{domain}' - EXIT!") return blocklist diff --git a/fba/networks/pleroma.py b/fba/networks/pleroma.py index c7963bb..bfcf68a 100644 --- a/fba/networks/pleroma.py +++ b/fba/networks/pleroma.py @@ -563,7 +563,7 @@ def fetch_blocks_from_about(domain: str) -> dict: "html.parser", ) - logger.debug("doc[]='%'", type(doc)) + logger.debug("doc[]='%s'", type(doc)) if doc.find("h2") is not None: logger.debug(f"Found 'h2' header in path='{path}' - BREAK!") break @@ -580,7 +580,7 @@ def fetch_blocks_from_about(domain: str) -> dict: "Silenced servers" : [], } - logger.debug("doc[]='%'", type(doc)) + logger.debug("doc[]='%s'", type(doc)) if doc is None: logger.warning(f"Cannot fetch any /about pages for domain='{domain}' - EXIT!") return blocklist -- 2.39.5