From: Roland Häder Date: Wed, 20 Aug 2025 22:22:34 +0000 (+0200) Subject: Continued: X-Git-Url: https://git.mxchange.org/?a=commitdiff_plain;h=98a3d3aa5687f8c2e25522613c5c630d82f9142c;p=fba.git Continued: - first checking `instances.is_registered()` and then when it fails (means not registered) checking if domain has recently being crawled doesn't make logically sense --- diff --git a/fba/commands.py b/fba/commands.py index 7561348..a1f1cce 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -169,9 +169,6 @@ def fetch_pixelfed_api(args: argparse.Namespace) -> int: elif instances.is_registered(domain): logger.debug("domain='%s' is already registered - SKIPPED!", domain) continue - elif instances.is_recent(domain): - logger.debug("domain='%s' has recently been crawled - SKIPPED!", domain) - continue logger.debug("Fetching instances from domain='%s' ...", domain) federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) @@ -244,9 +241,6 @@ def fetch_bkali(args: argparse.Namespace) -> int: elif instances.is_registered(entry["domain"]): logger.debug("entry[domain]='%s' is already registered - SKIPPED!", entry["domain"]) continue - elif instances.is_recent(entry["domain"]): - logger.debug("entry[domain]='%s' has recently been crawled - SKIPPED!", entry["domain"]) - continue logger.debug("Adding domain='%s' ...", entry["domain"]) domains.append(entry["domain"]) @@ -621,9 +615,6 @@ def fetch_observer(args: argparse.Namespace) -> int: elif instances.is_registered(domain): logger.debug("domain='%s' is already registered - SKIPPED!", domain) continue - elif instances.is_recent(domain): - logger.debug("domain='%s' has recently been crawled - SKIPPED!", domain) - continue logger.info("Fetching instances for domain='%s',software='%s' ...", domain, software) federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) @@ -894,9 +885,6 @@ def fetch_fba_rss(args: argparse.Namespace) -> int: elif instances.is_registered(domain): logger.debug("domain='%s' is already registered - SKIPPED!", domain) continue - elif instances.is_recent(domain): - logger.debug("domain='%s' has recently been crawled - SKIPPED!", domain) - continue logger.debug("Adding domain='%s'", domain) domains.append(domain) @@ -986,9 +974,6 @@ def fetch_fbabot_atom(args: argparse.Namespace) -> int: elif instances.is_registered(domain): logger.debug("domain='%s' is already registered - SKIPPED!", domain) continue - elif instances.is_recent(domain): - logger.debug("domain='%s' has recently been crawled - SKIPPED!", domain) - continue logger.debug("Adding domain='%s',domains()=%d", domain, len(domains)) domains.append(domain) @@ -1247,9 +1232,6 @@ def fetch_fedipact(args: argparse.Namespace) -> int: elif instances.is_registered(domain): logger.debug("domain='%s' is already registered - SKIPPED!", domain) continue - elif instances.is_recent(domain): - logger.debug("domain='%s' has recently been crawled - SKIPPED!", domain) - continue logger.info("Fetching domain='%s' ...", domain) federation.fetch_instances(domain, "beach.city", None, inspect.currentframe().f_code.co_name) @@ -1294,9 +1276,6 @@ def fetch_joinmobilizon(args: argparse.Namespace) -> int: elif instances.is_registered(row["host"]): logger.debug("row[host]='%s' is already registered - SKIPPED!", row["host"]) continue - elif instances.is_recent(row["host"]): - logger.debug("row[host]='%s' has recently been crawled - SKIPPED!", row["host"]) - continue logger.info("Fetching row[host]='%s' ...", row["host"]) federation.fetch_instances(row["host"], "demo.mobilizon.org", None, inspect.currentframe().f_code.co_name) @@ -1338,9 +1317,6 @@ def fetch_joinmisskey(args: argparse.Namespace) -> int: elif instances.is_registered(row["url"]): logger.debug("row[url]='%s' is already registered - SKIPPED!", row["url"]) continue - elif instances.is_recent(row["url"]): - logger.debug("row[url]='%s' has recently been crawled - SKIPPED!", row["url"]) - continue logger.info("Fetching row[url]='%s' ...", row["url"]) federation.fetch_instances(row["url"], "misskey.io", None, inspect.currentframe().f_code.co_name) @@ -1712,9 +1688,6 @@ def fetch_instances_social(args: argparse.Namespace) -> int: elif instances.is_registered(domain): logger.debug("domain='%s' is already registered - SKIPPED!", domain) continue - elif instances.is_recent(domain): - logger.debug("domain='%s' has recently been crawled - SKIPPED!", domain) - continue logger.info("Fetching instances from domain='%s' ...", domain) federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) @@ -1776,9 +1749,6 @@ def fetch_relaylist(args: argparse.Namespace) -> int: elif instances.is_registered(domain): logger.debug("domain='%s' is already registered - SKIPPED!", domain) continue - elif instances.is_recent(domain): - logger.debug("domain='%s' has recently been crawled - SKIPPED!", domain) - continue logger.info("Fetching instances from domain='%s'", domain) federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) @@ -2007,9 +1977,6 @@ def fetch_relays(args: argparse.Namespace) -> int: elif instances.is_registered(row["domain"]): logger.debug("row[domain]='%s' is already registered - SKIPPED!", row["domain"]) continue - elif instances.is_recent(row["domain"]): - logger.debug("row[domain]='%s' has recently been crawled - SKIPPED!", row["domain"]) - continue logger.info("Fetching row[domain]='%s',row[origin]='%s' ...", row["domain"], row["origin"]) federation.fetch_instances(row["domain"], row["origin"], None, inspect.currentframe().f_code.co_name)