From 18ef4e2fb7bea7b4c758cc74bed74c9cdee99795 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Wed, 16 Aug 2023 15:27:44 +0200 Subject: [PATCH] Continued: - chaos.social isn't part of oliphant, so it still requires being handled separately - fetch software/origin from local database instead software from remote nodeinfo (saves some requests to their servers) --- fba/commands.py | 22 +++++++++++++++++----- fba/http/federation.py | 6 ++++-- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/fba/commands.py b/fba/commands.py index 0c89b33..58e8686 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -321,7 +321,7 @@ def fetch_blocks(args: argparse.Namespace) -> int: blocking = list() - if not blocklists.is_excluded(blocker): + if blocker != "chaos.social" and not blocklists.is_excluded(blocker): logger.debug("blocker='%s',software='%s'", blocker, software) if software == "pleroma": logger.info("blocker='%s',software='%s'", blocker, software) @@ -349,7 +349,7 @@ def fetch_blocks(args: argparse.Namespace) -> int: logger.debug("Invoking instances.set_total_blocks(%s, %d) ...", blocker, len(blocking)) instances.set_total_blocks(blocker, blocking) else: - logger.debug("Skipping chaos.social, run ./fba.py fetch_cs instead!") + logger.debug("Skipping blocker='%s', run ./fba.py fetch_cs or fetch_oliphant instead!", blocker) logger.info("Checking %d entries from blocker='%s',software='%s' ...", len(blocking), blocker, software) blockdict = list() @@ -924,10 +924,21 @@ def fetch_instances(args: argparse.Namespace) -> int: logger.debug("Invoking locking.acquire() ...") locking.acquire() + # Initialize values + domain = tidyup.domain(args.domain) + origin = software = None + + # Fetch record + database.cursor.execute("SELECT origin, software FROM instances WHERE domain = ? LIMIT 1", [args.domain]) + row = database.cursor.fetchone() + if row is not None: + origin = row["origin"] + software = row["software"] + # Initial fetch try: - logger.info("Fetching instances from args.domain='%s' ...", args.domain) - federation.fetch_instances(args.domain, None, None, inspect.currentframe().f_code.co_name) + logger.info("Fetching instances from args.domain='%s',origin='%s',software='%s' ...", domain, origin, software) + federation.fetch_instances(domain, origin, software, inspect.currentframe().f_code.co_name) except network.exceptions as exception: logger.warning("Exception '%s' during fetching instances (fetch_instances) from args.domain='%s'", type(exception), args.domain) instances.set_last_error(args.domain, exception) @@ -1539,8 +1550,9 @@ def recheck_obfuscation(args: argparse.Namespace) -> int: logger.warning("Unknown software: domain='%s',software='%s'", row["domain"], row["software"]) logger.debug("row[domain]='%s'", row["domain"]) + # chaos.social requires special care ... - if not blocklists.is_excluded(row["domain"]): + if row["domain"] != "chaos.social" and not blocklists.is_excluded(row["domain"]): logger.debug("Invoking instances.set_total_blocks(%s, %d) ...", row["domain"], len(blocking)) instances.set_last_blocked(row["domain"]) instances.set_total_blocks(row["domain"], blocking) diff --git a/fba/http/federation.py b/fba/http/federation.py index ff831e5..f4c9529 100644 --- a/fba/http/federation.py +++ b/fba/http/federation.py @@ -48,7 +48,6 @@ logger = logging.getLogger(__name__) def fetch_instances(domain: str, origin: str, software: str, command: str, path: str = None): global _DEPTH logger.debug("domain='%s',origin='%s',software='%s',command='%s',path='%s',_DEPTH=%d - CALLED!", domain, origin, software, command, path, _DEPTH) - _DEPTH = _DEPTH + 1 domain_helper.raise_on(domain) if not isinstance(origin, str) and origin is not None: @@ -61,7 +60,7 @@ def fetch_instances(domain: str, origin: str, software: str, command: str, path: raise ValueError(f"Parameter command='{command}' but origin is None, please fix invoking this function.") elif not isinstance(path, str) and path is not None: raise ValueError(f"Parameter path[]='{type(path)}' is not of type 'str'") - elif instances.is_recent(domain, "last_instance_fetch"): + elif _DEPTH > 0 and instances.is_recent(domain, "last_instance_fetch"): raise ValueError(f"domain='{domain}' has recently been fetched but function was invoked") elif software is None and not instances.is_recent(domain, "last_nodeinfo"): try: @@ -77,6 +76,9 @@ def fetch_instances(domain: str, origin: str, software: str, command: str, path: elif not isinstance(software, str): raise ValueError(f"Parameter software[]='{type(software)}' is not of type 'str'") + # Increase depth + _DEPTH = _DEPTH + 1 + logger.debug("Checking if domain='%s' is registered ...", domain) if not instances.is_registered(domain): logger.debug("Adding new domain='%s',origin='%s',command='%s',path='%s',software='%s'", domain, origin, command, path, software) -- 2.39.5