From bf2a2f8af7971d483078e43ad511c759405d07ed Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Sun, 14 Jan 2024 02:25:10 +0100 Subject: [PATCH] WIP: - rewritten fetch_instances towars allowing also --software=foo as an alternative parameter --- fba/commands.py | 82 +++++++++++++++++++++++++++---------------------- 1 file changed, 45 insertions(+), 37 deletions(-) diff --git a/fba/commands.py b/fba/commands.py index ef9d746..8d176af 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -955,53 +955,58 @@ def fetch_fbabot_atom(args: argparse.Namespace) -> int: def fetch_instances(args: argparse.Namespace) -> int: logger.debug("args[]='%s' - CALLED!", type(args)) - logger.debug("args.domain='%s' - checking ...", args.domain) - if not validators.domain(args.domain): - logger.warning("args.domain='%s' is not valid.", args.domain) - return 100 - elif blacklist.is_blacklisted(args.domain): - logger.warning("args.domain='%s' is blacklisted, won't check it!", args.domain) - return 101 - logger.debug("Invoking locking.acquire() ...") locking.acquire() - # Initialize values - domain = tidyup.domain(args.domain) - origin = software = None + # Is domain or software set? + if args.domain != "": + logger.debug("args.domain='%s' - checking ...", args.domain) + if not validators.domain(args.domain): + logger.warning("args.domain='%s' is not valid.", args.domain) + return 100 + elif blacklist.is_blacklisted(args.domain): + logger.warning("args.domain='%s' is blacklisted, won't check it!", args.domain) + return 101 - # Fetch record - database.cursor.execute("SELECT origin, software FROM instances WHERE domain = ? LIMIT 1", [args.domain]) - row = database.cursor.fetchone() - if row is not None: - origin = row["origin"] - software = row["software"] + logger.debug("args.domain='%s' - BEFORE!", args.domain) + domain = tidyup.domain(args.domain) + logger.debug("domain='%s' - AFTER!", domain) - logger.debug("software='%s'", software) - if software is None: - logger.warning("args.domain='%s' has no software detected. You can try to run ./fba.py update_nodeinfo --domain=%s --force to get it updated.", args.domain, args.domain) - return 102 - elif software_helper.is_relay(software): - logger.warning("args.domain='%s' is of software type '%s' which is not supported by this command. Please invoke fetch_relays instead.", args.domain, software) - return 103 + # Fetch record + database.cursor.execute("SELECT domain, origin, software FROM instances WHERE domain = ? LIMIT 1", [domain]) - # Initial fetch - try: - logger.info("Fetching instances from args.domain='%s',origin='%s',software='%s' ...", domain, origin, software) - federation.fetch_instances(domain, origin, software, inspect.currentframe().f_code.co_name) - except network.exceptions as exception: - logger.warning("Exception '%s' during fetching instances (fetch_instances) from args.domain='%s'", type(exception), args.domain) - instances.set_last_error(args.domain, exception) - instances.update(args.domain) - return 104 + rows = database.cursor.fetchall() + logger.info("Checking %d entries ...", len(rows)) + for row in rows: + logger.debug("row[domain]='%s',row[origin]='%s',row[software]='%s'", row["domain"], row["origin"], row["software"]) + if row["software"] is None: + logger.warning("row[domain]='%s' has no software detected. You can try to run ./fba.py update_nodeinfo --domain=%s --force to get it updated - SKIPPED!", row["domain"], row["domain"]) + continue + elif software_helper.is_relay(row["software"]): + logger.warning("row[domain]='%s' is of software type '%s' which is not supported by this command. Please invoke fetch_relays instead - SKIPPED!", row["domain"], row["software"]) + continue - if args.single: - logger.debug("Not fetching more instances - EXIT!") - return 0 + # Initial fetch + try: + logger.info("Fetching instances from row[domain]='%s',row[origin]='%s',row[software]='%s' ...", row["domain"], row["origin"], row["software"]) + federation.fetch_instances(row["domain"], row["origin"], row["software"], inspect.currentframe().f_code.co_name) + except network.exceptions as exception: + logger.warning("Exception '%s' during fetching instances (fetch_instances) from row[domain]='%s'", type(exception), row["domain"]) + instances.set_last_error(row["domain"], exception) + instances.update(row["domain"]) + raise exception + + if args.single: + logger.debug("Not fetching more instances - BREAK!") + break # Loop through some instances database.cursor.execute( - "SELECT domain, origin, software FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'lemmy', 'peertube', 'takahe', 'gotosocial', 'brighteon', 'wildebeest', 'bookwyrm', 'mitra', 'areionskey', 'mammuthus', 'neodb') AND (last_instance_fetch IS NULL OR last_instance_fetch < ?) ORDER BY total_peers DESC, last_response_time ASC, last_updated ASC", [time.time() - config.get("recheck_instance")] + "SELECT domain, origin, software \ +FROM instances \ +WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'lemmy', 'peertube', 'takahe', 'gotosocial', 'brighteon', 'wildebeest', 'bookwyrm', 'mitra', 'areionskey', 'mammuthus', 'neodb') \ +AND (last_instance_fetch IS NULL OR last_instance_fetch < ?) \ +ORDER BY total_peers DESC, last_response_time ASC, last_updated ASC", [time.time() - config.get("recheck_instance")] ) rows = database.cursor.fetchall() @@ -1280,10 +1285,13 @@ def recheck_obfuscation(args: argparse.Namespace) -> int: locking.acquire() if isinstance(args.domain, str) and args.domain != "" and domain_helper.is_wanted(args.domain): + logger.debug("Fetching record for args.domain='%s' ...", args.domain) database.cursor.execute("SELECT domain, software, nodeinfo_url FROM instances WHERE (has_obfuscation = 1 OR has_obfuscation IS NULL) AND domain = ?", [args.domain]) elif isinstance(args.software, str) and args.software != "" and validators.domain(args.software) == args.software: + logger.debug("Fetching records for args.software='%s' ...", args.software) database.cursor.execute("SELECT domain, software, nodeinfo_url FROM instances WHERE (has_obfuscation = 1 OR has_obfuscation IS NULL) AND software = ?", [args.software]) else: + logger.debug("Fetching records where domains have obfuscated block entries ...") database.cursor.execute("SELECT domain, software, nodeinfo_url FROM instances WHERE has_obfuscation = 1 OR has_obfuscation IS NULL") rows = database.cursor.fetchall() -- 2.39.2