From a21887c7885b1fa6085794712f3f415bd91ce0a8 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Wed, 10 Jul 2024 01:09:45 +0200 Subject: [PATCH] Continued: - added 'wordpress' and 'oolong' as peer sources - some APIs may return a dictionary (dict) for an instance entry --- daemon.py | 2 +- fba/commands.py | 2 +- fba/http/federation.py | 11 +++++++++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/daemon.py b/daemon.py index 0d34158..9c76df9 100755 --- a/daemon.py +++ b/daemon.py @@ -56,7 +56,7 @@ templates = Jinja2Templates(directory="templates") @router.get(config.get("base_url") + "/api/info.json", response_class=JSONResponse) def api_info(): - database.cursor.execute("SELECT (SELECT COUNT(domain) FROM instances) AS total_websites, (SELECT COUNT(domain) FROM instances WHERE software IN ('pleroma', 'mastodon', 'lemmy', 'friendica', 'misskey', 'peertube', 'takahe', 'gotosocial', 'brighteon', 'wildebeest', 'bookwyrm', 'mitra', 'areionskey', 'mammuthus', 'neodb', 'smithereen', 'vebinet', 'toki', 'snac', 'biblioreads')) AS supported_instances, (SELECT COUNT(blocker) FROM blocks) AS total_blocks, (SELECT COUNT(domain) FROM instances WHERE last_error_details IS NOT NULL) AS erroneous_instances") + database.cursor.execute("SELECT (SELECT COUNT(domain) FROM instances) AS total_websites, (SELECT COUNT(domain) FROM instances WHERE software IN ('pleroma', 'mastodon', 'lemmy', 'friendica', 'misskey', 'peertube', 'takahe', 'gotosocial', 'brighteon', 'wildebeest', 'bookwyrm', 'mitra', 'areionskey', 'mammuthus', 'neodb', 'smithereen', 'vebinet', 'toki', 'snac', 'biblioreads', 'wordpress', 'oolong')) AS supported_instances, (SELECT COUNT(blocker) FROM blocks) AS total_blocks, (SELECT COUNT(domain) FROM instances WHERE last_error_details IS NOT NULL) AS erroneous_instances") row = database.cursor.fetchone() diff --git a/fba/commands.py b/fba/commands.py index 8643c30..36c1b2e 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -1031,7 +1031,7 @@ def fetch_instances(args: argparse.Namespace) -> int: database.cursor.execute( "SELECT domain, origin, software \ FROM instances \ -WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'lemmy', 'peertube', 'takahe', 'gotosocial', 'brighteon', 'wildebeest', 'bookwyrm', 'mitra', 'areionskey', 'mammuthus', 'neodb', 'smithereen', 'vebinet', 'toki', 'snac', 'biblioreads', 'wordpress') \ +WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'lemmy', 'peertube', 'takahe', 'gotosocial', 'brighteon', 'wildebeest', 'bookwyrm', 'mitra', 'areionskey', 'mammuthus', 'neodb', 'smithereen', 'vebinet', 'toki', 'snac', 'biblioreads', 'wordpress', 'oolong') \ ORDER BY total_peers DESC, last_response_time ASC, last_updated ASC" ) diff --git a/fba/http/federation.py b/fba/http/federation.py index 6fea3ea..8c1523c 100644 --- a/fba/http/federation.py +++ b/fba/http/federation.py @@ -148,6 +148,17 @@ def fetch_instances(domain: str, origin: str, software: str, command: str, path: if instance in [None, ""]: logger.debug("instance[%s]='%s' is either None or empty - SKIPPED!", type(instance), instance) continue + elif isinstance(instance, dict) and "url" in instance: + logger.debug("Found instance[url]='%s', extracting domain/host name ...", instance["url"]) + if not validators.url(instance["url"]): + logger.warning("instance[url]='%s' is not a valid URL - SKIPPED!", instance["url"]) + continue + + components = urlparse(instance["url"]) + logger.debug("components[%s]()=%d", type(components), len(components)) + + instance = components.netloc.lower().split(":")[0] + logger.debug("instance='%s'", instance) logger.debug("instance='%s' - BEFORE!", instance) instance = tidyup.domain(instance) if isinstance(instance, str) and instance != "" else None -- 2.39.5