From 95c5a926aeac455c476dd3a9e7baa429d6494f84 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Sat, 5 Aug 2023 23:54:19 +0200 Subject: [PATCH] Continued: - added network 'mammuthus[ experimental]' for retriving peers - moved software-related (not version number) functions to software.py - strip off " experimental", so you can enter e.g. 'mammuthus' easier --- daemon.py | 2 +- fba/commands.py | 2 +- fba/helpers/blacklist.py | 1 + fba/helpers/software.py | 79 ++++++++++++++++++++++++++++++++++++++-- fba/helpers/version.py | 68 ---------------------------------- fba/http/federation.py | 10 ++--- 6 files changed, 83 insertions(+), 79 deletions(-) diff --git a/daemon.py b/daemon.py index bfede50..cef815e 100755 --- a/daemon.py +++ b/daemon.py @@ -56,7 +56,7 @@ templates = Jinja2Templates(directory="templates") @router.get(config.get("base_url") + "/api/info.json", response_class=JSONResponse) def api_info(): - database.cursor.execute("SELECT (SELECT COUNT(domain) FROM instances), (SELECT COUNT(domain) FROM instances WHERE software IN ('pleroma', 'mastodon', 'lemmy', 'friendica', 'misskey', 'peertube', 'takahe', 'gotosocial', 'brighteon', 'wildebeest', 'bookwyrm', 'mitra', 'areionskey')), (SELECT COUNT(blocker) FROM blocks), (SELECT COUNT(domain) FROM instances WHERE last_error_details IS NOT NULL)") + database.cursor.execute("SELECT (SELECT COUNT(domain) FROM instances), (SELECT COUNT(domain) FROM instances WHERE software IN ('pleroma', 'mastodon', 'lemmy', 'friendica', 'misskey', 'peertube', 'takahe', 'gotosocial', 'brighteon', 'wildebeest', 'bookwyrm', 'mitra', 'areionskey', 'mammuthus')), (SELECT COUNT(blocker) FROM blocks), (SELECT COUNT(domain) FROM instances WHERE last_error_details IS NOT NULL)") row = database.cursor.fetchone() return JSONResponse(status_code=200, content={ diff --git a/fba/commands.py b/fba/commands.py index f35c096..bc8017a 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -938,7 +938,7 @@ def fetch_instances(args: argparse.Namespace) -> int: # Loop through some instances database.cursor.execute( - "SELECT domain, origin, software, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'lemmy', 'peertube', 'takahe', 'gotosocial', 'brighteon', 'wildebeest', 'bookwyrm', 'mitra', 'areionskey') AND (last_instance_fetch IS NULL OR last_instance_fetch < ?) ORDER BY rowid DESC", [time.time() - config.get("recheck_instance")] + "SELECT domain, origin, software, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'lemmy', 'peertube', 'takahe', 'gotosocial', 'brighteon', 'wildebeest', 'bookwyrm', 'mitra', 'areionskey', 'mammuthus') AND (last_instance_fetch IS NULL OR last_instance_fetch < ?) ORDER BY rowid DESC", [time.time() - config.get("recheck_instance")] ) rows = database.cursor.fetchall() diff --git a/fba/helpers/blacklist.py b/fba/helpers/blacklist.py index 282628c..2a65fbc 100644 --- a/fba/helpers/blacklist.py +++ b/fba/helpers/blacklist.py @@ -39,6 +39,7 @@ _blacklist = { "everyoneattack.com" : "Floods federation tables with fake nodes", "vercel.app" : "Floods federation tables with fake nodes", "run.app" : "Floods federation tables with fake nodes", + "denden.world" : "Looks like a valid Mastodon instance, but return exactly (!) 5k instances with trash domains", "fnaf.stream" : "Trolls with over-long sub-domain name(s)", "ngrok.io" : "Testing/developing instances shouldn't be part of public instances", "ngrok.app" : "Testing/developing instances shouldn't be part of public instances", diff --git a/fba/helpers/software.py b/fba/helpers/software.py index 646442b..858954a 100644 --- a/fba/helpers/software.py +++ b/fba/helpers/software.py @@ -81,17 +81,20 @@ def alias(software: str) -> str: software = software.split("|")[0] elif "powered by" in software: logger.debug("software='%s' has 'powered by' in it", software) - software = version.strip_powered_by(software) + software = strip_powered_by(software) + elif software.endswith(" experimental"): + logger.debug("software='%s' ends with 'experimental", software) + software = strip_until("experimental") if isinstance(software, str) and " by " in software: logger.debug("software='%s' has ' by ' in it", software) - software = version.strip_until(software, " by ") + software = strip_until(software, " by ") elif isinstance(software, str) and " - " in software: logger.debug("software='%s' has ' - ' in it", software) - software = version.strip_until(software, " - ") + software = strip_until(software, " - ") elif isinstance(software, str) and " see " in software: logger.debug("software='%s' has ' see ' in it", software) - software = version.strip_until(software, " see ") + software = strip_until(software, " see ") logger.debug("software['%s']='%s'", type(software), software) if software == "": @@ -103,3 +106,71 @@ def alias(software: str) -> str: logger.debug("software[%s]='%s' - EXIT!", type(software), software) return software + +def strip_hosted_on(software: str) -> str: + logger.debug("software='%s' - CALLED!", software) + if not isinstance(software, str): + raise ValueError(f"Parameter software[]='{type(software)}' is not of type 'str'") + elif software == "": + raise ValueError("Parameter 'software' is empty") + elif "hosted on" not in software: + logger.warning("Cannot find 'hosted on' in software='%s'!", software) + return software + + end = software.find("hosted on ") + logger.debug("end[%s]=%d", type(end), end) + + software = software[0:end].strip() + logger.debug("software[%s]='%s'", type(software), software) + + if " - " in software: + software = strip_until(software, " - ") + + logger.debug("software='%s' - EXIT!", software) + return software + +def strip_powered_by(software: str) -> str: + logger.debug("software='%s' - CALLED!", software) + if not isinstance(software, str): + raise ValueError(f"Parameter software[]='{type(software)}' is not of type 'str'") + elif software == "": + raise ValueError("Parameter 'software' is empty") + elif "powered by" not in software: + logger.warning("Cannot find 'powered by' in software='%s'!", software) + return software + + start = software.find("powered by ") + logger.debug("start[%s]=%d", type(start), start) + + software = software[start + 11:].strip() + logger.debug("software='%s'", software) + + if " - " in software: + software = strip_until(software, " - ") + + logger.debug("software='%s' - EXIT!", software) + return software + +def strip_until(software: str, until: str) -> str: + logger.debug("software='%s',until='%s' - CALLED!", software, until) + if not isinstance(software, str): + raise ValueError(f"Parameter software[]='{type(software)}' is not of type 'str'") + elif software == "": + raise ValueError("Parameter 'software' is empty") + elif not isinstance(until, str): + raise ValueError(f"Parameter until[]='{type(until)}' is not of type 'str'") + elif until == "": + raise ValueError("Parameter 'until' is empty") + elif not until in software: + logger.warning("Cannot find until='%s' in software='%s'!", until, software) + return software + + # Next, strip until part + end = software.find(until) + + logger.debug("end[%s]=%d", type(end), end) + if end > 0: + software = software[0:end].strip() + + logger.debug("software='%s' - EXIT!", software) + return software diff --git a/fba/helpers/version.py b/fba/helpers/version.py index bb8dc9a..1396b4e 100644 --- a/fba/helpers/version.py +++ b/fba/helpers/version.py @@ -84,71 +84,3 @@ def remove(software: str) -> str: logger.debug("software='%s' - EXIT!", software) return software - -def strip_powered_by(software: str) -> str: - logger.debug("software='%s' - CALLED!", software) - if not isinstance(software, str): - raise ValueError(f"Parameter software[]='{type(software)}' is not of type 'str'") - elif software == "": - raise ValueError("Parameter 'software' is empty") - elif "powered by" not in software: - logger.warning("Cannot find 'powered by' in software='%s'!", software) - return software - - start = software.find("powered by ") - logger.debug("start[%s]=%d", type(start), start) - - software = software[start + 11:].strip() - logger.debug("software='%s'", software) - - if " - " in software: - software = strip_until(software, " - ") - - logger.debug("software='%s' - EXIT!", software) - return software - -def strip_hosted_on(software: str) -> str: - logger.debug("software='%s' - CALLED!", software) - if not isinstance(software, str): - raise ValueError(f"Parameter software[]='{type(software)}' is not of type 'str'") - elif software == "": - raise ValueError("Parameter 'software' is empty") - elif "hosted on" not in software: - logger.warning("Cannot find 'hosted on' in software='%s'!", software) - return software - - end = software.find("hosted on ") - logger.debug("end[%s]=%d", type(end), end) - - software = software[0:end].strip() - logger.debug("software[%s]='%s'", type(software), software) - - if " - " in software: - software = strip_until(software, " - ") - - logger.debug("software='%s' - EXIT!", software) - return software - -def strip_until(software: str, until: str) -> str: - logger.debug("software='%s',until='%s' - CALLED!", software, until) - if not isinstance(software, str): - raise ValueError(f"Parameter software[]='{type(software)}' is not of type 'str'") - elif software == "": - raise ValueError("Parameter 'software' is empty") - elif not isinstance(until, str): - raise ValueError(f"Parameter until[]='{type(until)}' is not of type 'str'") - elif until == "": - raise ValueError("Parameter 'until' is empty") - elif not until in software: - logger.warning("Cannot find until='%s' in software='%s'!", until, software) - return software - - # Next, strip until part - end = software.find(until) - - logger.debug("end[%s]=%d", type(end), end) - if end > 0: - software = software[0:end].strip() - - logger.debug("software='%s' - EXIT!", software) - return software diff --git a/fba/http/federation.py b/fba/http/federation.py index 3b0e990..1c8f329 100644 --- a/fba/http/federation.py +++ b/fba/http/federation.py @@ -319,16 +319,16 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str: logger.debug("software[]='%s'", type(software)) if isinstance(software, str) and "powered by " in software: logger.debug("software='%s' has 'powered by' in it", software) - software = version.remove(version.strip_powered_by(software)) + software = version.remove(software_helper.strip_powered_by(software)) elif isinstance(software, str) and " hosted on " in software: logger.debug("software='%s' has 'hosted on' in it", software) - software = version.remove(version.strip_hosted_on(software)) + software = version.remove(software_helper.strip_hosted_on(software)) elif isinstance(software, str) and " by " in software: logger.debug("software='%s' has ' by ' in it", software) - software = version.strip_until(software, " by ") + software = software_helper.strip_until(software, " by ") elif isinstance(software, str) and " see " in software: logger.debug("software='%s' has ' see ' in it", software) - software = version.strip_until(software, " see ") + software = software_helper.strip_until(software, " see ") logger.debug("software='%s' - EXIT!", software) return software @@ -415,7 +415,7 @@ def determine_software(domain: str, path: str = None) -> str: logger.debug("software[]='%s'", type(software)) if isinstance(software, str) and "powered by" in software: logger.debug("software='%s' has 'powered by' in it", software) - software = version.remove(version.strip_powered_by(software)) + software = version.remove(software_helper.strip_powered_by(software)) logger.debug("software='%s' - EXIT!", software) return software -- 2.39.5