Continued:

- added network 'mammuthus[ experimental]' for retriving peers
- moved software-related (not version number) functions to software.py
- strip off " experimental", so you can enter e.g. 'mammuthus' easier

diff --git a/daemon.py b/daemon.py
index bfede5053fd4d0b566d77089092de532ee12a3cc..cef815e6ca3255a4e3202a776a574503e236b4df 100755 (executable)
--- a/daemon.py
+++ b/daemon.py
@@ -56,7 +56,7 @@ templates = Jinja2Templates(directory="templates")
 
 @router.get(config.get("base_url") + "/api/info.json", response_class=JSONResponse)
 def api_info():
-    database.cursor.execute("SELECT (SELECT COUNT(domain) FROM instances), (SELECT COUNT(domain) FROM instances WHERE software IN ('pleroma', 'mastodon', 'lemmy', 'friendica', 'misskey', 'peertube', 'takahe', 'gotosocial', 'brighteon', 'wildebeest', 'bookwyrm', 'mitra', 'areionskey')), (SELECT COUNT(blocker) FROM blocks), (SELECT COUNT(domain) FROM instances WHERE last_error_details IS NOT NULL)")
+    database.cursor.execute("SELECT (SELECT COUNT(domain) FROM instances), (SELECT COUNT(domain) FROM instances WHERE software IN ('pleroma', 'mastodon', 'lemmy', 'friendica', 'misskey', 'peertube', 'takahe', 'gotosocial', 'brighteon', 'wildebeest', 'bookwyrm', 'mitra', 'areionskey', 'mammuthus')), (SELECT COUNT(blocker) FROM blocks), (SELECT COUNT(domain) FROM instances WHERE last_error_details IS NOT NULL)")
     row = database.cursor.fetchone()
 
     return JSONResponse(status_code=200, content={

diff --git a/fba/commands.py b/fba/commands.py
index f35c0961571d79551f6291eb6f3aea5c023cade4..bc8017abfbbbae44868afbe692ed0b9ad7e347fd 100644 (file)
--- a/fba/commands.py
+++ b/fba/commands.py
@@ -938,7 +938,7 @@ def fetch_instances(args: argparse.Namespace) -> int:
 
     # Loop through some instances
     database.cursor.execute(
-        "SELECT domain, origin, software, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'lemmy', 'peertube', 'takahe', 'gotosocial', 'brighteon', 'wildebeest', 'bookwyrm', 'mitra', 'areionskey') AND (last_instance_fetch IS NULL OR last_instance_fetch < ?) ORDER BY rowid DESC", [time.time() - config.get("recheck_instance")]
+        "SELECT domain, origin, software, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'lemmy', 'peertube', 'takahe', 'gotosocial', 'brighteon', 'wildebeest', 'bookwyrm', 'mitra', 'areionskey', 'mammuthus') AND (last_instance_fetch IS NULL OR last_instance_fetch < ?) ORDER BY rowid DESC", [time.time() - config.get("recheck_instance")]
     )
 
     rows = database.cursor.fetchall()

diff --git a/fba/helpers/blacklist.py b/fba/helpers/blacklist.py
index 282628c07de254ab0e547aa9bc6f4d021b1da283..2a65fbc5037ff24a225907bcc625305ade62c790 100644 (file)
--- a/fba/helpers/blacklist.py
+++ b/fba/helpers/blacklist.py
@@ -39,6 +39,7 @@ _blacklist = {
     "everyoneattack.com"  : "Floods federation tables with fake nodes",
     "vercel.app"          : "Floods federation tables with fake nodes",
     "run.app"             : "Floods federation tables with fake nodes",
+    "denden.world"        : "Looks like a valid Mastodon instance, but return exactly (!) 5k instances with trash domains",
     "fnaf.stream"         : "Trolls with over-long sub-domain name(s)",
     "ngrok.io"            : "Testing/developing instances shouldn't be part of public instances",
     "ngrok.app"           : "Testing/developing instances shouldn't be part of public instances",

diff --git a/fba/helpers/software.py b/fba/helpers/software.py
index 646442bad0ad96dcfe4b60efd8f3ff484e5b45f3..858954a9a07d594de6b7acf59614b84de7c80306 100644 (file)
--- a/fba/helpers/software.py
+++ b/fba/helpers/software.py
@@ -81,17 +81,20 @@ def alias(software: str) -> str:
         software = software.split("|")[0]
     elif "powered by" in software:
         logger.debug("software='%s' has 'powered by' in it", software)
-        software = version.strip_powered_by(software)
+        software = strip_powered_by(software)
+    elif software.endswith(" experimental"):
+        logger.debug("software='%s' ends with 'experimental", software)
+        software = strip_until("experimental")
 
     if isinstance(software, str) and " by " in software:
         logger.debug("software='%s' has ' by ' in it", software)
-        software = version.strip_until(software, " by ")
+        software = strip_until(software, " by ")
     elif isinstance(software, str) and " - " in software:
         logger.debug("software='%s' has ' - ' in it", software)
-        software = version.strip_until(software, " - ")
+        software = strip_until(software, " - ")
     elif isinstance(software, str) and " see " in software:
         logger.debug("software='%s' has ' see ' in it", software)
-        software = version.strip_until(software, " see ")
+        software = strip_until(software, " see ")
 
     logger.debug("software['%s']='%s'", type(software), software)
     if software == "":
@@ -103,3 +106,71 @@ def alias(software: str) -> str:
 
     logger.debug("software[%s]='%s' - EXIT!", type(software), software)
     return software
+
+def strip_hosted_on(software: str) -> str:
+    logger.debug("software='%s' - CALLED!", software)
+    if not isinstance(software, str):
+        raise ValueError(f"Parameter software[]='{type(software)}' is not of type 'str'")
+    elif software == "":
+        raise ValueError("Parameter 'software' is empty")
+    elif "hosted on" not in software:
+        logger.warning("Cannot find 'hosted on' in software='%s'!", software)
+        return software
+
+    end = software.find("hosted on ")
+    logger.debug("end[%s]=%d", type(end), end)
+
+    software = software[0:end].strip()
+    logger.debug("software[%s]='%s'", type(software), software)
+
+    if " - " in software:
+        software = strip_until(software, " - ")
+
+    logger.debug("software='%s' - EXIT!", software)
+    return software
+
+def strip_powered_by(software: str) -> str:
+    logger.debug("software='%s' - CALLED!", software)
+    if not isinstance(software, str):
+        raise ValueError(f"Parameter software[]='{type(software)}' is not of type 'str'")
+    elif software == "":
+        raise ValueError("Parameter 'software' is empty")
+    elif "powered by" not in software:
+        logger.warning("Cannot find 'powered by' in software='%s'!", software)
+        return software
+
+    start = software.find("powered by ")
+    logger.debug("start[%s]=%d", type(start), start)
+
+    software = software[start + 11:].strip()
+    logger.debug("software='%s'", software)
+
+    if " - " in software:
+        software = strip_until(software, " - ")
+
+    logger.debug("software='%s' - EXIT!", software)
+    return software
+
+def strip_until(software: str, until: str) -> str:
+    logger.debug("software='%s',until='%s' - CALLED!", software, until)
+    if not isinstance(software, str):
+        raise ValueError(f"Parameter software[]='{type(software)}' is not of type 'str'")
+    elif software == "":
+        raise ValueError("Parameter 'software' is empty")
+    elif not isinstance(until, str):
+        raise ValueError(f"Parameter until[]='{type(until)}' is not of type 'str'")
+    elif until == "":
+        raise ValueError("Parameter 'until' is empty")
+    elif not until in software:
+        logger.warning("Cannot find until='%s' in software='%s'!", until, software)
+        return software
+
+    # Next, strip until part
+    end = software.find(until)
+
+    logger.debug("end[%s]=%d", type(end), end)
+    if end > 0:
+        software = software[0:end].strip()
+
+    logger.debug("software='%s' - EXIT!", software)
+    return software

diff --git a/fba/helpers/version.py b/fba/helpers/version.py
index bb8dc9aa614ce4d81e98d0007f1179e09196c149..1396b4e27cb444948cffc9a6ccac0cdcda62b543 100644 (file)
--- a/fba/helpers/version.py
+++ b/fba/helpers/version.py
@@ -84,71 +84,3 @@ def remove(software: str) -> str:
 
     logger.debug("software='%s' - EXIT!", software)
     return software
-
-def strip_powered_by(software: str) -> str:
-    logger.debug("software='%s' - CALLED!", software)
-    if not isinstance(software, str):
-        raise ValueError(f"Parameter software[]='{type(software)}' is not of type 'str'")
-    elif software == "":
-        raise ValueError("Parameter 'software' is empty")
-    elif "powered by" not in software:
-        logger.warning("Cannot find 'powered by' in software='%s'!", software)
-        return software
-
-    start = software.find("powered by ")
-    logger.debug("start[%s]=%d", type(start), start)
-
-    software = software[start + 11:].strip()
-    logger.debug("software='%s'", software)
-
-    if " - " in software:
-        software = strip_until(software, " - ")
-
-    logger.debug("software='%s' - EXIT!", software)
-    return software
-
-def strip_hosted_on(software: str) -> str:
-    logger.debug("software='%s' - CALLED!", software)
-    if not isinstance(software, str):
-        raise ValueError(f"Parameter software[]='{type(software)}' is not of type 'str'")
-    elif software == "":
-        raise ValueError("Parameter 'software' is empty")
-    elif "hosted on" not in software:
-        logger.warning("Cannot find 'hosted on' in software='%s'!", software)
-        return software
-
-    end = software.find("hosted on ")
-    logger.debug("end[%s]=%d", type(end), end)
-
-    software = software[0:end].strip()
-    logger.debug("software[%s]='%s'", type(software), software)
-
-    if " - " in software:
-        software = strip_until(software, " - ")
-
-    logger.debug("software='%s' - EXIT!", software)
-    return software
-
-def strip_until(software: str, until: str) -> str:
-    logger.debug("software='%s',until='%s' - CALLED!", software, until)
-    if not isinstance(software, str):
-        raise ValueError(f"Parameter software[]='{type(software)}' is not of type 'str'")
-    elif software == "":
-        raise ValueError("Parameter 'software' is empty")
-    elif not isinstance(until, str):
-        raise ValueError(f"Parameter until[]='{type(until)}' is not of type 'str'")
-    elif until == "":
-        raise ValueError("Parameter 'until' is empty")
-    elif not until in software:
-        logger.warning("Cannot find until='%s' in software='%s'!", until, software)
-        return software
-
-    # Next, strip until part
-    end = software.find(until)
-
-    logger.debug("end[%s]=%d", type(end), end)
-    if end > 0:
-        software = software[0:end].strip()
-
-    logger.debug("software='%s' - EXIT!", software)
-    return software

diff --git a/fba/http/federation.py b/fba/http/federation.py
index 3b0e990c5f077c10177fbaba30f8563a2e204de9..1c8f3298150d39c5f323edda8d790bd915e4b98a 100644 (file)
--- a/fba/http/federation.py
+++ b/fba/http/federation.py
@@ -319,16 +319,16 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str:
     logger.debug("software[]='%s'", type(software))
     if isinstance(software, str) and "powered by " in software:
         logger.debug("software='%s' has 'powered by' in it", software)
-        software = version.remove(version.strip_powered_by(software))
+        software = version.remove(software_helper.strip_powered_by(software))
     elif isinstance(software, str) and " hosted on " in software:
         logger.debug("software='%s' has 'hosted on' in it", software)
-        software = version.remove(version.strip_hosted_on(software))
+        software = version.remove(software_helper.strip_hosted_on(software))
     elif isinstance(software, str) and " by " in software:
         logger.debug("software='%s' has ' by ' in it", software)
-        software = version.strip_until(software, " by ")
+        software = software_helper.strip_until(software, " by ")
     elif isinstance(software, str) and " see " in software:
         logger.debug("software='%s' has ' see ' in it", software)
-        software = version.strip_until(software, " see ")
+        software = software_helper.strip_until(software, " see ")
 
     logger.debug("software='%s' - EXIT!", software)
     return software
@@ -415,7 +415,7 @@ def determine_software(domain: str, path: str = None) -> str:
     logger.debug("software[]='%s'", type(software))
     if isinstance(software, str) and "powered by" in software:
         logger.debug("software='%s' has 'powered by' in it", software)
-        software = version.remove(version.strip_powered_by(software))
+        software = version.remove(software_helper.strip_powered_by(software))
 
     logger.debug("software='%s' - EXIT!", software)
     return software