From: Roland Häder <roland@mxchange.org>
Date: Fri, 30 Jun 2023 22:49:36 +0000 (+0200)
Subject: Continued:
X-Git-Url: https://git.mxchange.org/?a=commitdiff_plain;h=af5f81581d2792cdb1e43a25809654932928fdb3;p=fba.git

Continued:
- rewrote fetch_observer() to dynamically load software list
- 'dolphin' is another alias for 'misskey'
---

diff --git a/fba/commands.py b/fba/commands.py
index 5caa65f..58a39bc 100644
--- a/fba/commands.py
+++ b/fba/commands.py
@@ -370,41 +370,36 @@ def fetch_blocks(args: argparse.Namespace) -> int:
 
 def fetch_observer(args: argparse.Namespace) -> int:
     logger.debug("args[]='%s' - CALLED!", type(args))
-    types = [
-        "akkoma",
-        "birdsitelive",
-        "bookwyrm",
-        "calckey",
-        "diaspora",
-        "foundkey",
-        "friendica",
-        "funkwhale",
-        "gancio",
-        "gnusocial",
-        "gotosocial",
-        "hometown",
-        "hubzilla",
-        "kbin",
-        "ktistec",
-        "lemmy",
-        "mastodon",
-        "microblogpub",
-        "misskey",
-        "mitra",
-        "mobilizon",
-        "owncast",
-        "peertube",
-        "pixelfed",
-        "pleroma",
-        "plume",
-        "snac",
-        "takahe",
-        "wildebeest",
-        "writefreely"
-    ]
 
+    # Acquire lock
     locking.acquire()
 
+    logger.info("Fetching software list ...")
+    raw = utils.fetch_url(
+        "https://fediverse.observer",
+        network.web_headers,
+        (config.get("connection_timeout"), config.get("read_timeout"))
+    ).text
+    logger.debug("raw[%s]()=%d", type(raw), len(raw))
+
+    doc = bs4.BeautifulSoup(raw, features="html.parser")
+    logger.debug("doc[]='%s'", type(doc))
+
+    items = doc.find("div", {"aria-labelledby": "navbarDropdownMenuSoftwares"}).findAll("a", {"class": "dropdown-item"})
+    logger.debug("items[]='%s'", type(items))
+
+    types = list()
+
+    logger.info("Checking %d menu items ...", len(items))
+    for item in items:
+        logger.debug("item[%s]='%s'", type(item), item)
+        if item.text.lower() == "all":
+            logger.debug("Skipping 'All' menu entry ...")
+            continue
+
+        logger.debug("Appending item.text='%s' ...", item.text)
+        types.append(tidyup.domain(item.text))
+
     logger.info("Fetching %d different table data ...", len(types))
     for software in types:
         logger.debug("software='%s' - BEFORE!", software)
@@ -422,7 +417,7 @@ def fetch_observer(args: argparse.Namespace) -> int:
             ).text
             logger.debug("raw[%s]()=%d", type(raw), len(raw))
 
-            doc = bs4.BeautifulSoup(raw, features='html.parser')
+            doc = bs4.BeautifulSoup(raw, features="html.parser")
             logger.debug("doc[]='%s'", type(doc))
         except network.exceptions as exception:
             logger.warning("Cannot fetch software='%s' from fediverse.observer: '%s'", software, type(exception))
@@ -543,7 +538,7 @@ def fetch_cs(args: argparse.Namespace):
     raw = utils.fetch_url("https://raw.githubusercontent.com/chaossocial/meta/master/federation.md", network.web_headers, (config.get("connection_timeout"), config.get("read_timeout"))).text
     logger.debug("raw()=%d,raw[]='%s'", len(raw), type(raw))
 
-    doc = bs4.BeautifulSoup(markdown.markdown(raw, extensions=extensions), features='html.parser')
+    doc = bs4.BeautifulSoup(markdown.markdown(raw, extensions=extensions), features="html.parser")
     logger.debug("doc()=%d[]='%s'", len(doc), type(doc))
 
     silenced = doc.find("h2", {"id": "silenced-instances"}).findNext("table").find("tbody")
diff --git a/fba/helpers/software.py b/fba/helpers/software.py
index 30e4118..f4e53ca 100644
--- a/fba/helpers/software.py
+++ b/fba/helpers/software.py
@@ -37,7 +37,7 @@ def alias(software: str) -> str:
     elif software in ["hometown", "ecko"]:
         logger.debug("Setting mastodon: software='%s'", software)
         software = "mastodon"
-    elif software in ["slipfox calckey", "calckey", "groundpolis", "foundkey", "cherrypick", "meisskey", "magnetar", "keybump"]:
+    elif software in ["slipfox calckey", "calckey", "groundpolis", "foundkey", "cherrypick", "meisskey", "magnetar", "keybump", "dolphin"]:
         logger.debug("Setting misskey: software='%s'", software)
         software = "misskey"
     elif software == "runtube.re":