Continued:

- don't use tidyup_domain() on header text
- also need to check /about/more - they often "hide" there the block list

diff --git a/fba/fba.py b/fba/fba.py
index 640f9fd2d75a9253d14b7d5c3e67b23abc722174..6d93fe018ed76966d79da5a7a647910e2371a34f 100644 (file)
--- a/fba/fba.py
+++ b/fba/fba.py
@@ -1242,7 +1242,7 @@ def get_mastodon_blocks(domain: str) -> dict:
 
     try:
         doc = bs4.BeautifulSoup(
-            get_response(domain, "/about", headers, (config.get("connection_timeout"), config.get("read_timeout"))).text,
+            get_response(domain, "/about/more", headers, (config.get("connection_timeout"), config.get("read_timeout"))).text,
             "html.parser",
         )
     except BaseException as e:
@@ -1251,8 +1251,9 @@ def get_mastodon_blocks(domain: str) -> dict:
         return {}
 
     for header in doc.find_all("h3"):
-        header_text = tidyup_domain(header.text)
+        header_text = tidyup_reason(header.text)
 
+        # DEBUG: print(f"DEBUG: header_text='{header_text}'")
         if header_text in language_mapping:
             # DEBUG: print(f"DEBUG: header_text='{header_text}'")
             header_text = language_mapping[header_text]