Continued:

- 'misskey' is also a software using obfuscation of blocked instances
- let us better combine None/empty string

diff --git a/daemon.py b/daemon.py
index f5bfd7f7852390af4bfea976aac1fc541f493f65..ff68610febec2415e866f67ff29c73c54be17251 100755 (executable)
--- a/daemon.py
+++ b/daemon.py
@@ -92,7 +92,7 @@ def api_scoreboard(mode: str, amount: int):
     elif mode == "obfuscator":
         database.cursor.execute("SELECT software, COUNT(domain) AS cnt FROM instances WHERE has_obfuscation = 1 GROUP BY software ORDER BY cnt DESC LIMIT ?", [amount])
     elif mode == "obfuscation":
-        database.cursor.execute("SELECT has_obfuscation, COUNT(domain) AS cnt FROM instances WHERE software IN ('pleroma', 'lemmy', 'mastodon', 'friendica') GROUP BY has_obfuscation ORDER BY cnt DESC LIMIT ?", [amount])
+        database.cursor.execute("SELECT has_obfuscation, COUNT(domain) AS cnt FROM instances WHERE software IN ('pleroma', 'lemmy', 'mastodon', 'misskey', 'friendica') GROUP BY has_obfuscation ORDER BY cnt DESC LIMIT ?", [amount])
     elif mode == "block_level":
         database.cursor.execute("SELECT block_level, COUNT(rowid) AS cnt FROM blocks GROUP BY block_level ORDER BY cnt DESC LIMIT ?", [amount])
     else:

diff --git a/fba/commands.py b/fba/commands.py
index 186619a8a61405965c5cd9c49b58348560545db6..4b41ac47d6b5e7207012e7a90fcac46666ad7284 100644 (file)
--- a/fba/commands.py
+++ b/fba/commands.py
@@ -148,11 +148,8 @@ def fetch_pixelfed_api(args: argparse.Namespace) -> int:
             if "domain" not in row:
                 logger.warning("row='%s' does not contain element 'domain' - SKIPPED!", row)
                 continue
-            elif row["domain"] is None:
-                logger.debug("row[domain] is None - SKIPPED!")
-                continue
-            elif row["domain"] == "":
-                logger.debug("row[domain] is empty - SKIPPED!")
+            elif row["domain"] is None or row["domain"] == "":
+                logger.debug("row[domain]='%s' is empty - SKIPPED!", row["domain"])
                 continue
 
             logger.debug("row[domain]='%s' - BEFORE!", row["domain"])
@@ -227,8 +224,8 @@ def fetch_bkali(args: argparse.Namespace) -> int:
             if "domain" not in entry:
                 logger.warning("entry()=%d does not contain 'domain' - SKIPPED!", len(entry))
                 continue
-            elif entry["domain"] == "":
-                logger.debug("entry[domain] is empty - SKIPPED!")
+            elif entry["domain"] is None or entry["domain"] == "":
+                logger.debug("entry[domain]='%s' is empty - SKIPPED!", entry["domain"])
                 continue
             elif not domain_helper.is_wanted(entry["domain"]):
                 logger.debug("entry[domain]='%s' is not wanted - SKIPPED!", entry["domain"])
@@ -368,8 +365,8 @@ def fetch_blocks(args: argparse.Namespace) -> int:
             block["reason"]  = tidyup.reason(block["reason"]) if block["reason"] is not None and block["reason"] != "" else None
             logger.debug("blocked='%s',reason='%s' - AFTER!", block["blocked"], block["reason"])
 
-            if block["blocked"] == "":
-                logger.warning("blocked is empty, blocker='%s'", blocker)
+            if block["blocked"] is None or block["blocked"] == "":
+                logger.warning("block[blocked]='%s' is empty, blocker='%s'", block["blocked"], blocker)
                 continue
             elif block["blocked"].endswith(".onion"):
                 logger.debug("blocked='%s' is a TOR .onion domain - SKIPPED", block["blocked"])
@@ -415,9 +412,9 @@ def fetch_blocks(args: argparse.Namespace) -> int:
                 origin           = row["origin"]
                 nodeinfo_url     = row["nodeinfo_url"]
 
-            logger.debug("Looking up instance by domainm, blocked='%s'", block["blocked"])
-            if block["blocked"] == "":
-                logger.debug("block[blocked] is empty - SKIPPED!")
+            logger.debug("Looking up instance by domain, blocked='%s'", block["blocked"])
+            if block["blocked"] is None or block["blocked"] == "":
+                logger.debug("block[blocked]='%s' is empty - SKIPPED!", block["blocked"])
                 continue
 
             logger.debug("block[blocked]='%s' - BEFORE!", block["blocked"])
@@ -544,6 +541,7 @@ def fetch_observer(args: argparse.Namespace) -> int:
         for item in items:
             logger.debug("item[]='%s'", type(item))
             domain = item.decode_contents()
+            logger.debug("domain[%s]='%s'", type(domain), domain)
             domain = tidyup.domain(domain) if domain not in [None, ""] else None
             logger.debug("domain='%s' - AFTER!", domain)
 
@@ -881,6 +879,7 @@ def fetch_fbabot_atom(args: argparse.Namespace) -> int:
             logger.debug("entry[]='%s'", type(entry))
             doc = bs4.BeautifulSoup(entry.content.value, "html.parser")
             logger.debug("doc[]='%s'", type(doc))
+
             for element in doc.findAll("a"):
                 logger.debug("element[]='%s'", type(element))
                 for href in element["href"].split(","):
@@ -1362,6 +1361,7 @@ def recheck_obfuscation(args: argparse.Namespace) -> int:
                     })
 
         logger.debug("Setting obfuscated=%d for row[domain]='%s' ...", obfuscated, row["domain"])
+        instances.set_has_obfuscation(row["domain"], (obfuscated > 0))
         instances.set_obfuscated_blocks(row["domain"], obfuscated)
 
         logger.info("domain='%s' has %d obfuscated domain(s)", row["domain"], obfuscated)
@@ -1548,8 +1548,8 @@ def fetch_instances_social(args: argparse.Namespace) -> int:
     fetched = network.get_json_api(
         source_domain,
         "/api/1.0/instances/list?count=0&sort_by=name",
-        headers,
-        (config.get("connection_timeout"), config.get("read_timeout"))
+        headers=headers,
+        timeout=(config.get("connection_timeout"), config.get("read_timeout"))
     )
     logger.debug("fetched[]='%s'", type(fetched))
 

diff --git a/fba/networks/lemmy.py b/fba/networks/lemmy.py
index 9b0351773b1f6405c3ebf4d4115ebbb30c352564..bfe1e805f0eed78685d3fa7669de0bb966aca2ac 100644 (file)
--- a/fba/networks/lemmy.py
+++ b/fba/networks/lemmy.py
@@ -164,7 +164,7 @@ def fetch_blocks(domain: str) -> list:
                             content = str(header.contents[0])
                         logger.debug("content[%s]='%s' - AFTER!", type(content), content)
 
-                        if content is None:
+                        if content is None or content == "":
                             logger.debug("domain='%s' has returned empty header='%s' - SKIPPED!", domain, header)
                             continue
                         elif not isinstance(content, str):

diff --git a/fba/networks/pleroma.py b/fba/networks/pleroma.py
index 3b580560cba02ddf154c3b5dae1ad95f30d4fc67..e26cabe60144ffb1e4a57ee07dd7e3c4078e7676 100644 (file)
--- a/fba/networks/pleroma.py
+++ b/fba/networks/pleroma.py
@@ -198,11 +198,8 @@ def fetch_blocks(domain: str) -> list:
             block_level = tidyup.domain(block_level) if block_level != "" else None
             logger.debug("block_level='%s' - AFTER!", block_level)
 
-            if block_level is None:
-                logger.warning("block_level is now None!")
-                continue
-            elif block_level == "":
-                logger.warning("block_level is now empty!")
+            if block_level is None or block_level == "":
+                logger.warning("block_level='%s' is now empty!", block_level)
                 continue
             elif block_level == "accept":
                 logger.debug("domain='%s': Skipping block_level='%s' ...", domain, block_level)