From 9584067c49bcf277009aaf41f52c19f6425fd9ec Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Mon, 20 Nov 2023 05:27:56 +0100 Subject: [PATCH] Continued: - 'misskey' is also a software using obfuscation of blocked instances - let us better combine None/empty string --- daemon.py | 2 +- fba/commands.py | 28 ++++++++++++++-------------- fba/networks/lemmy.py | 2 +- fba/networks/pleroma.py | 7 ++----- 4 files changed, 18 insertions(+), 21 deletions(-) diff --git a/daemon.py b/daemon.py index f5bfd7f..ff68610 100755 --- a/daemon.py +++ b/daemon.py @@ -92,7 +92,7 @@ def api_scoreboard(mode: str, amount: int): elif mode == "obfuscator": database.cursor.execute("SELECT software, COUNT(domain) AS cnt FROM instances WHERE has_obfuscation = 1 GROUP BY software ORDER BY cnt DESC LIMIT ?", [amount]) elif mode == "obfuscation": - database.cursor.execute("SELECT has_obfuscation, COUNT(domain) AS cnt FROM instances WHERE software IN ('pleroma', 'lemmy', 'mastodon', 'friendica') GROUP BY has_obfuscation ORDER BY cnt DESC LIMIT ?", [amount]) + database.cursor.execute("SELECT has_obfuscation, COUNT(domain) AS cnt FROM instances WHERE software IN ('pleroma', 'lemmy', 'mastodon', 'misskey', 'friendica') GROUP BY has_obfuscation ORDER BY cnt DESC LIMIT ?", [amount]) elif mode == "block_level": database.cursor.execute("SELECT block_level, COUNT(rowid) AS cnt FROM blocks GROUP BY block_level ORDER BY cnt DESC LIMIT ?", [amount]) else: diff --git a/fba/commands.py b/fba/commands.py index 186619a..4b41ac4 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -148,11 +148,8 @@ def fetch_pixelfed_api(args: argparse.Namespace) -> int: if "domain" not in row: logger.warning("row='%s' does not contain element 'domain' - SKIPPED!", row) continue - elif row["domain"] is None: - logger.debug("row[domain] is None - SKIPPED!") - continue - elif row["domain"] == "": - logger.debug("row[domain] is empty - SKIPPED!") + elif row["domain"] is None or row["domain"] == "": + logger.debug("row[domain]='%s' is empty - SKIPPED!", row["domain"]) continue logger.debug("row[domain]='%s' - BEFORE!", row["domain"]) @@ -227,8 +224,8 @@ def fetch_bkali(args: argparse.Namespace) -> int: if "domain" not in entry: logger.warning("entry()=%d does not contain 'domain' - SKIPPED!", len(entry)) continue - elif entry["domain"] == "": - logger.debug("entry[domain] is empty - SKIPPED!") + elif entry["domain"] is None or entry["domain"] == "": + logger.debug("entry[domain]='%s' is empty - SKIPPED!", entry["domain"]) continue elif not domain_helper.is_wanted(entry["domain"]): logger.debug("entry[domain]='%s' is not wanted - SKIPPED!", entry["domain"]) @@ -368,8 +365,8 @@ def fetch_blocks(args: argparse.Namespace) -> int: block["reason"] = tidyup.reason(block["reason"]) if block["reason"] is not None and block["reason"] != "" else None logger.debug("blocked='%s',reason='%s' - AFTER!", block["blocked"], block["reason"]) - if block["blocked"] == "": - logger.warning("blocked is empty, blocker='%s'", blocker) + if block["blocked"] is None or block["blocked"] == "": + logger.warning("block[blocked]='%s' is empty, blocker='%s'", block["blocked"], blocker) continue elif block["blocked"].endswith(".onion"): logger.debug("blocked='%s' is a TOR .onion domain - SKIPPED", block["blocked"]) @@ -415,9 +412,9 @@ def fetch_blocks(args: argparse.Namespace) -> int: origin = row["origin"] nodeinfo_url = row["nodeinfo_url"] - logger.debug("Looking up instance by domainm, blocked='%s'", block["blocked"]) - if block["blocked"] == "": - logger.debug("block[blocked] is empty - SKIPPED!") + logger.debug("Looking up instance by domain, blocked='%s'", block["blocked"]) + if block["blocked"] is None or block["blocked"] == "": + logger.debug("block[blocked]='%s' is empty - SKIPPED!", block["blocked"]) continue logger.debug("block[blocked]='%s' - BEFORE!", block["blocked"]) @@ -544,6 +541,7 @@ def fetch_observer(args: argparse.Namespace) -> int: for item in items: logger.debug("item[]='%s'", type(item)) domain = item.decode_contents() + logger.debug("domain[%s]='%s'", type(domain), domain) domain = tidyup.domain(domain) if domain not in [None, ""] else None logger.debug("domain='%s' - AFTER!", domain) @@ -881,6 +879,7 @@ def fetch_fbabot_atom(args: argparse.Namespace) -> int: logger.debug("entry[]='%s'", type(entry)) doc = bs4.BeautifulSoup(entry.content.value, "html.parser") logger.debug("doc[]='%s'", type(doc)) + for element in doc.findAll("a"): logger.debug("element[]='%s'", type(element)) for href in element["href"].split(","): @@ -1362,6 +1361,7 @@ def recheck_obfuscation(args: argparse.Namespace) -> int: }) logger.debug("Setting obfuscated=%d for row[domain]='%s' ...", obfuscated, row["domain"]) + instances.set_has_obfuscation(row["domain"], (obfuscated > 0)) instances.set_obfuscated_blocks(row["domain"], obfuscated) logger.info("domain='%s' has %d obfuscated domain(s)", row["domain"], obfuscated) @@ -1548,8 +1548,8 @@ def fetch_instances_social(args: argparse.Namespace) -> int: fetched = network.get_json_api( source_domain, "/api/1.0/instances/list?count=0&sort_by=name", - headers, - (config.get("connection_timeout"), config.get("read_timeout")) + headers=headers, + timeout=(config.get("connection_timeout"), config.get("read_timeout")) ) logger.debug("fetched[]='%s'", type(fetched)) diff --git a/fba/networks/lemmy.py b/fba/networks/lemmy.py index 9b03517..bfe1e80 100644 --- a/fba/networks/lemmy.py +++ b/fba/networks/lemmy.py @@ -164,7 +164,7 @@ def fetch_blocks(domain: str) -> list: content = str(header.contents[0]) logger.debug("content[%s]='%s' - AFTER!", type(content), content) - if content is None: + if content is None or content == "": logger.debug("domain='%s' has returned empty header='%s' - SKIPPED!", domain, header) continue elif not isinstance(content, str): diff --git a/fba/networks/pleroma.py b/fba/networks/pleroma.py index 3b58056..e26cabe 100644 --- a/fba/networks/pleroma.py +++ b/fba/networks/pleroma.py @@ -198,11 +198,8 @@ def fetch_blocks(domain: str) -> list: block_level = tidyup.domain(block_level) if block_level != "" else None logger.debug("block_level='%s' - AFTER!", block_level) - if block_level is None: - logger.warning("block_level is now None!") - continue - elif block_level == "": - logger.warning("block_level is now empty!") + if block_level is None or block_level == "": + logger.warning("block_level='%s' is now empty!", block_level) continue elif block_level == "accept": logger.debug("domain='%s': Skipping block_level='%s' ...", domain, block_level) -- 2.39.5