From 302197e14894d7951138edb6cbfcd6310190300f Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Sat, 11 Jan 2025 20:12:13 +0100 Subject: [PATCH] Continued: - rewrote multiple variables to array - renamed row -> obfuscated to make space for above change --- fba/commands.py | 138 ++++++++++++++++++++++++------------------------ 1 file changed, 70 insertions(+), 68 deletions(-) diff --git a/fba/commands.py b/fba/commands.py index 54bd55e..ca921e2 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -300,66 +300,68 @@ def fetch_blocks(args: argparse.Namespace) -> int: # Re-check after "timeout" (aka. minimum interval) database.cursor.execute("SELECT domain, software, origin, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'lemmy', 'friendica', 'misskey', 'piefed', 'typecho') AND nodeinfo_url IS NOT NULL ORDER BY last_blocked ASC, total_blocks DESC") + # Load all rows rows = database.cursor.fetchall() + logger.info("Checking %d entries ...", len(rows)) - for blocker, software, origin, nodeinfo_url in rows: - logger.debug("blocker='%s',software='%s',origin='%s',nodeinfo_url='%s'", blocker, software, origin, nodeinfo_url) + for row in rows: + logger.debug("row[blocker]='%s',row[software]='%s',row[software]='%s',nodeinfo_url='%s'", row["blocker"], row["software"], row["origin"], row["nodeinfo_url"]) - if not domain_helper.is_wanted(blocker): - logger.warning("blocker='%s' is not wanted - SKIPPED!", blocker) + if not domain_helper.is_wanted(row["blocker"]): + logger.warning("row[blocker]='%s' is not wanted - SKIPPED!", row["blocker"]) continue - elif not args.force_all and instances.is_recent(blocker, "last_blocked"): - logger.debug("blocker='%s' has recently been crawled - SKIPPED!", blocker) + elif not args.force_all and instances.is_recent(row["blocker"], "last_blocked"): + logger.debug("row[blocker]='%s' has recently been crawled - SKIPPED!", row["blocker"]) continue - elif software is None: - logger.warning("blocker='%s' has no software set - SKIPPED!", blocker) + elif row["software"] is None: + logger.warning("row[blocker]='%s' has no software set - SKIPPED!", row["blocker"]) continue - logger.debug("Setting last_blocked,has_obfuscation=false for blocker='%s' ...", blocker) - instances.set_last_blocked(blocker) - instances.set_has_obfuscation(blocker, False) + logger.debug("Setting last_blocked,has_obfuscation=false for row[blocker]='%s' ...", row["blocker"]) + instances.set_last_blocked(row["blocker"]) + instances.set_has_obfuscation(row["blocker"], False) # c.s isn't part of oliphant's "hidden" blocklists - if blocker == "chaos.social" or software_helper.is_relay(software) or blocklists.has(blocker): - logger.debug("Skipping blocker='%s', run ./fba.py fetch_cs, fetch_oliphant, fetch_csv instead!", blocker) + if row["blocker"] == "chaos.social" or software_helper.is_relay(row["software"]) or blocklists.has(row["blocker"]): + logger.debug("Skipping row[blocker]='%s', run ./fba.py fetch_cs, fetch_oliphant, fetch_csv instead!", row["blocker"]) continue - logger.debug("Invoking federation.fetch_blocks(%s) ...", blocker) - blocking = federation.fetch_blocks(blocker) + logger.debug("Invoking federation.fetch_blocks(%s) ...", row["blocker"]) + blocking = federation.fetch_blocks(row["blocker"]) - logger.debug("blocker='%s',software='%s',blocking()=%d", blocker, software, len(blocking)) + logger.debug("row[blocker]='%s',row[software]='%s',blocking()=%d", row["blocker"], row["software"], len(blocking)) if len(blocking) == 0: - logger.debug("blocker='%s',software='%s' - fetching blocklist ...", blocker, software) - if software == "pleroma": - blocking = pleroma.fetch_blocks(blocker) - logger.debug("blocker='%s' returned %d entries,software='%s'", blocker, len(blocking), software) - elif software == "mastodon": - blocking = mastodon.fetch_blocks(blocker) - logger.debug("blocker='%s' returned %d entries,software='%s'", blocker, len(blocking), software) - elif software == "lemmy": - blocking = lemmy.fetch_blocks(blocker) - logger.debug("blocker='%s' returned %d entries,software='%s'", blocker, len(blocking), software) - elif software == "friendica": - blocking = friendica.fetch_blocks(blocker) - logger.debug("blocker='%s' returned %d entries,software='%s'", blocker, len(blocking), software) - elif software == "misskey": - blocking = misskey.fetch_blocks(blocker) - logger.debug("blocker='%s' returned %d entries,software='%s'", blocker, len(blocking), software) + logger.debug("row[blocker]='%s',row[software]='%s' - fetching blocklist ...", row["blocker"], row["software"]) + if row["software"] == "pleroma": + blocking = pleroma.fetch_blocks(row["blocker"]) + logger.debug("row[blocker]='%s' returned %d entries,row[software]='%s'", row["blocker"], len(blocking), row["software"]) + elif row["software"] == "mastodon": + blocking = mastodon.fetch_blocks(row["blocker"]) + logger.debug("row[blocker]='%s' returned %d entries,row[software]='%s'", row["blocker"], len(blocking), row["software"]) + elif row["software"] == "lemmy": + blocking = lemmy.fetch_blocks(row["blocker"]) + logger.debug("row[blocker]='%s' returned %d entries,row[software]='%s'", row["blocker"], len(blocking), row["software"]) + elif row["software"] == "friendica": + blocking = friendica.fetch_blocks(row["blocker"]) + logger.debug("row[blocker]='%s' returned %d entries,row[software]='%s'", row["blocker"], len(blocking), row["software"]) + elif row["software"] == "misskey": + blocking = misskey.fetch_blocks(row["blocker"]) + logger.debug("row[blocker]='%s' returned %d entries,row[software]='%s'", row["blocker"], len(blocking), row["software"]) else: - logger.warning("Unknown software: blocker='%s',software='%s'", blocker, software) + logger.warning("Unknown software: row[blocker]='%s',row[software]='%s'", row["blocker"], row["software"]) - logger.debug("Invoking instances.set_total_blocks(%s, %d) ...", blocker, len(blocking)) - instances.set_total_blocks(blocker, blocking) + logger.debug("Invoking instances.set_total_blocks(%s, %d) ...", row["blocker"], len(blocking)) + instances.set_total_blocks(row["blocker"], blocking) blockdict = list() deobfuscated = obfuscated = 0 - logger.info("Checking %d entries from blocker='%s',software='%s' ...", len(blocking), blocker, software) + logger.info("Checking %d entries from row[blocker]='%s',row[software]='%s' ...", len(blocking), row["blocker"], row["software"]) for block in blocking: logger.debug("blocked='%s',block_level='%s',reason='%s'", block["blocked"], block["block_level"], block["reason"]) if block["block_level"] in [None, ""]: - logger.warning("block_level is empty, blocker='%s',blocked='%s'", block["blocker"], block["blocked"]) + logger.warning("block_level is empty, row[blocker]='%s',blocked='%s'", block["blocker"], block["blocked"]) continue logger.debug("blocked='%s',reason='%s' - BEFORE!", block["blocked"], block["reason"]) @@ -368,22 +370,22 @@ def fetch_blocks(args: argparse.Namespace) -> int: logger.debug("blocked='%s',reason='%s' - AFTER!", block["blocked"], block["reason"]) if block["blocked"] in [None, ""]: - logger.warning("block[blocked]='%s' is empty, blocker='%s'", block["blocked"], blocker) + logger.warning("block[blocked]='%s' is empty, row[blocker]='%s'", block["blocked"], row["blocker"]) continue elif not domain_helper.is_tld_wanted(block["blocked"]): logger.warning("block[blocked]='%s' has an unwanted TLD - SKIPPED!", block["blocked"]) continue elif block["blocked"].find("*") >= 0: - logger.debug("blocker='%s' uses '*' for obfuscating domains", blocker) - instances.set_has_obfuscation(blocker, True) + logger.debug("row[blocker]='%s' uses '*' for obfuscating domains", row["blocker"]) + instances.set_has_obfuscation(row["blocker"], True) obfuscated = obfuscated + 1 # Some friendica servers also obscure domains without hash - row = instances.deobfuscate("*", block["blocked"], block["digest"] if "digest" in block else None) + obfuscated = instances.deobfuscate("*", block["blocked"], block["digest"] if "digest" in block else None) - logger.debug("row[]='%s'", type(row)) - if row is None: - logger.warning("Cannot deobfuscate block[blocked]='%s',blocker='%s',software='%s' - SKIPPED!", block["blocked"], blocker, software) + logger.debug("obfuscated[]='%s'", type(obfuscated)) + if obfuscated is None: + logger.warning("Cannot deobfuscate block[blocked]='%s',row[blocker]='%s',row[software]='%s' - SKIPPED!", block["blocked"], row["blocker"], row["software"]) if not obfuscation.is_added(block["blocked"]): logger.debug("Invoking obfuscation.add(%s) ...", block["blocked"]) @@ -395,20 +397,20 @@ def fetch_blocks(args: argparse.Namespace) -> int: continue deobfuscated = deobfuscated + 1 - block["blocked"] = row["domain"] - origin = row["origin"] - nodeinfo_url = row["nodeinfo_url"] + block["blocked"] = obfuscated["domain"] + row["origin"] = obfuscated["origin"] + row["nodeinfo_url"] = obfuscated["nodeinfo_url"] elif block["blocked"].find("?") >= 0: - logger.debug("blocker='%s' uses '?' for obfuscating domains", blocker) - instances.set_has_obfuscation(blocker, True) + logger.debug("row[blocker]='%s' uses '?' for obfuscating domains", row["blocker"]) + instances.set_has_obfuscation(row["blocker"], True) obfuscated = obfuscated + 1 # Some obscure them with question marks, not sure if that's dependent on version or not - row = instances.deobfuscate("?", block["blocked"], block["digest"] if "digest" in block else None) + obfuscated = instances.deobfuscate("?", block["blocked"], block["digest"] if "digest" in block else None) - logger.debug("row[]='%s'", type(row)) - if row is None: - logger.warning("Cannot deobfuscate block[blocked]='%s',blocker='%s',software='%s' - SKIPPED!", block["blocked"], blocker, software) + logger.debug("obfuscated[]='%s'", type(obfuscated)) + if obfuscated is None: + logger.warning("Cannot deobfuscate block[blocked]='%s',row[blocker]='%s',row[software]='%s' - SKIPPED!", block["blocked"], row["blocker"], row["software"]) if not obfuscation.is_added(block["blocked"]): logger.debug("Invoking obfuscation.add(%s) ...", block["blocked"]) @@ -420,9 +422,9 @@ def fetch_blocks(args: argparse.Namespace) -> int: continue deobfuscated = deobfuscated + 1 - block["blocked"] = row["domain"] - origin = row["origin"] - nodeinfo_url = row["nodeinfo_url"] + block["blocked"] = obfuscated["domain"] + row["origin"] = obfuscated["origin"] + row["nodeinfo_url"] = obfuscated["nodeinfo_url"] elif not validators.domain(block["blocked"], rfc_2782=True): logger.warning("block[blocked]='%s' is not a valid domain - SKIPPED!", block["blocked"]) continue @@ -449,15 +451,15 @@ def fetch_blocks(args: argparse.Namespace) -> int: logger.debug("block[blocked]='%s' is accepted, not wanted here - SKIPPED!", block["blocked"]) continue elif not instances.is_registered(block["blocked"]): - logger.info("Fetching new instance block[blocked]'%s' for blocker='%s' ...", block["blocked"], blocker) - federation.fetch_instances(block["blocked"], blocker, None, inspect.currentframe().f_code.co_name) + logger.info("Fetching new instance block[blocked]'%s' for row[blocker]='%s' ...", block["blocked"], row["blocker"]) + federation.fetch_instances(block["blocked"], row["blocker"], None, inspect.currentframe().f_code.co_name) logger.debug("Invoking blocks_helper.alias_block_level('%s') ... - BEFORE!", block["block_level"]) block["block_level"] = blocks_helper.alias_block_level(block["block_level"]) logger.debug("block[block_level]='%s' - AFTER!", block["block_level"]) - if processing.block(blocker, block["blocked"], block["reason"], block["block_level"]) and block["block_level"] in ["rejected", "suspended"] and _bot_enabled: - logger.debug("Appending block[blocked]'%s',reason='%s' for blocker='%s' ...", block["blocked"], block["block_level"], blocker) + if processing.block(row["blocker"], block["blocked"], block["reason"], block["block_level"]) and block["block_level"] in ["rejected", "suspended"] and _bot_enabled: + logger.debug("Appending block[blocked]'%s',reason='%s' for row[blocker]='%s' ...", block["blocked"], block["block_level"], row["blocker"]) blockdict.append({ "blocked": block["blocked"], "reason" : block["reason"], @@ -466,22 +468,22 @@ def fetch_blocks(args: argparse.Namespace) -> int: logger.debug("Invoking cookies.clear(%s) ...", block["blocked"]) cookies.clear(block["blocked"]) - logger.info("blocker='%s' has %d obfuscated domain(s) and %d of them could be deobfuscated.", blocker, obfuscated, deobfuscated) - instances.set_obfuscated_blocks(blocker, obfuscated) + logger.info("row[blocker]='%s' has %d obfuscated domain(s) and %d of them could be deobfuscated.", row["blocker"], obfuscated, deobfuscated) + instances.set_obfuscated_blocks(row["blocker"], obfuscated) - logger.debug("Flushing updates for blocker='%s' ...", blocker) - instances.update(blocker) + logger.debug("Flushing updates for row[blocker]='%s' ...", row["blocker"]) + instances.update(row["blocker"]) logger.debug("Invoking commit() ...") database.connection.commit() - logger.debug("Invoking cookies.clear(%s) ...", blocker) - cookies.clear(blocker) + logger.debug("Invoking cookies.clear(%s) ...", row["blocker"]) + cookies.clear(row["blocker"]) logger.debug("_bot_enabled='%s',blockdict()=%d'", _bot_enabled, len(blockdict)) if _bot_enabled and len(blockdict) > 0: - logger.info("Sending bot POST for blocker='%s',blockdict()=%d ...", blocker, len(blockdict)) - network.send_bot_post(blocker, blockdict) + logger.info("Sending bot POST for row[blocker]='%s',blockdict()=%d ...", row["blocker"], len(blockdict)) + network.send_bot_post(row["blocker"], blockdict) logger.debug("Success! - EXIT!") return 0 -- 2.39.5