Continued:

- rewrote multiple variables to array
- renamed row -> obfuscated to make space for above change

diff --git a/fba/commands.py b/fba/commands.py
index 54bd55e4877807c37b612e54833ef19f19213d35..ca921e292500dc3d92b70ea89fa8334b8ae7b729 100644 (file)
--- a/fba/commands.py
+++ b/fba/commands.py
@@ -300,66 +300,68 @@ def fetch_blocks(args: argparse.Namespace) -> int:
         # Re-check after "timeout" (aka. minimum interval)
         database.cursor.execute("SELECT domain, software, origin, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'lemmy', 'friendica', 'misskey', 'piefed', 'typecho') AND nodeinfo_url IS NOT NULL ORDER BY last_blocked ASC, total_blocks DESC")
 
+    # Load all rows
     rows = database.cursor.fetchall()
+
     logger.info("Checking %d entries ...", len(rows))
-    for blocker, software, origin, nodeinfo_url in rows:
-        logger.debug("blocker='%s',software='%s',origin='%s',nodeinfo_url='%s'", blocker, software, origin, nodeinfo_url)
+    for row in rows:
+        logger.debug("row[blocker]='%s',row[software]='%s',row[software]='%s',nodeinfo_url='%s'", row["blocker"], row["software"], row["origin"], row["nodeinfo_url"])
 
-        if not domain_helper.is_wanted(blocker):
-            logger.warning("blocker='%s' is not wanted - SKIPPED!", blocker)
+        if not domain_helper.is_wanted(row["blocker"]):
+            logger.warning("row[blocker]='%s' is not wanted - SKIPPED!", row["blocker"])
             continue
-        elif not args.force_all and instances.is_recent(blocker, "last_blocked"):
-            logger.debug("blocker='%s' has recently been crawled - SKIPPED!", blocker)
+        elif not args.force_all and instances.is_recent(row["blocker"], "last_blocked"):
+            logger.debug("row[blocker]='%s' has recently been crawled - SKIPPED!", row["blocker"])
             continue
-        elif software is None:
-            logger.warning("blocker='%s' has no software set - SKIPPED!", blocker)
+        elif row["software"] is None:
+            logger.warning("row[blocker]='%s' has no software set - SKIPPED!", row["blocker"])
             continue
 
-        logger.debug("Setting last_blocked,has_obfuscation=false for blocker='%s' ...", blocker)
-        instances.set_last_blocked(blocker)
-        instances.set_has_obfuscation(blocker, False)
+        logger.debug("Setting last_blocked,has_obfuscation=false for row[blocker]='%s' ...", row["blocker"])
+        instances.set_last_blocked(row["blocker"])
+        instances.set_has_obfuscation(row["blocker"], False)
 
         # c.s isn't part of oliphant's "hidden" blocklists
-        if blocker == "chaos.social" or software_helper.is_relay(software) or blocklists.has(blocker):
-            logger.debug("Skipping blocker='%s', run ./fba.py fetch_cs, fetch_oliphant, fetch_csv instead!", blocker)
+        if row["blocker"] == "chaos.social" or software_helper.is_relay(row["software"]) or blocklists.has(row["blocker"]):
+            logger.debug("Skipping row[blocker]='%s', run ./fba.py fetch_cs, fetch_oliphant, fetch_csv instead!", row["blocker"])
             continue
 
-        logger.debug("Invoking federation.fetch_blocks(%s) ...", blocker)
-        blocking = federation.fetch_blocks(blocker)
+        logger.debug("Invoking federation.fetch_blocks(%s) ...", row["blocker"])
+        blocking = federation.fetch_blocks(row["blocker"])
 
-        logger.debug("blocker='%s',software='%s',blocking()=%d", blocker, software, len(blocking))
+        logger.debug("row[blocker]='%s',row[software]='%s',blocking()=%d", row["blocker"], row["software"], len(blocking))
         if len(blocking) == 0:
-            logger.debug("blocker='%s',software='%s' - fetching blocklist ...", blocker, software)
-            if software == "pleroma":
-                blocking = pleroma.fetch_blocks(blocker)
-                logger.debug("blocker='%s' returned %d entries,software='%s'", blocker, len(blocking), software)
-            elif software == "mastodon":
-                blocking = mastodon.fetch_blocks(blocker)
-                logger.debug("blocker='%s' returned %d entries,software='%s'", blocker, len(blocking), software)
-            elif software == "lemmy":
-                blocking = lemmy.fetch_blocks(blocker)
-                logger.debug("blocker='%s' returned %d entries,software='%s'", blocker, len(blocking), software)
-            elif software == "friendica":
-                blocking = friendica.fetch_blocks(blocker)
-                logger.debug("blocker='%s' returned %d entries,software='%s'", blocker, len(blocking), software)
-            elif software == "misskey":
-                blocking = misskey.fetch_blocks(blocker)
-                logger.debug("blocker='%s' returned %d entries,software='%s'", blocker, len(blocking), software)
+            logger.debug("row[blocker]='%s',row[software]='%s' - fetching blocklist ...", row["blocker"], row["software"])
+            if row["software"] == "pleroma":
+                blocking = pleroma.fetch_blocks(row["blocker"])
+                logger.debug("row[blocker]='%s' returned %d entries,row[software]='%s'", row["blocker"], len(blocking), row["software"])
+            elif row["software"] == "mastodon":
+                blocking = mastodon.fetch_blocks(row["blocker"])
+                logger.debug("row[blocker]='%s' returned %d entries,row[software]='%s'", row["blocker"], len(blocking), row["software"])
+            elif row["software"] == "lemmy":
+                blocking = lemmy.fetch_blocks(row["blocker"])
+                logger.debug("row[blocker]='%s' returned %d entries,row[software]='%s'", row["blocker"], len(blocking), row["software"])
+            elif row["software"] == "friendica":
+                blocking = friendica.fetch_blocks(row["blocker"])
+                logger.debug("row[blocker]='%s' returned %d entries,row[software]='%s'", row["blocker"], len(blocking), row["software"])
+            elif row["software"] == "misskey":
+                blocking = misskey.fetch_blocks(row["blocker"])
+                logger.debug("row[blocker]='%s' returned %d entries,row[software]='%s'", row["blocker"], len(blocking), row["software"])
             else:
-                logger.warning("Unknown software: blocker='%s',software='%s'", blocker, software)
+                logger.warning("Unknown software: row[blocker]='%s',row[software]='%s'", row["blocker"], row["software"])
 
-        logger.debug("Invoking instances.set_total_blocks(%s, %d) ...", blocker, len(blocking))
-        instances.set_total_blocks(blocker, blocking)
+        logger.debug("Invoking instances.set_total_blocks(%s, %d) ...", row["blocker"], len(blocking))
+        instances.set_total_blocks(row["blocker"], blocking)
 
         blockdict = list()
         deobfuscated = obfuscated = 0
 
-        logger.info("Checking %d entries from blocker='%s',software='%s' ...", len(blocking), blocker, software)
+        logger.info("Checking %d entries from row[blocker]='%s',row[software]='%s' ...", len(blocking), row["blocker"], row["software"])
         for block in blocking:
             logger.debug("blocked='%s',block_level='%s',reason='%s'", block["blocked"], block["block_level"], block["reason"])
 
             if block["block_level"] in [None, ""]:
-                logger.warning("block_level is empty, blocker='%s',blocked='%s'", block["blocker"], block["blocked"])
+                logger.warning("block_level is empty, row[blocker]='%s',blocked='%s'", block["blocker"], block["blocked"])
                 continue
 
             logger.debug("blocked='%s',reason='%s' - BEFORE!", block["blocked"], block["reason"])
@@ -368,22 +370,22 @@ def fetch_blocks(args: argparse.Namespace) -> int:
             logger.debug("blocked='%s',reason='%s' - AFTER!", block["blocked"], block["reason"])
 
             if block["blocked"] in [None, ""]:
-                logger.warning("block[blocked]='%s' is empty, blocker='%s'", block["blocked"], blocker)
+                logger.warning("block[blocked]='%s' is empty, row[blocker]='%s'", block["blocked"], row["blocker"])
                 continue
             elif not domain_helper.is_tld_wanted(block["blocked"]):
                 logger.warning("block[blocked]='%s' has an unwanted TLD - SKIPPED!", block["blocked"])
                 continue
             elif block["blocked"].find("*") >= 0:
-                logger.debug("blocker='%s' uses '*' for obfuscating domains", blocker)
-                instances.set_has_obfuscation(blocker, True)
+                logger.debug("row[blocker]='%s' uses '*' for obfuscating domains", row["blocker"])
+                instances.set_has_obfuscation(row["blocker"], True)
                 obfuscated = obfuscated + 1
 
                 # Some friendica servers also obscure domains without hash
-                row = instances.deobfuscate("*", block["blocked"], block["digest"] if "digest" in block else None)
+                obfuscated = instances.deobfuscate("*", block["blocked"], block["digest"] if "digest" in block else None)
 
-                logger.debug("row[]='%s'", type(row))
-                if row is None:
-                    logger.warning("Cannot deobfuscate block[blocked]='%s',blocker='%s',software='%s' - SKIPPED!", block["blocked"], blocker, software)
+                logger.debug("obfuscated[]='%s'", type(obfuscated))
+                if obfuscated is None:
+                    logger.warning("Cannot deobfuscate block[blocked]='%s',row[blocker]='%s',row[software]='%s' - SKIPPED!", block["blocked"], row["blocker"], row["software"])
 
                     if not obfuscation.is_added(block["blocked"]):
                         logger.debug("Invoking obfuscation.add(%s) ...", block["blocked"])
@@ -395,20 +397,20 @@ def fetch_blocks(args: argparse.Namespace) -> int:
                     continue
 
                 deobfuscated = deobfuscated + 1
-                block["blocked"] = row["domain"]
-                origin           = row["origin"]
-                nodeinfo_url     = row["nodeinfo_url"]
+                block["blocked"]    = obfuscated["domain"]
+                row["origin"]       = obfuscated["origin"]
+                row["nodeinfo_url"] = obfuscated["nodeinfo_url"]
             elif block["blocked"].find("?") >= 0:
-                logger.debug("blocker='%s' uses '?' for obfuscating domains", blocker)
-                instances.set_has_obfuscation(blocker, True)
+                logger.debug("row[blocker]='%s' uses '?' for obfuscating domains", row["blocker"])
+                instances.set_has_obfuscation(row["blocker"], True)
                 obfuscated = obfuscated + 1
 
                 # Some obscure them with question marks, not sure if that's dependent on version or not
-                row = instances.deobfuscate("?", block["blocked"], block["digest"] if "digest" in block else None)
+                obfuscated = instances.deobfuscate("?", block["blocked"], block["digest"] if "digest" in block else None)
 
-                logger.debug("row[]='%s'", type(row))
-                if row is None:
-                    logger.warning("Cannot deobfuscate block[blocked]='%s',blocker='%s',software='%s' - SKIPPED!", block["blocked"], blocker, software)
+                logger.debug("obfuscated[]='%s'", type(obfuscated))
+                if obfuscated is None:
+                    logger.warning("Cannot deobfuscate block[blocked]='%s',row[blocker]='%s',row[software]='%s' - SKIPPED!", block["blocked"], row["blocker"], row["software"])
 
                     if not obfuscation.is_added(block["blocked"]):
                         logger.debug("Invoking obfuscation.add(%s) ...", block["blocked"])
@@ -420,9 +422,9 @@ def fetch_blocks(args: argparse.Namespace) -> int:
                     continue
 
                 deobfuscated = deobfuscated + 1
-                block["blocked"] = row["domain"]
-                origin           = row["origin"]
-                nodeinfo_url     = row["nodeinfo_url"]
+                block["blocked"]    = obfuscated["domain"]
+                row["origin"]       = obfuscated["origin"]
+                row["nodeinfo_url"] = obfuscated["nodeinfo_url"]
             elif not validators.domain(block["blocked"], rfc_2782=True):
                 logger.warning("block[blocked]='%s' is not a valid domain - SKIPPED!", block["blocked"])
                 continue
@@ -449,15 +451,15 @@ def fetch_blocks(args: argparse.Namespace) -> int:
                 logger.debug("block[blocked]='%s' is accepted, not wanted here - SKIPPED!", block["blocked"])
                 continue
             elif not instances.is_registered(block["blocked"]):
-                logger.info("Fetching new instance block[blocked]'%s' for blocker='%s' ...", block["blocked"], blocker)
-                federation.fetch_instances(block["blocked"], blocker, None, inspect.currentframe().f_code.co_name)
+                logger.info("Fetching new instance block[blocked]'%s' for row[blocker]='%s' ...", block["blocked"], row["blocker"])
+                federation.fetch_instances(block["blocked"], row["blocker"], None, inspect.currentframe().f_code.co_name)
 
             logger.debug("Invoking blocks_helper.alias_block_level('%s') ... - BEFORE!", block["block_level"])
             block["block_level"] = blocks_helper.alias_block_level(block["block_level"])
             logger.debug("block[block_level]='%s' - AFTER!", block["block_level"])
 
-            if processing.block(blocker, block["blocked"], block["reason"], block["block_level"]) and block["block_level"] in ["rejected", "suspended"] and _bot_enabled:
-                logger.debug("Appending block[blocked]'%s',reason='%s' for blocker='%s' ...", block["blocked"], block["block_level"], blocker)
+            if processing.block(row["blocker"], block["blocked"], block["reason"], block["block_level"]) and block["block_level"] in ["rejected", "suspended"] and _bot_enabled:
+                logger.debug("Appending block[blocked]'%s',reason='%s' for row[blocker]='%s' ...", block["blocked"], block["block_level"], row["blocker"])
                 blockdict.append({
                     "blocked": block["blocked"],
                     "reason" : block["reason"],
@@ -466,22 +468,22 @@ def fetch_blocks(args: argparse.Namespace) -> int:
             logger.debug("Invoking cookies.clear(%s) ...", block["blocked"])
             cookies.clear(block["blocked"])
 
-        logger.info("blocker='%s' has %d obfuscated domain(s) and %d of them could be deobfuscated.", blocker, obfuscated, deobfuscated)
-        instances.set_obfuscated_blocks(blocker, obfuscated)
+        logger.info("row[blocker]='%s' has %d obfuscated domain(s) and %d of them could be deobfuscated.", row["blocker"], obfuscated, deobfuscated)
+        instances.set_obfuscated_blocks(row["blocker"], obfuscated)
 
-        logger.debug("Flushing updates for blocker='%s' ...", blocker)
-        instances.update(blocker)
+        logger.debug("Flushing updates for row[blocker]='%s' ...", row["blocker"])
+        instances.update(row["blocker"])
 
         logger.debug("Invoking commit() ...")
         database.connection.commit()
 
-        logger.debug("Invoking cookies.clear(%s) ...", blocker)
-        cookies.clear(blocker)
+        logger.debug("Invoking cookies.clear(%s) ...", row["blocker"])
+        cookies.clear(row["blocker"])
 
         logger.debug("_bot_enabled='%s',blockdict()=%d'", _bot_enabled, len(blockdict))
         if _bot_enabled and len(blockdict) > 0:
-            logger.info("Sending bot POST for blocker='%s',blockdict()=%d ...", blocker, len(blockdict))
-            network.send_bot_post(blocker, blockdict)
+            logger.info("Sending bot POST for row[blocker]='%s',blockdict()=%d ...", row["blocker"], len(blockdict))
+            network.send_bot_post(row["blocker"], blockdict)
 
     logger.debug("Success! - EXIT!")
     return 0