Continued:

- update total blocks in any command that handles instance blocks

diff --git a/fba/commands.py b/fba/commands.py
index 9f5b50962b882f9605d543f6c329782f5c343375..938f8c88c2643dc34804cf837aa9e1e6746faa6d 100644 (file)
--- a/fba/commands.py
+++ b/fba/commands.py
@@ -511,6 +511,12 @@ def fetch_todon_wiki(args: argparse.Namespace) -> int:
     logger.info("Checking %d suspended entries ...", len(suspended))
     blocklist["reject"] = utils.find_domains(suspended, "div")
 
+    blocking = blocklist["silenced"] + blocklist["reject"]
+    blocker = "todon.eu"
+
+    logger.debug("Invoking instances.set_total_blocks(%s, %d) ...", blocker, len(blocking))
+    instances.set_total_blocks(blocker, blocking)
+
     blockdict = list()
     for block_level in blocklist:
         blockers = blocklist[block_level]
@@ -522,18 +528,18 @@ def fetch_todon_wiki(args: argparse.Namespace) -> int:
             if not instances.is_registered(blocked):
                 try:
                     logger.info("Fetching instances from domain='%s' ...", blocked)
-                    federation.fetch_instances(blocked, 'chaos.social', None, inspect.currentframe().f_code.co_name)
+                    federation.fetch_instances(blocked, blocker, None, inspect.currentframe().f_code.co_name)
                 except network.exceptions as exception:
                     logger.warning("Exception '%s' during fetching instances (fetch_cs) from blocked='%s'", type(exception), blocked)
                     instances.set_last_error(blocked, exception)
 
-            if blocks.is_instance_blocked("todon.eu", blocked, block_level):
+            if blocks.is_instance_blocked(blocker, blocked, block_level):
                 logger.debug("blocked='%s',block_level='%s' is already blocked - SKIPPED!", blocked, block_level)
                 continue
 
             logger.info("Adding new block: blocked='%s',block_level='%s'", blocked, block_level)
-            if utils.process_block("todon.eu", blocked, None, block_level) and block_level == "reject" and config.get("bot_enabled"):
-                logger.debug("Appending blocked='%s',reason='%s' for blocker='todon.eu' ...", blocked, block_level)
+            if utils.process_block(blocker, blocked, None, block_level) and block_level == "reject" and config.get("bot_enabled"):
+                logger.debug("Appending blocked='%s',reason='%s' for blocker='%s' ...", blocked, block_level, blocker)
                 blockdict.append({
                     "blocked": blocked,
                     "reason" : None,
@@ -544,8 +550,13 @@ def fetch_todon_wiki(args: argparse.Namespace) -> int:
 
         logger.debug("config.get(bot_enabled)='%s',blockdict()=%d", config.get("bot_enabled"), len(blockdict))
         if config.get("bot_enabled") and len(blockdict) > 0:
-            logger.info("Sending bot POST for blocker='todon.eu',blockdict()=%d ...", len(blockdict))
-            network.send_bot_post("todon.eu", blockdict)
+            logger.info("Sending bot POST for blocker='%s',blockdict()=%d ...", blocker, len(blockdict))
+            network.send_bot_post(blocker, blockdict)
+
+    logger.debug("Checking if blocker='%s' has pending updates ...", blocker)
+    if instances.has_pending(blocker):
+        logger.debug("Flushing updates for blocker='%s' ...", blocker)
+        instances.update_data(blocker)
 
     logger.debug("Success! - EXIT!")
     return 0
@@ -591,6 +602,12 @@ def fetch_cs(args: argparse.Namespace):
     logger.debug("blocked[%s]()=%d", type(blocked), len(blocked))
     domains["reject"] = federation.find_domains(blocked)
 
+    blocking = blocklist["silenced"] + blocklist["reject"]
+    blocker = "chaos.social"
+
+    logger.debug("Invoking instances.set_total_blocks(%s, %d) ...", blocker, len(blocking))
+    instances.set_total_blocks(blocker, blocking)
+
     logger.debug("domains[silenced]()=%d,domains[reject]()=%d", len(domains["silenced"]), len(domains["reject"]))
     blockdict = list()
     if len(domains) > 0:
@@ -607,13 +624,13 @@ def fetch_cs(args: argparse.Namespace):
                 elif not instances.is_registered(row["domain"]):
                     try:
                         logger.info("Fetching instances from domain='%s' ...", row["domain"])
-                        federation.fetch_instances(row["domain"], 'chaos.social', None, inspect.currentframe().f_code.co_name)
+                        federation.fetch_instances(row["domain"], blocker, None, inspect.currentframe().f_code.co_name)
                     except network.exceptions as exception:
                         logger.warning("Exception '%s' during fetching instances (fetch_cs) from row[domain]='%s'", type(exception), row["domain"])
                         instances.set_last_error(row["domain"], exception)
 
-                if utils.process_block("chaos.social", row["domain"], row["reason"], block_level) and block_level == "reject" and config.get("bot_enabled"):
-                    logger.debug("Appending blocked='%s',reason='%s' for blocker='chaos.social' ...", row["domain"], block_level)
+                if utils.process_block(blocker, row["domain"], row["reason"], block_level) and block_level == "reject" and config.get("bot_enabled"):
+                    logger.debug("Appending blocked='%s',reason='%s' for blocker='%s' ...", row["domain"], block_level, blocker)
                     blockdict.append({
                         "blocked": row["domain"],
                         "reason" : row["reason"],
@@ -624,8 +641,13 @@ def fetch_cs(args: argparse.Namespace):
 
         logger.debug("config.get(bot_enabled)='%s',blockdict()=%d", config.get("bot_enabled"), len(blockdict))
         if config.get("bot_enabled") and len(blockdict) > 0:
-            logger.info("Sending bot POST for blocker='chaos.social',blockdict()=%d ...", len(blockdict))
-            network.send_bot_post("chaos.social", blockdict)
+            logger.info("Sending bot POST for blocker='%s',blockdict()=%d ...", blocker, len(blockdict))
+            network.send_bot_post(blocker, blockdict)
+
+    logger.debug("Checking if blocker='%s' has pending updates ...", blocker)
+    if instances.has_pending(blocker):
+        logger.debug("Flushing updates for blocker='%s' ...", blocker)
+        instances.update_data(blocker)
 
     logger.debug("Success! - EXIT!")
     return 0
@@ -735,6 +757,7 @@ def fetch_fbabot_atom(args: argparse.Namespace) -> int:
 
         logger.info("Adding %d new instances ...", len(domains))
         for domain in domains:
+            logger.debug("domain='%s'", domain)
             try:
                 logger.info("Fetching instances from domain='%s' ...", domain)
                 federation.fetch_instances(domain, "ryona.agency", None, inspect.currentframe().f_code.co_name)
@@ -865,12 +888,15 @@ def fetch_oliphant(args: argparse.Namespace) -> int:
         logger.debug("Fetched %d Bytes, parsing CSV ...", len(response.content))
         reader = csv.DictReader(response.content.decode('utf-8').splitlines(), dialect="unix")
 
-        logger.debug("reader[]='%s'", type(reader))
         blockdict = list()
+
+        logger.info("Processing %d rows ...", len(reader))
+        cnt = 0
         for row in reader:
             logger.debug("row[%s]='%s'", type(row), row)
             domain = severity = None
             reject_media = reject_reports = False
+
             if "#domain" in row:
                 domain = row["#domain"]
             elif "domain" in row:
@@ -897,6 +923,7 @@ def fetch_oliphant(args: argparse.Namespace) -> int:
             elif "reject_reports" in row and row["reject_reports"].lower() == "true":
                 reject_reports = True
 
+            cnt = cnt + 1
             logger.debug("domain='%s',severity='%s',reject_media='%s',reject_reports='%s'", domain, severity, reject_media, reject_reports)
             if domain == "":
                 logger.debug("domain is empty - SKIPPED!")
@@ -924,6 +951,14 @@ def fetch_oliphant(args: argparse.Namespace) -> int:
             if reject_reports:
                 utils.process_block(block["blocker"], domain, None, "reject_reports")
 
+        logger.debug("Invoking instances.set_total_blocks(%s, %d) ...", block["blocker"], cnt)
+        instances.set_total_blocks(block["blocker"], cnt)
+
+        logger.debug("Checking if blocker='%s' has pending updates ...", block["blocker"])
+        if instances.has_pending(blocker):
+            logger.debug("Flushing updates for block[blocker]='%s' ...", block["blocker"])
+            instances.update_data(block["blocker"])
+
         logger.debug("Invoking commit() ...")
         database.connection.commit()