From 9c5e3f299782d092da577c3e81dff1212934a9f1 Mon Sep 17 00:00:00 2001
From: =?utf8?q?Roland=20H=C3=A4der?= <roland@mxchange.org>
Date: Sun, 10 Dec 2023 05:50:06 +0100
Subject: [PATCH] Continued: - command 'fetch_txt' supports now --force
 parameter - check against blacklist (wasn't in is_registered()) - some debug
 messages added

---
 fba/boot.py               |  1 +
 fba/commands.py           | 14 +++++---------
 fba/helpers/processing.py | 12 +++++++-----
 fba/models/instances.py   |  4 +++-
 4 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/fba/boot.py b/fba/boot.py
index 2ffdf16..ea31d7e 100644
--- a/fba/boot.py
+++ b/fba/boot.py
@@ -153,6 +153,7 @@ def init_parser():
         help="Fetches text/plain files as simple domain lists",
     )
     parser.set_defaults(command=commands.fetch_txt)
+    parser.add_argument("--force", action="store_true", help="Forces update of data, no matter what.")
 
     ### Fetch blocks from joinfediverse.wiki ###
     #parser = subparser_command.add_parser(
diff --git a/fba/commands.py b/fba/commands.py
index de14596..29e78ba 100644
--- a/fba/commands.py
+++ b/fba/commands.py
@@ -1080,25 +1080,21 @@ def fetch_txt(args: argparse.Namespace) -> int:
             for domain in domains:
                 logger.debug("domain='%s' - BEFORE!", domain)
                 domain = tidyup.domain(domain) if domain not in[None, ""] else None
-
                 logger.debug("domain='%s' - AFTER!", domain)
+
                 if domain in [None, ""]:
                     logger.debug("domain='%s' is empty after tidyup.domain() - SKIPPED!", domain)
                     continue
                 elif not domain_helper.is_wanted(domain):
                     logger.debug("domain='%s' is not wanted - SKIPPED!", domain)
                     continue
-                elif instances.is_recent(domain):
-                    logger.debug("domain='%s' has been recently crawled - SKIPPED!", domain)
+                elif not args.force and instances.is_registered(domain):
+                    logger.debug("domain='%s' is already registered - SKIPPED!", domain)
                     continue
 
-                logger.debug("Processing domain='%s',row[blocker]='%s'", domain, row["blocker"])
-                processed = processing.instance(domain, row["blocker"], inspect.currentframe().f_code.co_name)
-
+                logger.debug("Processing domain='%s',row[blocker]='%s' ...", domain, row["blocker"])
+                processed = processing.instance(domain, row["blocker"], inspect.currentframe().f_code.co_name, force=args.force)
                 logger.debug("processed='%s'", processed)
-                if not processed:
-                    logger.debug("domain='%s' was not generically processed - SKIPPED!", domain)
-                    continue
 
     logger.debug("Success! - EXIT!")
     return 0
diff --git a/fba/helpers/processing.py b/fba/helpers/processing.py
index 37a4706..d0df8aa 100644
--- a/fba/helpers/processing.py
+++ b/fba/helpers/processing.py
@@ -36,8 +36,8 @@ from fba.models import instances
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
-def instance(blocked: str, blocker: str, command: str) -> bool:
-    logger.debug("blocked='%s',blocker='%s',command='%s' - CALLED!", blocked, blocker, command)
+def instance(blocked: str, blocker: str, command: str, force: bool = False) -> bool:
+    logger.debug("blocked='%s',blocker='%s',command='%s',force='%s' - CALLED!", blocked, blocker, command, force)
     domain_helper.raise_on(blocked)
     domain_helper.raise_on(blocker)
 
@@ -61,16 +61,18 @@ def instance(blocked: str, blocker: str, command: str) -> bool:
 
     logger.debug("Checking blocked='%s' if wanted and recent ...", blocked)
     if not domain_helper.is_wanted(blocked):
-        logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
+        logger.debug("blocked='%s' is not wanted - EXIT!", blocked)
         return False
-    elif instances.is_recent(blocked):
-        logger.debug("blocked='%s' has been recently checked - SKIPPED!", blocked)
+    elif not force and instances.is_recent(blocked):
+        logger.debug("blocked='%s' has been recently checked - EXIT!", blocked)
         return False
 
     processed = False
     try:
         logger.info("Fetching instances for blocked='%s',blocker='%s',command='%s' ...", blocked, blocker, command)
         federation.fetch_instances(blocked, blocker, None, command)
+
+        logger.debug("Setting processed=True for blocked='%s',blocker='%s' ...", blocked, blocker)
         processed = True
     except network.exceptions as exception:
         logger.warning("Exception '%s' during fetching instances (%s) from blocked='%s'", type(exception), command, blocked)
diff --git a/fba/models/instances.py b/fba/models/instances.py
index 8a4103e..e861281 100644
--- a/fba/models/instances.py
+++ b/fba/models/instances.py
@@ -308,7 +308,9 @@ def is_registered(domain: str, skip_raise = False) -> bool:
     logger.debug("domain='%s',skip_raise='%s' - CALLED!", domain, skip_raise)
     domain_helper.raise_on(domain)
 
-    if not isinstance(skip_raise, bool):
+    if blacklist.is_blacklisted(domain):
+        raise Exception(f"domain='{domain}' is blacklisted but function has been invoked")
+    elif not isinstance(skip_raise, bool):
         raise ValueError(f"skip_raise[]='{type(skip_raise)}' is not type of 'bool'")
 
     if not skip_raise:
-- 
2.39.5