Continued:

- command 'fetch_txt' supports now --force parameter
- check against blacklist (wasn't in is_registered())
- some debug messages added

diff --git a/fba/boot.py b/fba/boot.py
index 2ffdf16a4b44e2a4f0dfd72f4dce0382d5ab9a99..ea31d7efe9ad465b0bce58164152990f99a64b37 100644 (file)
--- a/fba/boot.py
+++ b/fba/boot.py
@@ -153,6 +153,7 @@ def init_parser():
         help="Fetches text/plain files as simple domain lists",
     )
     parser.set_defaults(command=commands.fetch_txt)
+    parser.add_argument("--force", action="store_true", help="Forces update of data, no matter what.")
 
     ### Fetch blocks from joinfediverse.wiki ###
     #parser = subparser_command.add_parser(

diff --git a/fba/commands.py b/fba/commands.py
index de14596f2686dc8ab984bc2a23c4421cbec4a602..29e78ba84f79effdcc2414ae972ceb994bb0589e 100644 (file)
--- a/fba/commands.py
+++ b/fba/commands.py
@@ -1080,25 +1080,21 @@ def fetch_txt(args: argparse.Namespace) -> int:
             for domain in domains:
                 logger.debug("domain='%s' - BEFORE!", domain)
                 domain = tidyup.domain(domain) if domain not in[None, ""] else None
-
                 logger.debug("domain='%s' - AFTER!", domain)
+
                 if domain in [None, ""]:
                     logger.debug("domain='%s' is empty after tidyup.domain() - SKIPPED!", domain)
                     continue
                 elif not domain_helper.is_wanted(domain):
                     logger.debug("domain='%s' is not wanted - SKIPPED!", domain)
                     continue
-                elif instances.is_recent(domain):
-                    logger.debug("domain='%s' has been recently crawled - SKIPPED!", domain)
+                elif not args.force and instances.is_registered(domain):
+                    logger.debug("domain='%s' is already registered - SKIPPED!", domain)
                     continue
 
-                logger.debug("Processing domain='%s',row[blocker]='%s'", domain, row["blocker"])
-                processed = processing.instance(domain, row["blocker"], inspect.currentframe().f_code.co_name)
-
+                logger.debug("Processing domain='%s',row[blocker]='%s' ...", domain, row["blocker"])
+                processed = processing.instance(domain, row["blocker"], inspect.currentframe().f_code.co_name, force=args.force)
                 logger.debug("processed='%s'", processed)
-                if not processed:
-                    logger.debug("domain='%s' was not generically processed - SKIPPED!", domain)
-                    continue
 
     logger.debug("Success! - EXIT!")
     return 0

diff --git a/fba/helpers/processing.py b/fba/helpers/processing.py
index 37a47065be200e2e2b01f1dacb5ced8de506c837..d0df8aab081e3116d18d097781e40a27dc0db42e 100644 (file)
--- a/fba/helpers/processing.py
+++ b/fba/helpers/processing.py
@@ -36,8 +36,8 @@ from fba.models import instances
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
-def instance(blocked: str, blocker: str, command: str) -> bool:
-    logger.debug("blocked='%s',blocker='%s',command='%s' - CALLED!", blocked, blocker, command)
+def instance(blocked: str, blocker: str, command: str, force: bool = False) -> bool:
+    logger.debug("blocked='%s',blocker='%s',command='%s',force='%s' - CALLED!", blocked, blocker, command, force)
     domain_helper.raise_on(blocked)
     domain_helper.raise_on(blocker)
 
@@ -61,16 +61,18 @@ def instance(blocked: str, blocker: str, command: str) -> bool:
 
     logger.debug("Checking blocked='%s' if wanted and recent ...", blocked)
     if not domain_helper.is_wanted(blocked):
-        logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
+        logger.debug("blocked='%s' is not wanted - EXIT!", blocked)
         return False
-    elif instances.is_recent(blocked):
-        logger.debug("blocked='%s' has been recently checked - SKIPPED!", blocked)
+    elif not force and instances.is_recent(blocked):
+        logger.debug("blocked='%s' has been recently checked - EXIT!", blocked)
         return False
 
     processed = False
     try:
         logger.info("Fetching instances for blocked='%s',blocker='%s',command='%s' ...", blocked, blocker, command)
         federation.fetch_instances(blocked, blocker, None, command)
+
+        logger.debug("Setting processed=True for blocked='%s',blocker='%s' ...", blocked, blocker)
         processed = True
     except network.exceptions as exception:
         logger.warning("Exception '%s' during fetching instances (%s) from blocked='%s'", type(exception), command, blocked)

diff --git a/fba/models/instances.py b/fba/models/instances.py
index 8a4103ef518d45bf01221270541c6c76a507a5bc..e86128181f8dd487a299d40412f6a4502e9c039b 100644 (file)
--- a/fba/models/instances.py
+++ b/fba/models/instances.py
@@ -308,7 +308,9 @@ def is_registered(domain: str, skip_raise = False) -> bool:
     logger.debug("domain='%s',skip_raise='%s' - CALLED!", domain, skip_raise)
     domain_helper.raise_on(domain)
 
-    if not isinstance(skip_raise, bool):
+    if blacklist.is_blacklisted(domain):
+        raise Exception(f"domain='{domain}' is blacklisted but function has been invoked")
+    elif not isinstance(skip_raise, bool):
         raise ValueError(f"skip_raise[]='{type(skip_raise)}' is not type of 'bool'")
 
     if not skip_raise: