From 9c5e3f299782d092da577c3e81dff1212934a9f1 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Sun, 10 Dec 2023 05:50:06 +0100 Subject: [PATCH] Continued: - command 'fetch_txt' supports now --force parameter - check against blacklist (wasn't in is_registered()) - some debug messages added --- fba/boot.py | 1 + fba/commands.py | 14 +++++--------- fba/helpers/processing.py | 12 +++++++----- fba/models/instances.py | 4 +++- 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/fba/boot.py b/fba/boot.py index 2ffdf16..ea31d7e 100644 --- a/fba/boot.py +++ b/fba/boot.py @@ -153,6 +153,7 @@ def init_parser(): help="Fetches text/plain files as simple domain lists", ) parser.set_defaults(command=commands.fetch_txt) + parser.add_argument("--force", action="store_true", help="Forces update of data, no matter what.") ### Fetch blocks from joinfediverse.wiki ### #parser = subparser_command.add_parser( diff --git a/fba/commands.py b/fba/commands.py index de14596..29e78ba 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -1080,25 +1080,21 @@ def fetch_txt(args: argparse.Namespace) -> int: for domain in domains: logger.debug("domain='%s' - BEFORE!", domain) domain = tidyup.domain(domain) if domain not in[None, ""] else None - logger.debug("domain='%s' - AFTER!", domain) + if domain in [None, ""]: logger.debug("domain='%s' is empty after tidyup.domain() - SKIPPED!", domain) continue elif not domain_helper.is_wanted(domain): logger.debug("domain='%s' is not wanted - SKIPPED!", domain) continue - elif instances.is_recent(domain): - logger.debug("domain='%s' has been recently crawled - SKIPPED!", domain) + elif not args.force and instances.is_registered(domain): + logger.debug("domain='%s' is already registered - SKIPPED!", domain) continue - logger.debug("Processing domain='%s',row[blocker]='%s'", domain, row["blocker"]) - processed = processing.instance(domain, row["blocker"], inspect.currentframe().f_code.co_name) - + logger.debug("Processing domain='%s',row[blocker]='%s' ...", domain, row["blocker"]) + processed = processing.instance(domain, row["blocker"], inspect.currentframe().f_code.co_name, force=args.force) logger.debug("processed='%s'", processed) - if not processed: - logger.debug("domain='%s' was not generically processed - SKIPPED!", domain) - continue logger.debug("Success! - EXIT!") return 0 diff --git a/fba/helpers/processing.py b/fba/helpers/processing.py index 37a4706..d0df8aa 100644 --- a/fba/helpers/processing.py +++ b/fba/helpers/processing.py @@ -36,8 +36,8 @@ from fba.models import instances logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) -def instance(blocked: str, blocker: str, command: str) -> bool: - logger.debug("blocked='%s',blocker='%s',command='%s' - CALLED!", blocked, blocker, command) +def instance(blocked: str, blocker: str, command: str, force: bool = False) -> bool: + logger.debug("blocked='%s',blocker='%s',command='%s',force='%s' - CALLED!", blocked, blocker, command, force) domain_helper.raise_on(blocked) domain_helper.raise_on(blocker) @@ -61,16 +61,18 @@ def instance(blocked: str, blocker: str, command: str) -> bool: logger.debug("Checking blocked='%s' if wanted and recent ...", blocked) if not domain_helper.is_wanted(blocked): - logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked) + logger.debug("blocked='%s' is not wanted - EXIT!", blocked) return False - elif instances.is_recent(blocked): - logger.debug("blocked='%s' has been recently checked - SKIPPED!", blocked) + elif not force and instances.is_recent(blocked): + logger.debug("blocked='%s' has been recently checked - EXIT!", blocked) return False processed = False try: logger.info("Fetching instances for blocked='%s',blocker='%s',command='%s' ...", blocked, blocker, command) federation.fetch_instances(blocked, blocker, None, command) + + logger.debug("Setting processed=True for blocked='%s',blocker='%s' ...", blocked, blocker) processed = True except network.exceptions as exception: logger.warning("Exception '%s' during fetching instances (%s) from blocked='%s'", type(exception), command, blocked) diff --git a/fba/models/instances.py b/fba/models/instances.py index 8a4103e..e861281 100644 --- a/fba/models/instances.py +++ b/fba/models/instances.py @@ -308,7 +308,9 @@ def is_registered(domain: str, skip_raise = False) -> bool: logger.debug("domain='%s',skip_raise='%s' - CALLED!", domain, skip_raise) domain_helper.raise_on(domain) - if not isinstance(skip_raise, bool): + if blacklist.is_blacklisted(domain): + raise Exception(f"domain='{domain}' is blacklisted but function has been invoked") + elif not isinstance(skip_raise, bool): raise ValueError(f"skip_raise[]='{type(skip_raise)}' is not type of 'bool'") if not skip_raise: -- 2.39.5