- command 'fetch_txt' supports now --force parameter
- check against blacklist (wasn't in is_registered())
- some debug messages added
help="Fetches text/plain files as simple domain lists",
)
parser.set_defaults(command=commands.fetch_txt)
help="Fetches text/plain files as simple domain lists",
)
parser.set_defaults(command=commands.fetch_txt)
+ parser.add_argument("--force", action="store_true", help="Forces update of data, no matter what.")
### Fetch blocks from joinfediverse.wiki ###
#parser = subparser_command.add_parser(
### Fetch blocks from joinfediverse.wiki ###
#parser = subparser_command.add_parser(
for domain in domains:
logger.debug("domain='%s' - BEFORE!", domain)
domain = tidyup.domain(domain) if domain not in[None, ""] else None
for domain in domains:
logger.debug("domain='%s' - BEFORE!", domain)
domain = tidyup.domain(domain) if domain not in[None, ""] else None
logger.debug("domain='%s' - AFTER!", domain)
logger.debug("domain='%s' - AFTER!", domain)
if domain in [None, ""]:
logger.debug("domain='%s' is empty after tidyup.domain() - SKIPPED!", domain)
continue
elif not domain_helper.is_wanted(domain):
logger.debug("domain='%s' is not wanted - SKIPPED!", domain)
continue
if domain in [None, ""]:
logger.debug("domain='%s' is empty after tidyup.domain() - SKIPPED!", domain)
continue
elif not domain_helper.is_wanted(domain):
logger.debug("domain='%s' is not wanted - SKIPPED!", domain)
continue
- elif instances.is_recent(domain):
- logger.debug("domain='%s' has been recently crawled - SKIPPED!", domain)
+ elif not args.force and instances.is_registered(domain):
+ logger.debug("domain='%s' is already registered - SKIPPED!", domain)
- logger.debug("Processing domain='%s',row[blocker]='%s'", domain, row["blocker"])
- processed = processing.instance(domain, row["blocker"], inspect.currentframe().f_code.co_name)
-
+ logger.debug("Processing domain='%s',row[blocker]='%s' ...", domain, row["blocker"])
+ processed = processing.instance(domain, row["blocker"], inspect.currentframe().f_code.co_name, force=args.force)
logger.debug("processed='%s'", processed)
logger.debug("processed='%s'", processed)
- if not processed:
- logger.debug("domain='%s' was not generically processed - SKIPPED!", domain)
- continue
logger.debug("Success! - EXIT!")
return 0
logger.debug("Success! - EXIT!")
return 0
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
-def instance(blocked: str, blocker: str, command: str) -> bool:
- logger.debug("blocked='%s',blocker='%s',command='%s' - CALLED!", blocked, blocker, command)
+def instance(blocked: str, blocker: str, command: str, force: bool = False) -> bool:
+ logger.debug("blocked='%s',blocker='%s',command='%s',force='%s' - CALLED!", blocked, blocker, command, force)
domain_helper.raise_on(blocked)
domain_helper.raise_on(blocker)
domain_helper.raise_on(blocked)
domain_helper.raise_on(blocker)
logger.debug("Checking blocked='%s' if wanted and recent ...", blocked)
if not domain_helper.is_wanted(blocked):
logger.debug("Checking blocked='%s' if wanted and recent ...", blocked)
if not domain_helper.is_wanted(blocked):
- logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
+ logger.debug("blocked='%s' is not wanted - EXIT!", blocked)
- elif instances.is_recent(blocked):
- logger.debug("blocked='%s' has been recently checked - SKIPPED!", blocked)
+ elif not force and instances.is_recent(blocked):
+ logger.debug("blocked='%s' has been recently checked - EXIT!", blocked)
return False
processed = False
try:
logger.info("Fetching instances for blocked='%s',blocker='%s',command='%s' ...", blocked, blocker, command)
federation.fetch_instances(blocked, blocker, None, command)
return False
processed = False
try:
logger.info("Fetching instances for blocked='%s',blocker='%s',command='%s' ...", blocked, blocker, command)
federation.fetch_instances(blocked, blocker, None, command)
+
+ logger.debug("Setting processed=True for blocked='%s',blocker='%s' ...", blocked, blocker)
processed = True
except network.exceptions as exception:
logger.warning("Exception '%s' during fetching instances (%s) from blocked='%s'", type(exception), command, blocked)
processed = True
except network.exceptions as exception:
logger.warning("Exception '%s' during fetching instances (%s) from blocked='%s'", type(exception), command, blocked)
logger.debug("domain='%s',skip_raise='%s' - CALLED!", domain, skip_raise)
domain_helper.raise_on(domain)
logger.debug("domain='%s',skip_raise='%s' - CALLED!", domain, skip_raise)
domain_helper.raise_on(domain)
- if not isinstance(skip_raise, bool):
+ if blacklist.is_blacklisted(domain):
+ raise Exception(f"domain='{domain}' is blacklisted but function has been invoked")
+ elif not isinstance(skip_raise, bool):
raise ValueError(f"skip_raise[]='{type(skip_raise)}' is not type of 'bool'")
if not skip_raise:
raise ValueError(f"skip_raise[]='{type(skip_raise)}' is not type of 'bool'")
if not skip_raise: