From 3b4d0299b7609758af0efd33cc279cb35ae8b14b Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Sun, 2 Jul 2023 10:42:23 +0200 Subject: [PATCH] Continued: - update total blocks in any command that handles instance blocks --- fba/commands.py | 59 +++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 47 insertions(+), 12 deletions(-) diff --git a/fba/commands.py b/fba/commands.py index 9f5b509..938f8c8 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -511,6 +511,12 @@ def fetch_todon_wiki(args: argparse.Namespace) -> int: logger.info("Checking %d suspended entries ...", len(suspended)) blocklist["reject"] = utils.find_domains(suspended, "div") + blocking = blocklist["silenced"] + blocklist["reject"] + blocker = "todon.eu" + + logger.debug("Invoking instances.set_total_blocks(%s, %d) ...", blocker, len(blocking)) + instances.set_total_blocks(blocker, blocking) + blockdict = list() for block_level in blocklist: blockers = blocklist[block_level] @@ -522,18 +528,18 @@ def fetch_todon_wiki(args: argparse.Namespace) -> int: if not instances.is_registered(blocked): try: logger.info("Fetching instances from domain='%s' ...", blocked) - federation.fetch_instances(blocked, 'chaos.social', None, inspect.currentframe().f_code.co_name) + federation.fetch_instances(blocked, blocker, None, inspect.currentframe().f_code.co_name) except network.exceptions as exception: logger.warning("Exception '%s' during fetching instances (fetch_cs) from blocked='%s'", type(exception), blocked) instances.set_last_error(blocked, exception) - if blocks.is_instance_blocked("todon.eu", blocked, block_level): + if blocks.is_instance_blocked(blocker, blocked, block_level): logger.debug("blocked='%s',block_level='%s' is already blocked - SKIPPED!", blocked, block_level) continue logger.info("Adding new block: blocked='%s',block_level='%s'", blocked, block_level) - if utils.process_block("todon.eu", blocked, None, block_level) and block_level == "reject" and config.get("bot_enabled"): - logger.debug("Appending blocked='%s',reason='%s' for blocker='todon.eu' ...", blocked, block_level) + if utils.process_block(blocker, blocked, None, block_level) and block_level == "reject" and config.get("bot_enabled"): + logger.debug("Appending blocked='%s',reason='%s' for blocker='%s' ...", blocked, block_level, blocker) blockdict.append({ "blocked": blocked, "reason" : None, @@ -544,8 +550,13 @@ def fetch_todon_wiki(args: argparse.Namespace) -> int: logger.debug("config.get(bot_enabled)='%s',blockdict()=%d", config.get("bot_enabled"), len(blockdict)) if config.get("bot_enabled") and len(blockdict) > 0: - logger.info("Sending bot POST for blocker='todon.eu',blockdict()=%d ...", len(blockdict)) - network.send_bot_post("todon.eu", blockdict) + logger.info("Sending bot POST for blocker='%s',blockdict()=%d ...", blocker, len(blockdict)) + network.send_bot_post(blocker, blockdict) + + logger.debug("Checking if blocker='%s' has pending updates ...", blocker) + if instances.has_pending(blocker): + logger.debug("Flushing updates for blocker='%s' ...", blocker) + instances.update_data(blocker) logger.debug("Success! - EXIT!") return 0 @@ -591,6 +602,12 @@ def fetch_cs(args: argparse.Namespace): logger.debug("blocked[%s]()=%d", type(blocked), len(blocked)) domains["reject"] = federation.find_domains(blocked) + blocking = blocklist["silenced"] + blocklist["reject"] + blocker = "chaos.social" + + logger.debug("Invoking instances.set_total_blocks(%s, %d) ...", blocker, len(blocking)) + instances.set_total_blocks(blocker, blocking) + logger.debug("domains[silenced]()=%d,domains[reject]()=%d", len(domains["silenced"]), len(domains["reject"])) blockdict = list() if len(domains) > 0: @@ -607,13 +624,13 @@ def fetch_cs(args: argparse.Namespace): elif not instances.is_registered(row["domain"]): try: logger.info("Fetching instances from domain='%s' ...", row["domain"]) - federation.fetch_instances(row["domain"], 'chaos.social', None, inspect.currentframe().f_code.co_name) + federation.fetch_instances(row["domain"], blocker, None, inspect.currentframe().f_code.co_name) except network.exceptions as exception: logger.warning("Exception '%s' during fetching instances (fetch_cs) from row[domain]='%s'", type(exception), row["domain"]) instances.set_last_error(row["domain"], exception) - if utils.process_block("chaos.social", row["domain"], row["reason"], block_level) and block_level == "reject" and config.get("bot_enabled"): - logger.debug("Appending blocked='%s',reason='%s' for blocker='chaos.social' ...", row["domain"], block_level) + if utils.process_block(blocker, row["domain"], row["reason"], block_level) and block_level == "reject" and config.get("bot_enabled"): + logger.debug("Appending blocked='%s',reason='%s' for blocker='%s' ...", row["domain"], block_level, blocker) blockdict.append({ "blocked": row["domain"], "reason" : row["reason"], @@ -624,8 +641,13 @@ def fetch_cs(args: argparse.Namespace): logger.debug("config.get(bot_enabled)='%s',blockdict()=%d", config.get("bot_enabled"), len(blockdict)) if config.get("bot_enabled") and len(blockdict) > 0: - logger.info("Sending bot POST for blocker='chaos.social',blockdict()=%d ...", len(blockdict)) - network.send_bot_post("chaos.social", blockdict) + logger.info("Sending bot POST for blocker='%s',blockdict()=%d ...", blocker, len(blockdict)) + network.send_bot_post(blocker, blockdict) + + logger.debug("Checking if blocker='%s' has pending updates ...", blocker) + if instances.has_pending(blocker): + logger.debug("Flushing updates for blocker='%s' ...", blocker) + instances.update_data(blocker) logger.debug("Success! - EXIT!") return 0 @@ -735,6 +757,7 @@ def fetch_fbabot_atom(args: argparse.Namespace) -> int: logger.info("Adding %d new instances ...", len(domains)) for domain in domains: + logger.debug("domain='%s'", domain) try: logger.info("Fetching instances from domain='%s' ...", domain) federation.fetch_instances(domain, "ryona.agency", None, inspect.currentframe().f_code.co_name) @@ -865,12 +888,15 @@ def fetch_oliphant(args: argparse.Namespace) -> int: logger.debug("Fetched %d Bytes, parsing CSV ...", len(response.content)) reader = csv.DictReader(response.content.decode('utf-8').splitlines(), dialect="unix") - logger.debug("reader[]='%s'", type(reader)) blockdict = list() + + logger.info("Processing %d rows ...", len(reader)) + cnt = 0 for row in reader: logger.debug("row[%s]='%s'", type(row), row) domain = severity = None reject_media = reject_reports = False + if "#domain" in row: domain = row["#domain"] elif "domain" in row: @@ -897,6 +923,7 @@ def fetch_oliphant(args: argparse.Namespace) -> int: elif "reject_reports" in row and row["reject_reports"].lower() == "true": reject_reports = True + cnt = cnt + 1 logger.debug("domain='%s',severity='%s',reject_media='%s',reject_reports='%s'", domain, severity, reject_media, reject_reports) if domain == "": logger.debug("domain is empty - SKIPPED!") @@ -924,6 +951,14 @@ def fetch_oliphant(args: argparse.Namespace) -> int: if reject_reports: utils.process_block(block["blocker"], domain, None, "reject_reports") + logger.debug("Invoking instances.set_total_blocks(%s, %d) ...", block["blocker"], cnt) + instances.set_total_blocks(block["blocker"], cnt) + + logger.debug("Checking if blocker='%s' has pending updates ...", block["blocker"]) + if instances.has_pending(blocker): + logger.debug("Flushing updates for block[blocker]='%s' ...", block["blocker"]) + instances.update_data(block["blocker"]) + logger.debug("Invoking commit() ...") database.connection.commit() -- 2.39.5