From d3e01415130d10ba7da6b268cf92a0171e34808c Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Tue, 3 Jun 2025 02:07:58 +0200 Subject: [PATCH] Continued: - `pleroma.envs.net` does now publish a more detailed blocklist than its CSV file - but still it can be used as peer source (key `only_peers` set to `True`) - exception messages improved --- fba/commands.py | 8 +++- fba/helpers/blocklists.py | 34 +++++++++------ fba/helpers/config.py | 2 +- fba/helpers/dicts.py | 2 +- fba/helpers/domain.py | 8 ++-- fba/helpers/processing.py | 90 +++++++++++++++++++++++++++++++++++++-- fba/helpers/software.py | 12 +++--- fba/helpers/tidyup.py | 2 +- fba/http/federation.py | 10 ++--- fba/http/network.py | 10 ++--- fba/models/blocks.py | 14 +++--- fba/models/instances.py | 20 ++++----- fba/networks/lemmy.py | 2 +- fba/networks/misskey.py | 11 +++-- fba/networks/peertube.py | 1 + fba/networks/pleroma.py | 6 +-- fba/utils.py | 8 ++-- 17 files changed, 169 insertions(+), 71 deletions(-) diff --git a/fba/commands.py b/fba/commands.py index 9dc0d2c..cd931ea 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -1103,8 +1103,12 @@ def fetch_csv(args: argparse.Namespace) -> int: logger.debug("block[blocker]='%s' does not match args.domain='%s' - SKIPPED!", block["blocker"], args.domain) continue - logger.debug("Invoking processing.csv_block(%s, %s, fetch_csv) ...", block["blocker"], block["csv_url"]) - processing.csv_block(block["blocker"], block["csv_url"], inspect.currentframe().f_code.co_name) + if block["only_peers"]: + logger.debug("block[blocker]='%s' is not registered, adding instance ...", block["blocker"]) + processing.csv_instance(block["blocker"], block["csv_url"], inspect.currentframe().f_code.co_name) + else: + logger.debug("Invoking processing.csv_block(%s, %s, fetch_csv) ...", block["blocker"], block["csv_url"]) + processing.csv_block(block["blocker"], block["csv_url"], inspect.currentframe().f_code.co_name) logger.debug("Success - EXIT!") return 0 diff --git a/fba/helpers/blocklists.py b/fba/helpers/blocklists.py index 90cf174..86612de 100644 --- a/fba/helpers/blocklists.py +++ b/fba/helpers/blocklists.py @@ -26,17 +26,21 @@ logger = logging.getLogger(__name__) # Blocklists hosted by oliphant oliphant_blocklists = ( { - "blocker": "mastodon.online", - "csv_url": "mastodon/mastodon.online.csv", + "blocker" : "mastodon.online", + "csv_url" : "mastodon/mastodon.online.csv", + "only_peers": False, },{ - "blocker": "mastodon.social", - "csv_url": "mastodon/mastodon.social.csv", + "blocker" : "mastodon.social", + "csv_url" : "mastodon/mastodon.social.csv", + "only_peers": False, },{ - "blocker": "mastodon.social", - "csv_url": "other/missing-tier0-mastodon.social.csv", + "blocker" : "mastodon.social", + "csv_url" : "other/missing-tier0-mastodon.social.csv", + "only_peers": False, },{ - "blocker": "seirdy.one", - "csv_url": "mastodon/seirdy-tier0.csv", + "blocker" : "seirdy.one", + "csv_url" : "mastodon/seirdy-tier0.csv", + "only_peers": False, }, ) @@ -66,11 +70,13 @@ txt_files = ( # Other CSV files csv_files = ( { - "blocker": "pleroma.envs.net", - "csv_url": "https://seirdy.one/pb/pleroma.envs.net.csv", + "blocker" : "pleroma.envs.net", + "csv_url" : "https://seirdy.one/pb/pleroma.envs.net.csv", + "only_peers": True, },{ - "blocker": "social.mooneyed.de", - "csv_url": "https://git.bka.li/kromonos/fedi-health/-/raw/master/_unified_tier3_blocklist.csv", + "blocker" : "social.mooneyed.de", + "csv_url" : "https://git.bka.li/kromonos/fedi-health/-/raw/master/_unified_tier3_blocklist.csv", + "only_peers": False, }, ) @@ -84,8 +90,8 @@ def has(domain: str) -> bool: # Default is not found found = False for row in oliphant_blocklists + csv_files: - logger.debug("row[blocker]='%s',domain='%s'", row["blocker"], domain) - if row["blocker"] == domain: + logger.debug("row[blocker]='%s',row[only_peers]='%s',domain='%s'", row["blocker"], row["only_peers"], domain) + if row["blocker"] == domain and not row["only_peers"]: found = True logger.debug("domain='%s' is found and excluded from regular fetch_blocks command - BREAK!", domain) break diff --git a/fba/helpers/config.py b/fba/helpers/config.py index 0e7b12c..6788182 100644 --- a/fba/helpers/config.py +++ b/fba/helpers/config.py @@ -46,7 +46,7 @@ def get(key: str) -> any: if not isinstance(key, str): raise TypeError(f"Parameter key[]='{type(key)}' has not expected type 'str'") elif key == "": - raise ValueError("Parameter 'key' is empty") + raise ValueError("Parameter 'key' is an empty string") elif not key in _config: raise KeyError(f"key='{key}' does not exist in _config array") diff --git a/fba/helpers/dicts.py b/fba/helpers/dicts.py index 7c97e1d..f867672 100644 --- a/fba/helpers/dicts.py +++ b/fba/helpers/dicts.py @@ -28,7 +28,7 @@ def has_key(lists: list, key: str, value: any) -> bool: elif not isinstance(key, str): raise TypeError(f"Parameter key[]='{type(key)}' has not expected type 'str'") elif key == "": - raise ValueError("Parameter 'key' is empty") + raise ValueError("Parameter 'key' is an empty string") has = False logger.debug("Checking lists()=%d ...", len(lists)) diff --git a/fba/helpers/domain.py b/fba/helpers/domain.py index 6948fb8..6d91303 100644 --- a/fba/helpers/domain.py +++ b/fba/helpers/domain.py @@ -39,7 +39,7 @@ def raise_on(domain: str) -> None: if not isinstance(domain, str): raise TypeError(f"Parameter domain[]='{type(domain)}' has not expected type 'str'") elif domain == "": - raise ValueError("Parameter 'domain' is empty") + raise ValueError("Parameter 'domain' is an empty string") elif domain.lower() != domain: raise ValueError(f"Parameter domain='{domain}' must be all lower-case") elif "?" in domain: @@ -67,7 +67,7 @@ def is_in_url(domain: str, url: str) -> bool: elif not isinstance(url, str): raise TypeError(f"Parameter url[]='{type(url)}' has not expected type 'str'") elif url == "": - raise ValueError("Parameter 'url' is empty") + raise ValueError("Parameter 'url' is an empty string") elif not validators.url(url): raise RuntimeError(f"Parameter url='{url}' is not a valid URL but function was invoked") @@ -89,7 +89,7 @@ def is_tld_wanted(domain: str) -> bool: if not isinstance(domain, str): raise TypeError(f"Parameter domain[]='{type(domain)}' has not expected type 'str'") elif domain == "": - raise ValueError("Parameter 'domain' is empty") + raise ValueError("Parameter 'domain' is an empty string") wanted = True @@ -116,7 +116,7 @@ def is_wanted(domain: str) -> bool: if not isinstance(domain, str): raise TypeError(f"Parameter domain[]='{type(domain)}' has not expected type 'str'") elif domain == "": - raise ValueError("Parameter 'domain' is empty") + raise ValueError("Parameter 'domain' is an empty string") wanted = True if domain.lower() != domain: diff --git a/fba/helpers/processing.py b/fba/helpers/processing.py index b11b53e..f21ff53 100644 --- a/fba/helpers/processing.py +++ b/fba/helpers/processing.py @@ -49,7 +49,7 @@ def instance(blocked: str, blocker: str, command: str, force: bool = False) -> b if not isinstance(command, str): raise TypeError(f"Parameter command[]='{type(command)}' has not expected type 'str'") elif command == "": - raise ValueError("Parameter 'command' is empty") + raise ValueError("Parameter 'command' is an empty string") elif blacklist.is_blacklisted(blocked): raise RuntimeError(f"blocked='{blocked}' is blacklisted but function was invoked") elif blacklist.is_blacklisted(blocker): @@ -132,13 +132,13 @@ def csv_block(blocker: str, url: str, command: str) -> None: if not isinstance(url, str): raise TypeError(f"url[]='{url}' has not expected type 'str'") elif url in [None, ""]: - raise ValueError("Parameter 'url' is empty") + raise ValueError("Parameter 'url' is an empty string") elif not validators.url(url): raise ValueError(f"Parameter url='{url}' is not a valid URL") elif not isinstance(command, str): raise TypeError(f"command[]='{command}' has not expected type 'str'") elif command == "": - raise ValueError("Parameter 'command' is empty") + raise ValueError("Parameter 'command' is an empty string") elif blacklist.is_blacklisted(blocker): raise RuntimeError(f"blocker='{blocker}' is blacklisted but function was invoked") @@ -264,3 +264,87 @@ def csv_block(blocker: str, url: str, command: str) -> None: network.send_bot_post(blocker, blockdict) logger.debug("EXIT!") + +def csv_instance(instance: str, url: str, command: str) -> None: + logger.debug("instance='%s',url='%s',command='%s' - CALLED!", instance, url, command) + domain_helper.raise_on(instance) + + if not isinstance(url, str): + raise TypeError(f"url[]='{url}' has not expected type 'str'") + elif url in [None, ""]: + raise ValueError("Parameter 'url' is an empty string") + elif not validators.url(url): + raise ValueError(f"Parameter url='{url}' is not a valid URL") + elif not isinstance(command, str): + raise TypeError(f"command[]='{command}' has not expected type 'str'") + elif command == "": + raise ValueError("Parameter 'command' is an empty string") + elif blacklist.is_blacklisted(instance): + raise RuntimeError(f"instance='{instance}' is blacklisted but function was invoked") + + # Init local variables + domains = [] + + logger.debug("Setting last_instance_fetch for instance='%s' ...", instance) + instances.set_last_instance_fetch(instance) + + # Fetch this URL + logger.info("Fetching url='%s' for instance='%s' ...", url, instance) + rows = network.fetch_csv_rows(url) + + logger.info("Checking %d CSV lines ...", len(rows)) + for row in rows: + logger.debug("row[%s]='%s'", type(row), row) + domain = None + + if "#domain" in row and row["#domain"] not in [None, ""]: + domain = tidyup.domain(row["#domain"]) + elif "domain" in row and row["domain"] not in [None, ""]: + domain = tidyup.domain(row["domain"]) + elif "Domain" in row and row["Domain"] not in [None, ""]: + domain = tidyup.domain(row["Domain"]) + else: + logger.warning("row='%s' does not contain domain column - SKIPPED!", row) + continue + + logger.debug("domain='%s'", domain) + if domain in [None, ""]: + logger.debug("domain='%s' is empty - SKIPPED!", domain) + continue + elif not domain_helper.is_tld_wanted(domain): + logger.debug("domain='%s' has an unwanted TLD - SKIPPED!", domain) + continue + elif domain.find("*") >= 0 or domain.find("?") >= 0: + logger.debug("domain='%s' is obfuscated - Invoking utils.deobfuscate(%s, %s) ...", domain, domain, instance) + domain = utils.deobfuscate(domain, instance) + logger.debug("domain='%s' - AFTER!", domain) + + logger.debug("Marking domain='%s' as handled ...", domain) + domains.append(domain) + + if not validators.domain(domain, rfc_2782=True): + logger.warning("domain='%s' is not a valid domain - SKIPPED!", domain) + continue + elif blacklist.is_blacklisted(domain): + logger.debug("domain='%s' is blacklisted - SKIPPED!", domain) + continue + elif instances.is_registered(domain): + logger.debug("domain='%s' is already registered - SKIPPED!", domain) + continue + + logger.debug("Processing domain='%s',instance='%s',command='%s' ...", domain, instance, command) + processed = instance(domain, instance, command) + logger.debug("processed='%s'", processed) + + logger.debug("Invoking commit() ...") + database.connection.commit() + + logger.debug("Invoking instances.set_total_instances(%s, domains()=%d) ...", instance, len(domains)) + instances.set_total_instances(instance, domains) + + logger.debug("Checking if instance='%s' has pending updates ...", instance) + if instances.has_pending(instance): + logger.debug("Flushing updates for instance='%s' ...", instance) + instances.update(instance) + + logger.debug("EXIT!") diff --git a/fba/helpers/software.py b/fba/helpers/software.py index a2ad615..f7565e1 100644 --- a/fba/helpers/software.py +++ b/fba/helpers/software.py @@ -113,7 +113,7 @@ def alias(software: str) -> str: if not isinstance(software, str) and software is not None: raise TypeError(f"software[]='{type(software)}' is not type 'str'") elif software == "": - raise ValueError("Parameter 'software' is empty") + raise ValueError("Parameter 'software' is an empty string") elif software.startswith("re:"): logger.debug("Cutting prefix 're:' from software='%s' ...", software) software = software.split(":")[1] @@ -226,7 +226,7 @@ def strip_hosted_on(software: str) -> str: if not isinstance(software, str): raise TypeError(f"Parameter software[]='{type(software)}' has not expected type 'str'") elif software == "": - raise ValueError("Parameter 'software' is empty") + raise ValueError("Parameter 'software' is an empty string") elif "hosted on" not in software: logger.warning("Cannot find 'hosted on' in software='%s'!", software) return software @@ -251,7 +251,7 @@ def strip_powered_by(software: str) -> str: if not isinstance(software, str): raise TypeError(f"Parameter software[]='{type(software)}' has not expected type 'str'") elif software == "": - raise ValueError("Parameter 'software' is empty") + raise ValueError("Parameter 'software' is an empty string") elif "powered by" not in software: logger.warning("Cannot find 'powered by' in software='%s'!", software) return software @@ -276,11 +276,11 @@ def strip_until(software: str, until: str) -> str: if not isinstance(software, str): raise TypeError(f"Parameter software[]='{type(software)}' has not expected type 'str'") elif software == "": - raise ValueError("Parameter 'software' is empty") + raise ValueError("Parameter 'software' is an empty string") elif not isinstance(until, str): raise TypeError(f"Parameter until[]='{type(until)}' has not expected type 'str'") elif until == "": - raise ValueError("Parameter 'until' is empty") + raise ValueError("Parameter 'until' is an empty string") elif not until in software: logger.warning("Cannot find until='%s' in software='%s'!", until, software) return software @@ -301,7 +301,7 @@ def is_relay(software: str) -> bool: if not isinstance(software, str): raise TypeError(f"software[]='{type(software)}' is not type 'str'") elif software == "": - raise ValueError("Parameter 'software' is empty") + raise ValueError("Parameter 'software' is an empty string") found = software in _relays diff --git a/fba/helpers/tidyup.py b/fba/helpers/tidyup.py index 2f80183..02932ca 100644 --- a/fba/helpers/tidyup.py +++ b/fba/helpers/tidyup.py @@ -39,7 +39,7 @@ def domain(string: str) -> str: if not isinstance(string, str): raise TypeError(f"Parameter string[]='{type(string)}' has not expected type 'str'") elif string == "": - raise ValueError("Parameter 'string' is empty") + raise ValueError("Parameter 'string' is an empty string") # All lower-case and strip spaces out + last dot string = string.lower().strip().rstrip(".").replace("..", ".") diff --git a/fba/http/federation.py b/fba/http/federation.py index c9cd446..7c996e9 100644 --- a/fba/http/federation.py +++ b/fba/http/federation.py @@ -69,7 +69,7 @@ def fetch_instances(domain: str, origin: str, software: str, command: str, path: elif not isinstance(command, str): raise TypeError(f"Parameter command[]='{type(command)}' has not expected type 'str'") elif command == "": - raise ValueError("Parameter 'command' is empty") + raise ValueError("Parameter 'command' is an empty string") elif command in ["fetch_blocks", "fetch_cs", "fetch_bkali", "fetch_relays", "fetch_fedipact", "fetch_joinmobilizon", "fetch_joinmisskey", "fetch_joinfediverse", "fetch_relaylist"] and origin is None: raise ValueError(f"Parameter command='{command}' but origin is None, please fix invoking this function.") elif not isinstance(path, str) and path is not None: @@ -229,13 +229,13 @@ def fetch_peers(domain: str, software: str, origin: str) -> list: elif not isinstance(software, str) and software is not None: raise TypeError(f"Parameter software[]='{type(software)}' has not expected type 'str'") elif isinstance(software, str) and software == "": - raise ValueError("Parameter 'software' is empty") + raise ValueError("Parameter 'software' is an empty string") elif software is not None and software_helper.is_relay(software): raise RuntimeError(f"domain='{domain}' is of software='{software}' and isn't supported here.") elif not isinstance(origin, str) and origin is not None: raise TypeError(f"Parameter origin[]='{type(origin)}' has not expected type 'str'") elif isinstance(origin, str) and origin == "": - raise ValueError("Parameter 'origin' is empty") + raise ValueError("Parameter 'origin' is an empty string") if software == "misskey": logger.debug("Invoking misskey.fetch_peers(%s) ...", domain) @@ -297,7 +297,7 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str: elif not isinstance(path, str): raise TypeError(f"path[]='{type(path)}' has not expected type 'str'") elif path == "": - raise ValueError("Parameter 'path' is empty") + raise ValueError("Parameter 'path' is an empty string") elif not path.startswith("/"): raise ValueError(f"path='{path}' does not start with / but should") @@ -562,7 +562,7 @@ def add_peers(rows: dict) -> list: if not isinstance(rows, dict): raise TypeError(f"Parameter rows[]='{type(rows)}' has not expected type 'dict'") elif len(rows) == 0: - raise ValueError("Parameter 'rows' is empty") + raise ValueError("Parameter 'rows' is an empty string") # Init variables peers = [] diff --git a/fba/http/network.py b/fba/http/network.py index 9f62316..56049a5 100644 --- a/fba/http/network.py +++ b/fba/http/network.py @@ -141,7 +141,7 @@ def fetch_api_url(url: str, timeout: tuple) -> dict: if not isinstance(url, str): raise TypeError(f"Parameter url[]='{type(url)}' has not expected type 'str'") elif url == "": - raise ValueError("Parameter 'url' is empty") + raise ValueError("Parameter 'url' is an empty string") elif not validators.url(url): raise ValueError(f"Parameter url='{url}' is not a valid URL") elif not isinstance(timeout, tuple): @@ -244,7 +244,7 @@ def send_bot_post(domain: str, blocklist: list) -> None: elif not isinstance(blocklist, list): raise TypeError(f"Parameter blocklist[]='{type(blocklist)}' has not expected type 'list'") elif len(blocklist) == 0: - raise ValueError("Parameter 'blocklist' is empty") + raise ValueError("Parameter 'blocklist' is an empty string") elif config.get("bot_token") == "": raise ValueError("config[bot_token] is not set") @@ -296,7 +296,7 @@ def _fetch_response(domain: str, path: str, headers: dict, timeout: tuple, allow elif not isinstance(path, str): raise TypeError(f"Parameter path[]='{type(path)}' has not expected type 'str'") elif path == "": - raise ValueError("Parameter 'path' is empty") + raise ValueError("Parameter 'path' is an empty string") elif not path.startswith("/"): raise ValueError(f"path='{path}' does not start with / but should") elif not isinstance(headers, dict): @@ -343,7 +343,7 @@ def fetch_url(url: str, headers: dict, timeout: tuple, allow_redirects: bool = T if not isinstance(url, str): raise TypeError(f"Parameter url[]='{type(url)}' has not expected type 'str'") elif url == "": - raise ValueError("Parameter 'url' is empty") + raise ValueError("Parameter 'url' is an empty string") elif not validators.url(url): raise ValueError(f"Parameter url='{url}' is not a valid URL") elif not isinstance(headers, dict): @@ -479,7 +479,7 @@ def get_generic(domain: str, path: str, allow_redirects: bool = False) -> reques elif not isinstance(path, str): raise TypeError(f"Parameter path[]='{type(path)}' has not expected type 'str'") elif path == "": - raise ValueError("Parameter 'path' is empty") + raise ValueError("Parameter 'path' is an empty string") elif not path.startswith("/"): raise ValueError(f"path='{path}' does not start with / but should") elif not isinstance(allow_redirects, bool): diff --git a/fba/models/blocks.py b/fba/models/blocks.py index b77c668..61db783 100644 --- a/fba/models/blocks.py +++ b/fba/models/blocks.py @@ -36,7 +36,7 @@ def get_reason(blocker: str, blocked: str, block_level: str) -> str: if not isinstance(block_level, str): raise TypeError(f"Parameter block_level[]='{type(block_level)}' has not expected type 'str'") elif block_level == "": - raise ValueError("Parameter 'block_level' is empty") + raise ValueError("Parameter 'block_level' is an empty string") elif block_level in ["accept", "reject", "suspend", "silence", "nsfw", "quarantined_instances"]: raise ValueError(f"block_level='{block_level}' is not wanted.") elif blacklist.is_blacklisted(blocker): @@ -71,7 +71,7 @@ def update_reason(reason: str, blocker: str, blocked: str, block_level: str) -> elif not isinstance(block_level, str): raise TypeError(f"Parameter block_level[]='{type(block_level)}' has not expected type 'str'") elif block_level == "": - raise ValueError("Parameter 'block_level' is empty") + raise ValueError("Parameter 'block_level' is an empty string") elif block_level in ["accept", "reject", "suspend", "silence", "nsfw", "quarantined_instances"]: raise ValueError(f"block_level='{block_level}' is not wanted.") elif blacklist.is_blacklisted(blocker): @@ -102,7 +102,7 @@ def update_last_seen(blocker: str, blocked: str, block_level: str) -> None: if not isinstance(block_level, str): raise TypeError(f"Parameter block_level[]='{type(block_level)}' has not expected type 'str'") elif block_level == "": - raise ValueError("Parameter 'block_level' is empty") + raise ValueError("Parameter 'block_level' is an empty string") elif block_level in ["accept", "reject", "suspend", "silence", "nsfw", "quarantined_instances"]: raise ValueError(f"blocked='{blocked}' has unwanted block_level='{block_level}'") elif blacklist.is_blacklisted(blocker): @@ -131,7 +131,7 @@ def is_instance_blocked(blocker: str, blocked: str, block_level: str = None) -> if not isinstance(block_level, str) and block_level is not None: raise TypeError(f"Parameter block_level[]='{type(block_level)}' has not expected type 'str'") elif block_level == "": - raise ValueError("Parameter 'block_level' is empty") + raise ValueError("Parameter 'block_level' is an empty string") elif block_level in ["accept", "reject", "suspend", "silence", "nsfw", "quarantined_instances"]: raise ValueError(f"blocked='{blocked}' has unwanted block_level='{block_level}'") elif blacklist.is_blacklisted(blocker): @@ -170,7 +170,7 @@ def add(blocker: str, blocked: str, reason: str, block_level: str) -> None: if not isinstance(block_level, str): raise TypeError(f"Parameter block_level[]='{type(block_level)}' has not expected type 'str'") elif block_level == "": - raise ValueError("Parameter 'block_level' is empty") + raise ValueError("Parameter 'block_level' is an empty string") elif block_level in ["accept", "reject", "suspend", "silence", "nsfw", "quarantined_instances"]: raise ValueError(f"blocked='{blocked}' has unwanted block_level='{block_level}'") elif blacklist.is_blacklisted(blocker): @@ -207,11 +207,11 @@ def valid(value: str, column: str) -> bool: if not isinstance(value, str): raise TypeError(f"Parameter value[]='{type(value)}' has not expected type 'str'") elif value == "": - raise ValueError("Parameter 'value' is empty") + raise ValueError("Parameter 'value' is an empty string") elif not isinstance(column, str): raise TypeError(f"Parameter column[]='{type(column)}' has not expected type 'str'") elif column == "": - raise ValueError("Parameter 'column' is empty") + raise ValueError("Parameter 'column' is an empty string") # Query database database.cursor.execute( diff --git a/fba/models/instances.py b/fba/models/instances.py index 19572f3..a7fabf0 100644 --- a/fba/models/instances.py +++ b/fba/models/instances.py @@ -389,11 +389,11 @@ def deobfuscate(char: str, domain: str, blocked_hash: str = None) -> tuple: if not isinstance(char, str): raise TypeError(f"Parameter char[]='{type(char)}' has not expected type 'str'") elif char == "": - raise ValueError("Parameter 'char' is empty") + raise ValueError("Parameter 'char' is an empty string") elif not isinstance(domain, str): raise TypeError(f"Parameter domain[]='{type(domain)}'") elif domain == "": - raise ValueError("Parameter 'domain' is empty") + raise ValueError("Parameter 'domain' is an empty string") elif not char in domain: raise ValueError(f"char='{char}' not found in domain='{domain}' but function invoked") elif not isinstance(blocked_hash, str) and blocked_hash is not None: @@ -514,7 +514,7 @@ def set_nodeinfo_url(domain: str, url: str) -> None: if not isinstance(url, str) and url is not None: raise TypeError(f"Parameter url[]='{type(url)}' has not expected type 'str'") elif url == "": - raise ValueError("Parameter 'url' is empty") + raise ValueError("Parameter 'url' is an empty string") elif url is not None and not validators.url(url): raise ValueError(f"Parameter url='{url}' is not a valid URL") @@ -529,7 +529,7 @@ def set_detection_mode(domain: str, mode: str) -> None: if not isinstance(mode, str) and mode is not None: raise TypeError(f"Parameter mode[]='{type(mode)}' has not expected type 'str'") elif mode == "": - raise ValueError("Parameter 'mode' is empty") + raise ValueError("Parameter 'mode' is an empty string") # Set timestamp _set_pending_data("detection_mode", domain, mode) @@ -553,7 +553,7 @@ def set_original_software(domain: str, software: str) -> None: if not isinstance(software, str) and software is not None: raise TypeError(f"Parameter software[]='{type(software)}' has not expected type 'str'") elif software == "": - raise ValueError("Parameter 'software' is empty") + raise ValueError("Parameter 'software' is an empty string") # Set original software _set_pending_data("original_software", domain, software) @@ -566,7 +566,7 @@ def set_software(domain: str, software: str) -> None: if not isinstance(software, str) and software is not None: raise TypeError(f"Parameter software[]='{type(software)}' has not expected type 'str'") elif software == "": - raise ValueError("Parameter 'software' is empty") + raise ValueError("Parameter 'software' is an empty string") # Set software (maybe aliased to generic name) _set_pending_data("software", domain, software) @@ -578,11 +578,11 @@ def valid(value: str, column: str) -> bool: if not isinstance(value, str): raise TypeError(f"Parameter value[]='{type(value)}' has not expected type 'str'") elif value == "": - raise ValueError("Parameter 'value' is empty") + raise ValueError("Parameter 'value' is an empty string") elif not isinstance(column, str): raise TypeError(f"Parameter column[]='{type(column)}' has not expected type 'str'") elif column == "": - raise ValueError("Parameter 'column' is empty") + raise ValueError("Parameter 'column' is an empty string") # Query database database.cursor.execute( @@ -600,7 +600,7 @@ def delete(search: str, column: str = "domain") -> None: if not isinstance(column, str): raise TypeError(f"Parameter column[]='{type(column)}' has not expected type 'str'") elif column == "": - raise ValueError("Parameter 'column' is empty") + raise ValueError("Parameter 'column' is an empty string") elif column in ["domain", "origin"]: domain_helper.raise_on(search) @@ -621,7 +621,7 @@ def translate_idnas(rows: list, column: str) -> None: elif not isinstance(column, str): raise TypeError(f"column='{type(column)}' has not expected type 'str'") elif column == "": - raise ValueError("Parameter 'column' is empty") + raise ValueError("Parameter 'column' is an empty string") elif column not in ["domain", "origin"]: raise ValueError(f"column='{column}' is not supported") diff --git a/fba/networks/lemmy.py b/fba/networks/lemmy.py index 4ba2392..648cee0 100644 --- a/fba/networks/lemmy.py +++ b/fba/networks/lemmy.py @@ -326,7 +326,7 @@ def parse_script(doc: bs4.BeautifulSoup, only: str = None) -> list: elif not isinstance(only, str) and only is not None: raise TypeError(f"Parameter only[]='{type(only)}' has not expected type 'str'") elif isinstance(only, str) and only == "": - raise ValueError("Parameter 'only' is empty") + raise ValueError("Parameter 'only' is an empty string") scripts = doc.find_all("script") peers = [] diff --git a/fba/networks/misskey.py b/fba/networks/misskey.py index 4ba88fd..8070d4d 100644 --- a/fba/networks/misskey.py +++ b/fba/networks/misskey.py @@ -112,8 +112,8 @@ def fetch_peers(domain: str) -> list: elif not isinstance(row["host"], str): logger.warning("row[host][]='%s' has not expected type 'str' - SKIPPED!", type(row["host"])) continue - elif row["host"] in [None, ""]: - logger.warning("row[host]='%s' is empty,domain='%s' - SKIPPED!", row["host"], domain) + elif row["host"] == "": + logger.warning("row[host] is an empty string,domain='%s' - SKIPPED!", domain) continue elif row["host"] in peers: logger.debug("Not adding row[host]='%s', already found - SKIPPED!", row["host"]) @@ -214,8 +214,11 @@ def fetch_blocks(domain: str) -> list: if "host" not in instance: logger.warning("instance(%d)='%s' has no key 'host' - SKIPPED!", len(instance), instance) continue - elif instance["host"] in [None, ""]: - logger.debug("instance[host]='%s' is None or empty - SKIPPED!", instance["host"]) + elif not isinstance(instance["host"], str): + logger.warning("instance[host][]='%s' has not expected type 'str' - SKIPPED!", type(instance["host"])) + continue + elif instance["host"] == "": + logger.warning("instance[host] is an empty string,domain='%s' - SKIPPED!", domain) continue logger.debug("instance[host]='%s' - BEFORE!", instance["host"]) diff --git a/fba/networks/peertube.py b/fba/networks/peertube.py index 06dafbd..f2c0540 100644 --- a/fba/networks/peertube.py +++ b/fba/networks/peertube.py @@ -89,6 +89,7 @@ def fetch_peers(domain: str) -> list: logger.debug("Appending mode2='%s',host='%s' ...", mode2, record[mode2]["host"]) peers.append(record[mode2]["host"]) + logger.debug("rows()=%d,peers()=%d,start=%d", len(rows), len(peers), start) if len(rows) < 100: logger.debug("Reached end of JSON response, domain='%s'", domain) break diff --git a/fba/networks/pleroma.py b/fba/networks/pleroma.py index 1d24b27..c18542f 100644 --- a/fba/networks/pleroma.py +++ b/fba/networks/pleroma.py @@ -110,7 +110,7 @@ def fetch_blocks(domain: str) -> list: logger.debug("block_level='%s' - AFTER!", block_level) if block_level in [None, ""]: - logger.warning("block_level is now empty!") + logger.warning("block_level='%s' is empty - SKIPPED!", block_level) continue elif block_level == "accept": logger.debug("domain='%s' skipping block_level='accept'", domain) @@ -265,7 +265,7 @@ def fetch_blocks(domain: str) -> list: else: logger.warning("Cannot find 'mrf_simple_info' or 'quarantined_instances_info' in JSON reply: domain='%s'", domain) - logger.debug("found='%s'", found) + logger.debug("found='%s',blockdict()=%d", found, len(blockdict)) if not found: logger.debug("Did not find any useable JSON elements, domain='%s', continuing with /about page ...", domain) blocklist = fetch_blocks_from_about(domain) @@ -379,7 +379,7 @@ def fetch_blocks_from_about(domain: str) -> dict: tds = line.find_all("td") logger.debug("tds[%s]()=%d", type(tds), len(tds)) - if len(tds) == 0: + if len(tds) < 2: logger.warning("No 'td' tag found in line[]='%s' - SKIPPED!", type(line)) continue diff --git a/fba/utils.py b/fba/utils.py index 78c5ffa..a28a965 100644 --- a/fba/utils.py +++ b/fba/utils.py @@ -51,7 +51,7 @@ def find_domains(tags: bs4.element.ResultSet, search: str) -> list: elif not isinstance(search, str): raise TypeError(f"Parameter search[]='{type(search)}' has not expected type 'str'") elif search == "": - raise ValueError("Parameter 'search' is empty") + raise ValueError("Parameter 'search' is an empty string") domains = [] logger.debug("Parsing %d tags ...", len(tags)) @@ -112,11 +112,11 @@ def add_all_to_list(domains: list, source: str, splitter: str) -> None: elif not isinstance(source, str): raise TypeError(f"Parameter source[]='{type(source)}' is not type 'list'") elif source == "": - raise ValueError("Parameter 'source' is empty") + raise ValueError("Parameter 'source' is an empty string") elif not isinstance(splitter, str): raise TypeError(f"Parameter splitter[]='{type(splitter)}' is not type 'list'") elif splitter == "": - raise ValueError("Parameter 'splitter' is empty") + raise ValueError("Parameter 'splitter' is an empty string") for domain in source.split(splitter): logger.debug("domain='%s' - LOOP!", domain) @@ -139,7 +139,7 @@ def deobfuscate(domain: str, blocker: str, domain_hash: str = None) -> str: elif not isinstance(domain_hash, str) and domain_hash is not None: raise TypeError(f"Parameter domain_hash[]='{type(domain_hash)}' has not expected type 'str'") elif domain_hash == "": - raise ValueError("Parameter 'domain_hash' is empty") + raise ValueError("Parameter 'domain_hash' is an empty string") logger.debug("Checking domain='%s' ...", domain) if domain.find("*") >= 0: -- 2.39.5