From c4708e0ce664e97a108135d8f5e1923075bfee1f Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Sat, 1 Feb 2025 22:34:01 +0100 Subject: [PATCH] Continued: - introduced network.fetch_csv_rows() - improved logger messages as parameter types are expected --- fba/commands.py | 25 +----------- fba/helpers/blocks.py | 2 +- fba/helpers/cache.py | 10 ++--- fba/helpers/config.py | 2 +- fba/helpers/cookies.py | 2 +- fba/helpers/dicts.py | 6 +-- fba/helpers/domain.py | 8 ++-- fba/helpers/processing.py | 32 +++------------ fba/helpers/software.py | 8 ++-- fba/helpers/tidyup.py | 4 +- fba/http/csrf.py | 2 +- fba/http/federation.py | 20 ++++----- fba/http/network.py | 86 ++++++++++++++++++++++++++++----------- fba/http/nodeinfo.py | 6 +-- fba/models/blocks.py | 20 ++++----- fba/models/instances.py | 46 ++++++++++----------- fba/models/obfuscation.py | 8 ++-- fba/networks/lemmy.py | 4 +- fba/networks/misskey.py | 6 +-- fba/utils.py | 6 +-- 20 files changed, 150 insertions(+), 153 deletions(-) diff --git a/fba/commands.py b/fba/commands.py index 30b8dbd..b2aac60 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -14,7 +14,6 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import csv import inspect import json import logging @@ -1518,29 +1517,7 @@ def fetch_fedilist(args: argparse.Namespace) -> int: url = f"http://{source_domain}/instance/csv?software={args.software}&onion=not" logger.info("Fetching url='%s' ...", url) - response = network.fetch_url( - url, - headers=network.web_headers, - timeout=config.timeout, - allow_redirects=False - ) - - logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text)) - if not response.ok or response.status_code > 200 or len(response.content) == 0: - logger.warning("Failed fetching url='%s': response.ok='%s',response.status_code=%d,response.content()=%d - EXIT!", url, response.ok, response.status_code, len(response.text)) - return 1 - - lines = response.content.decode("utf-8").splitlines() - - logger.debug("Reading %d lines, dialect=unix ...", len(lines)) - reader = csv.DictReader(lines, dialect="unix") - - logger.debug("reader[]='%s'", type(reader)) - if reader is None: - logger.warning("Failed parsing response.content()=%d as CSV content", len(response.content)) - return 2 - - rows = list(reader) + rows = network.fetch_csv_rows(url) logger.info("Checking %d rows ...", len(rows)) for row in rows: diff --git a/fba/helpers/blocks.py b/fba/helpers/blocks.py index 58836f1..196e8f0 100644 --- a/fba/helpers/blocks.py +++ b/fba/helpers/blocks.py @@ -27,7 +27,7 @@ def alias_block_level(block_level: str) -> str: logger.debug("block_level='%s' - CALLED!", block_level) if not isinstance(block_level, str): - raise TypeError(f"Parameter block_level[]='{type(block_level)}' is not of type 'str'") + raise TypeError(f"Parameter block_level[]='{type(block_level)}' has not expected type 'str'") elif block_level in ["accept", "accepted"]: raise RuntimeError(f"Parameter block_level='{block_level}' is 'accept(ed)' but function was invoked") elif block_level == "silence": diff --git a/fba/helpers/cache.py b/fba/helpers/cache.py index c9b234b..8cfe2c3 100644 --- a/fba/helpers/cache.py +++ b/fba/helpers/cache.py @@ -35,7 +35,7 @@ def set_all(key: str, rows: list, value: any) -> None: logger.debug("key='%s',rows()=%d,value[]='%s' - CALLED!", key, len(rows), type(value)) if not isinstance(key, str): - raise TypeError(f"Parameter key[]='{type(key)}' is not of type 'str'") + raise TypeError(f"Parameter key[]='{type(key)}' has not expected type 'str'") elif not key_exists(key): logger.debug("Cache for key='%s' not initialized.", key) _cache[key] = {} @@ -55,9 +55,9 @@ def set_sub_key(key: str, sub: str, value: any) -> None: logger.debug("key='%s',sub='%s',value[]='%s' - CALLED!", key, sub, type(value)) if not isinstance(key, str): - raise TypeError(f"Parameter key[]='{type(key)}' is not of type 'str'") + raise TypeError(f"Parameter key[]='{type(key)}' has not expected type 'str'") elif not isinstance(sub, str): - raise TypeError(f"Parameter sub[]='{type(sub)}' is not of type 'str'") + raise TypeError(f"Parameter sub[]='{type(sub)}' has not expected type 'str'") elif not key_exists(key): raise KeyError(f"Cache for key='{key}' is not initialized, but function invoked") @@ -70,9 +70,9 @@ def sub_key_exists(key: str, sub: str) -> bool: logger.debug("key='%s',sub='%s' - CALLED!", key, sub) if not isinstance(key, str): - raise TypeError(f"Parameter key[]='{type(key)}' is not of type 'str'") + raise TypeError(f"Parameter key[]='{type(key)}' has not expected type 'str'") elif not isinstance(sub, str): - raise TypeError(f"Parameter sub[]='{type(sub)}' is not of type 'str'") + raise TypeError(f"Parameter sub[]='{type(sub)}' has not expected type 'str'") elif not key_exists(key): raise KeyError(f"Cache for key='{key}' is not initialized, but function invoked") diff --git a/fba/helpers/config.py b/fba/helpers/config.py index ac45e04..0e7b12c 100644 --- a/fba/helpers/config.py +++ b/fba/helpers/config.py @@ -44,7 +44,7 @@ def get(key: str) -> any: logger.debug("key[%s]='%s' - CALLED!", type(key), key) if not isinstance(key, str): - raise TypeError(f"Parameter key[]='{type(key)}' is not of type 'str'") + raise TypeError(f"Parameter key[]='{type(key)}' has not expected type 'str'") elif key == "": raise ValueError("Parameter 'key' is empty") elif not key in _config: diff --git a/fba/helpers/cookies.py b/fba/helpers/cookies.py index 16c8711..253ccc6 100644 --- a/fba/helpers/cookies.py +++ b/fba/helpers/cookies.py @@ -33,7 +33,7 @@ def store(domain: str, cookies: dict): if blacklist.is_blacklisted(domain): raise RuntimeError(f"domain='{domain}' is blacklisted but function was invoked") elif not isinstance(cookies, dict): - raise TypeError(f"Parameter cookies[]='{type(cookies)}' is not of type 'dict'") + raise TypeError(f"Parameter cookies[]='{type(cookies)}' has not expected type 'dict'") _cookies[domain] = cookies diff --git a/fba/helpers/dicts.py b/fba/helpers/dicts.py index d18f30d..386a2f1 100644 --- a/fba/helpers/dicts.py +++ b/fba/helpers/dicts.py @@ -23,9 +23,9 @@ def has_key(lists: list, key: str, value: any) -> bool: logger.debug("lists()=%d,key='%s',value[]='%s' - CALLED!", len(lists), key, type(value)) if not isinstance(lists, list): - raise TypeError(f"Parameter lists[]='{type(lists)}' is not of type 'list'") + raise TypeError(f"Parameter lists[]='{type(lists)}' has not expected type 'list'") elif not isinstance(key, str): - raise TypeError(f"Parameter key[]='{type(key)}' is not of type 'str'") + raise TypeError(f"Parameter key[]='{type(key)}' has not expected type 'str'") elif key == "": raise ValueError("Parameter 'key' is empty") @@ -35,7 +35,7 @@ def has_key(lists: list, key: str, value: any) -> bool: logger.debug("row[%s]='%s", type(row), row) if not isinstance(row, dict): - raise TypeError(f"row[]='{type(row)}' is not of type 'dict'") + raise TypeError(f"row[]='{type(row)}' has not expected type 'dict'") elif not key in row: raise KeyError(f"Cannot find key='{key}'") elif row[key] == value: diff --git a/fba/helpers/domain.py b/fba/helpers/domain.py index d3e63ef..2daf419 100644 --- a/fba/helpers/domain.py +++ b/fba/helpers/domain.py @@ -34,7 +34,7 @@ def raise_on(domain: str) -> None: logger.debug("domain='%s' - CALLED!", domain) if not isinstance(domain, str): - raise TypeError(f"Parameter domain[]='{type(domain)}' is not of type 'str'") + raise TypeError(f"Parameter domain[]='{type(domain)}' has not expected type 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") elif domain.lower() != domain: @@ -62,7 +62,7 @@ def is_in_url(domain: str, url: str) -> bool: if blacklist.is_blacklisted(domain): raise RuntimeError(f"domain='{domain}' is blacklisted but function was invoked") elif not isinstance(url, str): - raise TypeError(f"Parameter url[]='{type(url)}' is not of type 'str'") + raise TypeError(f"Parameter url[]='{type(url)}' has not expected type 'str'") elif url == "": raise ValueError("Parameter 'url' is empty") elif not validators.url(url): @@ -84,7 +84,7 @@ def is_tld_wanted(domain: str) -> bool: logger.debug("domain='%s' - CALLED!", domain) if not isinstance(domain, str): - raise TypeError(f"Parameter domain[]='{type(domain)}' is not of type 'str'") + raise TypeError(f"Parameter domain[]='{type(domain)}' has not expected type 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") @@ -111,7 +111,7 @@ def is_wanted(domain: str) -> bool: logger.debug("domain='%s' - CALLED!", domain) if not isinstance(domain, str): - raise TypeError(f"Parameter domain[]='{type(domain)}' is not of type 'str'") + raise TypeError(f"Parameter domain[]='{type(domain)}' has not expected type 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") diff --git a/fba/helpers/processing.py b/fba/helpers/processing.py index 9349694..b9a4a88 100644 --- a/fba/helpers/processing.py +++ b/fba/helpers/processing.py @@ -13,7 +13,6 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import csv import logging import validators @@ -44,7 +43,7 @@ def instance(blocked: str, blocker: str, command: str, force: bool = False) -> b domain_helper.raise_on(blocker) if not isinstance(command, str): - raise TypeError(f"Parameter command[]='{type(command)}' is not of type 'str'") + raise TypeError(f"Parameter command[]='{type(command)}' has not expected type 'str'") elif command == "": raise ValueError("Parameter 'command' is empty") elif blacklist.is_blacklisted(blocked): @@ -94,9 +93,9 @@ def block(blocker: str, blocked: str, reason: str, block_level: str) -> bool: domain_helper.raise_on(blocked) if not isinstance(reason, str) and reason is not None: - raise TypeError(f"Parameter reason[]='{type(reason)}' is not of type 'str'") + raise TypeError(f"Parameter reason[]='{type(reason)}' has not expected type 'str'") elif not isinstance(block_level, str): - raise TypeError(f"Parameter block_level[]='{type(block_level)}' is not of type 'str'") + raise TypeError(f"Parameter block_level[]='{type(block_level)}' has not expected type 'str'") elif block_level == "": raise ValueError("Parameter block_level is empty") elif block_level in ["reject", "suspend", "accept", "silence", "nsfw", "quarantined_instances"]: @@ -127,13 +126,13 @@ def csv_block(blocker: str, url: str, command: str) -> None: domain_helper.raise_on(blocker) if not isinstance(url, str): - raise TypeError(f"url[]='{url}' is not of type 'str'") + raise TypeError(f"url[]='{url}' has not expected type 'str'") elif url in [None, ""]: raise ValueError("Parameter 'url' is empty") elif not validators.url(url): raise ValueError(f"Parameter url='{url}' is not a valid URL") elif not isinstance(command, str): - raise TypeError(f"command[]='{command}' is not of type 'str'") + raise TypeError(f"command[]='{command}' has not expected type 'str'") elif command == "": raise ValueError("Parameter 'command' is empty") elif blacklist.is_blacklisted(blocker): @@ -144,26 +143,7 @@ def csv_block(blocker: str, url: str, command: str) -> None: # Fetch this URL logger.info("Fetching url='%s' for blocker='%s' ...", url, blocker) - response = network.fetch_url( - url, - headers=network.web_headers, - timeout=config.timeout - ) - - logger.debug("response.ok='%s',response.status_code=%d,response.content()=%d", response.ok, response.status_code, len(response.content)) - if not response.ok or response.status_code > 200 or response.content == "": - logger.warning("Could not fetch url='%s' for blocker='%s' - EXIT!", url, blocker) - return - - logger.debug("Fetched %d Bytes, parsing CSV ...", len(response.content)) - reader = csv.DictReader(response.content.decode("utf-8").splitlines(), dialect="unix") - - logger.debug("reader[]='%s'", type(reader)) - rows = list(reader) - - # Init local variables - blockdict = domains = [] - cnt = 0 + rows = network.fetch_csv_rows(url) logger.info("Checking %d CSV lines ...", len(rows)) for row in rows: diff --git a/fba/helpers/software.py b/fba/helpers/software.py index 2a64cc7..342ee51 100644 --- a/fba/helpers/software.py +++ b/fba/helpers/software.py @@ -219,7 +219,7 @@ def strip_hosted_on(software: str) -> str: logger.debug("software='%s' - CALLED!", software) if not isinstance(software, str): - raise TypeError(f"Parameter software[]='{type(software)}' is not of type 'str'") + raise TypeError(f"Parameter software[]='{type(software)}' has not expected type 'str'") elif software == "": raise ValueError("Parameter 'software' is empty") elif "hosted on" not in software: @@ -244,7 +244,7 @@ def strip_powered_by(software: str) -> str: logger.debug("software='%s' - CALLED!", software) if not isinstance(software, str): - raise TypeError(f"Parameter software[]='{type(software)}' is not of type 'str'") + raise TypeError(f"Parameter software[]='{type(software)}' has not expected type 'str'") elif software == "": raise ValueError("Parameter 'software' is empty") elif "powered by" not in software: @@ -269,11 +269,11 @@ def strip_until(software: str, until: str) -> str: logger.debug("software='%s',until='%s' - CALLED!", software, until) if not isinstance(software, str): - raise TypeError(f"Parameter software[]='{type(software)}' is not of type 'str'") + raise TypeError(f"Parameter software[]='{type(software)}' has not expected type 'str'") elif software == "": raise ValueError("Parameter 'software' is empty") elif not isinstance(until, str): - raise TypeError(f"Parameter until[]='{type(until)}' is not of type 'str'") + raise TypeError(f"Parameter until[]='{type(until)}' has not expected type 'str'") elif until == "": raise ValueError("Parameter 'until' is empty") elif not until in software: diff --git a/fba/helpers/tidyup.py b/fba/helpers/tidyup.py index 2bd16c2..37134f2 100644 --- a/fba/helpers/tidyup.py +++ b/fba/helpers/tidyup.py @@ -24,7 +24,7 @@ def reason(string: str) -> str: logger.debug("string='%s' - CALLED!", string) if not isinstance(string, str): - raise TypeError(f"Parameter string[]='{type(string)}' is not of type 'str'") + raise TypeError(f"Parameter string[]='{type(string)}' has not expected type 'str'") # Strip string string = string.strip() @@ -36,7 +36,7 @@ def domain(string: str) -> str: logger.debug("string='%s' - CALLED!", string) if not isinstance(string, str): - raise TypeError(f"Parameter string[]='{type(string)}' is not of type 'str'") + raise TypeError(f"Parameter string[]='{type(string)}' has not expected type 'str'") elif string == "": raise ValueError("Parameter 'string' is empty") diff --git a/fba/http/csrf.py b/fba/http/csrf.py index 3ddfe40..f2871c9 100644 --- a/fba/http/csrf.py +++ b/fba/http/csrf.py @@ -39,7 +39,7 @@ def determine(domain: str, headers: dict) -> dict: if blacklist.is_blacklisted(domain): raise RuntimeError(f"domain='{domain}' is blacklisted but function was invoked") elif not isinstance(headers, dict): - raise TypeError(f"Parameter headers[]='{type(headers)}' is not of type 'dict'") + raise TypeError(f"Parameter headers[]='{type(headers)}' has not expected type 'dict'") # Default headers with no CSRF reqheaders = headers diff --git a/fba/http/federation.py b/fba/http/federation.py index 5fac558..49fc55e 100644 --- a/fba/http/federation.py +++ b/fba/http/federation.py @@ -65,15 +65,15 @@ def fetch_instances(domain: str, origin: str, software: str, command: str, path: if blacklist.is_blacklisted(domain): raise RuntimeError(f"domain='{domain}' is blacklisted but function was invoked") elif not isinstance(origin, str) and origin is not None: - raise TypeError(f"Parameter origin[]='{type(origin)}' is not of type 'str'") + raise TypeError(f"Parameter origin[]='{type(origin)}' has not expected type 'str'") elif not isinstance(command, str): - raise TypeError(f"Parameter command[]='{type(command)}' is not of type 'str'") + raise TypeError(f"Parameter command[]='{type(command)}' has not expected type 'str'") elif command == "": raise ValueError("Parameter 'command' is empty") elif command in ["fetch_blocks", "fetch_cs", "fetch_bkali", "fetch_relays", "fetch_fedipact", "fetch_joinmobilizon", "fetch_joinmisskey", "fetch_joinfediverse", "fetch_relaylist"] and origin is None: raise ValueError(f"Parameter command='{command}' but origin is None, please fix invoking this function.") elif not isinstance(path, str) and path is not None: - raise TypeError(f"Parameter path[]='{type(path)}' is not of type 'str'") + raise TypeError(f"Parameter path[]='{type(path)}' has not expected type 'str'") elif path is not None and not path.startswith("/"): raise ValueError(f"path='{path}' does not start with a slash") elif _DEPTH > 0 and instances.is_recent(domain, "last_instance_fetch"): @@ -90,7 +90,7 @@ def fetch_instances(domain: str, origin: str, software: str, command: str, path: elif software is None: logger.debug("domain='%s' has unknown software or nodeinfo has recently being fetched", domain) elif not isinstance(software, str): - raise TypeError(f"Parameter software[]='{type(software)}' is not of type 'str'") + raise TypeError(f"Parameter software[]='{type(software)}' has not expected type 'str'") # Increase depth _DEPTH = _DEPTH + 1 @@ -227,13 +227,13 @@ def fetch_peers(domain: str, software: str, origin: str) -> list: if blacklist.is_blacklisted(domain): raise RuntimeError(f"domain='{domain}' is blacklisted but function was invoked") elif not isinstance(software, str) and software is not None: - raise TypeError(f"Parameter software[]='{type(software)}' is not of type 'str'") + raise TypeError(f"Parameter software[]='{type(software)}' has not expected type 'str'") elif isinstance(software, str) and software == "": raise ValueError("Parameter 'software' is empty") elif software is not None and software_helper.is_relay(software): raise RuntimeError(f"domain='{domain}' is of software='{software}' and isn't supported here.") elif not isinstance(origin, str) and origin is not None: - raise TypeError(f"Parameter origin[]='{type(origin)}' is not of type 'str'") + raise TypeError(f"Parameter origin[]='{type(origin)}' has not expected type 'str'") elif isinstance(origin, str) and origin == "": raise ValueError("Parameter 'origin' is empty") @@ -279,7 +279,7 @@ def fetch_peers(domain: str, software: str, origin: str) -> list: break if not isinstance(peers, list): - logger.warning("peers[]='%s' is not of type 'list', maybe bad API response?", type(peers)) + logger.warning("peers[]='%s' has not expected type 'list', maybe bad API response?", type(peers)) peers = [] logger.debug("Invoking instances.set_total_peers(%s,%d) ...", domain, len(peers)) @@ -295,7 +295,7 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str: if blacklist.is_blacklisted(domain): raise RuntimeError(f"domain='{domain}' is blacklisted but function was invoked") elif not isinstance(path, str): - raise TypeError(f"path[]='{type(path)}' is not of type 'str'") + raise TypeError(f"path[]='{type(path)}' has not expected type 'str'") elif path == "": raise ValueError("Parameter 'path' is empty") elif not path.startswith("/"): @@ -398,7 +398,7 @@ def determine_software(domain: str, path: str = None) -> str: if blacklist.is_blacklisted(domain): raise RuntimeError(f"domain='{domain}' is blacklisted but function was invoked") elif not isinstance(path, str) and path is not None: - raise TypeError(f"Parameter path[]='{type(path)}' is not of type 'str'") + raise TypeError(f"Parameter path[]='{type(path)}' has not expected type 'str'") elif path is not None and not path.startswith("/"): raise ValueError(f"path='{path}' does not start with a slash") @@ -559,7 +559,7 @@ def add_peers(rows: dict) -> list: logger.debug("rows[]='%s' - CALLED!", type(rows)) if not isinstance(rows, dict): - raise TypeError(f"Parameter rows[]='{type(rows)}' is not of type 'dict'") + raise TypeError(f"Parameter rows[]='{type(rows)}' has not expected type 'dict'") elif len(rows) == 0: raise ValueError("Parameter 'rows' is empty") diff --git a/fba/http/network.py b/fba/http/network.py index fe2cacd..bf121be 100644 --- a/fba/http/network.py +++ b/fba/http/network.py @@ -14,6 +14,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import csv import logging import time @@ -72,15 +73,15 @@ def post_json_api(domain: str, path: str, data: str = "", headers: dict = {}) -> if blacklist.is_blacklisted(domain): raise RuntimeError(f"domain='{domain}' is blacklisted but function was invoked") elif not isinstance(path, str): - raise TypeError(f"path[]='{type(path)}' is not of type 'str'") + raise TypeError(f"path[]='{type(path)}' has not expected type 'str'") elif path == "": raise ValueError("Parameter 'path' is empty") elif not path.startswith("/"): raise ValueError(f"path='{path}' does not start with / but should") elif not isinstance(data, str): - raise TypeError(f"data[]='{type(data)}' is not of type 'str'") + raise TypeError(f"data[]='{type(data)}' has not expected type 'str'") elif headers is not None and not isinstance(headers, dict): - raise ValueError(f"headers[]='{type(headers)}' is not of type 'dict'") + raise ValueError(f"headers[]='{type(headers)}' has not expected type 'dict'") json_reply = { "status_code": 200, @@ -132,13 +133,13 @@ def fetch_api_url(url: str, timeout: tuple) -> dict: logger.debug("url='%s',timeout()=%d - CALLED!", url, len(timeout)) if not isinstance(url, str): - raise TypeError(f"Parameter url[]='{type(url)}' is not of type 'str'") + raise TypeError(f"Parameter url[]='{type(url)}' has not expected type 'str'") elif url == "": raise ValueError("Parameter 'url' is empty") elif not validators.url(url): raise ValueError(f"Parameter url='{url}' is not a valid URL") elif not isinstance(timeout, tuple): - raise TypeError(f"timeout[]='{type(timeout)}' is not of type 'tuple'") + raise TypeError(f"timeout[]='{type(timeout)}' has not expected type 'tuple'") json_reply = { "status_code": 200, @@ -176,15 +177,15 @@ def get_json_api(domain: str, path: str, headers: dict, timeout: tuple) -> dict: if blacklist.is_blacklisted(domain): raise RuntimeError(f"domain='{domain}' is blacklisted but function was invoked") elif not isinstance(path, str): - raise TypeError(f"path[]='{type(path)}' is not of type 'str'") + raise TypeError(f"path[]='{type(path)}' has not expected type 'str'") elif path == "": raise ValueError("Parameter 'path' is empty") elif not path.startswith("/"): raise ValueError(f"path='{path}' does not start with / but should") elif not isinstance(headers, dict): - raise TypeError(f"headers[]='{type(headers)}' is not of type 'list'") + raise TypeError(f"headers[]='{type(headers)}' has not expected type 'list'") elif not isinstance(timeout, tuple): - raise TypeError(f"timeout[]='{type(timeout)}' is not of type 'tuple'") + raise TypeError(f"timeout[]='{type(timeout)}' has not expected type 'tuple'") json_reply = { "status_code": 200, @@ -235,7 +236,7 @@ def send_bot_post(domain: str, blocklist: list) -> None: if blacklist.is_blacklisted(domain): raise RuntimeError(f"domain='{domain}' is blacklisted but function was invoked") elif not isinstance(blocklist, list): - raise TypeError(f"Parameter blocklist[]='{type(blocklist)}' is not of type 'list'") + raise TypeError(f"Parameter blocklist[]='{type(blocklist)}' has not expected type 'list'") elif len(blocklist) == 0: raise ValueError("Parameter 'blocklist' is empty") elif config.get("bot_token") == "": @@ -287,17 +288,17 @@ def _fetch_response(domain: str, path: str, headers: dict, timeout: tuple, allow if blacklist.is_blacklisted(domain): raise RuntimeError(f"domain='{domain}' is blacklisted but function was invoked") elif not isinstance(path, str): - raise TypeError(f"Parameter path[]='{type(path)}' is not of type 'str'") + raise TypeError(f"Parameter path[]='{type(path)}' has not expected type 'str'") elif path == "": raise ValueError("Parameter 'path' is empty") elif not path.startswith("/"): raise ValueError(f"path='{path}' does not start with / but should") elif not isinstance(headers, dict): - raise TypeError(f"headers[]='{type(headers)}' is not of type 'dict'") + raise TypeError(f"headers[]='{type(headers)}' has not expected type 'dict'") elif not isinstance(timeout, tuple): - raise TypeError(f"timeout[]='{type(timeout)}' is not of type 'tuple'") + raise TypeError(f"timeout[]='{type(timeout)}' has not expected type 'tuple'") elif not isinstance(allow_redirects, bool): - raise TypeError(f"allow_redirects[]='{type(allow_redirects)}' is not of type 'bool'") + raise TypeError(f"allow_redirects[]='{type(allow_redirects)}' has not expected type 'bool'") start = 0 try: @@ -334,17 +335,17 @@ def fetch_url(url: str, headers: dict, timeout: tuple, allow_redirects: bool = T logger.debug("url='%s',headers()=%d,timeout(%d)='%s',allow_redirects='%s' - CALLED!", url, len(headers), len(timeout), timeout, allow_redirects) if not isinstance(url, str): - raise TypeError(f"Parameter url[]='{type(url)}' is not of type 'str'") + raise TypeError(f"Parameter url[]='{type(url)}' has not expected type 'str'") elif url == "": raise ValueError("Parameter 'url' is empty") elif not validators.url(url): raise ValueError(f"Parameter url='{url}' is not a valid URL") elif not isinstance(headers, dict): - raise TypeError(f"Parameter headers[]='{type(headers)}' is not of type 'dict'") + raise TypeError(f"Parameter headers[]='{type(headers)}' has not expected type 'dict'") elif not isinstance(timeout, tuple): - raise TypeError(f"Parameter timeout[]='{type(timeout)}' is not of type 'tuple'") + raise TypeError(f"Parameter timeout[]='{type(timeout)}' has not expected type 'tuple'") elif not isinstance(allow_redirects, bool): - raise TypeError(f"Parameter allow_redirects[]='{type(allow_redirects)}' is not of type 'bool'") + raise TypeError(f"Parameter allow_redirects[]='{type(allow_redirects)}' has not expected type 'bool'") logger.debug("Parsing url='%s' ...", url) components = urlparse(url) @@ -377,21 +378,21 @@ def fetch_json_rows(hostname: str, path: str, headers: dict = {}, rows_key: str logger.debug("hostname='%s',path='%s',headers()=%d,rows_key='%s' - CALLED!", hostname, path, len(headers), rows_key) if not isinstance(hostname, str): - raise TypeError(f"hostname[]='{type(hostname)}' is not of type 'str'") + raise TypeError(f"hostname[]='{type(hostname)}' has not expected type 'str'") elif hostname == "": raise ValueError("Parameter 'hostname' is an empty string") elif not validators.hostname(hostname): raise ValueError(f"hostname='{hostname}' is not a valid hostname") elif not isinstance(path, str): - raise TypeError(f"path[]='{type(path)}' is not of type 'str'") + raise TypeError(f"path[]='{type(path)}' has not expected type 'str'") elif path == "": raise ValueError("Parameter 'path' is an empty string") elif not path.startswith("/"): raise ValueError(f"path='{path}' does not start with a slash") elif headers is not None and not isinstance(headers, dict): - raise ValueError(f"headers[]='{type(headers)}' is not of type 'dict'") + raise ValueError(f"headers[]='{type(headers)}' has not expected type 'dict'") elif not isinstance(rows_key, str) and rows_key is not None: - raise TypeError(f"rows_key[]='{type(rows_key)}' is not of type 'str'") + raise TypeError(f"rows_key[]='{type(rows_key)}' has not expected type 'str'") elif rows_key is not None and rows_key == "": raise ValueError("Parameter 'rows_key' is an empty string") @@ -424,6 +425,45 @@ def fetch_json_rows(hostname: str, path: str, headers: dict = {}, rows_key: str logger.debug("rows()=%d - EXIT!", len(rows)) return rows +def fetch_csv_rows (url: str) -> list: + logger.debug("url='%s' - CALLED!", url) + + if not isinstance(url, str): + raise TypeError(f"url[]='{type(url)}' has not expected type 'str'") + elif url == "": + raise ValueError("Parameter 'url' is an empty string") + elif not validators.url(url): + raise ValueError(f"Parameter url='{url}' is not a valid URL") + + logger.debug("Fetching url='%s' ...", url) + response = fetch_url( + url, + headers=web_headers, + timeout=config.timeout, + allow_redirects=False + ) + + logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text)) + if not response.ok or response.status_code > 200 or len(response.content) == 0: + logger.warning("Failed fetching url='%s': response.ok='%s',response.status_code=%d,response.content()=%d - EXIT!", url, response.ok, response.status_code, len(response.text)) + raise RuntimeException(f"response.ok='{response.ok}',response.status_code={response.status_code},response.content()={len(response.content)} is unexpected") + + lines = response.content.decode("utf-8").splitlines() + logger.debug("Reading %d lines, dialect=unix ...", len(lines)) + + reader = csv.DictReader(lines, dialect="unix") + logger.debug("reader[]='%s'", type(reader)) + + if reader is None: + logger.warning("Failed parsing response.content()=%d as CSV content", len(response.content)) + raise RuntimeException(f"reader is None after parsing {len(lines)} CSV lines") + + # Init rows from CSV reader + rows = list(reader) + + logger.debug("rows()=%d - EXIT!", len(rows)) + return rows + def get_generic(domain: str, path: str, allow_redirects: bool = False) -> requests.models.Response: logger.debug("domain='%s',path='%s',allow_redirects='%s' - CALLED!", domain, path, allow_redirects) domain_helper.raise_on(domain) @@ -431,13 +471,13 @@ def get_generic(domain: str, path: str, allow_redirects: bool = False) -> reques if blacklist.is_blacklisted(domain): raise RuntimeError(f"domain='{domain}' is blacklisted but function was invoked") elif not isinstance(path, str): - raise TypeError(f"Parameter path[]='{type(path)}' is not of type 'str'") + raise TypeError(f"Parameter path[]='{type(path)}' has not expected type 'str'") elif path == "": raise ValueError("Parameter 'path' is empty") elif not path.startswith("/"): raise ValueError(f"path='{path}' does not start with / but should") elif not isinstance(allow_redirects, bool): - raise TypeError(f"allow_redirects[]='{type(allow_redirects)}' is not of type 'bool'") + raise TypeError(f"allow_redirects[]='{type(allow_redirects)}' has not expected type 'bool'") logger.debug("Fetching path='%s' from domain='%s' ...", path, domain) response = _fetch_response( diff --git a/fba/http/nodeinfo.py b/fba/http/nodeinfo.py index 59259b2..6d0995e 100644 --- a/fba/http/nodeinfo.py +++ b/fba/http/nodeinfo.py @@ -67,11 +67,11 @@ def fetch(domain: str, path: str = None, update_mode: bool = True) -> dict: if blacklist.is_blacklisted(domain): raise RuntimeError(f"domain='{domain}' is blacklisted but function was invoked") elif not isinstance(path, str) and path is not None: - raise TypeError(f"Parameter path[]='{type(path)}' is not of type 'str'") + raise TypeError(f"Parameter path[]='{type(path)}' has not expected type 'str'") elif path is not None and not path.startswith("/"): raise ValueError(f"path='{path}' does not start with a slash") elif not isinstance(update_mode, bool) and update_mode is not None: - raise TypeError(f"Parameter update_mode[]='{type(update_mode)}' is not of type 'bool'") + raise TypeError(f"Parameter update_mode[]='{type(update_mode)}' has not expected type 'bool'") if path is None and update_mode: logger.debug("Fetching well-known nodeinfo from domain='%s' ...", domain) @@ -207,7 +207,7 @@ def fetch_wellknown_nodeinfo(domain: str) -> dict: for link in infos["links"]: logger.debug("link[%s]='%s'", type(link), link) if not isinstance(link, dict) or not "rel" in link: - logger.debug("link[]='%s' is not of type 'dict' or no element 'rel' found - SKIPPED!", type(link)) + logger.debug("link[]='%s' has not expected type 'dict' or no element 'rel' found - SKIPPED!", type(link)) continue elif link["rel"] != niid: logger.debug("link[re]='%s' does not matched niid='%s' - SKIPPED!", link["rel"], niid) diff --git a/fba/models/blocks.py b/fba/models/blocks.py index dd4979a..38f4950 100644 --- a/fba/models/blocks.py +++ b/fba/models/blocks.py @@ -34,7 +34,7 @@ def get_reason(blocker: str, blocked: str, block_level: str) -> str: domain_helper.raise_on(blocked) if not isinstance(block_level, str): - raise TypeError(f"Parameter block_level[]='{type(block_level)}' is not of type 'str'") + raise TypeError(f"Parameter block_level[]='{type(block_level)}' has not expected type 'str'") elif block_level == "": raise ValueError("Parameter 'block_level' is empty") elif block_level in ["accept", "reject", "suspend", "silence", "nsfw", "quarantined_instances"]: @@ -67,9 +67,9 @@ def update_reason(reason: str, blocker: str, blocked: str, block_level: str) -> domain_helper.raise_on(blocked) if not isinstance(reason, str) and reason is not None: - raise TypError(f"Parameter reason[]='{type(reason)}' is not of type 'str'") + raise TypError(f"Parameter reason[]='{type(reason)}' has not expected type 'str'") elif not isinstance(block_level, str): - raise TypeError(f"Parameter block_level[]='{type(block_level)}' is not of type 'str'") + raise TypeError(f"Parameter block_level[]='{type(block_level)}' has not expected type 'str'") elif block_level == "": raise ValueError("Parameter 'block_level' is empty") elif block_level in ["accept", "reject", "suspend", "silence", "nsfw", "quarantined_instances"]: @@ -100,7 +100,7 @@ def update_last_seen(blocker: str, blocked: str, block_level: str) -> None: domain_helper.raise_on(blocked) if not isinstance(block_level, str): - raise TypeError(f"Parameter block_level[]='{type(block_level)}' is not of type 'str'") + raise TypeError(f"Parameter block_level[]='{type(block_level)}' has not expected type 'str'") elif block_level == "": raise ValueError("Parameter 'block_level' is empty") elif block_level in ["accept", "reject", "suspend", "silence", "nsfw", "quarantined_instances"]: @@ -129,7 +129,7 @@ def is_instance_blocked(blocker: str, blocked: str, block_level: str = None) -> domain_helper.raise_on(blocked) if not isinstance(block_level, str) and block_level is not None: - raise TypeError(f"Parameter block_level[]='{type(block_level)}' is not of type 'str'") + raise TypeError(f"Parameter block_level[]='{type(block_level)}' has not expected type 'str'") elif block_level == "": raise ValueError("Parameter 'block_level' is empty") elif block_level in ["accept", "reject", "suspend", "silence", "nsfw", "quarantined_instances"]: @@ -168,7 +168,7 @@ def add(blocker: str, blocked: str, reason: str, block_level: str) -> None: domain_helper.raise_on(blocked) if not isinstance(block_level, str): - raise TypeError(f"Parameter block_level[]='{type(block_level)}' is not of type 'str'") + raise TypeError(f"Parameter block_level[]='{type(block_level)}' has not expected type 'str'") elif block_level == "": raise ValueError("Parameter 'block_level' is empty") elif block_level in ["accept", "reject", "suspend", "silence", "nsfw", "quarantined_instances"]: @@ -205,11 +205,11 @@ def valid(value: str, column: str) -> bool: logger.debug("value='%s',column='%s' - CALLED!", value, column) if not isinstance(value, str): - raise TypeError(f"Parameter value[]='{type(value)}' is not of type 'str'") + raise TypeError(f"Parameter value[]='{type(value)}' has not expected type 'str'") elif value == "": raise ValueError("Parameter 'value' is empty") elif not isinstance(column, str): - raise TypeError(f"Parameter column[]='{type(column)}' is not of type 'str'") + raise TypeError(f"Parameter column[]='{type(column)}' has not expected type 'str'") elif column == "": raise ValueError("Parameter 'column' is empty") @@ -227,11 +227,11 @@ def translate_idnas(rows: list, column: str) -> None: logger.debug("rows[]='%s',column='%s' - CALLED!", type(rows), column) if not isinstance(rows, list): - raise TypeError(f"rows[]='{type(rows)}' is not of type 'list'") + raise TypeError(f"rows[]='{type(rows)}' has not expected type 'list'") elif len(rows) == 0: raise ValueError("Parameter 'rows' is an empty list") elif not isinstance(column, str): - raise TypeError(f"column='{type(column)}' is not of type 'str'") + raise TypeError(f"column='{type(column)}' has not expected type 'str'") elif column not in ["blocker", "blocked"]: raise ValueError(f"column='{column}' is not supported") diff --git a/fba/models/instances.py b/fba/models/instances.py index f9e23eb..3ac2cac 100644 --- a/fba/models/instances.py +++ b/fba/models/instances.py @@ -89,7 +89,7 @@ def _set_data(key: str, domain: str, value: any) -> None: domain_helper.raise_on(domain) if not isinstance(key, str): - raise TypeError(f"Parameter key[]='{type(key)}' is not of type 'str'") + raise TypeError(f"Parameter key[]='{type(key)}' has not expected type 'str'") elif key == "": raise ValueError("Parameter 'key' is empty") elif not key in _pending: @@ -186,21 +186,21 @@ def add(domain: str, origin: str, command: str, path: str = None, software: str domain_helper.raise_on(domain) if not isinstance(origin, str) and origin is not None: - raise TypeError(f"origin[]='{type(origin)}' is not of type 'str'") + raise TypeError(f"origin[]='{type(origin)}' has not expected type 'str'") elif origin == "": raise ValueError("Parameter 'origin' is empty") elif not isinstance(command, str): - raise TypeError(f"command[]='{type(command)}' is not of type 'str'") + raise TypeError(f"command[]='{type(command)}' has not expected type 'str'") elif command == "": raise ValueError("Parameter 'command' is empty") elif not isinstance(path, str) and path is not None: - raise TypeError(f"path[]='{type(path)}' is not of type 'str'") + raise TypeError(f"path[]='{type(path)}' has not expected type 'str'") elif path == "": raise ValueError("Parameter 'path' is empty") elif path is not None and not path.startswith("/"): raise ValueError(f"path='{path}' does not start with / but should") elif not isinstance(software, str) and software is not None: - raise TypeError(f"software[]='{type(software)}' is not of type 'str'") + raise TypeError(f"software[]='{type(software)}' has not expected type 'str'") elif software == "": raise ValueError("Parameter 'software' is empty") elif origin is not None and not validators.domain(origin.split("/")[0], rfc_2782=True): @@ -351,7 +351,7 @@ def is_recent(domain: str, column: str = "last_instance_fetch") -> bool: domain_helper.raise_on(domain) if not isinstance(column, str): - raise TypeError(f"Parameter column[]='{type(column)}' is not of type 'str'") + raise TypeError(f"Parameter column[]='{type(column)}' has not expected type 'str'") elif not column.startswith("last_"): raise ValueError(f"Parameter column='{column}' is not expected") elif blacklist.is_blacklisted(domain): @@ -383,7 +383,7 @@ def deobfuscate(char: str, domain: str, blocked_hash: str = None) -> tuple: logger.debug("char='%s',domain='%s',blocked_hash='%s' - CALLED!", char, domain, blocked_hash) if not isinstance(char, str): - raise TypeError(f"Parameter char[]='{type(char)}' is not of type 'str'") + raise TypeError(f"Parameter char[]='{type(char)}' has not expected type 'str'") elif char == "": raise ValueError("Parameter 'char' is empty") elif not isinstance(domain, str): @@ -393,7 +393,7 @@ def deobfuscate(char: str, domain: str, blocked_hash: str = None) -> tuple: elif not char in domain: raise ValueError(f"char='{char}' not found in domain='{domain}' but function invoked") elif not isinstance(blocked_hash, str) and blocked_hash is not None: - raise TypeError(f"Parameter blocked_hash[]='{type(blocked_hash)}' is not of type 'str'") + raise TypeError(f"Parameter blocked_hash[]='{type(blocked_hash)}' has not expected type 'str'") # Init variables row = None @@ -445,7 +445,7 @@ def set_last_response_time(domain: str, response_time: float) -> None: domain_helper.raise_on(domain) if not isinstance(response_time, float): - raise TypeError(f"response_time[]='{type(response_time)}' is not of type 'float'") + raise TypeError(f"response_time[]='{type(response_time)}' has not expected type 'float'") elif response_time < 0: raise ValueError(f"response_time={response_time} is below zero") @@ -458,7 +458,7 @@ def set_last_requested_path(domain: str, path: float) -> None: domain_helper.raise_on(domain) if not isinstance(path, str): - raise TypeError(f"path[]='{type(path)}' is not of type 'str'") + raise TypeError(f"path[]='{type(path)}' has not expected type 'str'") elif path == "": raise ValueError(f"path='{path}' is an empty string") elif not path.startswith("/"): @@ -473,7 +473,7 @@ def set_total_peers(domain: str, peers: list) -> None: domain_helper.raise_on(domain) if not isinstance(peers, list): - raise TypeError(f"Parameter peers[]='{type(peers)}' is not of type 'list'") + raise TypeError(f"Parameter peers[]='{type(peers)}' has not expected type 'list'") # Set timestamp _set_data("total_peers", domain, len(peers)) @@ -484,7 +484,7 @@ def set_total_blocks(domain: str, blocks: list) -> None: domain_helper.raise_on(domain) if not isinstance(blocks, list): - raise TypeError(f"Parameter blocks[]='{type(blocks)}' is not of type 'list'") + raise TypeError(f"Parameter blocks[]='{type(blocks)}' has not expected type 'list'") # Set timestamp _set_data("total_blocks", domain, len(blocks)) @@ -495,7 +495,7 @@ def set_obfuscated_blocks(domain: str, obfuscated: int) -> None: domain_helper.raise_on(domain) if not isinstance(obfuscated, int): - raise TypeError(f"Parameter obfuscated[]='{type(obfuscated)}' is not of type 'int'") + raise TypeError(f"Parameter obfuscated[]='{type(obfuscated)}' has not expected type 'int'") elif obfuscated < 0: raise ValueError(f"Parameter obfuscated={obfuscated} is not valid") @@ -508,7 +508,7 @@ def set_nodeinfo_url(domain: str, url: str) -> None: domain_helper.raise_on(domain) if not isinstance(url, str) and url is not None: - raise TypeError(f"Parameter url[]='{type(url)}' is not of type 'str'") + raise TypeError(f"Parameter url[]='{type(url)}' has not expected type 'str'") elif url == "": raise ValueError("Parameter 'url' is empty") elif url is not None and not validators.url(url): @@ -523,7 +523,7 @@ def set_detection_mode(domain: str, mode: str) -> None: domain_helper.raise_on(domain) if not isinstance(mode, str) and mode is not None: - raise TypeError(f"Parameter mode[]='{type(mode)}' is not of type 'str'") + raise TypeError(f"Parameter mode[]='{type(mode)}' has not expected type 'str'") elif mode == "": raise ValueError("Parameter 'mode' is empty") @@ -536,7 +536,7 @@ def set_has_obfuscation(domain: str, has_obfuscation: bool) -> None: domain_helper.raise_on(domain) if not isinstance(has_obfuscation, bool): - raise TypeError(f"Parameter has_obfuscation[]='{type(has_obfuscation)}' is not of type 'bool'") + raise TypeError(f"Parameter has_obfuscation[]='{type(has_obfuscation)}' has not expected type 'bool'") # Set timestamp _set_data("has_obfuscation", domain, has_obfuscation) @@ -547,7 +547,7 @@ def set_original_software(domain: str, software: str) -> None: domain_helper.raise_on(domain) if not isinstance(software, str) and software is not None: - raise TypeError(f"Parameter software[]='{type(software)}' is not of type 'str'") + raise TypeError(f"Parameter software[]='{type(software)}' has not expected type 'str'") elif software == "": raise ValueError("Parameter 'software' is empty") @@ -560,7 +560,7 @@ def set_software(domain: str, software: str) -> None: domain_helper.raise_on(domain) if not isinstance(software, str) and software is not None: - raise TypeError(f"Parameter software[]='{type(software)}' is not of type 'str'") + raise TypeError(f"Parameter software[]='{type(software)}' has not expected type 'str'") elif software == "": raise ValueError("Parameter 'software' is empty") @@ -572,11 +572,11 @@ def valid(value: str, column: str) -> bool: logger.debug("value='%s',column='%s' - CALLED!", value, column) if not isinstance(value, str): - raise TypeError(f"Parameter value[]='{type(value)}' is not of type 'str'") + raise TypeError(f"Parameter value[]='{type(value)}' has not expected type 'str'") elif value == "": raise ValueError("Parameter 'value' is empty") elif not isinstance(column, str): - raise TypeError(f"Parameter column[]='{type(column)}' is not of type 'str'") + raise TypeError(f"Parameter column[]='{type(column)}' has not expected type 'str'") elif column == "": raise ValueError("Parameter 'column' is empty") @@ -594,7 +594,7 @@ def delete(search: str, column: str = "domain") -> None: logger.debug("search='%s, column='%s'' - CALLED!", search, column) if not isinstance(column, str): - raise TypeError(f"Parameter column[]='{type(column)}' is not of type 'str'") + raise TypeError(f"Parameter column[]='{type(column)}' has not expected type 'str'") elif column == "": raise ValueError("Parameter 'column' is empty") elif column in ["domain", "origin"]: @@ -611,11 +611,11 @@ def translate_idnas(rows: list, column: str) -> None: logger.debug("rows[]='%s',column='%s' - CALLED!", type(rows), column) if not isinstance(rows, list): - raise TypeError("rows[]='{type(rows)}' is not of type 'list'") + raise TypeError("rows[]='{type(rows)}' has not expected type 'list'") elif len(rows) == 0: raise ValueError("Parameter 'rows' is an empty list") elif not isinstance(column, str): - raise TypeError(f"column='{type(column)}' is not of type 'str'") + raise TypeError(f"column='{type(column)}' has not expected type 'str'") elif column == "": raise ValueError("Parameter 'column' is empty") elif column not in ["domain", "origin"]: diff --git a/fba/models/obfuscation.py b/fba/models/obfuscation.py index e9dc660..958f3a3 100644 --- a/fba/models/obfuscation.py +++ b/fba/models/obfuscation.py @@ -28,7 +28,7 @@ def has(pattern: str) -> bool: logger.debug("pattern='%s' - CALLED!", pattern) if not isinstance(pattern, str): - raise TypeError(f"pattern[]='{type(pattern)}' is not of type 'str'") + raise TypeError(f"pattern[]='{type(pattern)}' has not expected type 'str'") elif pattern == "": raise ValueError("Parametern 'pattern' is an empty string") @@ -46,7 +46,7 @@ def add(pattern: str) -> None: logger.debug("pattern='%s' - CALLED!", pattern) if not isinstance(pattern, str): - raise TypeError(f"pattern[]='{type(pattern)}' is not of type 'str'") + raise TypeError(f"pattern[]='{type(pattern)}' has not expected type 'str'") elif pattern == "": raise ValueError("Parametern 'pattern' is an empty string") elif has(pattern): @@ -63,7 +63,7 @@ def update (pattern: str) -> None: logger.debug("pattern='%s' - CALLED!", pattern) if not isinstance(pattern, str): - raise TypeError(f"pattern[]='{type(pattern)}' is not of type 'str'") + raise TypeError(f"pattern[]='{type(pattern)}' has not expected type 'str'") elif pattern == "": raise ValueError("Parametern 'pattern' is an empty string") elif not has(pattern): @@ -80,7 +80,7 @@ def delete (pattern: str) -> None: logger.debug("pattern='%s' - CALLED!", pattern) if not isinstance(pattern, str): - raise TypeError(f"pattern[]='{type(pattern)}' is not of type 'str'") + raise TypeError(f"pattern[]='{type(pattern)}' has not expected type 'str'") elif pattern == "": raise ValueError("Parametern 'pattern' is an empty string") elif not has(pattern): diff --git a/fba/networks/lemmy.py b/fba/networks/lemmy.py index 6b5d739..da30014 100644 --- a/fba/networks/lemmy.py +++ b/fba/networks/lemmy.py @@ -322,9 +322,9 @@ def parse_script(doc: bs4.BeautifulSoup, only: str = None) -> list: logger.debug("doc[]='%s',only='%s' - CALLED!", type(doc), only) if not isinstance(doc, bs4.BeautifulSoup): - raise TypeError(f"Parameter doc[]='{type(only)}' is not of type 'bs4.BeautifulSoup'") + raise TypeError(f"Parameter doc[]='{type(only)}' has not expected type 'bs4.BeautifulSoup'") elif not isinstance(only, str) and only is not None: - raise TypeError(f"Parameter only[]='{type(only)}' is not of type 'str'") + raise TypeError(f"Parameter only[]='{type(only)}' has not expected type 'str'") elif isinstance(only, str) and only == "": raise ValueError("Parameter 'only' is empty") diff --git a/fba/networks/misskey.py b/fba/networks/misskey.py index d958e83..5630cdb 100644 --- a/fba/networks/misskey.py +++ b/fba/networks/misskey.py @@ -110,7 +110,7 @@ def fetch_peers(domain: str) -> list: logger.warning("row()=%d does not contain key 'host': row='%s',domain='%s' - SKIPPED!", len(row), row, domain) continue elif not isinstance(row["host"], str): - logger.warning("row[host][]='%s' is not of type 'str' - SKIPPED!", type(row['host'])) + logger.warning("row[host][]='%s' has not expected type 'str' - SKIPPED!", type(row['host'])) continue elif row["host"] in peers: logger.debug("Not adding row[host]='%s', already found - SKIPPED!", row['host']) @@ -299,13 +299,13 @@ def fetch_blocks(domain: str) -> list: # Is it there? logger.debug("instance[]='%s'", type(instance)) if not isinstance(instance, dict): - logger.warning("instance[]='%s' is not of type 'dict' - SKIPPED!", type(instance)) + logger.warning("instance[]='%s' has not expected type 'dict' - SKIPPED!", type(instance)) continue elif "host" not in instance: logger.warning("instance()=%d has no element 'host' - SKIPPED!", len(instance)) continue elif not isinstance(instance["host"], str): - logger.warning("instance[host][]='%s' is not of type 'str' - SKIPPED!", type(instance["host"])) + logger.warning("instance[host][]='%s' has not expected type 'str' - SKIPPED!", type(instance["host"])) continue logger.debug("instance[host]='%s' - BEFORE!", instance["host"]) diff --git a/fba/utils.py b/fba/utils.py index fcb09f0..4c7440a 100644 --- a/fba/utils.py +++ b/fba/utils.py @@ -46,9 +46,9 @@ def find_domains(tags: bs4.element.ResultSet, search: str) -> list: logger.debug("tags[%s]()=%d,search='%s' - CALLED!", type(tags), len(tags), search) if not isinstance(tags, bs4.element.ResultSet): - raise TypeError(f"Parameter tags[]='{type(tags)}' is not of type 'ResultSet'") + raise TypeError(f"Parameter tags[]='{type(tags)}' has not expected type 'ResultSet'") elif not isinstance(search, str): - raise TypeError(f"Parameter search[]='{type(search)}' is not of type 'str'") + raise TypeError(f"Parameter search[]='{type(search)}' has not expected type 'str'") elif search == "": raise ValueError("Parameter 'search' is empty") @@ -90,7 +90,7 @@ def deobfuscate(domain: str, blocker: str, domain_hash: str = None) -> str: if validators.domain(domain, rfc_2782=True) and blacklist.is_blacklisted(domain): raise RuntimeError(f"domain='{domain}' is blacklisted but function was invoked") elif not isinstance(domain_hash, str) and domain_hash is not None: - raise TypeError(f"Parameter domain_hash[]='{type(domain_hash)}' is not of type 'str'") + raise TypeError(f"Parameter domain_hash[]='{type(domain_hash)}' has not expected type 'str'") elif domain_hash == "": raise ValueError("Parameter 'domain_hash' is empty") -- 2.39.5