Continued:

- `pleroma.envs.net` does now publish a more detailed blocklist than its CSV file
- but still it can be used as peer source (key `only_peers` set to `True`)
- exception messages improved

diff --git a/fba/commands.py b/fba/commands.py
index 9dc0d2c1ba76fc45133e85168ed7bbc2c8e37f6d..cd931eafac2f84b0424ff9b9057519b0f1617423 100644 (file)
--- a/fba/commands.py
+++ b/fba/commands.py
@@ -1103,8 +1103,12 @@ def fetch_csv(args: argparse.Namespace) -> int:
             logger.debug("block[blocker]='%s' does not match args.domain='%s' - SKIPPED!", block["blocker"], args.domain)
             continue
 
-        logger.debug("Invoking processing.csv_block(%s, %s, fetch_csv) ...", block["blocker"], block["csv_url"])
-        processing.csv_block(block["blocker"], block["csv_url"], inspect.currentframe().f_code.co_name)
+        if block["only_peers"]:
+            logger.debug("block[blocker]='%s' is not registered, adding instance ...", block["blocker"])
+            processing.csv_instance(block["blocker"], block["csv_url"], inspect.currentframe().f_code.co_name)
+        else:
+            logger.debug("Invoking processing.csv_block(%s, %s, fetch_csv) ...", block["blocker"], block["csv_url"])
+            processing.csv_block(block["blocker"], block["csv_url"], inspect.currentframe().f_code.co_name)
 
     logger.debug("Success - EXIT!")
     return 0

diff --git a/fba/helpers/blocklists.py b/fba/helpers/blocklists.py
index 90cf1741eae836af3da47a6335efa7fe0a30cd3a..86612de4affb61a3267097b555cd875a2a325b02 100644 (file)
--- a/fba/helpers/blocklists.py
+++ b/fba/helpers/blocklists.py
@@ -26,17 +26,21 @@ logger = logging.getLogger(__name__)
 # Blocklists hosted by oliphant
 oliphant_blocklists = (
     {
-        "blocker": "mastodon.online",
-        "csv_url": "mastodon/mastodon.online.csv",
+        "blocker"   : "mastodon.online",
+        "csv_url"   : "mastodon/mastodon.online.csv",
+        "only_peers": False,
     },{
-        "blocker": "mastodon.social",
-        "csv_url": "mastodon/mastodon.social.csv",
+        "blocker"   : "mastodon.social",
+        "csv_url"   : "mastodon/mastodon.social.csv",
+        "only_peers": False,
     },{
-        "blocker": "mastodon.social",
-        "csv_url": "other/missing-tier0-mastodon.social.csv",
+        "blocker"   : "mastodon.social",
+        "csv_url"   : "other/missing-tier0-mastodon.social.csv",
+        "only_peers": False,
     },{
-        "blocker": "seirdy.one",
-        "csv_url": "mastodon/seirdy-tier0.csv",
+        "blocker"   : "seirdy.one",
+        "csv_url"   : "mastodon/seirdy-tier0.csv",
+        "only_peers": False,
     },
 )
 
@@ -66,11 +70,13 @@ txt_files = (
 # Other CSV files
 csv_files = (
     {
-        "blocker": "pleroma.envs.net",
-        "csv_url": "https://seirdy.one/pb/pleroma.envs.net.csv",
+        "blocker"   : "pleroma.envs.net",
+        "csv_url"   : "https://seirdy.one/pb/pleroma.envs.net.csv",
+        "only_peers": True,
     },{
-        "blocker": "social.mooneyed.de",
-        "csv_url": "https://git.bka.li/kromonos/fedi-health/-/raw/master/_unified_tier3_blocklist.csv",
+        "blocker"   : "social.mooneyed.de",
+        "csv_url"   : "https://git.bka.li/kromonos/fedi-health/-/raw/master/_unified_tier3_blocklist.csv",
+        "only_peers": False,
     },
 )
 
@@ -84,8 +90,8 @@ def has(domain: str) -> bool:
     # Default is not found
     found = False
     for row in oliphant_blocklists + csv_files:
-        logger.debug("row[blocker]='%s',domain='%s'", row["blocker"], domain)
-        if row["blocker"] == domain:
+        logger.debug("row[blocker]='%s',row[only_peers]='%s',domain='%s'", row["blocker"], row["only_peers"], domain)
+        if row["blocker"] == domain and not row["only_peers"]:
             found = True
             logger.debug("domain='%s' is found and excluded from regular fetch_blocks command - BREAK!", domain)
             break

diff --git a/fba/helpers/config.py b/fba/helpers/config.py
index 0e7b12c6e10abf1eb129f23fbacc9c1983c49b36..6788182e593ab6537fd81817b9d6a268b18a5b80 100644 (file)
--- a/fba/helpers/config.py
+++ b/fba/helpers/config.py
@@ -46,7 +46,7 @@ def get(key: str) -> any:
     if not isinstance(key, str):
         raise TypeError(f"Parameter key[]='{type(key)}' has not expected type 'str'")
     elif key == "":
-        raise ValueError("Parameter 'key' is empty")
+        raise ValueError("Parameter 'key' is an empty string")
     elif not key in _config:
         raise KeyError(f"key='{key}' does not exist in _config array")
 

diff --git a/fba/helpers/dicts.py b/fba/helpers/dicts.py
index 7c97e1dfd0cd31d45bd864bfc993280f195ef406..f8676721f7790988c4105b6b9f24156ed953d487 100644 (file)
--- a/fba/helpers/dicts.py
+++ b/fba/helpers/dicts.py
@@ -28,7 +28,7 @@ def has_key(lists: list, key: str, value: any) -> bool:
     elif not isinstance(key, str):
         raise TypeError(f"Parameter key[]='{type(key)}' has not expected type 'str'")
     elif key == "":
-        raise ValueError("Parameter 'key' is empty")
+        raise ValueError("Parameter 'key' is an empty string")
 
     has = False
     logger.debug("Checking lists()=%d ...", len(lists))

diff --git a/fba/helpers/domain.py b/fba/helpers/domain.py
index 6948fb81bf077a97ae776e327e95d1c75789a559..6d9130354a7a70eb84bd98f4432af24b3b80d90b 100644 (file)
--- a/fba/helpers/domain.py
+++ b/fba/helpers/domain.py
@@ -39,7 +39,7 @@ def raise_on(domain: str) -> None:
     if not isinstance(domain, str):
         raise TypeError(f"Parameter domain[]='{type(domain)}' has not expected type 'str'")
     elif domain == "":
-        raise ValueError("Parameter 'domain' is empty")
+        raise ValueError("Parameter 'domain' is an empty string")
     elif domain.lower() != domain:
         raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
     elif "?" in domain:
@@ -67,7 +67,7 @@ def is_in_url(domain: str, url: str) -> bool:
     elif not isinstance(url, str):
         raise TypeError(f"Parameter url[]='{type(url)}' has not expected type 'str'")
     elif url == "":
-        raise ValueError("Parameter 'url' is empty")
+        raise ValueError("Parameter 'url' is an empty string")
     elif not validators.url(url):
         raise RuntimeError(f"Parameter url='{url}' is not a valid URL but function was invoked")
 
@@ -89,7 +89,7 @@ def is_tld_wanted(domain: str) -> bool:
     if not isinstance(domain, str):
         raise TypeError(f"Parameter domain[]='{type(domain)}' has not expected type 'str'")
     elif domain == "":
-        raise ValueError("Parameter 'domain' is empty")
+        raise ValueError("Parameter 'domain' is an empty string")
 
     wanted = True
 
@@ -116,7 +116,7 @@ def is_wanted(domain: str) -> bool:
     if not isinstance(domain, str):
         raise TypeError(f"Parameter domain[]='{type(domain)}' has not expected type 'str'")
     elif domain == "":
-        raise ValueError("Parameter 'domain' is empty")
+        raise ValueError("Parameter 'domain' is an empty string")
 
     wanted = True
     if domain.lower() != domain:

diff --git a/fba/helpers/processing.py b/fba/helpers/processing.py
index b11b53e06e525cca42741275988aebad0b9658ac..f21ff535cb17f67f57408bffabffbf0c3eaf2331 100644 (file)
--- a/fba/helpers/processing.py
+++ b/fba/helpers/processing.py
@@ -49,7 +49,7 @@ def instance(blocked: str, blocker: str, command: str, force: bool = False) -> b
     if not isinstance(command, str):
         raise TypeError(f"Parameter command[]='{type(command)}' has not expected type 'str'")
     elif command == "":
-        raise ValueError("Parameter 'command' is empty")
+        raise ValueError("Parameter 'command' is an empty string")
     elif blacklist.is_blacklisted(blocked):
         raise RuntimeError(f"blocked='{blocked}' is blacklisted but function was invoked")
     elif blacklist.is_blacklisted(blocker):
@@ -132,13 +132,13 @@ def csv_block(blocker: str, url: str, command: str) -> None:
     if not isinstance(url, str):
         raise TypeError(f"url[]='{url}' has not expected type 'str'")
     elif url in [None, ""]:
-        raise ValueError("Parameter 'url' is empty")
+        raise ValueError("Parameter 'url' is an empty string")
     elif not validators.url(url):
         raise ValueError(f"Parameter url='{url}' is not a valid URL")
     elif not isinstance(command, str):
         raise TypeError(f"command[]='{command}' has not expected type 'str'")
     elif command == "":
-        raise ValueError("Parameter 'command' is empty")
+        raise ValueError("Parameter 'command' is an empty string")
     elif blacklist.is_blacklisted(blocker):
         raise RuntimeError(f"blocker='{blocker}' is blacklisted but function was invoked")
 
@@ -264,3 +264,87 @@ def csv_block(blocker: str, url: str, command: str) -> None:
         network.send_bot_post(blocker, blockdict)
 
     logger.debug("EXIT!")
+
+def csv_instance(instance: str, url: str, command: str) -> None:
+    logger.debug("instance='%s',url='%s',command='%s' - CALLED!", instance, url, command)
+    domain_helper.raise_on(instance)
+
+    if not isinstance(url, str):
+        raise TypeError(f"url[]='{url}' has not expected type 'str'")
+    elif url in [None, ""]:
+        raise ValueError("Parameter 'url' is an empty string")
+    elif not validators.url(url):
+        raise ValueError(f"Parameter url='{url}' is not a valid URL")
+    elif not isinstance(command, str):
+        raise TypeError(f"command[]='{command}' has not expected type 'str'")
+    elif command == "":
+        raise ValueError("Parameter 'command' is an empty string")
+    elif blacklist.is_blacklisted(instance):
+        raise RuntimeError(f"instance='{instance}' is blacklisted but function was invoked")
+
+    # Init local variables
+    domains = []
+
+    logger.debug("Setting last_instance_fetch for instance='%s' ...", instance)
+    instances.set_last_instance_fetch(instance)
+
+    # Fetch this URL
+    logger.info("Fetching url='%s' for instance='%s' ...", url, instance)
+    rows = network.fetch_csv_rows(url)
+
+    logger.info("Checking %d CSV lines ...", len(rows))
+    for row in rows:
+        logger.debug("row[%s]='%s'", type(row), row)
+        domain = None
+
+        if "#domain" in row and row["#domain"] not in [None, ""]:
+            domain = tidyup.domain(row["#domain"])
+        elif "domain" in row and row["domain"] not in [None, ""]:
+            domain = tidyup.domain(row["domain"])
+        elif "Domain" in row and row["Domain"] not in [None, ""]:
+            domain = tidyup.domain(row["Domain"])
+        else:
+            logger.warning("row='%s' does not contain domain column - SKIPPED!", row)
+            continue
+
+        logger.debug("domain='%s'", domain)
+        if domain in [None, ""]:
+            logger.debug("domain='%s' is empty - SKIPPED!", domain)
+            continue
+        elif not domain_helper.is_tld_wanted(domain):
+            logger.debug("domain='%s' has an unwanted TLD - SKIPPED!", domain)
+            continue
+        elif domain.find("*") >= 0 or domain.find("?") >= 0:
+            logger.debug("domain='%s' is obfuscated - Invoking utils.deobfuscate(%s, %s) ...", domain, domain, instance)
+            domain = utils.deobfuscate(domain, instance)
+            logger.debug("domain='%s' - AFTER!", domain)
+
+        logger.debug("Marking domain='%s' as handled ...", domain)
+        domains.append(domain)
+
+        if not validators.domain(domain, rfc_2782=True):
+            logger.warning("domain='%s' is not a valid domain - SKIPPED!", domain)
+            continue
+        elif blacklist.is_blacklisted(domain):
+            logger.debug("domain='%s' is blacklisted - SKIPPED!", domain)
+            continue
+        elif instances.is_registered(domain):
+            logger.debug("domain='%s' is already registered - SKIPPED!", domain)
+            continue
+
+        logger.debug("Processing domain='%s',instance='%s',command='%s' ...", domain, instance, command)
+        processed = instance(domain, instance, command)
+        logger.debug("processed='%s'", processed)
+
+        logger.debug("Invoking commit() ...")
+        database.connection.commit()
+
+    logger.debug("Invoking instances.set_total_instances(%s, domains()=%d) ...", instance, len(domains))
+    instances.set_total_instances(instance, domains)
+
+    logger.debug("Checking if instance='%s' has pending updates ...", instance)
+    if instances.has_pending(instance):
+        logger.debug("Flushing updates for instance='%s' ...", instance)
+        instances.update(instance)
+
+    logger.debug("EXIT!")

diff --git a/fba/helpers/software.py b/fba/helpers/software.py
index a2ad615666ea9a73e44f6ff1e2110ed5156d70a1..f7565e1dbbb3da83fd497b2a0da1e5232225cd79 100644 (file)
--- a/fba/helpers/software.py
+++ b/fba/helpers/software.py
@@ -113,7 +113,7 @@ def alias(software: str) -> str:
     if not isinstance(software, str) and software is not None:
         raise TypeError(f"software[]='{type(software)}' is not type 'str'")
     elif software == "":
-        raise ValueError("Parameter 'software' is empty")
+        raise ValueError("Parameter 'software' is an empty string")
     elif software.startswith("re:"):
         logger.debug("Cutting prefix 're:' from software='%s' ...", software)
         software = software.split(":")[1]
@@ -226,7 +226,7 @@ def strip_hosted_on(software: str) -> str:
     if not isinstance(software, str):
         raise TypeError(f"Parameter software[]='{type(software)}' has not expected type 'str'")
     elif software == "":
-        raise ValueError("Parameter 'software' is empty")
+        raise ValueError("Parameter 'software' is an empty string")
     elif "hosted on" not in software:
         logger.warning("Cannot find 'hosted on' in software='%s'!", software)
         return software
@@ -251,7 +251,7 @@ def strip_powered_by(software: str) -> str:
     if not isinstance(software, str):
         raise TypeError(f"Parameter software[]='{type(software)}' has not expected type 'str'")
     elif software == "":
-        raise ValueError("Parameter 'software' is empty")
+        raise ValueError("Parameter 'software' is an empty string")
     elif "powered by" not in software:
         logger.warning("Cannot find 'powered by' in software='%s'!", software)
         return software
@@ -276,11 +276,11 @@ def strip_until(software: str, until: str) -> str:
     if not isinstance(software, str):
         raise TypeError(f"Parameter software[]='{type(software)}' has not expected type 'str'")
     elif software == "":
-        raise ValueError("Parameter 'software' is empty")
+        raise ValueError("Parameter 'software' is an empty string")
     elif not isinstance(until, str):
         raise TypeError(f"Parameter until[]='{type(until)}' has not expected type 'str'")
     elif until == "":
-        raise ValueError("Parameter 'until' is empty")
+        raise ValueError("Parameter 'until' is an empty string")
     elif not until in software:
         logger.warning("Cannot find until='%s' in software='%s'!", until, software)
         return software
@@ -301,7 +301,7 @@ def is_relay(software: str) -> bool:
     if not isinstance(software, str):
         raise TypeError(f"software[]='{type(software)}' is not type 'str'")
     elif software == "":
-        raise ValueError("Parameter 'software' is empty")
+        raise ValueError("Parameter 'software' is an empty string")
 
     found = software in _relays
 

diff --git a/fba/helpers/tidyup.py b/fba/helpers/tidyup.py
index 2f801836aebd520d31fc89c03221bc1800db6016..02932ca6d65e74920dd75988ba5384c46c115d98 100644 (file)
--- a/fba/helpers/tidyup.py
+++ b/fba/helpers/tidyup.py
@@ -39,7 +39,7 @@ def domain(string: str) -> str:
     if not isinstance(string, str):
         raise TypeError(f"Parameter string[]='{type(string)}' has not expected type 'str'")
     elif string == "":
-        raise ValueError("Parameter 'string' is empty")
+        raise ValueError("Parameter 'string' is an empty string")
 
     # All lower-case and strip spaces out + last dot
     string = string.lower().strip().rstrip(".").replace("..", ".")

diff --git a/fba/http/federation.py b/fba/http/federation.py
index c9cd446174cf3a24086daf3c8e9cb0b8aaf4a8a7..7c996e944246e0d06c373bc9fdaf583dac33fd84 100644 (file)
--- a/fba/http/federation.py
+++ b/fba/http/federation.py
@@ -69,7 +69,7 @@ def fetch_instances(domain: str, origin: str, software: str, command: str, path:
     elif not isinstance(command, str):
         raise TypeError(f"Parameter command[]='{type(command)}' has not expected type 'str'")
     elif command == "":
-        raise ValueError("Parameter 'command' is empty")
+        raise ValueError("Parameter 'command' is an empty string")
     elif command in ["fetch_blocks", "fetch_cs", "fetch_bkali", "fetch_relays", "fetch_fedipact", "fetch_joinmobilizon", "fetch_joinmisskey", "fetch_joinfediverse", "fetch_relaylist"] and origin is None:
         raise ValueError(f"Parameter command='{command}' but origin is None, please fix invoking this function.")
     elif not isinstance(path, str) and path is not None:
@@ -229,13 +229,13 @@ def fetch_peers(domain: str, software: str, origin: str) -> list:
     elif not isinstance(software, str) and software is not None:
         raise TypeError(f"Parameter software[]='{type(software)}' has not expected type 'str'")
     elif isinstance(software, str) and software == "":
-        raise ValueError("Parameter 'software' is empty")
+        raise ValueError("Parameter 'software' is an empty string")
     elif software is not None and software_helper.is_relay(software):
         raise RuntimeError(f"domain='{domain}' is of software='{software}' and isn't supported here.")
     elif not isinstance(origin, str) and origin is not None:
         raise TypeError(f"Parameter origin[]='{type(origin)}' has not expected type 'str'")
     elif isinstance(origin, str) and origin == "":
-        raise ValueError("Parameter 'origin' is empty")
+        raise ValueError("Parameter 'origin' is an empty string")
 
     if software == "misskey":
         logger.debug("Invoking misskey.fetch_peers(%s) ...", domain)
@@ -297,7 +297,7 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str:
     elif not isinstance(path, str):
         raise TypeError(f"path[]='{type(path)}' has not expected type 'str'")
     elif path == "":
-        raise ValueError("Parameter 'path' is empty")
+        raise ValueError("Parameter 'path' is an empty string")
     elif not path.startswith("/"):
         raise ValueError(f"path='{path}' does not start with / but should")
 
@@ -562,7 +562,7 @@ def add_peers(rows: dict) -> list:
     if not isinstance(rows, dict):
         raise TypeError(f"Parameter rows[]='{type(rows)}' has not expected type 'dict'")
     elif len(rows) == 0:
-        raise ValueError("Parameter 'rows' is empty")
+        raise ValueError("Parameter 'rows' is an empty string")
 
     # Init variables
     peers = []

diff --git a/fba/http/network.py b/fba/http/network.py
index 9f62316aba501486216de8cc7f8ef835817f2fd8..56049a5a419e1fa61a8ea3446d0637bf426b3287 100644 (file)
--- a/fba/http/network.py
+++ b/fba/http/network.py
@@ -141,7 +141,7 @@ def fetch_api_url(url: str, timeout: tuple) -> dict:
     if not isinstance(url, str):
         raise TypeError(f"Parameter url[]='{type(url)}' has not expected type 'str'")
     elif url == "":
-        raise ValueError("Parameter 'url' is empty")
+        raise ValueError("Parameter 'url' is an empty string")
     elif not validators.url(url):
         raise ValueError(f"Parameter url='{url}' is not a valid URL")
     elif not isinstance(timeout, tuple):
@@ -244,7 +244,7 @@ def send_bot_post(domain: str, blocklist: list) -> None:
     elif not isinstance(blocklist, list):
         raise TypeError(f"Parameter blocklist[]='{type(blocklist)}' has not expected type 'list'")
     elif len(blocklist) == 0:
-        raise ValueError("Parameter 'blocklist' is empty")
+        raise ValueError("Parameter 'blocklist' is an empty string")
     elif config.get("bot_token") == "":
         raise ValueError("config[bot_token] is not set")
 
@@ -296,7 +296,7 @@ def _fetch_response(domain: str, path: str, headers: dict, timeout: tuple, allow
     elif not isinstance(path, str):
         raise TypeError(f"Parameter path[]='{type(path)}' has not expected type 'str'")
     elif path == "":
-        raise ValueError("Parameter 'path' is empty")
+        raise ValueError("Parameter 'path' is an empty string")
     elif not path.startswith("/"):
         raise ValueError(f"path='{path}' does not start with / but should")
     elif not isinstance(headers, dict):
@@ -343,7 +343,7 @@ def fetch_url(url: str, headers: dict, timeout: tuple, allow_redirects: bool = T
     if not isinstance(url, str):
         raise TypeError(f"Parameter url[]='{type(url)}' has not expected type 'str'")
     elif url == "":
-        raise ValueError("Parameter 'url' is empty")
+        raise ValueError("Parameter 'url' is an empty string")
     elif not validators.url(url):
         raise ValueError(f"Parameter url='{url}' is not a valid URL")
     elif not isinstance(headers, dict):
@@ -479,7 +479,7 @@ def get_generic(domain: str, path: str, allow_redirects: bool = False) -> reques
     elif not isinstance(path, str):
         raise TypeError(f"Parameter path[]='{type(path)}' has not expected type 'str'")
     elif path == "":
-        raise ValueError("Parameter 'path' is empty")
+        raise ValueError("Parameter 'path' is an empty string")
     elif not path.startswith("/"):
         raise ValueError(f"path='{path}' does not start with / but should")
     elif not isinstance(allow_redirects, bool):

diff --git a/fba/models/blocks.py b/fba/models/blocks.py
index b77c6680209c0d213d030ee4bffb91bdc6398fd7..61db78363ba096526ac593a19eaabbf4cd6b7323 100644 (file)
--- a/fba/models/blocks.py
+++ b/fba/models/blocks.py
@@ -36,7 +36,7 @@ def get_reason(blocker: str, blocked: str, block_level: str) -> str:
     if not isinstance(block_level, str):
         raise TypeError(f"Parameter block_level[]='{type(block_level)}' has not expected type 'str'")
     elif block_level == "":
-        raise ValueError("Parameter 'block_level' is empty")
+        raise ValueError("Parameter 'block_level' is an empty string")
     elif block_level in ["accept", "reject", "suspend", "silence", "nsfw", "quarantined_instances"]:
         raise ValueError(f"block_level='{block_level}' is not wanted.")
     elif blacklist.is_blacklisted(blocker):
@@ -71,7 +71,7 @@ def update_reason(reason: str, blocker: str, blocked: str, block_level: str) ->
     elif not isinstance(block_level, str):
         raise TypeError(f"Parameter block_level[]='{type(block_level)}' has not expected type 'str'")
     elif block_level == "":
-        raise ValueError("Parameter 'block_level' is empty")
+        raise ValueError("Parameter 'block_level' is an empty string")
     elif block_level in ["accept", "reject", "suspend", "silence", "nsfw", "quarantined_instances"]:
         raise ValueError(f"block_level='{block_level}' is not wanted.")
     elif blacklist.is_blacklisted(blocker):
@@ -102,7 +102,7 @@ def update_last_seen(blocker: str, blocked: str, block_level: str) -> None:
     if not isinstance(block_level, str):
         raise TypeError(f"Parameter block_level[]='{type(block_level)}' has not expected type 'str'")
     elif block_level == "":
-        raise ValueError("Parameter 'block_level' is empty")
+        raise ValueError("Parameter 'block_level' is an empty string")
     elif block_level in ["accept", "reject", "suspend", "silence", "nsfw", "quarantined_instances"]:
         raise ValueError(f"blocked='{blocked}' has unwanted block_level='{block_level}'")
     elif blacklist.is_blacklisted(blocker):
@@ -131,7 +131,7 @@ def is_instance_blocked(blocker: str, blocked: str, block_level: str = None) ->
     if not isinstance(block_level, str) and block_level is not None:
         raise TypeError(f"Parameter block_level[]='{type(block_level)}' has not expected type 'str'")
     elif block_level == "":
-        raise ValueError("Parameter 'block_level' is empty")
+        raise ValueError("Parameter 'block_level' is an empty string")
     elif block_level in ["accept", "reject", "suspend", "silence", "nsfw", "quarantined_instances"]:
         raise ValueError(f"blocked='{blocked}' has unwanted block_level='{block_level}'")
     elif blacklist.is_blacklisted(blocker):
@@ -170,7 +170,7 @@ def add(blocker: str, blocked: str, reason: str, block_level: str) -> None:
     if not isinstance(block_level, str):
         raise TypeError(f"Parameter block_level[]='{type(block_level)}' has not expected type 'str'")
     elif block_level == "":
-        raise ValueError("Parameter 'block_level' is empty")
+        raise ValueError("Parameter 'block_level' is an empty string")
     elif block_level in ["accept", "reject", "suspend", "silence", "nsfw", "quarantined_instances"]:
         raise ValueError(f"blocked='{blocked}' has unwanted block_level='{block_level}'")
     elif blacklist.is_blacklisted(blocker):
@@ -207,11 +207,11 @@ def valid(value: str, column: str) -> bool:
     if not isinstance(value, str):
         raise TypeError(f"Parameter value[]='{type(value)}' has not expected type 'str'")
     elif value == "":
-        raise ValueError("Parameter 'value' is empty")
+        raise ValueError("Parameter 'value' is an empty string")
     elif not isinstance(column, str):
         raise TypeError(f"Parameter column[]='{type(column)}' has not expected type 'str'")
     elif column == "":
-        raise ValueError("Parameter 'column' is empty")
+        raise ValueError("Parameter 'column' is an empty string")
 
     # Query database
     database.cursor.execute(

diff --git a/fba/models/instances.py b/fba/models/instances.py
index 19572f3752c2769385cbf74dd4abe94dd3ea0eb4..a7fabf0917e3838e4f119fcda4de05741f21dcf3 100644 (file)
--- a/fba/models/instances.py
+++ b/fba/models/instances.py
@@ -389,11 +389,11 @@ def deobfuscate(char: str, domain: str, blocked_hash: str = None) -> tuple:
     if not isinstance(char, str):
         raise TypeError(f"Parameter char[]='{type(char)}' has not expected type 'str'")
     elif char == "":
-        raise ValueError("Parameter 'char' is empty")
+        raise ValueError("Parameter 'char' is an empty string")
     elif not isinstance(domain, str):
         raise TypeError(f"Parameter domain[]='{type(domain)}'")
     elif domain == "":
-        raise ValueError("Parameter 'domain' is empty")
+        raise ValueError("Parameter 'domain' is an empty string")
     elif not char in domain:
         raise ValueError(f"char='{char}' not found in domain='{domain}' but function invoked")
     elif not isinstance(blocked_hash, str) and blocked_hash is not None:
@@ -514,7 +514,7 @@ def set_nodeinfo_url(domain: str, url: str) -> None:
     if not isinstance(url, str) and url is not None:
         raise TypeError(f"Parameter url[]='{type(url)}' has not expected type 'str'")
     elif url == "":
-        raise ValueError("Parameter 'url' is empty")
+        raise ValueError("Parameter 'url' is an empty string")
     elif url is not None and not validators.url(url):
         raise ValueError(f"Parameter url='{url}' is not a valid URL")
 
@@ -529,7 +529,7 @@ def set_detection_mode(domain: str, mode: str) -> None:
     if not isinstance(mode, str) and mode is not None:
         raise TypeError(f"Parameter mode[]='{type(mode)}' has not expected type 'str'")
     elif mode == "":
-        raise ValueError("Parameter 'mode' is empty")
+        raise ValueError("Parameter 'mode' is an empty string")
 
     # Set timestamp
     _set_pending_data("detection_mode", domain, mode)
@@ -553,7 +553,7 @@ def set_original_software(domain: str, software: str) -> None:
     if not isinstance(software, str) and software is not None:
         raise TypeError(f"Parameter software[]='{type(software)}' has not expected type 'str'")
     elif software == "":
-        raise ValueError("Parameter 'software' is empty")
+        raise ValueError("Parameter 'software' is an empty string")
 
     # Set original software
     _set_pending_data("original_software", domain, software)
@@ -566,7 +566,7 @@ def set_software(domain: str, software: str) -> None:
     if not isinstance(software, str) and software is not None:
         raise TypeError(f"Parameter software[]='{type(software)}' has not expected type 'str'")
     elif software == "":
-        raise ValueError("Parameter 'software' is empty")
+        raise ValueError("Parameter 'software' is an empty string")
 
     # Set software (maybe aliased to generic name)
     _set_pending_data("software", domain, software)
@@ -578,11 +578,11 @@ def valid(value: str, column: str) -> bool:
     if not isinstance(value, str):
         raise TypeError(f"Parameter value[]='{type(value)}' has not expected type 'str'")
     elif value == "":
-        raise ValueError("Parameter 'value' is empty")
+        raise ValueError("Parameter 'value' is an empty string")
     elif not isinstance(column, str):
         raise TypeError(f"Parameter column[]='{type(column)}' has not expected type 'str'")
     elif column == "":
-        raise ValueError("Parameter 'column' is empty")
+        raise ValueError("Parameter 'column' is an empty string")
 
     # Query database
     database.cursor.execute(
@@ -600,7 +600,7 @@ def delete(search: str, column: str = "domain") -> None:
     if not isinstance(column, str):
         raise TypeError(f"Parameter column[]='{type(column)}' has not expected type 'str'")
     elif column == "":
-        raise ValueError("Parameter 'column' is empty")
+        raise ValueError("Parameter 'column' is an empty string")
     elif column in ["domain", "origin"]:
         domain_helper.raise_on(search)
 
@@ -621,7 +621,7 @@ def translate_idnas(rows: list, column: str) -> None:
     elif not isinstance(column, str):
         raise TypeError(f"column='{type(column)}' has not expected type 'str'")
     elif column == "":
-        raise ValueError("Parameter 'column' is empty")
+        raise ValueError("Parameter 'column' is an empty string")
     elif column not in ["domain", "origin"]:
         raise ValueError(f"column='{column}' is not supported")
 

diff --git a/fba/networks/lemmy.py b/fba/networks/lemmy.py
index 4ba2392dd9b4f43dff2be5232996e9a9f881510d..648cee0735f1362134a0cfd4c23a1c19b0c7787e 100644 (file)
--- a/fba/networks/lemmy.py
+++ b/fba/networks/lemmy.py
@@ -326,7 +326,7 @@ def parse_script(doc: bs4.BeautifulSoup, only: str = None) -> list:
     elif not isinstance(only, str) and only is not None:
         raise TypeError(f"Parameter only[]='{type(only)}' has not expected type 'str'")
     elif isinstance(only, str) and only == "":
-        raise ValueError("Parameter 'only' is empty")
+        raise ValueError("Parameter 'only' is an empty string")
 
     scripts = doc.find_all("script")
     peers = []

diff --git a/fba/networks/misskey.py b/fba/networks/misskey.py
index 4ba88fd21a191f4878915c5492b5301a490847ee..8070d4daafdc94e78c1c63cdf02af850ac767aec 100644 (file)
--- a/fba/networks/misskey.py
+++ b/fba/networks/misskey.py
@@ -112,8 +112,8 @@ def fetch_peers(domain: str) -> list:
             elif not isinstance(row["host"], str):
                 logger.warning("row[host][]='%s' has not expected type 'str' - SKIPPED!", type(row["host"]))
                 continue
-            elif row["host"] in [None, ""]:
-                logger.warning("row[host]='%s' is empty,domain='%s' - SKIPPED!", row["host"], domain)
+            elif row["host"] == "":
+                logger.warning("row[host] is an empty string,domain='%s' - SKIPPED!", domain)
                 continue
             elif row["host"] in peers:
                 logger.debug("Not adding row[host]='%s', already found - SKIPPED!", row["host"])
@@ -214,8 +214,11 @@ def fetch_blocks(domain: str) -> list:
                 if "host" not in instance:
                     logger.warning("instance(%d)='%s' has no key 'host' - SKIPPED!", len(instance), instance)
                     continue
-                elif instance["host"] in [None, ""]:
-                    logger.debug("instance[host]='%s' is None or empty - SKIPPED!", instance["host"])
+                elif not isinstance(instance["host"], str):
+                    logger.warning("instance[host][]='%s' has not expected type 'str' - SKIPPED!", type(instance["host"]))
+                    continue
+                elif instance["host"] == "":
+                    logger.warning("instance[host] is an empty string,domain='%s' - SKIPPED!", domain)
                     continue
 
                 logger.debug("instance[host]='%s' - BEFORE!", instance["host"])

diff --git a/fba/networks/peertube.py b/fba/networks/peertube.py
index 06dafbd546b02827c4f362ec1b9854dccd1c0f9c..f2c05403fe7a981fac095c30ce157a9bd6091200 100644 (file)
--- a/fba/networks/peertube.py
+++ b/fba/networks/peertube.py
@@ -89,6 +89,7 @@ def fetch_peers(domain: str) -> list:
                     logger.debug("Appending mode2='%s',host='%s' ...", mode2, record[mode2]["host"])
                     peers.append(record[mode2]["host"])
 
+            logger.debug("rows()=%d,peers()=%d,start=%d", len(rows), len(peers), start)
             if len(rows) < 100:
                 logger.debug("Reached end of JSON response, domain='%s'", domain)
                 break

diff --git a/fba/networks/pleroma.py b/fba/networks/pleroma.py
index 1d24b27efde849a7e9400a1d97be24a1741e0721..c18542f2866ee7aa5be45308e5892c6a260cfba8 100644 (file)
--- a/fba/networks/pleroma.py
+++ b/fba/networks/pleroma.py
@@ -110,7 +110,7 @@ def fetch_blocks(domain: str) -> list:
             logger.debug("block_level='%s' - AFTER!", block_level)
 
             if block_level in [None, ""]:
-                logger.warning("block_level is now empty!")
+                logger.warning("block_level='%s' is empty - SKIPPED!", block_level)
                 continue
             elif block_level == "accept":
                 logger.debug("domain='%s' skipping block_level='accept'", domain)
@@ -265,7 +265,7 @@ def fetch_blocks(domain: str) -> list:
     else:
         logger.warning("Cannot find 'mrf_simple_info' or 'quarantined_instances_info' in JSON reply: domain='%s'", domain)
 
-    logger.debug("found='%s'", found)
+    logger.debug("found='%s',blockdict()=%d", found, len(blockdict))
     if not found:
         logger.debug("Did not find any useable JSON elements, domain='%s', continuing with /about page ...", domain)
         blocklist = fetch_blocks_from_about(domain)
@@ -379,7 +379,7 @@ def fetch_blocks_from_about(domain: str) -> dict:
             tds = line.find_all("td")
 
             logger.debug("tds[%s]()=%d", type(tds), len(tds))
-            if len(tds) == 0:
+            if len(tds) < 2:
                 logger.warning("No 'td' tag found in line[]='%s' - SKIPPED!", type(line))
                 continue
 

diff --git a/fba/utils.py b/fba/utils.py
index 78c5ffa3b4e2e9d9f51e189b46d04956b8ed3907..a28a965993a385da1995d57b8db7a1068d5f970d 100644 (file)
--- a/fba/utils.py
+++ b/fba/utils.py
@@ -51,7 +51,7 @@ def find_domains(tags: bs4.element.ResultSet, search: str) -> list:
     elif not isinstance(search, str):
         raise TypeError(f"Parameter search[]='{type(search)}' has not expected type 'str'")
     elif search == "":
-        raise ValueError("Parameter 'search' is empty")
+        raise ValueError("Parameter 'search' is an empty string")
 
     domains = []
     logger.debug("Parsing %d tags ...", len(tags))
@@ -112,11 +112,11 @@ def add_all_to_list(domains: list, source: str, splitter: str) -> None:
     elif not isinstance(source, str):
         raise TypeError(f"Parameter source[]='{type(source)}' is not type 'list'")
     elif source == "":
-        raise ValueError("Parameter 'source' is empty")
+        raise ValueError("Parameter 'source' is an empty string")
     elif not isinstance(splitter, str):
         raise TypeError(f"Parameter splitter[]='{type(splitter)}' is not type 'list'")
     elif splitter == "":
-        raise ValueError("Parameter 'splitter' is empty")
+        raise ValueError("Parameter 'splitter' is an empty string")
 
     for domain in source.split(splitter):
         logger.debug("domain='%s' - LOOP!", domain)
@@ -139,7 +139,7 @@ def deobfuscate(domain: str, blocker: str, domain_hash: str = None) -> str:
     elif not isinstance(domain_hash, str) and domain_hash is not None:
         raise TypeError(f"Parameter domain_hash[]='{type(domain_hash)}' has not expected type 'str'")
     elif domain_hash == "":
-        raise ValueError("Parameter 'domain_hash' is empty")
+        raise ValueError("Parameter 'domain_hash' is an empty string")
 
     logger.debug("Checking domain='%s' ...", domain)
     if domain.find("*") >= 0: