Conntinued:

- added more checks on domain names

diff --git a/fba/csrf.py b/fba/csrf.py
index 5158ca5501bef9a7d691e8da0b447f2a7e5af311..b0cd258ed62d701511222d10b53ff02272c95650 100644 (file)
--- a/fba/csrf.py
+++ b/fba/csrf.py
@@ -30,6 +30,8 @@ def determine(domain: str, headers: dict) -> dict:
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
+    elif domain.lower() != domain:
+        raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
     elif not validators.domain(domain.split("/")[0]):
         raise ValueError(f"domain='{domain}' is not a valid domain")
     elif domain.endswith(".arpa"):

diff --git a/fba/fba.py b/fba/fba.py
index e945ffdb56930487a614a47904ba12a6552ab152..b31a99d1cce4ad7cf04aa0b2288b6ed61db423c3 100644 (file)
--- a/fba/fba.py
+++ b/fba/fba.py
@@ -45,6 +45,14 @@ def get_hash(domain: str) -> str:
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
+    elif domain.lower() != domain:
+        raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
+    elif not validators.domain(domain.split("/")[0]):
+        raise ValueError(f"domain='{domain}' is not a valid domain")
+    elif domain.endswith(".arpa"):
+        raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!")
+    elif domain.endswith(".tld"):
+        raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
 
     return hashlib.sha256(domain.encode("utf-8")).hexdigest()
 
@@ -78,6 +86,14 @@ def process_domain(domain: str, blocker: str, command: str) -> bool:
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
+    elif domain.lower() != domain:
+        raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
+    elif not validators.domain(domain.split("/")[0]):
+        raise ValueError(f"domain='{domain}' is not a valid domain")
+    elif domain.endswith(".arpa"):
+        raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!")
+    elif domain.endswith(".tld"):
+        raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
     elif not isinstance(blocker, str):
         raise ValueError(f"Parameter blocker[]='{type(blocker)}' is not 'str'")
     elif blocker == "":

diff --git a/fba/helpers/blacklist.py b/fba/helpers/blacklist.py
index bbe097b1f8a6d5fd0f45e33cdb086c5dc8bf7a49..3961d7d5dd432d66f7b5c93bb36306acfea42840 100644 (file)
--- a/fba/helpers/blacklist.py
+++ b/fba/helpers/blacklist.py
@@ -14,6 +14,8 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
+import validators
+
 # Don't check these, known trolls/flooders/testing/developing
 blacklist = [
     # Floods federation with fake nodes as "research" project
@@ -40,6 +42,14 @@ def is_blacklisted(domain: str) -> bool:
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
+    elif domain.lower() != domain:
+        raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
+    elif not validators.domain(domain.split("/")[0]):
+        raise ValueError(f"domain='{domain}' is not a valid domain")
+    elif domain.endswith(".arpa"):
+        raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!")
+    elif domain.endswith(".tld"):
+        raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
 
     blacklisted = False
     for peer in blacklist:

diff --git a/fba/helpers/cookies.py b/fba/helpers/cookies.py
index 2ed3f1d7a79763a07496ebb4280c9a4f894af77e..09d8034106e57818414e223cc41738f2db00eb69 100644 (file)
--- a/fba/helpers/cookies.py
+++ b/fba/helpers/cookies.py
@@ -14,6 +14,8 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
+import validators
+
 # Cookies stor
 _cookies = {}
 
@@ -23,6 +25,14 @@ def store (domain: str, cookies: dict):
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
+    elif domain.lower() != domain:
+        raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
+    elif not validators.domain(domain.split("/")[0]):
+        raise ValueError(f"domain='{domain}' is not a valid domain")
+    elif domain.endswith(".arpa"):
+        raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!")
+    elif domain.endswith(".tld"):
+        raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
     elif not isinstance(cookies, dict):
         raise ValueError(f"Parameter cookies[]='{type(cookies)}' is not 'dict'")
 
@@ -36,6 +46,14 @@ def get_all(domain: str) -> dict:
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
+    elif domain.lower() != domain:
+        raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
+    elif not validators.domain(domain.split("/")[0]):
+        raise ValueError(f"domain='{domain}' is not a valid domain")
+    elif domain.endswith(".arpa"):
+        raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!")
+    elif domain.endswith(".tld"):
+        raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
     elif not has(domain):
         raise Exception(f"domain='{domain}' has no cookies stored, maybe invoke store() first?")
 
@@ -48,6 +66,8 @@ def has (domain: str) -> bool:
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
+    elif domain.lower() != domain:
+        raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
 
     has_cookies = domain in _cookies
 
@@ -60,6 +80,14 @@ def clear (domain: str):
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
+    elif domain.lower() != domain:
+        raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
+    elif not validators.domain(domain.split("/")[0]):
+        raise ValueError(f"domain='{domain}' is not a valid domain")
+    elif domain.endswith(".arpa"):
+        raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!")
+    elif domain.endswith(".tld"):
+        raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
 
     if has(domain):
         # DEBUG: print(f"DEBUG: Removing cookies for domain='{domain}' ...")

diff --git a/fba/http/federation.py b/fba/http/federation.py
index 054fc3f163bf21645f825155e1fa932b2037ee1f..28b04a26de065d372695565ad81978120396b5cc 100644 (file)
--- a/fba/http/federation.py
+++ b/fba/http/federation.py
@@ -51,6 +51,8 @@ def fetch_instances(domain: str, origin: str, software: str, command: str, path:
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
+    elif domain.lower() != domain:
+        raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
     elif not validators.domain(domain.split("/")[0]):
         raise ValueError(f"domain='{domain}' is not a valid domain")
     elif domain.endswith(".arpa"):
@@ -143,6 +145,8 @@ def fetch_peers(domain: str, software: str) -> list:
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
+    elif domain.lower() != domain:
+        raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
     elif not validators.domain(domain.split("/")[0]):
         raise ValueError(f"domain='{domain}' is not a valid domain")
     elif domain.endswith(".arpa"):
@@ -223,6 +227,8 @@ def fetch_nodeinfo(domain: str, path: str = None) -> dict:
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
+    elif domain.lower() != domain:
+        raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
     elif not validators.domain(domain.split("/")[0]):
         raise ValueError(f"domain='{domain}' is not a valid domain")
     elif domain.endswith(".arpa"):
@@ -299,6 +305,8 @@ def fetch_wellknown_nodeinfo(domain: str) -> dict:
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
+    elif domain.lower() != domain:
+        raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
     elif not validators.domain(domain.split("/")[0]):
         raise ValueError(f"domain='{domain}' is not a valid domain")
     elif domain.endswith(".arpa"):
@@ -384,6 +392,8 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str:
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
+    elif domain.lower() != domain:
+        raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
     elif not validators.domain(domain.split("/")[0]):
         raise ValueError(f"domain='{domain}' is not a valid domain")
     elif domain.endswith(".arpa"):
@@ -457,6 +467,14 @@ def determine_software(domain: str, path: str = None) -> str:
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
+    elif domain.lower() != domain:
+        raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
+    elif not validators.domain(domain.split("/")[0]):
+        raise ValueError(f"domain='{domain}' is not a valid domain")
+    elif domain.endswith(".arpa"):
+        raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!")
+    elif domain.endswith(".tld"):
+        raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
     elif not isinstance(path, str) and path is not None:
         raise ValueError(f"Parameter path[]='{type(path)}' is not 'str'")
 

diff --git a/fba/http/network.py b/fba/http/network.py
index 33ae3c69e0efd6f4a63902532c2df648065ec97f..a8c62f0814845e25c9684ccf71c1f3b85f7f11d4 100644 (file)
--- a/fba/http/network.py
+++ b/fba/http/network.py
@@ -56,6 +56,8 @@ def post_json_api(domain: str, path: str, data: str = "", headers: dict = {}) ->
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
+    elif domain.lower() != domain:
+        raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
     elif not validators.domain(domain.split("/")[0]):
         raise ValueError(f"domain='{domain}' is not a valid domain")
     elif domain.endswith(".arpa"):
@@ -146,6 +148,8 @@ def get_json_api(domain: str, path: str, headers: dict, timeout: tuple) -> dict:
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
+    elif domain.lower() != domain:
+        raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
     elif not validators.domain(domain.split("/")[0]):
         raise ValueError(f"domain='{domain}' is not a valid domain")
     elif domain.endswith(".arpa"):
@@ -201,6 +205,8 @@ def send_bot_post(domain: str, blocklist: dict):
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
+    elif domain.lower() != domain:
+        raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
     elif not validators.domain(domain.split("/")[0]):
         raise ValueError(f"domain='{domain}' is not a valid domain")
     elif domain.endswith(".arpa"):
@@ -252,6 +258,8 @@ def fetch_response(domain: str, path: str, headers: dict, timeout: tuple) -> req
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
+    elif domain.lower() != domain:
+        raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
     elif not validators.domain(domain.split("/")[0]):
         raise ValueError(f"domain='{domain}' is not a valid domain")
     elif domain.endswith(".arpa"):

diff --git a/fba/models/blocks.py b/fba/models/blocks.py
index 0858d523b7591a0d014c94b91b9af88b84c33eca..7e67979ab728fa304cedce869bdca4019ffdba0d 100644 (file)
--- a/fba/models/blocks.py
+++ b/fba/models/blocks.py
@@ -30,10 +30,14 @@ def update_reason(reason: str, blocker: str, blocked: str, block_level: str):
         raise ValueError(f"Parameter blocker[]='{type(blocker)}' is not 'str'")
     elif blocker == "":
         raise ValueError("Parameter 'blocker' is empty")
+    elif blocker.lower() != blocker:
+        raise ValueError(f"Parameter blocker='{blocker}' must be all lower-case")
     elif not isinstance(blocked, str):
         raise ValueError(f"Parameter blocked[]='{type(blocked)}' is not 'str'")
     elif blocked == "":
         raise ValueError("Parameter 'blocked' is empty")
+    elif blocked.lower() != blocked:
+        raise ValueError(f"Parameter blocked='{blocked}' must be all lower-case")
     elif not isinstance(block_level, str):
         raise ValueError(f"Parameter block_level[]='{type(block_level)}' is not 'str'")
     elif block_level == "":
@@ -60,10 +64,14 @@ def update_last_seen(blocker: str, blocked: str, block_level: str):
         raise ValueError(f"Parameter blocker[]='{type(blocker)}' is not 'str'")
     elif blocker == "":
         raise ValueError("Parameter 'blocker' is empty")
+    elif blocker.lower() != blocker:
+        raise ValueError(f"Parameter blocker='{blocker}' must be all lower-case")
     elif not isinstance(blocked, str):
         raise ValueError(f"Parameter blocked[]='{type(blocked)}' is not 'str'")
     elif blocked == "":
         raise ValueError("Parameter 'blocked' is empty")
+    elif blocked.lower() != blocked:
+        raise ValueError(f"Parameter blocked='{blocked}' must be all lower-case")
     elif not isinstance(block_level, str):
         raise ValueError(f"Parameter block_level[]='{type(block_level)}' is not 'str'")
     elif block_level == "":
@@ -88,10 +96,14 @@ def is_instance_blocked(blocker: str, blocked: str, block_level: str) -> bool:
         raise ValueError(f"Parameter blocker[]='{type(blocker)}' is not of type 'str'")
     elif blocker == "":
         raise ValueError("Parameter 'blocker' is empty")
+    elif blocker.lower() != blocker:
+        raise ValueError(f"Parameter blocker='{blocker}' must be all lower-case")
     elif not isinstance(blocked, str):
         raise ValueError(f"Parameter blocked[]='{type(blocked)}' is not of type 'str'")
     elif blocked == "":
         raise ValueError("Parameter 'blocked' is empty")
+    elif blocked.lower() != blocked:
+        raise ValueError(f"Parameter blocked='{blocked}' must be all lower-case")
     elif not isinstance(block_level, str):
         raise ValueError(f"Parameter block_level[]='{type(block_level)}' is not of type 'str'")
     elif block_level == "":
@@ -119,12 +131,16 @@ def add_instance(blocker: str, blocked: str, reason: str, block_level: str):
         raise ValueError(f"Parameter blocker[]='{type(blocker)}' is not 'str'")
     elif blocker == "":
         raise ValueError("Parameter 'blocker' is empty")
+    elif blocker.lower() != blocker:
+        raise ValueError(f"Parameter blocker='{blocker}' must be all lower-case")
     elif not validators.domain(blocker.split("/")[0]):
         raise ValueError(f"Bad blocker='{blocker}'")
     elif not isinstance(blocked, str):
         raise ValueError(f"Parameter blocked[]='{type(blocked)}' is not 'str'")
     elif blocked == "":
         raise ValueError("Parameter 'blocked' is empty")
+    elif blocked.lower() != blocked:
+        raise ValueError(f"Parameter blocked='{blocked}' must be all lower-case")
     elif not isinstance(block_level, str):
         raise ValueError(f"Parameter block_level[]='{type(block_level)}' is not of type 'str'")
     elif block_level == "":

diff --git a/fba/models/error_log.py b/fba/models/error_log.py
index 68c5407a1b2644329296ad883285fb29d4ce3c15..b6844082827ec532bd6c682dd29d1a2eb1bd5773 100644 (file)
--- a/fba/models/error_log.py
+++ b/fba/models/error_log.py
@@ -16,6 +16,8 @@
 import json
 import time
 
+import validators
+
 from fba import fba
 
 from fba.helpers import config
@@ -26,6 +28,14 @@ def add(domain: str, error: dict):
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
+    elif domain.lower() != domain:
+        raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
+    elif not validators.domain(domain.split("/")[0]):
+        raise ValueError(f"domain='{domain}' is not a valid domain")
+    elif domain.endswith(".arpa"):
+        raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!")
+    elif domain.endswith(".tld"):
+        raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
     elif config.get("write_error_log").lower() != "true":
         # DEBUG: print(f"DEBUG: Writing to error_log is disabled in configuruation file - EXIT!")
         return

diff --git a/fba/models/instances.py b/fba/models/instances.py
index 571475f9fc9e63bad22d4bbc1b86242369ad87a1..0be35ddda09c9f2ef1c75d1ef80f275f3c5096a8 100644 (file)
--- a/fba/models/instances.py
+++ b/fba/models/instances.py
@@ -66,6 +66,14 @@ def _set_data(key: str, domain: str, value: any):
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
+    elif domain.lower() != domain:
+        raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
+    elif not validators.domain(domain.split("/")[0]):
+        raise ValueError(f"domain='{domain}' is not a valid domain")
+    elif domain.endswith(".arpa"):
+        raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!")
+    elif domain.endswith(".tld"):
+        raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
     elif not key in _pending:
         raise ValueError(f"key='{key}' not found in _pending")
     elif not fba.is_primitive(value):
@@ -82,6 +90,14 @@ def has_pending(domain: str) -> bool:
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
+    elif domain.lower() != domain:
+        raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
+    elif not validators.domain(domain.split("/")[0]):
+        raise ValueError(f"domain='{domain}' is not a valid domain")
+    elif domain.endswith(".arpa"):
+        raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!")
+    elif domain.endswith(".tld"):
+        raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
 
     has = False
     for key in _pending:
@@ -99,6 +115,14 @@ def update_data(domain: str):
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
+    elif domain.lower() != domain:
+        raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
+    elif not validators.domain(domain.split("/")[0]):
+        raise ValueError(f"domain='{domain}' is not a valid domain")
+    elif domain.endswith(".arpa"):
+        raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!")
+    elif domain.endswith(".tld"):
+        raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
     elif not has_pending(domain):
         raise Exception(f"domain='{domain}' has no pending instance data, but function invoked")
     elif not is_registered(domain):
@@ -152,6 +176,14 @@ def add(domain: str, origin: str, command: str, path: str = None, software: str
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
+    elif domain.lower() != domain:
+        raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
+    elif not validators.domain(domain.split("/")[0]):
+        raise ValueError(f"domain='{domain}' is not a valid domain")
+    elif domain.endswith(".arpa"):
+        raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!")
+    elif domain.endswith(".tld"):
+        raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
     elif not isinstance(origin, str) and origin is not None:
         raise ValueError(f"origin[]='{type(origin)}' is not 'str'")
     elif origin == "":
@@ -224,6 +256,14 @@ def set_last_nodeinfo(domain: str):
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
+    elif domain.lower() != domain:
+        raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
+    elif not validators.domain(domain.split("/")[0]):
+        raise ValueError(f"domain='{domain}' is not a valid domain")
+    elif domain.endswith(".arpa"):
+        raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!")
+    elif domain.endswith(".tld"):
+        raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
 
     # DEBUG: print("DEBUG: Updating last_nodeinfo for domain:", domain)
     _set_data("last_nodeinfo", domain, time.time())
@@ -240,6 +280,14 @@ def set_last_error(domain: str, error: dict):
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
+    elif domain.lower() != domain:
+        raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
+    elif not validators.domain(domain.split("/")[0]):
+        raise ValueError(f"domain='{domain}' is not a valid domain")
+    elif domain.endswith(".arpa"):
+        raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!")
+    elif domain.endswith(".tld"):
+        raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
 
     # DEBUG: print("DEBUG: BEFORE error[]:", type(error))
     if isinstance(error, (BaseException, json.decoder.JSONDecodeError)):
@@ -275,6 +323,14 @@ def is_registered(domain: str) -> bool:
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
+    elif domain.lower() != domain:
+        raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
+    elif not validators.domain(domain.split("/")[0]):
+        raise ValueError(f"domain='{domain}' is not a valid domain")
+    elif domain.endswith(".arpa"):
+        raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!")
+    elif domain.endswith(".tld"):
+        raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
 
     # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
     if not cache.key_exists("is_registered"):
@@ -296,6 +352,14 @@ def is_recent(domain: str) -> bool:
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
+    elif domain.lower() != domain:
+        raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
+    elif not validators.domain(domain.split("/")[0]):
+        raise ValueError(f"domain='{domain}' is not a valid domain")
+    elif domain.endswith(".arpa"):
+        raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!")
+    elif domain.endswith(".tld"):
+        raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
     elif not is_registered(domain):
         # DEBUG: print(f"DEBUG: domain='{domain}' is not registered, returning False - EXIT!")
         return False
@@ -322,6 +386,14 @@ def deobscure(char: str, domain: str, blocked_hash: str = None) -> tuple:
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
+    elif domain.lower() != domain:
+        raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
+    elif not validators.domain(domain.split("/")[0]):
+        raise ValueError(f"domain='{domain}' is not a valid domain")
+    elif domain.endswith(".arpa"):
+        raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!")
+    elif domain.endswith(".tld"):
+        raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
     elif not isinstance(blocked_hash, str) and blocked_hash is not None:
         raise ValueError(f"Parameter blocked_hash[]='{type(blocked_hash)}' is not 'str'")
 
@@ -355,6 +427,14 @@ def set_last_blocked(domain: str):
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
+    elif domain.lower() != domain:
+        raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
+    elif not validators.domain(domain.split("/")[0]):
+        raise ValueError(f"domain='{domain}' is not a valid domain")
+    elif domain.endswith(".arpa"):
+        raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!")
+    elif domain.endswith(".tld"):
+        raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
 
     # Set timestamp
     _set_data("last_blocked", domain, time.time())
@@ -366,6 +446,14 @@ def set_last_instance_fetch(domain: str):
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
+    elif domain.lower() != domain:
+        raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
+    elif not validators.domain(domain.split("/")[0]):
+        raise ValueError(f"domain='{domain}' is not a valid domain")
+    elif domain.endswith(".arpa"):
+        raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!")
+    elif domain.endswith(".tld"):
+        raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
 
     # Set timestamp
     _set_data("last_instance_fetch", domain, time.time())
@@ -377,6 +465,14 @@ def set_total_peers(domain: str, peers: list):
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
+    elif domain.lower() != domain:
+        raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
+    elif not validators.domain(domain.split("/")[0]):
+        raise ValueError(f"domain='{domain}' is not a valid domain")
+    elif domain.endswith(".arpa"):
+        raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!")
+    elif domain.endswith(".tld"):
+        raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
     elif not isinstance(peers, list):
         raise ValueError(f"Parameter peers[]='{type(peers)}' is not 'list'")
 
@@ -390,6 +486,14 @@ def set_nodeinfo_url(domain: str, url: str):
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
+    elif domain.lower() != domain:
+        raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
+    elif not validators.domain(domain.split("/")[0]):
+        raise ValueError(f"domain='{domain}' is not a valid domain")
+    elif domain.endswith(".arpa"):
+        raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!")
+    elif domain.endswith(".tld"):
+        raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
     elif not isinstance(url, str):
         raise ValueError("Parameter url[]='{type(url)}' is not 'list'")
     elif url == "":
@@ -405,6 +509,14 @@ def set_detection_mode(domain: str, mode: str):
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
+    elif domain.lower() != domain:
+        raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
+    elif not validators.domain(domain.split("/")[0]):
+        raise ValueError(f"domain='{domain}' is not a valid domain")
+    elif domain.endswith(".arpa"):
+        raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!")
+    elif domain.endswith(".tld"):
+        raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
     elif not isinstance(mode, str):
         raise ValueError("Parameter mode[]='{type(mode)}' is not 'list'")
     elif mode == "":

diff --git a/fba/networks/friendica.py b/fba/networks/friendica.py
index ffce628cd65b6928ba63613d445a9b34f0f8c537..13f3583e6d7c51380fa294a0490a0e442f69a644 100644 (file)
--- a/fba/networks/friendica.py
+++ b/fba/networks/friendica.py
@@ -31,6 +31,14 @@ def fetch_blocks(domain: str) -> dict:
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
+    elif domain.lower() != domain:
+        raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
+    elif not validators.domain(domain.split("/")[0]):
+        raise ValueError(f"domain='{domain}' is not a valid domain")
+    elif domain.endswith(".arpa"):
+        raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!")
+    elif domain.endswith(".tld"):
+        raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
 
     blocklist = list()
     block_tag = None

diff --git a/fba/networks/lemmy.py b/fba/networks/lemmy.py
index 7dd8f1d83024d58431b8a6c8dde3ded4db212f09..3cd96d1fde64e0fb3db4d0b40c55b30759e07afa 100644 (file)
--- a/fba/networks/lemmy.py
+++ b/fba/networks/lemmy.py
@@ -38,6 +38,14 @@ def fetch_peers(domain: str) -> list:
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
+    elif domain.lower() != domain:
+        raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
+    elif not validators.domain(domain.split("/")[0]):
+        raise ValueError(f"domain='{domain}' is not a valid domain")
+    elif domain.endswith(".arpa"):
+        raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!")
+    elif domain.endswith(".tld"):
+        raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
 
     peers = list()
 
@@ -89,6 +97,14 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str):
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
+    elif domain.lower() != domain:
+        raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
+    elif not validators.domain(domain.split("/")[0]):
+        raise ValueError(f"domain='{domain}' is not a valid domain")
+    elif domain.endswith(".arpa"):
+        raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!")
+    elif domain.endswith(".tld"):
+        raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
     elif not isinstance(origin, str) and origin is not None:
         raise ValueError(f"Parameter origin[]='{type(origin)}' is not 'str'")
     elif origin == "":

diff --git a/fba/networks/mastodon.py b/fba/networks/mastodon.py
index a440c04cd97d2ecb3ef0f772dd5691384fcee823..a6fccd014181525f3f0acf36bb85afa5b1b1ab4f 100644 (file)
--- a/fba/networks/mastodon.py
+++ b/fba/networks/mastodon.py
@@ -63,6 +63,14 @@ def fetch_blocks_from_about(domain: str) -> dict:
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
+    elif domain.lower() != domain:
+        raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
+    elif not validators.domain(domain.split("/")[0]):
+        raise ValueError(f"domain='{domain}' is not a valid domain")
+    elif domain.endswith(".arpa"):
+        raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!")
+    elif domain.endswith(".tld"):
+        raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
 
     # DEBUG: print("DEBUG: Fetching mastodon blocks from domain:", domain)
     doc = None
@@ -134,6 +142,14 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str):
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
+    elif domain.lower() != domain:
+        raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
+    elif not validators.domain(domain.split("/")[0]):
+        raise ValueError(f"domain='{domain}' is not a valid domain")
+    elif domain.endswith(".arpa"):
+        raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!")
+    elif domain.endswith(".tld"):
+        raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
     elif not isinstance(origin, str) and origin is not None:
         raise ValueError(f"Parameter origin[]='{type(origin)}' is not 'str'")
     elif origin == "":

diff --git a/fba/networks/misskey.py b/fba/networks/misskey.py
index 47a3de012440aadbdbdbd2fde0076bdc5a970c67..2cede8b704b505f226d0c150aad7a0b53e4d3cce 100644 (file)
--- a/fba/networks/misskey.py
+++ b/fba/networks/misskey.py
@@ -15,6 +15,7 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 import json
+import validators
 
 from fba import csrf
 
@@ -33,6 +34,14 @@ def fetch_peers(domain: str) -> list:
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
+    elif domain.lower() != domain:
+        raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
+    elif not validators.domain(domain.split("/")[0]):
+        raise ValueError(f"domain='{domain}' is not a valid domain")
+    elif domain.endswith(".arpa"):
+        raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!")
+    elif domain.endswith(".tld"):
+        raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
 
     # DEBUG: print(f"DEBUG: domain='{domain}' is misskey, sending API POST request ...")
     peers   = list()
@@ -130,6 +139,14 @@ def fetch_blocks(domain: str) -> dict:
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
+    elif domain.lower() != domain:
+        raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
+    elif not validators.domain(domain.split("/")[0]):
+        raise ValueError(f"domain='{domain}' is not a valid domain")
+    elif domain.endswith(".arpa"):
+        raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!")
+    elif domain.endswith(".tld"):
+        raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
 
     # DEBUG: print(f"DEBUG: Fetching misskey blocks from domain='{domain}'")
     blocklist = {

diff --git a/fba/networks/peertube.py b/fba/networks/peertube.py
index b9764c5104eb34cb21dfa0183395e57cb5eb6f23..a43315ff2a2c3f9824e5bc5f18505f8c85f040d7 100644 (file)
--- a/fba/networks/peertube.py
+++ b/fba/networks/peertube.py
@@ -14,6 +14,8 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
+import validators
+
 from fba import csrf
 
 from fba.helpers import config
@@ -28,6 +30,14 @@ def fetch_peers(domain: str) -> list:
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
+    elif domain.lower() != domain:
+        raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
+    elif not validators.domain(domain.split("/")[0]):
+        raise ValueError(f"domain='{domain}' is not a valid domain")
+    elif domain.endswith(".arpa"):
+        raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!")
+    elif domain.endswith(".tld"):
+        raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
 
     # DEBUG: print(f"DEBUG: domain='{domain}' is a PeerTube, fetching JSON ...")
     peers   = list()

diff --git a/fba/networks/pleroma.py b/fba/networks/pleroma.py
index 9d6dab0c6d3ecdd7d68196cb0755c3c690fad49e..8ba2f65e38bc0b90f866a0922291c2296749dba5 100644 (file)
--- a/fba/networks/pleroma.py
+++ b/fba/networks/pleroma.py
@@ -42,6 +42,14 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str):
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
+    elif domain.lower() != domain:
+        raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
+    elif not validators.domain(domain.split("/")[0]):
+        raise ValueError(f"domain='{domain}' is not a valid domain")
+    elif domain.endswith(".arpa"):
+        raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!")
+    elif domain.endswith(".tld"):
+        raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
     elif not isinstance(origin, str) and origin is not None:
         raise ValueError(f"Parameter origin[]='{type(origin)}' is not 'str'")
     elif origin == "":
@@ -515,6 +523,14 @@ def fetch_blocks_from_about(domain: str) -> dict:
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
+    elif domain.lower() != domain:
+        raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
+    elif not validators.domain(domain.split("/")[0]):
+        raise ValueError(f"domain='{domain}' is not a valid domain")
+    elif domain.endswith(".arpa"):
+        raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!")
+    elif domain.endswith(".tld"):
+        raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
 
     # DEBUG: print(f"DEBUG: Fetching mastodon blocks from domain='{domain}'")
     doc = None