Continued:

- Parameter 'path' need to start with a slash
- checked more against blacklist
- logger message after API returned JSON made similar

diff --git a/fba/commands.py b/fba/commands.py
index c85f5feb915934a595989075b8b86f6415bd6fff..3b890ad428a7fbae0c4bfed1c70077fd4f0e0f99 100644 (file)
--- a/fba/commands.py
+++ b/fba/commands.py
@@ -133,7 +133,7 @@ def fetch_pixelfed_api(args: argparse.Namespace) -> int:
             (config.get("connection_timeout"), config.get("read_timeout"))
         )
 
-        logger.debug("JSON API returned %d elements", len(fetched))
+        logger.debug("fetched(%d)[]='%s'", len(fetched), type(fetched))
         if "error_message" in fetched:
             logger.warning("API returned error_message='%s' - EXIT!", fetched["error_message"])
             return 101
@@ -1559,7 +1559,7 @@ def fetch_instances_social(args: argparse.Namespace) -> int:
         headers=headers,
         timeout=(config.get("connection_timeout"), config.get("read_timeout"))
     )
-    logger.debug("fetched[]='%s'", type(fetched))
+    logger.debug("fetched(%d)[]='%s'", len(fetched), type(fetched))
 
     if "error_message" in fetched:
         logger.warning("Error during fetching API result: '%s' - EXIT!", fetched["error_message"])
@@ -1632,7 +1632,7 @@ def fetch_relaylist(args: argparse.Namespace) -> int:
         {},
         (config.get("connection_timeout"), config.get("read_timeout"))
     )
-    logger.debug("fetched[]='%s'", type(fetched))
+    logger.debug("fetched(%d)[]='%s'", len(fetched), type(fetched))
 
     if "error_message" in fetched:
         logger.warning("Error during fetching API result: '%s' - EXIT!", fetched["error_message"])

diff --git a/fba/http/federation.py b/fba/http/federation.py
index fdb18285a5bf91fea0948a32c82a34d70a6ee3af..0523938e8fe8f30c96866f894fa7bc12d43ea633 100644 (file)
--- a/fba/http/federation.py
+++ b/fba/http/federation.py
@@ -21,6 +21,7 @@ import bs4
 import requests
 import validators
 
+from fba.helpers import blacklist
 from fba.helpers import config
 from fba.helpers import cookies
 from fba.helpers import domain as domain_helper
@@ -56,7 +57,9 @@ def fetch_instances(domain: str, origin: str, software: str, command: str, path:
     logger.debug("domain='%s',origin='%s',software='%s',command='%s',path='%s',_DEPTH=%d - CALLED!", domain, origin, software, command, path, _DEPTH)
     domain_helper.raise_on(domain)
 
-    if not isinstance(origin, str) and origin is not None:
+    if blacklist.is_blacklisted(domain):
+        raise Exception(f"domain='{domain}' is blacklisted but function was invoked")
+    elif not isinstance(origin, str) and origin is not None:
         raise ValueError(f"Parameter origin[]='{type(origin)}' is not of type 'str'")
     elif not isinstance(command, str):
         raise ValueError(f"Parameter command[]='{type(command)}' is not of type 'str'")
@@ -190,7 +193,9 @@ def fetch_peers(domain: str, software: str, origin: str) -> list:
     logger.debug("domain='%s',software='%s',origin='%s' - CALLED!", domain, software, origin)
     domain_helper.raise_on(domain)
 
-    if not isinstance(software, str) and software is not None:
+    if blacklist.is_blacklisted(domain):
+        raise Exception(f"domain='{domain}' is blacklisted but function was invoked")
+    elif not isinstance(software, str) and software is not None:
         raise ValueError(f"Parameter software[]='{type(software)}' is not of type 'str'")
     elif isinstance(software, str) and software == "":
         raise ValueError("Parameter 'software' is empty")
@@ -237,7 +242,7 @@ def fetch_peers(domain: str, software: str, origin: str) -> list:
             timeout=(config.get("connection_timeout"), config.get("read_timeout"))
         )
 
-        logger.debug("data[]='%s'", type(data))
+        logger.debug("data(%d)[]='%s'", len(data), type(data))
         if "error_message" in data:
             logger.debug("Was not able to fetch peers from path='%s',domain='%s' ...", path, domain)
             instances.set_last_error(domain, data)
@@ -263,10 +268,14 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str:
     logger.debug("domain='%s',path='%s' - CALLED!", domain, path)
     domain_helper.raise_on(domain)
 
-    if not isinstance(path, str):
+    if blacklist.is_blacklisted(domain):
+        raise Exception(f"domain='{domain}' is blacklisted but function was invoked")
+    elif not isinstance(path, str):
         raise ValueError(f"path[]='{type(path)}' is not of type 'str'")
     elif path == "":
         raise ValueError("Parameter 'path' is empty")
+    elif not path.startswith("/"):
+        raise ValueError("path='{path}' does not start with / but should")
 
     software = None
 
@@ -374,7 +383,9 @@ def determine_software(domain: str, path: str = None) -> str:
     logger.debug("domain='%s',path='%s' - CALLED!", domain, path)
     domain_helper.raise_on(domain)
 
-    if not isinstance(path, str) and path is not None:
+    if blacklist.is_blacklisted(domain):
+        raise Exception(f"domain='{domain}' is blacklisted but function was invoked")
+    elif not isinstance(path, str) and path is not None:
         raise ValueError(f"Parameter path[]='{type(path)}' is not of type 'str'")
 
     logger.debug("Fetching nodeinfo from domain='%s',path='%s' ...", domain, path)
@@ -551,6 +562,8 @@ def fetch_blocks(domain: str) -> list:
 
     if not instances.is_registered(domain):
         raise Exception(f"domain='{domain}' is not registered but function is invoked.")
+    elif blacklist.is_blacklisted(domain):
+        raise Exception(f"domain='{domain}' is blacklisted but function was invoked")
 
     # Init block list
     blocklist = list()
@@ -579,7 +592,7 @@ def fetch_blocks(domain: str) -> list:
         )
         rows = list()
 
-        logger.debug("data[]='%s'", type(data))
+        logger.debug("data(%d)[]='%s'", len(data), type(data))
         if "error_message" in data:
             logger.debug("Was not able to fetch domain_blocks from domain='%s': status_code=%d,error_message='%s'", domain, data['status_code'], data['error_message'])
             instances.set_last_error(domain, data)
@@ -599,7 +612,7 @@ def fetch_blocks(domain: str) -> list:
             logger.debug("Marking domain='%s' as successfully handled ...", domain)
             instances.set_success(domain)
 
-        logger.debug("rows[%s]()=%d", type(rows), len(rows))
+        logger.debug("rows(%d)[]='%s'", len(rows), type(rows))
         if len(rows) > 0:
             logger.debug("Checking %d entries from domain='%s' ...", len(rows), domain)
             for block in rows:
@@ -623,7 +636,7 @@ def fetch_blocks(domain: str) -> list:
 
                 reason = tidyup.reason(block["comment"]) if "comment" in block and block["comment"] is not None and block["comment"] != "" else None
 
-                logger.debug("Appending blocker='%s',blocked='%s',reason='%s',block_level='%s'", domain, block["domain"], reason, block["severity"])
+                logger.debug("Appending blocker='%s',blocked='%s',reason='%s',block_level='%s' ...", domain, block["domain"], reason, block["severity"])
                 blocklist.append({
                     "blocker"    : domain,
                     "blocked"    : block["domain"],

diff --git a/fba/http/network.py b/fba/http/network.py
index 91397f971e4cdb4fc0c01f79e530eeb46a048e04..26fa12667263e8c1e7047b1ca3beec457363450f 100644 (file)
--- a/fba/http/network.py
+++ b/fba/http/network.py
@@ -65,6 +65,8 @@ def post_json_api(domain: str, path: str, data: str = "", headers: dict = dict()
         raise ValueError(f"path[]='{type(path)}' is not of type 'str'")
     elif path == "":
         raise ValueError("Parameter 'path' is empty")
+    elif not path.startswith("/"):
+        raise ValueError("path='{path}' does not start with / but should")
     elif not isinstance(data, str):
         raise ValueError(f"data[]='{type(data)}' is not of type 'str'")
     elif not isinstance(headers, dict):
@@ -160,6 +162,8 @@ def get_json_api(domain: str, path: str, headers: dict, timeout: tuple) -> dict:
         raise ValueError(f"path[]='{type(path)}' is not of type 'str'")
     elif path == "":
         raise ValueError("Parameter 'path' is empty")
+    elif not path.startswith("/"):
+        raise ValueError("path='{path}' does not start with / but should")
     elif not isinstance(headers, dict):
         raise ValueError(f"headers[]='{type(headers)}' is not of type 'list'")
     elif not isinstance(timeout, tuple):
@@ -184,6 +188,7 @@ def get_json_api(domain: str, path: str, headers: dict, timeout: tuple) -> dict:
     if response.ok and response.status_code == 200:
         logger.debug("Parsing JSON response from domain='%s',path='%s' ...", domain, path)
         json_reply["json"] = json_helper.from_response(response)
+        logger.debug("json_reply[json][]='%s'", type(json_reply["json"]))
 
     logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text))
     if not response.ok or response.status_code > 200 or len(response.text) == 0:
@@ -253,6 +258,8 @@ def fetch_response(domain: str, path: str, headers: dict, timeout: tuple, allow_
         raise ValueError(f"Parameter path[]='{type(path)}' is not of type 'str'")
     elif path == "":
         raise ValueError("Parameter 'path' is empty")
+    elif not path.startswith("/"):
+        raise ValueError("path='{path}' does not start with / but should")
     elif not isinstance(headers, dict):
         raise ValueError(f"headers[]='{type(headers)}' is not of type 'dict'")
     elif not isinstance(timeout, tuple):
@@ -260,7 +267,7 @@ def fetch_response(domain: str, path: str, headers: dict, timeout: tuple, allow_
 
     start = 0
     try:
-        logger.debug("Sending GET request to '%s%s' ...", domain, path)
+        logger.debug("Sending GET request to 'https://%s%s' ...", domain, path)
         start = time.perf_counter()
         response = reqto.get(
             f"https://{domain}{path}",

diff --git a/fba/http/nodeinfo.py b/fba/http/nodeinfo.py
index a0418dca471d7faf90ef33ef0943fb4e9150f3f3..34f912e499ae8664f62f6ab76ce53fea84408d26 100644 (file)
--- a/fba/http/nodeinfo.py
+++ b/fba/http/nodeinfo.py
@@ -17,6 +17,7 @@ import logging
 
 from urllib.parse import urlparse
 
+from fba.helpers import blacklist
 from fba.helpers import config
 from fba.helpers import domain as domain_helper
 
@@ -55,7 +56,9 @@ def fetch(domain: str, path: str = None, update_mode: bool = True) -> dict:
     logger.debug("domain='%s',path='%s',update_mode='%s' - CALLED!", domain, path, update_mode)
     domain_helper.raise_on(domain)
 
-    if not isinstance(path, str) and path is not None:
+    if blacklist.is_blacklisted(domain):
+        raise Exception(f"domain='{domain}' is blacklisted but function was invoked")
+    elif not isinstance(path, str) and path is not None:
         raise ValueError(f"Parameter path[]='{type(path)}' is not of type 'str'")
     elif not isinstance(update_mode, bool) and update_mode is not None:
         raise ValueError(f"Parameter update_mode[]='{type(update_mode)}' is not of type 'bool'")
@@ -108,7 +111,7 @@ def fetch(domain: str, path: str = None, update_mode: bool = True) -> dict:
                 timeout=(config.get("nodeinfo_connection_timeout"), config.get("nodeinfo_read_timeout"))
             )
 
-            logger.debug("data[]='%s'", type(data))
+            logger.debug("data(%d)[]='%s'", len(data), type(data))
             if "error_message" not in data and "json" in data:
                 logger.debug("Updating last_nodeinfo for domain='%s' ....", domain)
                 instances.set_last_nodeinfo(domain)
@@ -133,6 +136,9 @@ def fetch_wellknown_nodeinfo(domain: str) -> dict:
     logger.debug("domain='%s' - CALLED!", domain)
     domain_helper.raise_on(domain)
 
+    if blacklist.is_blacklisted(domain):
+        raise Exception(f"domain='{domain}' is blacklisted but function was invoked")
+
     # No CSRF by default, you don't have to add network.api_headers by yourself here
     headers = tuple()
 
@@ -159,8 +165,8 @@ def fetch_wellknown_nodeinfo(domain: str) -> dict:
             headers,
             (config.get("nodeinfo_connection_timeout"), config.get("nodeinfo_read_timeout"))
         )
-        logger.debug("data[]='%s'", type(data))
 
+        logger.debug("data(%d)[]='%s'", len(data), type(data))
         if "error_message" not in data and "json" in data and len(data["json"]) > 0:
             logger.debug("path='%s' returned valid json()=%d - BREAK!", path, len(data["json"]))
             break

diff --git a/fba/models/instances.py b/fba/models/instances.py
index 69684d902fdf099ab07ebf9a96e1d9ff4ff965e9..91af96946d4a6cc9c64bf6c31db72996b6828020 100644 (file)
--- a/fba/models/instances.py
+++ b/fba/models/instances.py
@@ -178,6 +178,8 @@ def add(domain: str, origin: str, command: str, path: str = None, software: str
         raise ValueError(f"path[]='{type(path)}' is not of type 'str'")
     elif path == "":
         raise ValueError("Parameter 'path' is empty")
+    elif not path.startswith("/"):
+        raise ValueError("path='{path}' does not start with / but should")
     elif not isinstance(software, str) and software is not None:
         raise ValueError(f"software[]='{type(software)}' is not of type 'str'")
     elif software == "":