From 89e2575c2c5691d46a7743c5ba5f50a7e5f1fa2f Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Tue, 28 Nov 2023 23:04:46 +0100 Subject: [PATCH] Continued: - Parameter 'path' need to start with a slash - checked more against blacklist - logger message after API returned JSON made similar --- fba/commands.py | 6 +++--- fba/http/federation.py | 29 +++++++++++++++++++++-------- fba/http/network.py | 9 ++++++++- fba/http/nodeinfo.py | 12 +++++++++--- fba/models/instances.py | 2 ++ 5 files changed, 43 insertions(+), 15 deletions(-) diff --git a/fba/commands.py b/fba/commands.py index c85f5fe..3b890ad 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -133,7 +133,7 @@ def fetch_pixelfed_api(args: argparse.Namespace) -> int: (config.get("connection_timeout"), config.get("read_timeout")) ) - logger.debug("JSON API returned %d elements", len(fetched)) + logger.debug("fetched(%d)[]='%s'", len(fetched), type(fetched)) if "error_message" in fetched: logger.warning("API returned error_message='%s' - EXIT!", fetched["error_message"]) return 101 @@ -1559,7 +1559,7 @@ def fetch_instances_social(args: argparse.Namespace) -> int: headers=headers, timeout=(config.get("connection_timeout"), config.get("read_timeout")) ) - logger.debug("fetched[]='%s'", type(fetched)) + logger.debug("fetched(%d)[]='%s'", len(fetched), type(fetched)) if "error_message" in fetched: logger.warning("Error during fetching API result: '%s' - EXIT!", fetched["error_message"]) @@ -1632,7 +1632,7 @@ def fetch_relaylist(args: argparse.Namespace) -> int: {}, (config.get("connection_timeout"), config.get("read_timeout")) ) - logger.debug("fetched[]='%s'", type(fetched)) + logger.debug("fetched(%d)[]='%s'", len(fetched), type(fetched)) if "error_message" in fetched: logger.warning("Error during fetching API result: '%s' - EXIT!", fetched["error_message"]) diff --git a/fba/http/federation.py b/fba/http/federation.py index fdb1828..0523938 100644 --- a/fba/http/federation.py +++ b/fba/http/federation.py @@ -21,6 +21,7 @@ import bs4 import requests import validators +from fba.helpers import blacklist from fba.helpers import config from fba.helpers import cookies from fba.helpers import domain as domain_helper @@ -56,7 +57,9 @@ def fetch_instances(domain: str, origin: str, software: str, command: str, path: logger.debug("domain='%s',origin='%s',software='%s',command='%s',path='%s',_DEPTH=%d - CALLED!", domain, origin, software, command, path, _DEPTH) domain_helper.raise_on(domain) - if not isinstance(origin, str) and origin is not None: + if blacklist.is_blacklisted(domain): + raise Exception(f"domain='{domain}' is blacklisted but function was invoked") + elif not isinstance(origin, str) and origin is not None: raise ValueError(f"Parameter origin[]='{type(origin)}' is not of type 'str'") elif not isinstance(command, str): raise ValueError(f"Parameter command[]='{type(command)}' is not of type 'str'") @@ -190,7 +193,9 @@ def fetch_peers(domain: str, software: str, origin: str) -> list: logger.debug("domain='%s',software='%s',origin='%s' - CALLED!", domain, software, origin) domain_helper.raise_on(domain) - if not isinstance(software, str) and software is not None: + if blacklist.is_blacklisted(domain): + raise Exception(f"domain='{domain}' is blacklisted but function was invoked") + elif not isinstance(software, str) and software is not None: raise ValueError(f"Parameter software[]='{type(software)}' is not of type 'str'") elif isinstance(software, str) and software == "": raise ValueError("Parameter 'software' is empty") @@ -237,7 +242,7 @@ def fetch_peers(domain: str, software: str, origin: str) -> list: timeout=(config.get("connection_timeout"), config.get("read_timeout")) ) - logger.debug("data[]='%s'", type(data)) + logger.debug("data(%d)[]='%s'", len(data), type(data)) if "error_message" in data: logger.debug("Was not able to fetch peers from path='%s',domain='%s' ...", path, domain) instances.set_last_error(domain, data) @@ -263,10 +268,14 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str: logger.debug("domain='%s',path='%s' - CALLED!", domain, path) domain_helper.raise_on(domain) - if not isinstance(path, str): + if blacklist.is_blacklisted(domain): + raise Exception(f"domain='{domain}' is blacklisted but function was invoked") + elif not isinstance(path, str): raise ValueError(f"path[]='{type(path)}' is not of type 'str'") elif path == "": raise ValueError("Parameter 'path' is empty") + elif not path.startswith("/"): + raise ValueError("path='{path}' does not start with / but should") software = None @@ -374,7 +383,9 @@ def determine_software(domain: str, path: str = None) -> str: logger.debug("domain='%s',path='%s' - CALLED!", domain, path) domain_helper.raise_on(domain) - if not isinstance(path, str) and path is not None: + if blacklist.is_blacklisted(domain): + raise Exception(f"domain='{domain}' is blacklisted but function was invoked") + elif not isinstance(path, str) and path is not None: raise ValueError(f"Parameter path[]='{type(path)}' is not of type 'str'") logger.debug("Fetching nodeinfo from domain='%s',path='%s' ...", domain, path) @@ -551,6 +562,8 @@ def fetch_blocks(domain: str) -> list: if not instances.is_registered(domain): raise Exception(f"domain='{domain}' is not registered but function is invoked.") + elif blacklist.is_blacklisted(domain): + raise Exception(f"domain='{domain}' is blacklisted but function was invoked") # Init block list blocklist = list() @@ -579,7 +592,7 @@ def fetch_blocks(domain: str) -> list: ) rows = list() - logger.debug("data[]='%s'", type(data)) + logger.debug("data(%d)[]='%s'", len(data), type(data)) if "error_message" in data: logger.debug("Was not able to fetch domain_blocks from domain='%s': status_code=%d,error_message='%s'", domain, data['status_code'], data['error_message']) instances.set_last_error(domain, data) @@ -599,7 +612,7 @@ def fetch_blocks(domain: str) -> list: logger.debug("Marking domain='%s' as successfully handled ...", domain) instances.set_success(domain) - logger.debug("rows[%s]()=%d", type(rows), len(rows)) + logger.debug("rows(%d)[]='%s'", len(rows), type(rows)) if len(rows) > 0: logger.debug("Checking %d entries from domain='%s' ...", len(rows), domain) for block in rows: @@ -623,7 +636,7 @@ def fetch_blocks(domain: str) -> list: reason = tidyup.reason(block["comment"]) if "comment" in block and block["comment"] is not None and block["comment"] != "" else None - logger.debug("Appending blocker='%s',blocked='%s',reason='%s',block_level='%s'", domain, block["domain"], reason, block["severity"]) + logger.debug("Appending blocker='%s',blocked='%s',reason='%s',block_level='%s' ...", domain, block["domain"], reason, block["severity"]) blocklist.append({ "blocker" : domain, "blocked" : block["domain"], diff --git a/fba/http/network.py b/fba/http/network.py index 91397f9..26fa126 100644 --- a/fba/http/network.py +++ b/fba/http/network.py @@ -65,6 +65,8 @@ def post_json_api(domain: str, path: str, data: str = "", headers: dict = dict() raise ValueError(f"path[]='{type(path)}' is not of type 'str'") elif path == "": raise ValueError("Parameter 'path' is empty") + elif not path.startswith("/"): + raise ValueError("path='{path}' does not start with / but should") elif not isinstance(data, str): raise ValueError(f"data[]='{type(data)}' is not of type 'str'") elif not isinstance(headers, dict): @@ -160,6 +162,8 @@ def get_json_api(domain: str, path: str, headers: dict, timeout: tuple) -> dict: raise ValueError(f"path[]='{type(path)}' is not of type 'str'") elif path == "": raise ValueError("Parameter 'path' is empty") + elif not path.startswith("/"): + raise ValueError("path='{path}' does not start with / but should") elif not isinstance(headers, dict): raise ValueError(f"headers[]='{type(headers)}' is not of type 'list'") elif not isinstance(timeout, tuple): @@ -184,6 +188,7 @@ def get_json_api(domain: str, path: str, headers: dict, timeout: tuple) -> dict: if response.ok and response.status_code == 200: logger.debug("Parsing JSON response from domain='%s',path='%s' ...", domain, path) json_reply["json"] = json_helper.from_response(response) + logger.debug("json_reply[json][]='%s'", type(json_reply["json"])) logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text)) if not response.ok or response.status_code > 200 or len(response.text) == 0: @@ -253,6 +258,8 @@ def fetch_response(domain: str, path: str, headers: dict, timeout: tuple, allow_ raise ValueError(f"Parameter path[]='{type(path)}' is not of type 'str'") elif path == "": raise ValueError("Parameter 'path' is empty") + elif not path.startswith("/"): + raise ValueError("path='{path}' does not start with / but should") elif not isinstance(headers, dict): raise ValueError(f"headers[]='{type(headers)}' is not of type 'dict'") elif not isinstance(timeout, tuple): @@ -260,7 +267,7 @@ def fetch_response(domain: str, path: str, headers: dict, timeout: tuple, allow_ start = 0 try: - logger.debug("Sending GET request to '%s%s' ...", domain, path) + logger.debug("Sending GET request to 'https://%s%s' ...", domain, path) start = time.perf_counter() response = reqto.get( f"https://{domain}{path}", diff --git a/fba/http/nodeinfo.py b/fba/http/nodeinfo.py index a0418dc..34f912e 100644 --- a/fba/http/nodeinfo.py +++ b/fba/http/nodeinfo.py @@ -17,6 +17,7 @@ import logging from urllib.parse import urlparse +from fba.helpers import blacklist from fba.helpers import config from fba.helpers import domain as domain_helper @@ -55,7 +56,9 @@ def fetch(domain: str, path: str = None, update_mode: bool = True) -> dict: logger.debug("domain='%s',path='%s',update_mode='%s' - CALLED!", domain, path, update_mode) domain_helper.raise_on(domain) - if not isinstance(path, str) and path is not None: + if blacklist.is_blacklisted(domain): + raise Exception(f"domain='{domain}' is blacklisted but function was invoked") + elif not isinstance(path, str) and path is not None: raise ValueError(f"Parameter path[]='{type(path)}' is not of type 'str'") elif not isinstance(update_mode, bool) and update_mode is not None: raise ValueError(f"Parameter update_mode[]='{type(update_mode)}' is not of type 'bool'") @@ -108,7 +111,7 @@ def fetch(domain: str, path: str = None, update_mode: bool = True) -> dict: timeout=(config.get("nodeinfo_connection_timeout"), config.get("nodeinfo_read_timeout")) ) - logger.debug("data[]='%s'", type(data)) + logger.debug("data(%d)[]='%s'", len(data), type(data)) if "error_message" not in data and "json" in data: logger.debug("Updating last_nodeinfo for domain='%s' ....", domain) instances.set_last_nodeinfo(domain) @@ -133,6 +136,9 @@ def fetch_wellknown_nodeinfo(domain: str) -> dict: logger.debug("domain='%s' - CALLED!", domain) domain_helper.raise_on(domain) + if blacklist.is_blacklisted(domain): + raise Exception(f"domain='{domain}' is blacklisted but function was invoked") + # No CSRF by default, you don't have to add network.api_headers by yourself here headers = tuple() @@ -159,8 +165,8 @@ def fetch_wellknown_nodeinfo(domain: str) -> dict: headers, (config.get("nodeinfo_connection_timeout"), config.get("nodeinfo_read_timeout")) ) - logger.debug("data[]='%s'", type(data)) + logger.debug("data(%d)[]='%s'", len(data), type(data)) if "error_message" not in data and "json" in data and len(data["json"]) > 0: logger.debug("path='%s' returned valid json()=%d - BREAK!", path, len(data["json"])) break diff --git a/fba/models/instances.py b/fba/models/instances.py index 69684d9..91af969 100644 --- a/fba/models/instances.py +++ b/fba/models/instances.py @@ -178,6 +178,8 @@ def add(domain: str, origin: str, command: str, path: str = None, software: str raise ValueError(f"path[]='{type(path)}' is not of type 'str'") elif path == "": raise ValueError("Parameter 'path' is empty") + elif not path.startswith("/"): + raise ValueError("path='{path}' does not start with / but should") elif not isinstance(software, str) and software is not None: raise ValueError(f"software[]='{type(software)}' is not of type 'str'") elif software == "": -- 2.39.5