From 4a214fdfc4e32144039a3f5c94ffa4ec7ffba606 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Fri, 10 Jan 2025 09:38:38 +0100 Subject: [PATCH] Continued: - introduced network.get_generic() to encapsulate some repeated parameters --- fba/http/csrf.py | 6 ++---- fba/http/federation.py | 4 +--- fba/http/network.py | 28 ++++++++++++++++++++++++++++ fba/networks/friendica.py | 6 ++---- fba/networks/lemmy.py | 12 ++++-------- fba/networks/mastodon.py | 7 +++---- fba/networks/pleroma.py | 6 ++---- 7 files changed, 42 insertions(+), 27 deletions(-) diff --git a/fba/http/csrf.py b/fba/http/csrf.py index b004c16..f3b8c56 100644 --- a/fba/http/csrf.py +++ b/fba/http/csrf.py @@ -47,11 +47,9 @@ def determine(domain: str, headers: dict) -> dict: # Fetch / to check for meta tag indicating csrf logger.debug("Fetching / from domain='%s' for CSRF check ...", domain) - response = network.fetch_response( + response = network.get_generic( domain, - "/", - headers=network.web_headers, - timeout=(config.get("connection_timeout"), config.get("read_timeout")) + "/" ) logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text)) diff --git a/fba/http/federation.py b/fba/http/federation.py index dad20a8..aa05e37 100644 --- a/fba/http/federation.py +++ b/fba/http/federation.py @@ -305,11 +305,9 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str: software = None logger.debug("Fetching path='%s' from domain='%s' ...", path, domain) - response = network.fetch_response( + response = network.get_generic( domain, path, - headers=network.web_headers, - timeout=(config.get("connection_timeout"), config.get("read_timeout")), allow_redirects=True ) diff --git a/fba/http/network.py b/fba/http/network.py index 5237d87..c551957 100644 --- a/fba/http/network.py +++ b/fba/http/network.py @@ -294,6 +294,8 @@ def fetch_response(domain: str, path: str, headers: dict, timeout: tuple, allow_ raise ValueError(f"headers[]='{type(headers)}' is not of type 'dict'") elif not isinstance(timeout, tuple): raise ValueError(f"timeout[]='{type(timeout)}' is not of type 'tuple'") + elif not isinstance(allow_redirects, bool): + raise ValueError(f"allow_redirects[]='{type(allow_redirects)}' is not of type 'bool'") start = 0 try: @@ -415,3 +417,29 @@ def fetch_json_rows(hostname: str, path: str, headers: dict = {}, rows_key: str logger.debug("rows()=%d - EXIT!", len(rows)) return rows + +def get_generic(domain: str, path: str, allow_redirects: bool = False) -> requests.models.Response: + logger.debug("domain='%s',path='%s',allow_redirects='%s' - CALLED!", domain, path, allow_redirects) + domain_helper.raise_on(domain) + + if blacklist.is_blacklisted(domain): + raise ValueError(f"domain='{domain}' is blacklisted but function was invoked") + elif not isinstance(path, str): + raise ValueError(f"Parameter path[]='{type(path)}' is not of type 'str'") + elif path == "": + raise ValueError("Parameter 'path' is empty") + elif not path.startswith("/"): + raise ValueError(f"path='{path}' does not start with / but should") + elif not isinstance(allow_redirects, bool): + raise ValueError(f"allow_redirects[]='{type(allow_redirects)}' is not of type 'bool'") + + logger.debug("Fetching path='%s' from domain='%s' ...", path, domain) + response = fetch_response( + domain, + path, + headers=web_headers, + allow_redirects=allow_redirects + ) + + logger.debug("response[]='%s' - EXIT!", type(response)) + return response diff --git a/fba/networks/friendica.py b/fba/networks/friendica.py index abcda4c..2fd53b7 100644 --- a/fba/networks/friendica.py +++ b/fba/networks/friendica.py @@ -45,11 +45,9 @@ def fetch_blocks(domain: str) -> list: try: logger.debug("Fetching friendica blocks from domain='%s' ...", domain) - raw = network.fetch_response( + raw = network.get_generic( domain, - "/friendica", - network.web_headers, - (config.get("connection_timeout"), config.get("read_timeout")) + "/friendica" ).text logger.debug("Parsing %d Bytes ...", len(raw)) diff --git a/fba/networks/lemmy.py b/fba/networks/lemmy.py index 785e5b7..e899f55 100644 --- a/fba/networks/lemmy.py +++ b/fba/networks/lemmy.py @@ -140,11 +140,9 @@ def fetch_blocks(domain: str) -> list: try: # json endpoint for newer mastodongs logger.debug("Fetching /instances from domain='%s'", domain) - response = network.fetch_response( + response = network.get_generic( domain, - "/instances", - network.web_headers, - (config.get("connection_timeout"), config.get("read_timeout")) + "/instances" ) logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text)) @@ -251,11 +249,9 @@ def fetch_instances(domain: str, origin: str) -> list: try: # json endpoint for newer mastodongs logger.debug("Fetching /instances from domain='%s'", domain) - response = network.fetch_response( + response = network.get_generic( domain, - "/instances", - network.web_headers, - (config.get("connection_timeout"), config.get("read_timeout")) + "/instances" ) logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text)) diff --git a/fba/networks/mastodon.py b/fba/networks/mastodon.py index df4ba92..adc3055 100644 --- a/fba/networks/mastodon.py +++ b/fba/networks/mastodon.py @@ -75,14 +75,13 @@ def fetch_blocks_from_about(domain: str) -> dict: logger.info("Fetching mastodon blocks from domain='%s'", domain) for path in ["/about/more", "/about"]: + logger.debug("path='%s'", path) try: logger.debug("Fetching path='%s' from domain='%s' ...", path, domain) doc = bs4.BeautifulSoup( - network.fetch_response( + network.get_generic( domain, - path, - network.web_headers, - (config.get("connection_timeout"), config.get("read_timeout")) + path ).text, "html.parser", ) diff --git a/fba/networks/pleroma.py b/fba/networks/pleroma.py index 024a4f5..1abb751 100644 --- a/fba/networks/pleroma.py +++ b/fba/networks/pleroma.py @@ -302,11 +302,9 @@ def fetch_blocks_from_about(domain: str) -> dict: doc = None logger.debug("Fetching path='%s' from domain='%s' ...", path, domain) - response = network.fetch_response( + response = network.get_generic( domain, - path, - network.web_headers, - (config.get("connection_timeout"), config.get("read_timeout")) + path ) logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text)) -- 2.39.5