Continued:

- added more exceptions to catch and handle
- split long 2-statements lines into single lines for better error handling
  and debugging (if fedilist is one day back -> https://fedilist.com )

diff --git a/fba/commands.py b/fba/commands.py
index d6fad6bcbaf12a13ac8c1e62d30bd852afa423e6..cab214a79ab5e921cddd27ba94317702c80638bb 100644 (file)
--- a/fba/commands.py
+++ b/fba/commands.py
@@ -1543,7 +1543,10 @@ def fetch_fedilist(args: argparse.Namespace) -> int:
         logger.warning("Failed fetching url='%s': response.ok='%s',response.status_code=%d,response.content()=%d - EXIT!", url, response.ok, response.status_code, len(response.text))
         return 1
 
-    reader = csv.DictReader(response.content.decode("utf-8").splitlines(), dialect="unix")
+    lines = response.content.decode("utf-8").splitlines()
+
+    logger.debug("Reading %d lines, dialect=unix ...", len(lines))
+    reader = csv.DictReader(lines, dialect="unix")
 
     logger.debug("reader[]='%s'", type(reader))
     if reader is None:

diff --git a/fba/http/network.py b/fba/http/network.py
index 27490c131393fd3eec6254078ebe7cab37dbd524..97d3cfa1e7ef74746d575066e0623227be4e388a 100644 (file)
--- a/fba/http/network.py
+++ b/fba/http/network.py
@@ -51,12 +51,14 @@ exceptions = (
     requests.exceptions.ContentDecodingError,
     requests.exceptions.InvalidSchema,
     requests.exceptions.InvalidURL,
+    requests.exceptions.SSLError,
     requests.exceptions.Timeout,
     eventlet.timeout.Timeout,
     requests.exceptions.TooManyRedirects,
     UnicodeDecodeError,
     UnicodeEncodeError,
-    urllib3.exceptions.LocationParseError
+    urllib3.exceptions.LocationParseError,
+    urllib3.util.ssl_match_hostname.CertificateError,
 )
 
 logging.basicConfig(level=logging.INFO)
@@ -328,8 +330,8 @@ def _fetch_response(domain: str, path: str, headers: dict, timeout: tuple, allow
     logger.debug("response[]='%s' - EXIT!", type(response))
     return response
 
-def fetch_url(url: str, headers: dict, timeout: tuple) -> requests.models.Response:
-    logger.debug("url='%s',headers()=%d,timeout(%d)='%s' - CALLED!", url, len(headers), len(timeout), timeout)
+def fetch_url(url: str, headers: dict, timeout: tuple, allow_redirects: bool = True) -> requests.models.Response:
+    logger.debug("url='%s',headers()=%d,timeout(%d)='%s',allow_redirects='%s' - CALLED!", url, len(headers), len(timeout), timeout, allow_redirects)
 
     if not isinstance(url, str):
         raise ValueError(f"Parameter url[]='{type(url)}' is not of type 'str'")
@@ -341,6 +343,8 @@ def fetch_url(url: str, headers: dict, timeout: tuple) -> requests.models.Respon
         raise ValueError(f"Parameter headers[]='{type(headers)}' is not of type 'dict'")
     elif not isinstance(timeout, tuple):
         raise ValueError(f"Parameter timeout[]='{type(timeout)}' is not of type 'tuple'")
+    elif not isinstance(allow_redirects, bool):
+        raise ValueError(f"Parameter allow_redirects[]='{type(allow_redirects)}' is not of type 'bool'")
 
     logger.debug("Parsing url='%s' ...", url)
     components = urlparse(url)
@@ -353,7 +357,8 @@ def fetch_url(url: str, headers: dict, timeout: tuple) -> requests.models.Respon
             components.netloc.split(":")[0],
             f"{components.path}?{components.query}",
             headers=headers,
-            timeout=timeout
+            timeout=timeout,
+            allow_redirects=allow_redirects
         )
     else:
         logger.debug("Fetching path='%s' from netloc='%s' ...", components.path, components.netloc)
@@ -361,7 +366,8 @@ def fetch_url(url: str, headers: dict, timeout: tuple) -> requests.models.Respon
             components.netloc.split(":")[0],
             components.path if isinstance(components.path, str) and components.path != '' else '/',
             headers=headers,
-            timeout=timeout
+            timeout=timeout,
+            allow_redirects=allow_redirects
         )
 
     logger.debug("response[]='%s' - EXIT!", type(response))