Continued:

- removed old-lost code utils.fetch_url(), maybe a leftover from previous
  moving code to network module?
- rewrote loop (better error handling

diff --git a/fba/commands.py b/fba/commands.py
index 88f8216c065314ab9f692b0296973584c7e008d7..a9df12d6df0c16b14786381a1419f132b3054ec9 100644 (file)
--- a/fba/commands.py
+++ b/fba/commands.py
@@ -531,12 +531,19 @@ def fetch_observer(args: argparse.Namespace) -> int:
         logger.info("Checking %d menu items ...", len(items))
         for item in items:
             logger.debug("item[%s]='%s'", type(item), item)
-            if item.text.lower() == "all":
-                logger.debug("Skipping 'All' menu entry ...")
+            domain = item.text.lower()
+            domain = tidyup.domain(domain) if domain is not None and len(domain) > 0 else None
+
+            logger.debug("domain='%s'", domain)
+            if domain is None:
+                logger.debug("Skipping empty domain in tag='%s' - SKIPPED!", tag)
+                continue
+            elif domain == "all":
+                logger.debug("Skipping 'All' menu entry in tag='%s' - SKIPPED!", tag)
                 continue
 
-            logger.debug("Appending item.text='%s' ...", item.text)
-            types.append(tidyup.domain(item.text))
+            logger.debug("Appending domain='%s' ...", domain)
+            types.append(domain)
     else:
         logger.info("Adding args.software='%s' as type ...", args.software)
         types.append(args.software)

diff --git a/fba/utils.py b/fba/utils.py
index 92d2b1b3963eaaee74a0f616cd882b56942342f8..b0702da93ddfd5c0a4f0ae67e25af240241a8b59 100644 (file)
--- a/fba/utils.py
+++ b/fba/utils.py
@@ -46,45 +46,6 @@ def get_hash(domain: str) -> str:
 
     return hashlib.sha256(domain.encode("utf-8")).hexdigest()
 
-def fetch_url(url: str, headers: dict, timeout: tuple) -> requests.models.Response:
-    logger.debug("url='%s',headers()=%d,timeout(%d)='%s' - CALLED!", url, len(headers), len(timeout), timeout)
-
-    if not isinstance(url, str):
-        raise ValueError(f"Parameter url[]='{type(url)}' is not of type 'str'")
-    elif url == "":
-        raise ValueError("Parameter 'url' is empty")
-    elif not validators.url(url):
-        raise ValueError(f"Parameter url='{url}' is not a valid URL")
-    elif not isinstance(headers, dict):
-        raise ValueError(f"Parameter headers[]='{type(headers)}' is not of type 'dict'")
-    elif not isinstance(timeout, tuple):
-        raise ValueError(f"Parameter timeout[]='{type(timeout)}' is not of type 'tuple'")
-
-    logger.debug("Parsing url='%s' ...", url)
-    components = urlparse(url)
-
-    # Invoke other function, avoid trailing ?
-    logger.debug("components[%s]='%s'", type(components), components)
-    if components.query != "":
-        logger.debug("Fetching path='%s?%s' from netloc='%s' ...", components.path, components.query, components.netloc)
-        response = network.fetch_response(
-            components.netloc.split(":")[0],
-            f"{components.path}?{components.query}",
-            headers,
-            timeout
-        )
-    else:
-        logger.debug("Fetching path='%s' from netloc='%s' ...", components.path, components.netloc)
-        response = network.fetch_response(
-            components.netloc.split(":")[0],
-            components.path if isinstance(components.path, str) and components.path != '' else '/',
-            headers,
-            timeout
-        )
-
-    logger.debug("response[]='%s' - EXIT!", type(response))
-    return response
-
 def find_domains(tags: bs4.element.ResultSet, search: str) -> list:
     logger.debug("tags[%s]()=%d,search='%s' - CALLED!", type(tags), len(tags), search)