Continued:

[fba.git] / fba / utils.py

diff --git a/fba/utils.py b/fba/utils.py
index 6652c19261e7747d268f5213d3d220e207f1af14..7b87b1316eefe63acdd8599fd345991e9494478e 100644 (file)
--- a/fba/utils.py
+++ b/fba/utils.py
@@ -20,7 +20,9 @@ from urllib.parse import urlparse
 
 import bs4
 import requests
+import validators
 
+from fba.helpers import blacklist
 from fba.helpers import config
 from fba.helpers import domain as domain_helper
 from fba.helpers import tidyup
@@ -51,6 +53,8 @@ def fetch_url(url: str, headers: dict, timeout: tuple) -> requests.models.Respon
         raise ValueError(f"Parameter url[]='{type(url)}' is not of type 'str'")
     elif url == "":
         raise ValueError("Parameter 'url' is empty")
+    elif not validators.url(url):
+        raise ValueError(f"Parameter url='{url}' is not a valid URL")
     elif not isinstance(headers, dict):
         raise ValueError(f"Parameter headers[]='{type(headers)}' is not of type 'dict'")
     elif not isinstance(timeout, tuple):
@@ -62,9 +66,21 @@ def fetch_url(url: str, headers: dict, timeout: tuple) -> requests.models.Respon
     # Invoke other function, avoid trailing ?
     logger.debug("components[%s]='%s'", type(components), components)
     if components.query != "":
-        response = network.fetch_response(components.netloc.split(":")[0], f"{components.path}?{components.query}", headers, timeout)
+        logger.debug("Fetching path='%s?%s' from netloc='%s' ...", components.path, components.query, components.netloc)
+        response = network.fetch_response(
+            components.netloc.split(":")[0],
+            f"{components.path}?{components.query}",
+            headers,
+            timeout
+        )
     else:
-        response = network.fetch_response(components.netloc.split(":")[0], components.path if isinstance(components.path, str) and components.path != '' else '/', headers, timeout)
+        logger.debug("Fetching path='%s' from netloc='%s' ...", components.path, components.netloc)
+        response = network.fetch_response(
+            components.netloc.split(":")[0],
+            components.path if isinstance(components.path, str) and components.path != '' else '/',
+            headers,
+            timeout
+        )
 
     logger.debug("response[]='%s' - EXIT!", type(response))
     return response
@@ -91,6 +107,7 @@ def find_domains(tags: bs4.element.ResultSet, search: str) -> list:
             domain = tidyup.domain(tag.find("em").contents[0])
             logger.debug("domain='%s' - AFTER!", domain)
 
+        logger.debug("domain='%s' - AFTER2!", domain)
         if domain == "":
             logger.warning("Empty domain after checking search='%s' and <em> tags - SKIPPED!", search)
             continue
@@ -113,16 +130,15 @@ def deobfuscate(domain: str, blocker: str, domain_hash: str = None) -> str:
     logger.debug("domain='%s',blocker='%s',domain_hash='%s' - CALLED!", domain, blocker, domain_hash)
     domain_helper.raise_on(blocker)
 
-    if not isinstance(domain, str):
-        raise ValueError(f"Parameter domain[]='{type(domain)}' is not of type 'str'")
-    elif domain == "":
-        raise ValueError("Parameter domain is empty")
+    if validators.domain(domain) and blacklist.is_blacklisted(domain):
+        raise ValueError(f"domain='{domain}' is blacklisted but function was invoked")
     elif not isinstance(domain_hash, str) and domain_hash is not None:
         raise ValueError(f"Parameter domain_hash[]='{type(domain_hash)}' is not of type 'str'")
 
     logger.debug("Setting has_obfuscation=False for blocker='%s' ...", blocker)
     instances.set_has_obfuscation(blocker, False)
 
+    logger.debug("Checking domain='%s' ...", domain)
     if domain.find("*") >= 0:
         logger.debug("blocker='%s' uses obfuscated domains", blocker)
         instances.set_has_obfuscation(blocker, True)