import bs4
import requests
+import validators
+from fba.helpers import blacklist
from fba.helpers import config
from fba.helpers import domain as domain_helper
from fba.helpers import tidyup
raise ValueError(f"Parameter url[]='{type(url)}' is not of type 'str'")
elif url == "":
raise ValueError("Parameter 'url' is empty")
+ elif not validators.url(url):
+ raise ValueError(f"Parameter url='{url}' is not a valid URL")
elif not isinstance(headers, dict):
raise ValueError(f"Parameter headers[]='{type(headers)}' is not of type 'dict'")
elif not isinstance(timeout, tuple):
# Invoke other function, avoid trailing ?
logger.debug("components[%s]='%s'", type(components), components)
if components.query != "":
- response = network.fetch_response(components.netloc, f"{components.path}?{components.query}", headers, timeout)
+ response = network.fetch_response(
+ components.netloc.split(":")[0],
+ f"{components.path}?{components.query}",
+ headers,
+ timeout
+ )
else:
- response = network.fetch_response(components.netloc, components.path if isinstance(components.path, str) and components.path != '' else '/', headers, timeout)
+ response = network.fetch_response(
+ components.netloc.split(":")[0],
+ components.path if isinstance(components.path, str) and components.path != '' else '/',
+ headers,
+ timeout
+ )
logger.debug("response[]='%s' - EXIT!", type(response))
return response
logger.debug("domain='%s',blocker='%s',domain_hash='%s' - CALLED!", domain, blocker, domain_hash)
domain_helper.raise_on(blocker)
- if not isinstance(domain, str):
- raise ValueError(f"Parameter domain[]='{type(domain)}' is not of type 'str'")
- elif domain == "":
- raise ValueError("Parameter domain is empty")
+ if validators.domain(domain) and blacklist.is_blacklisted(domain):
+ raise ValueError(f"domain='{domain}' is blacklisted but function was invoked")
elif not isinstance(domain_hash, str) and domain_hash is not None:
raise ValueError(f"Parameter domain_hash[]='{type(domain_hash)}' is not of type 'str'")
+ logger.debug("Setting has_obfuscation=False for blocker='%s' ...", blocker)
+ instances.set_has_obfuscation(blocker, False)
+
if domain.find("*") >= 0:
logger.debug("blocker='%s' uses obfuscated domains", blocker)
+ instances.set_has_obfuscation(blocker, True)
# Obscured domain name with no hash
row = instances.deobfuscate("*", domain, domain_hash)
domain = row["domain"]
else:
logger.warning("blocker='%s' has domain='%s' that cannot be deobfuscated.", blocker, domain)
- instances.set_has_obfuscation(blocker, True)
elif domain.find("?") >= 0:
logger.debug("blocker='%s' uses obfuscated domains", blocker)
+ instances.set_has_obfuscation(blocker, True)
# Obscured domain name with no hash
row = instances.deobfuscate("?", domain, domain_hash)
domain = row["domain"]
else:
logger.warning("blocker='%s' has domain='%s' that cannot be deobfuscated.", blocker, domain)
- instances.set_has_obfuscation(blocker, True)
else:
logger.debug("domain='%s' is not obfuscated", domain)