Continued:

[fba.git] / fba / helpers / domain.py

diff --git a/fba/helpers/domain.py b/fba/helpers/domain.py
index ac32cf1797c0941a6782296a91601828a85984bb..dd4e214c62ad543fea15e09f05358980594fac59 100644 (file)
--- a/fba/helpers/domain.py
+++ b/fba/helpers/domain.py
@@ -16,15 +16,17 @@
 
 import logging
 
+from urllib.parse import urlparse
+
 import validators
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
 def raise_on(domain: str):
-    logger.debug("domain='%s' - CALLED!")
+    logger.debug("domain='%s' - CALLED!", domain)
     if not isinstance(domain, str):
-        raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
+        raise ValueError(f"Parameter domain[]='{type(domain)}' is not of type 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
     elif domain.lower() != domain:
@@ -33,7 +35,28 @@ def raise_on(domain: str):
         raise ValueError(f"domain='{domain}' is not a valid domain")
     elif domain.endswith(".arpa"):
         raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!")
+    elif domain.endswith(".onion"):
+        raise ValueError(f"domain='{domain}' is a TOR, please don't crawl them!")
     elif domain.endswith(".tld"):
         raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
 
     logger.debug("EXIT!")
+
+def is_in_url(domain: str, url: str) -> bool:
+    logger.debug("domain='%s',url='%s' - CALLED!", domain, url)
+    raise_on(domain)
+
+    if not isinstance(url, str):
+        raise ValueError(f"Parameter url[]='{type(url)}' is not of type 'str'")
+    elif url == "":
+        raise ValueError("Parameter 'url' is empty")
+
+    punycode = domain.encode("idna").decode("utf-8")
+
+    components = urlparse(url)
+    logger.debug("components[]='%s',punycode='%s'", type(components), punycode)
+
+    is_found = (punycode in [components.netloc, components.hostname])
+
+    logger.debug("is_found='%s' - EXIT!", is_found)
+    return is_found