elif "href" not in link:
logger.warning("link[rel]='%s' has no element 'href' - SKIPPED!", link["rel"])
continue
+ elif link["href"] is None:
+ logger.debug("link[href] is None, link[rel]='%s' - SKIPPED!", link["rel"])
+ continue
# Default is that 'href' has a complete URL, but some hosts don't send that
logger.debug("link[rel]='%s' matches niid='%s'", link["rel"], niid)
url = link["href"]
- components = urlparse(link["href"])
+ components = urlparse(url)
logger.debug("components[%s]='%s'", type(components), components)
if components.scheme == "" and components.netloc == "":
url = f"{components.scheme}://{domain}{components.path}"
components = urlparse(url)
+ logger.debug("components.netloc[]='%s'", type(components.netloc))
if not utils.is_domain_wanted(components.netloc):
logger.debug("components.netloc='%s' is not wanted - SKIPPED!", components.netloc)
continue
return data
def fetch_generator_from_path(domain: str, path: str = "/") -> str:
- logger.debug("domain(%d)='%s',path='%s' - CALLED!", len(domain), domain, path)
+ logger.debug("domain='%s',path='%s' - CALLED!", domain, path)
domain_helper.raise_on(domain)
if not isinstance(path, str):
return software
def determine_software(domain: str, path: str = None) -> str:
- logger.debug("domain(%d)='%s',path='%s' - CALLED!", len(domain), domain, path)
+ logger.debug("domain='%s',path='%s' - CALLED!", domain, path)
domain_helper.raise_on(domain)
if not isinstance(path, str) and path is not None:
logger.debug("EXIT!")
def set_has_obfuscation(domain: str, status: bool):
- logger.debug("domain(%d)='%s',status='%s' - CALLED!", len(domain), domain, status)
+ logger.debug("domain='%s',status='%s' - CALLED!", domain, status)
domain_helper.raise_on(domain)
if not isinstance(status, bool):