-def process_domain(domain: str, blocker: str, command: str) -> bool:
- logger.debug(f"domain='{domain}',blocker='{blocker}',command='{command}' - CALLED!")
- if not isinstance(domain, str):
- raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
- elif domain == "":
- raise ValueError("Parameter 'domain' is empty")
- elif domain.lower() != domain:
- raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
- elif not validators.domain(domain.split("/")[0]):
- raise ValueError(f"domain='{domain}' is not a valid domain")
- elif domain.endswith(".arpa"):
- raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!")
- elif domain.endswith(".tld"):
- raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
- elif not isinstance(blocker, str):
- raise ValueError(f"Parameter blocker[]='{type(blocker)}' is not 'str'")
- elif blocker == "":
- raise ValueError("Parameter 'blocker' is empty")
- elif not validators.domain(blocker.split("/")[0]):
- raise ValueError(f"blocker='{blocker}' is not a valid domain")
- elif blocker.endswith(".arpa"):
- raise ValueError(f"blocker='{blocker}' is a domain for reversed IP addresses, please don't crawl them!")
- elif blocker.endswith(".tld"):
- raise ValueError(f"blocker='{blocker}' is a fake domain, please don't crawl them!")
- elif not isinstance(command, str):
- raise ValueError(f"Parameter command[]='{type(command)}' is not 'str'")
- elif command == "":
- raise ValueError("Parameter 'command' is empty")
-
- if domain.find("*") > 0:
- # Try to de-obscure it
- row = instances.deobscure("*", domain)
-
- logger.debug(f"row[{type(row)}]='{row}'")
- if row is None:
- logger.warning(f"Cannot de-obfucate domain='{domain}' - SKIPPED!")
- return False
-
- logger.debug(f"domain='{domain}' de-obscured to '{row[0]}'")
- domain = row[0]
- elif domain.find("?") > 0:
- # Try to de-obscure it
- row = instances.deobscure("?", domain)
-
- logger.debug(f"row[{type(row)}]='{row}'")
- if row is None:
- logger.warning(f"Cannot de-obfucate domain='{domain}' - SKIPPED!")
- return False
-
- logger.debug(f"domain='{domain}' de-obscured to '{row[0]}'")
- domain = row[0]
-
- if not is_domain_wanted(domain)
- logger.debug("domain='%s' is not wanted - SKIPPED!", domain)
- return False
- elif instances.is_recent(domain):
- logger.debug(f"domain='{domain}' has been recently checked - SKIPPED!")
- return False
-
- processed = False
- try:
- logger.info("Fetching instances for domain='%s',blocker='%s',command='%s' ...", domain, blocker, command)
- federation.fetch_instances(domain, blocker, None, command)
- processed = True
-
- logger.debug("Invoking cookies.clear(%s) ...", domain)
- cookies.clear(domain)
- except network.exceptions as exception:
- logger.warning(f"Exception '{type(exception)}' during fetching instances (fetch_oliphant) from domain='{domain}'")
- instances.set_last_error(domain, exception)
-
- logger.debug(f"processed='{processed}' - EXIT!")
- return processed
-