import logging
-from urllib.parse import urlparse
-
import bs4
import reqto
import requests
domain_helper.raise_on(domain)
if not isinstance(headers, dict):
- raise ValueError(f"Parameter headers[]='{type(headers)}' is not 'dict'")
+ raise ValueError(f"Parameter headers[]='{type(headers)}' is not of type 'dict'")
# Default headers with no CSRF
reqheaders = headers
headers=network.web_headers,
timeout=(config.get("connection_timeout"), config.get("read_timeout"))
)
- components = urlparse(response.url)
logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text))
- if response.ok and response.status_code < 300 and response.text.strip() != "" and response.text.find("<html") > 0 and domain == components.netloc:
+ if response.ok and response.status_code < 300 and response.text.strip() != "" and response.text.find("<html") > 0 and domain_helper.is_in_url(domain, response.url):
# Save cookies
logger.debug("Parsing response.text()=%d Bytes ...", len(response.text))
cookies.store(domain, response.cookies.get_dict())
if tag is not None:
logger.debug("Adding CSRF token='%s' for domain='%s'", tag["content"], domain)
reqheaders["X-CSRF-Token"] = tag["content"]
- elif domain != components.netloc:
- logger.warning("domain='%s' doesn't match components.netloc='%s', maybe redirect to other domain?", domain, components.netloc)
- message = f"Redirect from domain='{domain}' to components.netloc='{components.netloc}'"
+ elif not domain_helper.is_in_url(domain, response.url):
+ logger.warning("domain='%s' doesn't match with response.url='%s', maybe redirect to other domain?", domain, response.url)
+ message = f"Redirect from domain='{domain}' to response.url='{response.url}'"
instances.set_last_error(domain, message)
raise requests.exceptions.TooManyRedirects(message)