import logging
+from urllib.parse import urlparse
+
import bs4
import reqto
from fba.http import network
+from fba.models import instances
+
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
response = reqto.get(
f"https://{domain}/",
headers=network.web_headers,
- timeout=(config.get("connection_timeout"), config.get("read_timeout")),
- allow_redirects=False
+ timeout=(config.get("connection_timeout"), config.get("read_timeout"))
)
+ components = urlparse(response.url)
logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text))
- if response.ok and response.status_code < 300 and response.text != "" and response.text.find("<html") > 0:
+ if response.ok and response.status_code < 300 and response.text.strip() != "" and response.text.find("<html") > 0 and domain == components.netloc:
# Save cookies
logger.debug("Parsing response.text()=%d Bytes ...", len(response.text))
cookies.store(domain, response.cookies.get_dict())
if tag is not None:
logger.debug("Adding CSRF token='%s' for domain='%s'", tag["content"], domain)
reqheaders["X-CSRF-Token"] = tag["content"]
+ elif domain != components.netloc:
+ logger.warning("domain='%s' doesn't match components.netloc='%s', maybe redirect to other domain?", domain, components.netloc)
+ instances.set_last_error(domain, f"Redirect from domain='{domain}' to components.netloc='{components.netloc}'")
logger.debug("reqheaders()=%d - EXIT!", len(reqheaders))
return reqheaders