1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
2 # Copyright (C) 2023 Free Software Foundation
4 # This program is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU Affero General Public License as published
6 # by the Free Software Foundation, either version 3 of the License, or
7 # (at your option) any later version.
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU Affero General Public License for more details.
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 from fba import config
22 from fba import network
24 from fba.helpers import cookies
26 def determine(domain: str, headers: dict) -> dict:
27 # DEBUG: print(f"DEBUG: domain='{domain}',headers()={len(headers)} - CALLED!")
28 if not isinstance(domain, str):
29 raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
31 raise ValueError("Parameter 'domain' is empty")
32 elif not validators.domain(domain.split("/")[0]):
33 raise ValueError(f"domain='{domain}' is not a valid domain")
34 elif domain.endswith(".arpa"):
35 raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!")
36 elif domain.endswith(".tld"):
37 raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
38 elif not isinstance(headers, dict):
39 raise ValueError(f"Parameter headers[]='{type(headers)}' is not 'dict'")
41 # Default headers with no CSRF
44 # Fetch / to check for meta tag indicating csrf
45 # DEBUG: print(f"DEBUG: Fetching / from domain='{domain}' for CSRF check ...")
48 headers=network.web_headers,
49 timeout=(config.get("connection_timeout"), config.get("read_timeout"))
52 # DEBUG: print(f"DEBUG: response.ok='{response.ok}',response.status_code={response.status_code},response.text()={len(response.text)}")
53 if response.ok and len(response.text) > 0:
55 cookies.store(domain, response.cookies.get_dict())
58 meta = bs4.BeautifulSoup(
62 # DEBUG: print(f"DEBUG: meta[]='{type(meta)}'")
63 tag = meta.find("meta", attrs={"name": "csrf-token"})
65 # DEBUG: print(f"DEBUG: tag={tag}")
67 # DEBUG: print(f"DEBUG: Adding CSRF token='{tag['content']}' for domain='{domain}'")
68 reqheaders["X-CSRF-Token"] = tag["content"]
70 # DEBUG: print(f"DEBUG: reqheaders()={len(reqheaders)} - EXIT!")