1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
2 # Copyright (C) 2023 Free Software Foundation
4 # This program is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU Affero General Public License as published
6 # by the Free Software Foundation, either version 3 of the License, or
7 # (at your option) any later version.
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU Affero General Public License for more details.
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <https://www.gnu.org/licenses/>.
19 from urllib.parse import urlparse
23 logging.basicConfig(level=logging.INFO)
24 logger = logging.getLogger(__name__)
26 def raise_on(domain: str):
27 logger.debug("domain='%s' - CALLED!", domain)
28 if not isinstance(domain, str):
29 raise ValueError(f"Parameter domain[]='{type(domain)}' is not of type 'str'")
31 raise ValueError("Parameter 'domain' is empty")
32 elif domain.lower() != domain:
33 raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
34 elif not validators.domain(domain.split("/")[0]):
35 raise ValueError(f"domain='{domain}' is not a valid domain")
36 elif domain.endswith(".arpa"):
37 raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!")
38 elif domain.endswith(".onion"):
39 raise ValueError(f"domain='{domain}' is a TOR, please don't crawl them!")
40 elif domain.endswith(".tld"):
41 raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
45 def is_in_url(domain: str, url: str) -> bool:
46 logger.debug("domain='%s',url='%s' - CALLED!", domain, url)
49 if not isinstance(url, str):
50 raise ValueError(f"Parameter url[]='{type(url)}' is not of type 'str'")
52 raise ValueError("Parameter 'url' is empty")
54 punycode = domain.encode("idna").decode("utf-8")
56 components = urlparse(url)
57 logger.debug("components[]='%s',punycode='%s'", type(components), punycode)
59 is_found = (punycode in [components.netloc, components.hostname])
61 logger.debug("is_found='%s' - EXIT!", is_found)