1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
2 # Copyright (C) 2023 Free Software Foundation
4 # This program is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU Affero General Public License as published
6 # by the Free Software Foundation, either version 3 of the License, or
7 # (at your option) any later version.
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU Affero General Public License for more details.
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <https://www.gnu.org/licenses/>.
22 from fba import config
25 from fba import federation
26 from fba import network
28 from fba.helpers import blacklist
29 from fba.helpers import tidyup
31 from fba.models import blocks
32 from fba.models import instances
34 def fetch_peers(domain: str) -> list:
35 # DEBUG: print(f"DEBUG: domain({len(domain)})='{domain}',software='lemmy' - CALLED!")
36 if not isinstance(domain, str):
37 raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
39 raise ValueError("Parameter 'domain' is empty")
43 # No CSRF by default, you don't have to add network.api_headers by yourself here
47 # DEBUG: print(f"DEBUG: Checking CSRF for domain='{domain}'")
48 headers = csrf.determine(domain, dict())
49 except network.exceptions as exception:
50 print(f"WARNING: Exception '{type(exception)}' during checking CSRF (fetch_peers,{__name__}) - EXIT!")
51 instances.set_last_error(domain, exception)
55 # DEBUG: print(f"DEBUG: domain='{domain}' is Lemmy, fetching JSON ...")
56 data = network.get_json_api(
60 (config.get("connection_timeout"), config.get("read_timeout"))
63 # DEBUG: print(f"DEBUG: data[]='{type(data)}'")
64 if "error_message" in data:
65 print("WARNING: Could not reach any JSON API:", domain)
66 instances.set_last_error(domain, data)
67 elif "federated_instances" in data["json"]:
68 # DEBUG: print(f"DEBUG: Found federated_instances for domain='{domain}'")
69 peers = peers + federation.add_peers(data["json"]["federated_instances"])
70 # DEBUG: print("DEBUG: Added instance(s) to peers")
72 print("WARNING: JSON response does not contain 'federated_instances':", domain)
73 instances.set_last_error(domain, data)
75 except network.exceptions as exception:
76 print(f"WARNING: Exception during fetching JSON: domain='{domain}',exception[{type(exception)}]:'{str(exception)}'")
77 instances.set_last_error(domain, exception)
79 # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'")
80 instances.set_total_peers(domain, peers)
82 # DEBUG: print("DEBUG: Returning peers[]:", type(peers))
85 def fetch_blocks(domain: str, origin: str, nodeinfo_url: str):
86 # DEBUG: print(f"DEBUG: domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}' - CALLED!")
87 if not isinstance(domain, str):
88 raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
90 raise ValueError("Parameter 'domain' is empty")
91 elif not isinstance(origin, str) and origin is not None:
92 raise ValueError(f"Parameter origin[]='{type(origin)}' is not 'str'")
94 raise ValueError("Parameter 'origin' is empty")
95 elif not isinstance(nodeinfo_url, str):
96 raise ValueError(f"Parameter nodeinfo_url[]='{type(nodeinfo_url)}' is not 'str'")
97 elif nodeinfo_url == "":
98 raise ValueError("Parameter 'nodeinfo_url' is empty")
102 "Instàncies bloquejades",
105 "Blokované instance",
106 "Geblokkeerde instanties",
107 "Blockerade instanser",
108 "Instàncias blocadas",
110 "Instances bloquées",
111 "Letiltott példányok",
112 "Instancias bloqueadas",
113 "Blokeatuta dauden instantziak",
115 "Peladen Yang Diblokir",
118 "Блокирани Инстанции",
119 "Blockierte Instanzen",
120 "Estetyt instanssit",
121 "Instâncias bloqueadas",
122 "Zablokowane instancje",
123 "Blokované inštancie",
125 "Užblokuoti serveriai",
127 "Блокированные Инстансы",
128 "Αποκλεισμένοι διακομιστές",
130 "Instâncias bloqueadas",
134 # json endpoint for newer mastodongs
135 found_blocks = list()
140 "media_removal" : [],
141 "followers_only": [],
142 "report_removal": [],
145 # DEBUG: print(f"DEBUG: Fetching /instances from domain='{domain}'")
146 response = network.fetch_response(
150 (config.get("connection_timeout"), config.get("read_timeout"))
153 # DEBUG: print(f"DEBUG: response.ok='{response.ok}',response.status_code={response.status_code},response.text()={len(response.text)}")
154 if response.ok and response.status_code < 300 and response.text != "":
155 # DEBUG: print(f"DEBUG: Parsing {len(response.text)} Bytes ...")
157 doc = bs4.BeautifulSoup(response.text, "html.parser")
158 # DEBUG: print(f"DEBUG: doc[]={type(doc)}")
160 headers = doc.findAll("h5")
162 # DEBUG: print(f"DEBUG: Search in {len(headers)} header(s) ...")
163 for header in headers:
164 # DEBUG: print(f"DEBUG: header[]={type(header)}")
165 content = header.contents[0]
167 # DEBUG: print(f"DEBUG: content='{content}'")
168 if content in translations:
169 # DEBUG: print("DEBUG: Found header with blocked instances - BREAK!")
173 # DEBUG: print(f"DEBUG: found[]='{type(found)}'")
175 # DEBUG: print(f"DEBUG: domain='{domain}' is not blocking any instances - EXIT!")
178 blocking = found.find_next("ul").findAll("a")
179 # DEBUG: print(f"DEBUG: Found {len(blocking)} blocked instance(s) ...")
181 # DEBUG: print(f"DEBUG: tag[]='{type(tag)}'")
182 blocked = tidyup.domain(tag.contents[0])
184 # DEBUG: print(f"DEBUG: blocked='{blocked}'")
185 if not validators.domain(blocked):
186 print(f"WARNING: blocked='{blocked}' is not a valid domain - SKIPPED!")
188 elif blocked.endswith(".arpa"):
189 print(f"WARNING: blocked='{blocked}' is a reversed .arpa domain and should not be used generally.")
191 elif blocked.endswith(".tld"):
192 print(f"WARNING: blocked='{blocked}' is a fake domain, please don't crawl them!")
194 elif blacklist.is_blacklisted(blocked):
195 # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - SKIPPED!")
197 elif not instances.is_registered(blocked):
198 # DEBUG: print("DEBUG: Hash wasn't found, adding:", blocked, domain)
199 instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url)
201 if not blocks.is_instance_blocked(domain, blocked, "reject"):
202 # DEBUG: print("DEBUG: Blocking:", domain, blocked)
203 blocks.add_instance(domain, blocked, None, "reject")
205 found_blocks.append({
210 # DEBUG: print(f"DEBUG: Updating block last seen for domain='{domain}',blocked='{blocked}' ...")
211 blocks.update_last_seen(domain, blocked, "reject")
213 # DEBUG: print("DEBUG: Committing changes ...")
214 fba.connection.commit()
215 except network.exceptions as exception:
216 print(f"ERROR: domain='{domain}',software='mastodon',exception[{type(exception)}]:'{str(exception)}'")
217 instances.set_last_error(domain, exception)
219 # DEBUG: print("DEBUG: EXIT!")