1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
2 # Copyright (C) 2023 Free Software Foundation
4 # This program is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU Affero General Public License as published
6 # by the Free Software Foundation, either version 3 of the License, or
7 # (at your option) any later version.
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU Affero General Public License for more details.
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <https://www.gnu.org/licenses/>.
24 from fba import blacklist
27 from fba import federation
29 # Found info from node, such as nodeinfo URL, detection mode that needs to be
30 # written to database. Both arrays must be filled at the same time or else
31 # update_data() will fail
33 # Detection mode: 'AUTO_DISCOVERY', 'STATIC_CHECKS' or 'GENERATOR'
34 # NULL means all detection methods have failed (maybe still reachable instance)
35 "detection_mode" : {},
40 # Last fetched instances
41 "last_instance_fetch": {},
46 # Last nodeinfo (fetched)
49 "last_status_code" : {},
51 "last_error_details" : {},
54 def set_data(key: str, domain: str, value: any):
55 # DEBUG: print(f"DEBUG: key='{key}',domain='{domain}',value[]='{type(value)}' - CALLED!")
56 if not isinstance(key, str):
57 raise ValueError("Parameter key[]='{type(key)}' is not 'str'")
59 raise ValueError("Parameter 'key' is empty")
60 elif not isinstance(domain, str):
61 raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
63 raise ValueError("Parameter 'domain' is empty")
64 elif not key in _pending:
65 raise ValueError(f"key='{key}' not found in _pending")
66 elif not fba.is_primitive(value):
67 raise ValueError(f"value[]='{type(value)}' is not a primitive type")
70 _pending[key][domain] = value
72 # DEBUG: print("DEBUG: EXIT!")
74 def has_pending_instance_data(domain: str) -> bool:
75 # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
76 if not isinstance(domain, str):
77 raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
79 raise ValueError("Parameter 'domain' is empty")
83 # DEBUG: print(f"DEBUG: key='{key}',domain='{domain}',_pending[key]()='{len(_pending[key])}'")
84 if domain in _pending[key]:
88 # DEBUG: print(f"DEBUG: has_pending='{has_pending}' - EXIT!")
91 def update_data(domain: str):
92 # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
93 if not isinstance(domain, str):
94 raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
96 raise ValueError("Parameter 'domain' is empty")
97 elif not has_pending_instance_data(domain):
98 raise Exception(f"Domain '{domain}' has no pending instance data, but function invoked")
100 # DEBUG: print(f"DEBUG: Updating instance data for domain='{domain}' ...")
104 # DEBUG: print("DEBUG: key:", key)
105 if domain in _pending[key]:
106 # DEBUG: print(f"DEBUG: Adding '{_pending[key][domain]}' for key='{key}' ...")
107 fields.append(_pending[key][domain])
108 sql_string += f" {key} = ?,"
110 fields.append(time.time())
111 fields.append(domain)
114 raise ValueError(f"No fields have been set, but method invoked, domain='{domain}'")
116 # DEBUG: print(f"DEBUG: sql_string='{sql_string}',fields()={len(fields)}")
117 sql_string = "UPDATE instances SET" + sql_string + " last_updated = ? WHERE domain = ? LIMIT 1"
118 # DEBUG: print("DEBUG: sql_string:", sql_string)
121 # DEBUG: print("DEBUG: Executing SQL:", sql_string)
122 fba.cursor.execute(sql_string, fields)
124 # DEBUG: print(f"DEBUG: Success! (rowcount={fba.cursor.rowcount })")
125 if fba.cursor.rowcount == 0:
126 # DEBUG: print(f"DEBUG: Did not update any rows: domain='{domain}',fields()={len(fields)} - EXIT!")
129 # DEBUG: print("DEBUG: Committing changes ...")
130 fba.connection.commit()
132 # DEBUG: print("DEBUG: Deleting _pending for domain:", domain)
135 # DEBUG: print("DEBUG: Deleting key:", key)
136 del _pending[key][domain]
140 except BaseException as exception:
141 print(f"ERROR: failed SQL query: domain='{domain}',sql_string='{sql_string}',exception[{type(exception)}]:'{str(exception)}'")
144 # DEBUG: print("DEBUG: EXIT!")
146 def update_last_instance_fetch(domain: str):
147 # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
148 if not isinstance(domain, str):
149 raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
151 raise ValueError("Parameter 'domain' is empty")
153 # DEBUG: print("DEBUG: Updating last_instance_fetch for domain:", domain)
154 set_data("last_instance_fetch", domain, time.time())
156 # Running pending updated
157 # DEBUG: print(f"DEBUG: Invoking update_data({domain}) ...")
160 # DEBUG: print("DEBUG: EXIT!")
162 def update_last_blocked(domain: str):
163 if not isinstance(domain, str):
164 raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
166 raise ValueError("Parameter 'domain' is empty")
168 # DEBUG: print("DEBUG: Updating last_blocked for domain", domain)
169 set_data("last_blocked", domain, time.time())
171 # Running pending updated
172 # DEBUG: print(f"DEBUG: Invoking update_data({domain}) ...")
175 # DEBUG: print("DEBUG: EXIT!")
177 def add(domain: str, origin: str, command: str, path: str = None):
178 # DEBUG: print(f"DEBUG: domain='{domain}',origin='{origin}',command='{command}',path='{path}' - CALLED!")
179 if not isinstance(domain, str):
180 raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
182 raise ValueError("Parameter 'domain' is empty")
183 elif not isinstance(origin, str) and origin is not None:
184 raise ValueError(f"origin[]='{type(origin)}' is not 'str'")
186 raise ValueError("Parameter 'origin' is empty")
187 elif not isinstance(command, str):
188 raise ValueError(f"command[]='{type(command)}' is not 'str'")
190 raise ValueError("Parameter 'command' is empty")
191 elif not validators.domain(domain.split("/")[0]):
192 raise ValueError(f"Bad domain name='{domain}'")
193 elif origin is not None and not validators.domain(origin.split("/")[0]):
194 raise ValueError(f"Bad origin name='{origin}'")
195 elif blacklist.is_blacklisted(domain):
196 raise Exception(f"domain='{domain}' is blacklisted, but method invoked")
197 elif domain.find("/profile/") > 0 or domain.find("/users/") > 0:
198 raise Exception(f"domain='{domain}' is a single user")
200 # DEBUG: print("DEBUG: domain,origin,command,path:", domain, origin, command, path)
201 software = federation.determine_software(domain, path)
203 # DEBUG: print("DEBUG: Determined software:", software)
204 if software == "lemmy" and domain.find("/c/") > 0:
205 domain = domain.split("/c/")[0]
206 if is_registered(domain):
207 print(f"WARNING: domain='{domain}' already registered after cutting off user part. - EXIT!")
210 print(f"INFO: Adding instance domain='{domain}' (origin='{origin}',software='{software}')")
213 "INSERT INTO instances (domain, origin, command, hash, software, first_seen) VALUES (?, ?, ?, ?, ?, ?)",
218 fba.get_hash(domain),
224 cache.set_sub_key("is_registered", domain, True)
226 if has_pending_instance_data(domain):
227 # DEBUG: print(f"DEBUG: domain='{domain}' has pending nodeinfo being updated ...")
228 set_data("last_status_code" , domain, None)
229 set_data("last_error_details", domain, None)
232 except BaseException as exception:
233 update_last_error(domain, exception)
234 raise Exception(f"ERROR: failed SQL query: domain='{domain}',exception[{type(exception)}]:'{str(exception)}'") from exception
236 # DEBUG: print("DEBUG: Updating nodeinfo for domain:", domain)
237 update_last_nodeinfo(domain)
239 # DEBUG: print("DEBUG: EXIT!")
241 def update_last_nodeinfo(domain: str):
242 # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
243 if not isinstance(domain, str):
244 raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
246 raise ValueError("Parameter 'domain' is empty")
248 # DEBUG: print("DEBUG: Updating last_nodeinfo for domain:", domain)
249 set_data("last_nodeinfo", domain, time.time())
250 set_data("last_updated" , domain, time.time())
252 # Running pending updated
253 # DEBUG: print(f"DEBUG: Invoking update_data({domain}) ...")
256 # DEBUG: print("DEBUG: EXIT!")
258 def update_last_error(domain: str, error: dict):
259 # DEBUG: print("DEBUG: domain,error[]:", domain, type(error))
260 if not isinstance(domain, str):
261 raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
263 raise ValueError("Parameter 'domain' is empty")
265 # DEBUG: print("DEBUG: BEFORE error[]:", type(error))
266 if isinstance(error, BaseException) or isinstance(error, json.decoder.JSONDecodeError):
267 error = f"error[{type(error)}]='{str(error)}'"
268 # DEBUG: print("DEBUG: AFTER error[]:", type(error))
270 if isinstance(error, str):
271 # DEBUG: print(f"DEBUG: Setting last_error_details='{error}'")
272 set_data("last_status_code" , domain, 999)
273 set_data("last_error_details", domain, error)
274 elif isinstance(error, requests.models.Response):
275 # DEBUG: print(f"DEBUG: Setting last_error_details='{error.reason}'")
276 set_data("last_status_code" , domain, error.status_code)
277 set_data("last_error_details", domain, error.reason)
278 elif "status_code" in error and "error_message" in error:
279 # DEBUG: print(f"DEBUG: Setting last_error_details='{error['error_message']}'")
280 set_data("last_status_code" , domain, error["status_code"])
281 set_data("last_error_details", domain, error["error_message"])
282 elif "json" in error and "error" in error["json"]:
283 set_data("last_status_code" , domain, error["status_code"])
284 set_data("last_error_details", domain, error["json"]["error"])
286 raise KeyError(f"Cannot handle keys in error[{type(error)}]='{error}'")
288 # Running pending updated
289 # DEBUG: print(f"DEBUG: Invoking update_data({domain}) ...")
292 fba.log_error(domain, error)
294 # DEBUG: print("DEBUG: EXIT!")
296 def is_registered(domain: str) -> bool:
297 # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
298 if not isinstance(domain, str):
299 raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
301 raise ValueError("Parameter 'domain' is empty")
303 # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
304 if not cache.key_exists("is_registered"):
305 # DEBUG: print("DEBUG: Cache for 'is_registered' not initialized, fetching all rows ...")
307 fba.cursor.execute("SELECT domain FROM instances")
310 cache.set_all("is_registered", fba.cursor.fetchall(), True)
311 except BaseException as exception:
312 update_last_error(domain, exception)
313 raise Exception(f"ERROR: failed SQL query: domain='{domain}',exception[{type(exception)}]:'{str(exception)}'") from exception
316 registered = cache.sub_key_exists("is_registered", domain)
318 # DEBUG: print(f"DEBUG: registered='{registered}' - EXIT!")