1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
2 # Copyright (C) 2023 Free Software Foundation
4 # This program is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU Affero General Public License as published
6 # by the Free Software Foundation, either version 3 of the License, or
7 # (at your option) any later version.
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU Affero General Public License for more details.
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <https://www.gnu.org/licenses/>.
24 from fba import blacklist
25 from fba import config
27 from fba import federation
28 from fba import network
30 from fba.helpers import cache
32 # Found info from node, such as nodeinfo URL, detection mode that needs to be
33 # written to database. Both arrays must be filled at the same time or else
34 # update_data() will fail
36 # Detection mode: 'AUTO_DISCOVERY', 'STATIC_CHECKS' or 'GENERATOR'
37 # NULL means all detection methods have failed (maybe still reachable instance)
38 "detection_mode" : {},
43 # Last fetched instances
44 "last_instance_fetch": {},
49 # Last nodeinfo (fetched)
52 "last_status_code" : {},
54 "last_error_details" : {},
57 def set_data(key: str, domain: str, value: any):
58 # DEBUG: print(f"DEBUG: key='{key}',domain='{domain}',value[]='{type(value)}' - CALLED!")
59 if not isinstance(key, str):
60 raise ValueError("Parameter key[]='{type(key)}' is not 'str'")
62 raise ValueError("Parameter 'key' is empty")
63 elif not isinstance(domain, str):
64 raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
66 raise ValueError("Parameter 'domain' is empty")
67 elif not key in _pending:
68 raise ValueError(f"key='{key}' not found in _pending")
69 elif not fba.is_primitive(value):
70 raise ValueError(f"value[]='{type(value)}' is not a primitive type")
73 _pending[key][domain] = value
75 # DEBUG: print("DEBUG: EXIT!")
77 def has_pending(domain: str) -> bool:
78 # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
79 if not isinstance(domain, str):
80 raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
82 raise ValueError("Parameter 'domain' is empty")
86 # DEBUG: print(f"DEBUG: key='{key}',domain='{domain}',_pending[key]()='{len(_pending[key])}'")
87 if domain in _pending[key]:
91 # DEBUG: print(f"DEBUG: has_pending='{has_pending}' - EXIT!")
94 def update_data(domain: str):
95 # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
96 if not isinstance(domain, str):
97 raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
99 raise ValueError("Parameter 'domain' is empty")
100 elif not has_pending(domain):
101 raise Exception(f"Domain '{domain}' has no pending instance data, but function invoked")
103 # DEBUG: print(f"DEBUG: Updating instance data for domain='{domain}' ...")
107 # DEBUG: print("DEBUG: key:", key)
108 if domain in _pending[key]:
109 # DEBUG: print(f"DEBUG: Adding '{_pending[key][domain]}' for key='{key}' ...")
110 fields.append(_pending[key][domain])
111 sql_string += f" {key} = ?,"
113 fields.append(time.time())
114 fields.append(domain)
117 raise ValueError(f"No fields have been set, but method invoked, domain='{domain}'")
119 # DEBUG: print(f"DEBUG: sql_string='{sql_string}',fields()={len(fields)}")
120 sql_string = "UPDATE instances SET" + sql_string + " last_updated = ? WHERE domain = ? LIMIT 1"
121 # DEBUG: print("DEBUG: sql_string:", sql_string)
124 # DEBUG: print("DEBUG: Executing SQL:", sql_string)
125 fba.cursor.execute(sql_string, fields)
127 # DEBUG: print(f"DEBUG: Success! (rowcount={fba.cursor.rowcount })")
128 if fba.cursor.rowcount == 0:
129 # DEBUG: print(f"DEBUG: Did not update any rows: domain='{domain}',fields()={len(fields)} - EXIT!")
132 # DEBUG: print("DEBUG: Committing changes ...")
133 fba.connection.commit()
135 # DEBUG: print(f"DEBUG: Deleting _pending for domain='{domain}'")
137 # DEBUG: print(f"DEBUG: domain='{domain}',key='{key}'")
138 if domain in _pending[key]:
139 del _pending[key][domain]
141 except BaseException as exception:
142 print(f"ERROR: failed SQL query: domain='{domain}',sql_string='{sql_string}',exception[{type(exception)}]:'{str(exception)}'")
145 # DEBUG: print("DEBUG: EXIT!")
147 def update_last_instance_fetch(domain: str):
148 # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
149 if not isinstance(domain, str):
150 raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
152 raise ValueError("Parameter 'domain' is empty")
154 # DEBUG: print("DEBUG: Updating last_instance_fetch for domain:", domain)
155 set_data("last_instance_fetch", domain, time.time())
157 # Running pending updated
158 # DEBUG: print(f"DEBUG: Invoking update_data({domain}) ...")
161 # DEBUG: print("DEBUG: EXIT!")
163 def update_last_blocked(domain: str):
164 # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
165 if not isinstance(domain, str):
166 raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
168 raise ValueError("Parameter 'domain' is empty")
170 # DEBUG: print("DEBUG: Updating last_blocked for domain", domain)
171 set_data("last_blocked", domain, time.time())
173 # Running pending updated
174 # DEBUG: print(f"DEBUG: Invoking update_data({domain}) ...")
177 # DEBUG: print("DEBUG: EXIT!")
179 def add(domain: str, origin: str, command: str, path: str = None):
180 # DEBUG: print(f"DEBUG: domain='{domain}',origin='{origin}',command='{command}',path='{path}' - CALLED!")
181 if not isinstance(domain, str):
182 raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
184 raise ValueError("Parameter 'domain' is empty")
185 elif not isinstance(origin, str) and origin is not None:
186 raise ValueError(f"origin[]='{type(origin)}' is not 'str'")
188 raise ValueError("Parameter 'origin' is empty")
189 elif not isinstance(command, str):
190 raise ValueError(f"command[]='{type(command)}' is not 'str'")
192 raise ValueError("Parameter 'command' is empty")
193 elif not validators.domain(domain.split("/")[0]) or domain.split(".")[-1] == "arpa":
194 raise ValueError(f"Bad domain name='{domain}'")
195 elif origin is not None and not validators.domain(origin.split("/")[0]):
196 raise ValueError(f"Bad origin name='{origin}'")
197 elif blacklist.is_blacklisted(domain):
198 raise Exception(f"domain='{domain}' is blacklisted, but method invoked")
199 elif domain.find("/profile/") > 0 or domain.find("/users/") > 0:
200 raise Exception(f"domain='{domain}' is a single user")
204 # DEBUG: print("DEBUG: domain,origin,command,path:", domain, origin, command, path)
205 software = federation.determine_software(domain, path)
206 except network.exceptions as exception:
207 print(f"WARNING Exception '{type(exception)}' during determining software type")
209 # DEBUG: print("DEBUG: Determined software:", software)
210 if software == "lemmy" and domain.find("/c/") > 0:
211 domain = domain.split("/c/")[0]
212 if is_registered(domain):
213 print(f"WARNING: domain='{domain}' already registered after cutting off user part. - EXIT!")
216 print(f"INFO: Adding instance domain='{domain}' (origin='{origin}',software='{software}')")
218 "INSERT INTO instances (domain, origin, command, hash, software, first_seen) VALUES (?, ?, ?, ?, ?, ?)",
223 fba.get_hash(domain),
229 cache.set_sub_key("is_registered", domain, True)
231 if has_pending(domain):
232 # DEBUG: print(f"DEBUG: domain='{domain}' has pending nodeinfo being updated ...")
233 set_data("last_status_code" , domain, None)
234 set_data("last_error_details", domain, None)
237 # DEBUG: print(f"DEBUG: Updating nodeinfo for domain='{domain}'")
238 update_last_nodeinfo(domain)
240 # DEBUG: print("DEBUG: EXIT!")
242 def update_last_nodeinfo(domain: str):
243 # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
244 if not isinstance(domain, str):
245 raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
247 raise ValueError("Parameter 'domain' is empty")
249 # DEBUG: print("DEBUG: Updating last_nodeinfo for domain:", domain)
250 set_data("last_nodeinfo", domain, time.time())
251 set_data("last_updated" , domain, time.time())
253 # Running pending updated
254 # DEBUG: print(f"DEBUG: Invoking update_data({domain}) ...")
257 # DEBUG: print("DEBUG: EXIT!")
259 def update_last_error(domain: str, error: dict):
260 # DEBUG: print("DEBUG: domain,error[]:", domain, type(error))
261 if not isinstance(domain, str):
262 raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
264 raise ValueError("Parameter 'domain' is empty")
266 # DEBUG: print("DEBUG: BEFORE error[]:", type(error))
267 if isinstance(error, (BaseException, json.decoder.JSONDecodeError)):
268 error = f"error[{type(error)}]='{str(error)}'"
269 # DEBUG: print("DEBUG: AFTER error[]:", type(error))
271 if isinstance(error, str):
272 # DEBUG: print(f"DEBUG: Setting last_error_details='{error}'")
273 set_data("last_status_code" , domain, 999)
274 set_data("last_error_details", domain, error)
275 elif isinstance(error, requests.models.Response):
276 # DEBUG: print(f"DEBUG: Setting last_error_details='{error.reason}'")
277 set_data("last_status_code" , domain, error.status_code)
278 set_data("last_error_details", domain, error.reason)
279 elif not isinstance(error, dict):
280 raise KeyError(f"Cannot handle keys in error[{type(error)}]='{error}'")
281 elif "status_code" in error and "error_message" in error:
282 # DEBUG: print(f"DEBUG: Setting last_error_details='{error['error_message']}'")
283 set_data("last_status_code" , domain, error["status_code"])
284 set_data("last_error_details", domain, error["error_message"])
285 elif "json" in error and "error" in error["json"]:
286 set_data("last_status_code" , domain, error["status_code"])
287 set_data("last_error_details", domain, error["json"]["error"])
289 # Running pending updated
290 # DEBUG: print(f"DEBUG: Invoking update_data({domain}) ...")
293 fba.log_error(domain, error)
295 # DEBUG: print("DEBUG: EXIT!")
297 def is_registered(domain: str) -> bool:
298 # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
299 if not isinstance(domain, str):
300 raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
302 raise ValueError("Parameter 'domain' is empty")
304 # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
305 if not cache.key_exists("is_registered"):
306 # DEBUG: print("DEBUG: Cache for 'is_registered' not initialized, fetching all rows ...")
307 fba.cursor.execute("SELECT domain FROM instances")
310 cache.set_all("is_registered", fba.cursor.fetchall(), True)
313 registered = cache.sub_key_exists("is_registered", domain)
315 # DEBUG: print(f"DEBUG: registered='{registered}' - EXIT!")
318 def is_recent(domain: str) -> bool:
319 # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
320 if not isinstance(domain, str):
321 raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
323 raise ValueError("Parameter 'domain' is empty")
324 elif not is_registered(domain):
325 # DEBUG: print(f"DEBUG: domain='{domain}' is not registered, returning False - EXIT!")
329 fba.cursor.execute("SELECT last_instance_fetch FROM instances WHERE domain = ? LIMIT 1", [domain])
332 fetched = fba.cursor.fetchone()[0]
334 # DEBUG: print(f"DEBUG: fetched[{type(fetched)}]='{fetched}'")
335 recently = isinstance(fetched, float) and time.time() - fetched <= config.get("recheck_instance")
337 # DEBUG: print(f"DEBUG: recently='{recently}' - EXIT!")
340 def deobscure(char: str, domain: str, blocked_hash: str = None) -> tuple:
341 #print(f"DEBUG: char='{char}',domain='{domain}',blocked_hash='{blocked_hash}' - CALLED!")
342 if not isinstance(char, str):
343 raise ValueError(f"Parameter char[]='{type(char)}' is not 'str'")
345 raise ValueError("Parameter 'char' is empty")
346 elif not isinstance(domain, str):
347 raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
349 raise ValueError("Parameter 'domain' is empty")
350 elif not isinstance(blocked_hash, str) and blocked_hash is not None:
351 raise ValueError(f"Parameter blocked_hash[]='{type(blocked_hash)}' is not 'str'")
353 if isinstance(blocked_hash, str):
355 "SELECT domain, origin, nodeinfo_url FROM instances WHERE hash = ? LIMIT 1", [blocked_hash]
358 if fba.cursor.fetchone() is None:
359 #print(f"DEBUG: blocked_hash='{blocked_hash}' not found, trying domain='{domain}' ...")
360 return deobscure(char, domain)
363 "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [domain.replace(char, "_")]
366 row = fba.cursor.fetchone()
368 #print(f"DEBUG: row[]='{type(row)}' - EXIT!")