1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
2 # Copyright (C) 2023 Free Software Foundation
4 # This program is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU Affero General Public License as published
6 # by the Free Software Foundation, either version 3 of the License, or
7 # (at your option) any later version.
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU Affero General Public License for more details.
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <https://www.gnu.org/licenses/>.
24 from fba import blacklist
28 # Found info from node, such as nodeinfo URL, detection mode that needs to be
29 # written to database. Both arrays must be filled at the same time or else
30 # update_data() will fail
32 # Detection mode: 'AUTO_DISCOVERY', 'STATIC_CHECKS' or 'GENERATOR'
33 # NULL means all detection methods have failed (maybe still reachable instance)
34 "detection_mode" : {},
39 # Last fetched instances
40 "last_instance_fetch": {},
45 # Last nodeinfo (fetched)
48 "last_status_code" : {},
50 "last_error_details" : {},
53 def set_data(key: str, domain: str, value: any):
54 # DEBUG: print(f"DEBUG: key='{key}',domain='{domain}',value[]='{type(value)}' - CALLED!")
55 if not isinstance(key, str):
56 raise ValueError("Parameter key[]='{type(key)}' is not 'str'")
58 raise ValueError("Parameter 'key' is empty")
59 elif not isinstance(domain, str):
60 raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
62 raise ValueError("Parameter 'domain' is empty")
63 elif not key in _pending:
64 raise ValueError(f"key='{key}' not found in _pending")
65 elif not fba.is_primitive(value):
66 raise ValueError(f"value[]='{type(value)}' is not a primitive type")
69 _pending[key][domain] = value
71 # DEBUG: print("DEBUG: EXIT!")
73 def has_pending_instance_data(domain: str) -> bool:
74 # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
75 if not isinstance(domain, str):
76 raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
78 raise ValueError("Parameter 'domain' is empty")
82 # DEBUG: print(f"DEBUG: key='{key}',domain='{domain}',_pending[key]()='{len(_pending[key])}'")
83 if domain in _pending[key]:
87 # DEBUG: print(f"DEBUG: has_pending='{has_pending}' - EXIT!")
90 def update_data(domain: str):
91 # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
92 if not isinstance(domain, str):
93 raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
95 raise ValueError("Parameter 'domain' is empty")
96 elif not has_pending_instance_data(domain):
97 raise Exception(f"Domain '{domain}' has no pending instance data, but function invoked")
99 # DEBUG: print(f"DEBUG: Updating instance data for domain='{domain}' ...")
103 # DEBUG: print("DEBUG: key:", key)
104 if domain in _pending[key]:
105 # DEBUG: print(f"DEBUG: Adding '{_pending[key][domain]}' for key='{key}' ...")
106 fields.append(_pending[key][domain])
107 sql_string += f" {key} = ?,"
109 fields.append(time.time())
110 fields.append(domain)
113 raise ValueError(f"No fields have been set, but method invoked, domain='{domain}'")
115 # DEBUG: print(f"DEBUG: sql_string='{sql_string}',fields()={len(fields)}")
116 sql_string = "UPDATE instances SET" + sql_string + " last_updated = ? WHERE domain = ? LIMIT 1"
117 # DEBUG: print("DEBUG: sql_string:", sql_string)
120 # DEBUG: print("DEBUG: Executing SQL:", sql_string)
121 fba.cursor.execute(sql_string, fields)
123 # DEBUG: print(f"DEBUG: Success! (rowcount={fba.cursor.rowcount })")
124 if fba.cursor.rowcount == 0:
125 # DEBUG: print(f"DEBUG: Did not update any rows: domain='{domain}',fields()={len(fields)} - EXIT!")
128 # DEBUG: print("DEBUG: Committing changes ...")
129 fba.connection.commit()
131 # DEBUG: print("DEBUG: Deleting _pending for domain:", domain)
134 # DEBUG: print("DEBUG: Deleting key:", key)
135 del _pending[key][domain]
139 except BaseException as exc:
140 print(f"ERROR: failed SQL query: domain='{domain}',sql_string='{sql_string}',exc[{type(exc)}]:'{str(exc)}'")
143 # DEBUG: print("DEBUG: EXIT!")
145 def update_last_instance_fetch(domain: str):
146 # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
147 if not isinstance(domain, str):
148 raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
150 raise ValueError("Parameter 'domain' is empty")
152 # DEBUG: print("DEBUG: Updating last_instance_fetch for domain:", domain)
153 set_data("last_instance_fetch", domain, time.time())
155 # Running pending updated
156 # DEBUG: print(f"DEBUG: Invoking update_data({domain}) ...")
159 # DEBUG: print("DEBUG: EXIT!")
161 def update_last_blocked(domain: str):
162 if not isinstance(domain, str):
163 raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
165 raise ValueError("Parameter 'domain' is empty")
167 # DEBUG: print("DEBUG: Updating last_blocked for domain", domain)
168 set_data("last_blocked", domain, time.time())
170 # Running pending updated
171 # DEBUG: print(f"DEBUG: Invoking update_data({domain}) ...")
174 # DEBUG: print("DEBUG: EXIT!")
176 def add(domain: str, origin: str, command: str, path: str = None):
177 # DEBUG: print(f"DEBUG: domain='{domain}',origin='{origin}',command='{command}',path='{path}' - CALLED!")
178 if not isinstance(domain, str):
179 raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
181 raise ValueError("Parameter 'domain' is empty")
182 elif not isinstance(origin, str) and origin is not None:
183 raise ValueError(f"origin[]={type(origin)} is not 'str'")
185 raise ValueError("Parameter 'origin' is empty")
186 elif not isinstance(command, str):
187 raise ValueError(f"command[]={type(command)} is not 'str'")
189 raise ValueError("Parameter 'command' is empty")
190 elif not validators.domain(domain.split("/")[0]):
191 raise ValueError(f"Bad domain name='{domain}'")
192 elif origin is not None and not validators.domain(origin.split("/")[0]):
193 raise ValueError(f"Bad origin name='{origin}'")
194 elif blacklist.is_blacklisted(domain):
195 raise Exception(f"domain='{domain}' is blacklisted, but method invoked")
196 elif domain.find("/profile/") > 0 or domain.find("/users/") > 0:
197 raise Exception(f"domain='{domain}' is a single user")
199 # DEBUG: print("DEBUG: domain,origin,command,path:", domain, origin, command, path)
200 software = fba.determine_software(domain, path)
201 # DEBUG: print("DEBUG: Determined software:", software)
202 if domain.find("/c/") > 0 and software == "lemmy":
203 domain = domain.split("/c/")[0]
204 if is_registered(domain):
205 print(f"WARNING: domain='{domain}' already registered after cutting off user part. - EXIT!")
208 print(f"INFO: Adding instance domain='{domain}' (origin='{origin}',software='{software}')")
211 "INSERT INTO instances (domain, origin, command, hash, software, first_seen) VALUES (?, ?, ?, ?, ?, ?)",
216 fba.get_hash(domain),
222 cache.set_sub_key("is_registered", domain, True)
224 if has_pending_instance_data(domain):
225 # DEBUG: print(f"DEBUG: domain='{domain}' has pending nodeinfo being updated ...")
226 set_data("last_status_code" , domain, None)
227 set_data("last_error_details", domain, None)
229 fba.remove_pending_error(domain)
231 if domain in fba.pending_errors:
232 # DEBUG: print("DEBUG: domain has pending error being updated:", domain)
233 update_last_error(domain, fba.pending_errors[domain])
234 fba.remove_pending_error(domain)
236 except BaseException as exc:
237 print(f"ERROR: failed SQL query: domain='{domain}',exc[{type(exc)}]:'{str(exc)}'")
240 # DEBUG: print("DEBUG: Updating nodeinfo for domain:", domain)
241 update_last_nodeinfo(domain)
243 # DEBUG: print("DEBUG: EXIT!")
245 def update_last_nodeinfo(domain: str):
246 # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
247 if not isinstance(domain, str):
248 raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
250 raise ValueError("Parameter 'domain' is empty")
252 # DEBUG: print("DEBUG: Updating last_nodeinfo for domain:", domain)
253 set_data("last_nodeinfo", domain, time.time())
254 set_data("last_updated" , domain, time.time())
256 # Running pending updated
257 # DEBUG: print(f"DEBUG: Invoking update_data({domain}) ...")
260 # DEBUG: print("DEBUG: EXIT!")
262 def update_last_error(domain: str, response: requests.models.Response):
263 # DEBUG: print("DEBUG: domain,response[]:", domain, type(response))
264 if not isinstance(domain, str):
265 raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
267 raise ValueError("Parameter 'domain' is empty")
269 # DEBUG: print("DEBUG: BEFORE response[]:", type(response))
270 if isinstance(response, BaseException) or isinstance(response, json.decoder.JSONDecodeError):
271 response = f"response[{type(response)}]='{str(response)}'"
273 # DEBUG: print("DEBUG: AFTER response[]:", type(response))
274 if isinstance(response, str):
275 # DEBUG: print(f"DEBUG: Setting last_error_details='{response}'")
276 set_data("last_status_code" , domain, 999)
277 set_data("last_error_details", domain, response)
279 # DEBUG: print(f"DEBUG: Setting last_error_details='{response.reason}'")
280 set_data("last_status_code" , domain, response.status_code)
281 set_data("last_error_details", domain, response.reason)
283 # Running pending updated
284 # DEBUG: print(f"DEBUG: Invoking update_data({domain}) ...")
287 fba.log_error(domain, response)
289 # DEBUG: print("DEBUG: EXIT!")
291 def is_registered(domain: str) -> bool:
292 # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
293 if not isinstance(domain, str):
294 raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
296 raise ValueError("Parameter 'domain' is empty")
298 # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
299 if not cache.key_exists("is_registered"):
300 # DEBUG: print("DEBUG: Cache for 'is_registered' not initialized, fetching all rows ...")
302 fba.cursor.execute("SELECT domain FROM instances")
305 cache.set_all("is_registered", fba.cursor.fetchall(), True)
306 except BaseException as exc:
307 print(f"ERROR: failed SQL query: domain='{domain}',exc[{type(exc)}]:'{str(exc)}'")
311 registered = cache.sub_key_exists("is_registered", domain)
313 # DEBUG: print(f"DEBUG: registered='{registered}' - EXIT!")