1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
2 # Copyright (C) 2023 Free Software Foundation
4 # This program is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU Affero General Public License as published
6 # by the Free Software Foundation, either version 3 of the License, or
7 # (at your option) any later version.
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU Affero General Public License for more details.
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <https://www.gnu.org/licenses/>.
23 from fba import blacklist
27 # Found info from node, such as nodeinfo URL, detection mode that needs to be
28 # written to database. Both arrays must be filled at the same time or else
29 # update_data() will fail
31 # Detection mode: 'AUTO_DISCOVERY', 'STATIC_CHECKS' or 'GENERATOR'
32 # NULL means all detection methods have failed (maybe still reachable instance)
33 "detection_mode" : {},
38 # Last fetched instances
39 "last_instance_fetch": {},
44 # Last nodeinfo (fetched)
47 "last_status_code" : {},
49 "last_error_details" : {},
52 def set(key: str, domain: str, value: any):
53 # DEBUG: print(f"DEBUG: key='{key}',domain='{domain}',value[]='{type(value)}' - CALLED!")
54 if not isinstance(key, str):
55 raise ValueError("Parameter key[]='{type(key)}' is not 'str'")
57 raise ValueError("Parameter 'key' is empty")
58 elif not isinstance(domain, str):
59 raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
61 raise ValueError("Parameter 'domain' is empty")
62 elif not key in _pending:
63 raise ValueError(f"key='{key}' not found in _pending")
64 elif not fba.is_primitive(value):
65 raise ValueError(f"value[]='{type(value)}' is not a primitive type")
68 _pending[key][domain] = value
70 # DEBUG: print("DEBUG: EXIT!")
72 def has_pending_instance_data(domain: str) -> bool:
73 # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
74 if not isinstance(domain, str):
75 raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
77 raise ValueError("Parameter 'domain' is empty")
81 # DEBUG: print(f"DEBUG: key='{key}',domain='{domain}',_pending[key]()='{len(_pending[key])}'")
82 if domain in _pending[key]:
86 # DEBUG: print(f"DEBUG: has_pending='{has_pending}' - EXIT!")
89 def update_data(domain: str):
90 # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
91 if not isinstance(domain, str):
92 raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
94 raise ValueError("Parameter 'domain' is empty")
95 elif not has_pending_instance_data(domain):
96 raise Exception(f"Domain '{domain}' has no pending instance data, but function invoked")
98 # DEBUG: print(f"DEBUG: Updating instance data for domain='{domain}' ...")
102 # DEBUG: print("DEBUG: key:", key)
103 if domain in _pending[key]:
104 # DEBUG: print(f"DEBUG: Adding '{_pending[key][domain]}' for key='{key}' ...")
105 fields.append(_pending[key][domain])
106 sql_string += f" {key} = ?,"
108 fields.append(time.time())
109 fields.append(domain)
112 raise ValueError(f"No fields have been set, but method invoked, domain='{domain}'")
114 # DEBUG: print(f"DEBUG: sql_string='{sql_string}',fields()={len(fields)}")
115 sql_string = "UPDATE instances SET" + sql_string + " last_updated = ? WHERE domain = ? LIMIT 1"
116 # DEBUG: print("DEBUG: sql_string:", sql_string)
119 # DEBUG: print("DEBUG: Executing SQL:", sql_string)
120 fba.cursor.execute(sql_string, fields)
122 # DEBUG: print(f"DEBUG: Success! (rowcount={fba.cursor.rowcount })")
123 if fba.cursor.rowcount == 0:
124 # DEBUG: print(f"DEBUG: Did not update any rows: domain='{domain}',fields()={len(fields)} - EXIT!")
127 # DEBUG: print("DEBUG: Committing changes ...")
128 fba.connection.commit()
130 # DEBUG: print("DEBUG: Deleting _pending for domain:", domain)
133 # DEBUG: print("DEBUG: Deleting key:", key)
134 del _pending[key][domain]
138 except BaseException as exception:
139 print(f"ERROR: failed SQL query: domain='{domain}',sql_string='{sql_string}',exception[{type(exception)}]:'{str(exception)}'")
142 # DEBUG: print("DEBUG: EXIT!")
144 def update_last_instance_fetch(domain: str):
145 # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
146 if not isinstance(domain, str):
147 raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
149 raise ValueError("Parameter 'domain' is empty")
151 # DEBUG: print("DEBUG: Updating last_instance_fetch for domain:", domain)
152 set("last_instance_fetch", domain, time.time())
154 # Running pending updated
155 # DEBUG: print(f"DEBUG: Invoking update_data({domain}) ...")
158 # DEBUG: print("DEBUG: EXIT!")
160 def update_last_blocked(domain: str):
161 if not isinstance(domain, str):
162 raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
164 raise ValueError("Parameter 'domain' is empty")
166 # DEBUG: print("DEBUG: Updating last_blocked for domain", domain)
167 set("last_blocked", domain, time.time())
169 # Running pending updated
170 # DEBUG: print(f"DEBUG: Invoking update_data({domain}) ...")
173 # DEBUG: print("DEBUG: EXIT!")
175 def add(domain: str, origin: str, command: str, path: str = None):
176 # DEBUG: print(f"DEBUG: domain='{domain}',origin='{origin}',command='{command}',path='{path}' - CALLED!")
177 if not isinstance(domain, str):
178 raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
180 raise ValueError("Parameter 'domain' is empty")
181 elif not isinstance(origin, str) and origin is not None:
182 raise ValueError(f"origin[]={type(origin)} is not 'str'")
184 raise ValueError("Parameter 'origin' is empty")
185 elif not isinstance(command, str):
186 raise ValueError(f"command[]={type(command)} is not 'str'")
188 raise ValueError("Parameter 'command' is empty")
189 elif not validators.domain(domain.split("/")[0]):
190 raise ValueError(f"Bad domain name='{domain}'")
191 elif origin is not None and not validators.domain(origin.split("/")[0]):
192 raise ValueError(f"Bad origin name='{origin}'")
193 elif blacklist.is_blacklisted(domain):
194 raise Exception(f"domain='{domain}' is blacklisted, but method invoked")
195 elif domain.find("/profile/") > 0 or domain.find("/users/") > 0:
196 raise Exception(f"domain='{domain}' is a single user")
198 # DEBUG: print("DEBUG: domain,origin,command,path:", domain, origin, command, path)
199 software = fba.determine_software(domain, path)
200 # DEBUG: print("DEBUG: Determined software:", software)
201 if domain.find("/c/") > 0 and software == "lemmy":
202 domain = domain.split("/c/")[0]
203 if is_registered(domain):
204 print(f"WARNING: domain='{domain}' already registered after cutting off user part. - EXIT!")
207 print(f"INFO: Adding instance domain='{domain}' (origin='{origin}',software='{software}')")
210 "INSERT INTO instances (domain, origin, command, hash, software, first_seen) VALUES (?, ?, ?, ?, ?, ?)",
215 fba.get_hash(domain),
221 cache.set_sub_key("is_registered", domain, True)
223 if has_pending_instance_data(domain):
224 # DEBUG: print(f"DEBUG: domain='{domain}' has pending nodeinfo being updated ...")
225 set("last_status_code" , domain, None)
226 set("last_error_details", domain, None)
228 fba.remove_pending_error(domain)
230 if domain in fba.pending_errors:
231 # DEBUG: print("DEBUG: domain has pending error being updated:", domain)
232 update_last_error(domain, fba.pending_errors[domain])
233 fba.remove_pending_error(domain)
235 except BaseException as exception:
236 print(f"ERROR: failed SQL query: domain='{domain}',exception[{type(exception)}]:'{str(exception)}'")
239 # DEBUG: print("DEBUG: Updating nodeinfo for domain:", domain)
240 update_last_nodeinfo(domain)
242 # DEBUG: print("DEBUG: EXIT!")
244 def update_last_nodeinfo(domain: str):
245 # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
246 if not isinstance(domain, str):
247 raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
249 raise ValueError("Parameter 'domain' is empty")
251 # DEBUG: print("DEBUG: Updating last_nodeinfo for domain:", domain)
252 set("last_nodeinfo", domain, time.time())
253 set("last_updated" , domain, time.time())
255 # Running pending updated
256 # DEBUG: print(f"DEBUG: Invoking update_data({domain}) ...")
259 # DEBUG: print("DEBUG: EXIT!")
261 def update_last_error(domain: str, response: requests.models.Response):
262 # DEBUG: print("DEBUG: domain,response[]:", domain, type(response))
263 if not isinstance(domain, str):
264 raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
266 raise ValueError("Parameter 'domain' is empty")
268 # DEBUG: print("DEBUG: BEFORE response[]:", type(response))
269 if isinstance(response, BaseException) or isinstance(response, json.decoder.JSONDecodeError):
270 response = f"{type}:str(response)"
272 # DEBUG: print("DEBUG: AFTER response[]:", type(response))
273 if isinstance(response, str):
274 # DEBUG: print(f"DEBUG: Setting last_error_details='{response}'");
275 set("last_status_code" , domain, 999)
276 set("last_error_details", domain, response)
278 # DEBUG: print(f"DEBUG: Setting last_error_details='{response.reason}'");
279 set("last_status_code" , domain, response.status_code)
280 set("last_error_details", domain, response.reason)
282 # Running pending updated
283 # DEBUG: print(f"DEBUG: Invoking update_data({domain}) ...")
286 fba.log_error(domain, response)
288 # DEBUG: print("DEBUG: EXIT!")
290 def is_registered(domain: str) -> bool:
291 # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
292 if not isinstance(domain, str):
293 raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
295 raise ValueError("Parameter 'domain' is empty")
297 # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
298 if not cache.key_exists("is_registered"):
299 # DEBUG: print(f"DEBUG: Cache for 'is_registered' not initialized, fetching all rows ...")
301 fba.cursor.execute("SELECT domain FROM instances")
304 cache.set_all("is_registered", fba.cursor.fetchall(), True)
305 except BaseException as exception:
306 print(f"ERROR: failed SQL query: domain='{domain}',exception[{type(exception)}]:'{str(exception)}'")
310 registered = cache.sub_key_exists("is_registered", domain)
312 # DEBUG: print(f"DEBUG: registered='{registered}' - EXIT!")