9 "SELECT domain, software FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial')"
12 for blocker, software in fba.c.fetchall():
13 # NOISY-DEBUG: print("DEBUG: BEFORE-blocker,software:", blocker, software)
15 blocker = fba.tidyup(blocker)
16 # NOISY-DEBUG: print("DEBUG: AFTER-blocker,software:", blocker, software)
19 print("WARNING: blocker is now empty!")
22 if software == "pleroma":
23 print("INFO: blocker:", blocker)
26 federation = reqto.get(
27 f"https://{blocker}/nodeinfo/2.1.json", headers=fba.headers, timeout=5
28 ).json()["metadata"]["federation"]
29 if "mrf_simple" in federation:
30 for block_level, blocks in (
31 {**federation["mrf_simple"],
32 **{"quarantined_instances": federation["quarantined_instances"]}}
34 # NOISY-DEBUG: print("DEBUG: block_level, blocks.items():", block_level, len(blocks.items()))
35 block_level = fba.tidyup(block_level)
36 # NOISY-DEBUG: print("DEBUG: BEFORE block_level:", block_level)
39 print("WARNING: block_level is now empty!")
42 for blocked in blocks:
43 # NOISY-DEBUG: print("DEBUG: BEFORE blocked:", blocked)
44 blocked = fba.tidyup(blocked)
45 # NOISY-DEBUG: print("DEBUG: AFTER blocked:", blocked)
48 print("WARNING: blocked is empty after fba.tidyup():", blocker, block_level)
51 if blocked.count("*") > 1:
52 # -ACK!-oma also started obscuring domains without hash
54 "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", (blocked.replace("*", "_"),)
56 searchres = fba.c.fetchone()
57 # NOISY-DEBUG: print("DEBUG: searchres[]:", type(searchres))
59 blocked = searchres[0]
60 # NOISY-DEBUG: print("DEBUG: Looked up domain:", blocked)
62 # NOISY-DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
64 "SELECT domain FROM instances WHERE domain = ?", (blocked,)
67 if fba.c.fetchone() == None:
68 # NOISY-DEBUG: print("DEBUG: Domain wasn't found, adding:", blocked)
69 fba.add_instance(blocked)
71 timestamp = int(time.time())
73 "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ?",
81 if fba.c.fetchone() == None:
82 # NOISY-DEBUG: print("DEBUG: Blocking:", blocker, blocked, block_level)
83 fba.block_instance(blocker, blocked, "unknown", block_level, timestamp, timestamp)
85 if block_level == "reject":
86 # NOISY-DEBUG: print("DEBUG: Adding to blockdict:", blocked)
93 # NOISY-DEBUG: print("DEBUG: Updating last_seen:", blocker, blocked, block_level)
94 fba.update_last_seen(timestamp, blocker, blocked, block_level)
99 if "mrf_simple_info" in federation:
100 # NOISY-DEBUG: print("DEBUG: Found mrf_simple_info:", blocker)
101 for block_level, info in (
102 {**federation["mrf_simple_info"],
103 **(federation["quarantined_instances_info"]
104 if "quarantined_instances_info" in federation
107 # NOISY-DEBUG: print("DEBUG: block_level, info.items():", block_level, len(info.items()))
108 block_level = fba.tidyup(block_level)
109 # NOISY-DEBUG: print("DEBUG: BEFORE block_level:", block_level)
111 if block_level == "":
112 print("WARNING: block_level is now empty!")
115 for blocked, reason in info.items():
116 # NOISY-DEBUG: print("DEBUG: BEFORE blocked:", blocked)
117 blocked = fba.tidyup(blocked)
118 # NOISY-DEBUG: print("DEBUG: AFTER blocked:", blocked)
121 print("WARNING: blocked is empty after fba.tidyup():", blocker, block_level)
123 elif blocked.count("*") > 1:
124 # same domain guess as above, but for reasons field
126 "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", (blocked.replace("*", "_"),)
128 searchres = fba.c.fetchone()
130 if searchres != None:
131 blocked = searchres[0]
133 # NOISY-DEBUG: print("DEBUG: Updating block reason:", blocker, blocked, reason["reason"])
134 fba.update_block_reason(reason["reason"], blocker, blocked, block_level)
136 for entry in blockdict:
137 if entry["blocked"] == blocked:
138 # NOISY-DEBUG: print("DEBUG: Updating entry reason:", blocked)
139 entry["reason"] = reason["reason"]
142 except Exception as e:
143 print("error:", e, blocker, software)
144 elif software == "mastodon":
145 print("INFO: blocker:", blocker)
147 # json endpoint for newer mastodongs
152 "followers_only": [],
156 # handling CSRF, I've saw at least one server requiring it to access the endpoint
157 # NOISY-DEBUG: print("DEBUG: Fetching meta:", blocker)
158 meta = bs4.BeautifulSoup(
159 reqto.get(f"https://{blocker}/about", headers=fba.headers, timeout=5).text,
163 csrf = meta.find("meta", attrs={"name": "csrf-token"})["content"]
164 # NOISY-DEBUG: print("DEBUG: Adding CSRF token:", blocker, csrf)
165 reqheaders = {**fba.headers, **{"x-csrf-token": csrf}}
167 # NOISY-DEBUG: print("DEBUG: No CSRF token found, using normal headers:", blocker)
168 reqheaders = fba.headers
170 # NOISY-DEBUG: print("DEBUG: Quering API domain_blocks:", blocker)
172 f"https://{blocker}/api/v1/instance/domain_blocks", headers=reqheaders, timeout=5
175 # NOISY-DEBUG: print("DEBUG: blocks():", len(blocks))
178 'domain': block['domain'],
179 'hash': block['digest'],
180 'reason': block['comment']
183 # NOISY-DEBUG: print("DEBUG: severity,domain,hash,comment:", block['severity'], block['domain'], block['digest'], block['comment'])
184 if block['severity'] == 'suspend':
185 json['reject'].append(entry)
186 elif block['severity'] == 'silence':
187 json['followers_only'].append(entry)
188 elif block['severity'] == 'reject_media':
189 json['media_removal'].append(entry)
190 elif block['severity'] == 'reject_reports':
191 json['report_removal'].append(entry)
193 print("WARNING: Unknown severity:", block['severity'], block['domain'])
195 # NOISY-DEBUG: print("DEBUG: Failed, Trying mastodon-specific fetches:", blocker)
196 json = fba.get_mastodon_blocks(blocker)
198 # NOISY-DEBUG: print("DEBUG: json.items():", blocker, len(json.items()))
199 for block_level, blocks in json.items():
200 # NOISY-DEBUG: print("DEBUG: blocker,block_level,blocks():", blocker, block_level, len(blocks))
201 block_level = fba.tidyup(block_level)
202 # NOISY-DEBUG: print("DEBUG: AFTER-block_level:", block_level)
203 if block_level == "":
204 print("WARNING: block_level is empty, blocker:", blocker)
207 for instance in blocks:
208 blocked, blocked_hash, reason = instance.values()
209 # NOISY-DEBUG: print("DEBUG: blocked,hash,reason:", blocked, blocked_hash, reason)
210 blocked = fba.tidyup(blocked)
211 # NOISY-DEBUG: print("DEBUG: AFTER-blocked:", blocked)
214 print("WARNING: blocked is empty:", blocker)
216 elif blocked.count("*") < 1:
217 # No obsfucation for this instance
219 "SELECT hash FROM instances WHERE domain = ? LIMIT 1", (blocked,)
222 if fba.c.fetchone() == None:
223 # NOISY-DEBUG: print("DEBUG: Hash wasn't found, adding:", blocked)
224 fba.add_instance(blocked)
226 # Doing the hash search for instance names as well to tidy up DB
228 "SELECT domain FROM instances WHERE hash = ? LIMIT 1", (blocked_hash,)
230 searchres = fba.c.fetchone()
232 if searchres != None:
233 # NOISY-DEBUG: print("DEBUG: Updating domain: ", searchres[0])
234 blocked = searchres[0]
236 timestamp = int(time.time())
238 "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ?",
241 blocked if blocked.count("*") <= 1 else blocked_hash,
246 if fba.c.fetchone() == None:
247 fba.block_instance(blocker, blocked if blocked.count("*") <= 1 else blocked_hash, reason, block_level, timestamp, timestamp)
249 if block_level == "reject":
256 fba.update_last_seen(timestamp, blocker, blocked if blocked.count("*") <= 1 else blocked_hash, block_level)
259 # NOISY-DEBUG: print("DEBUG: Updating block reason:", blocker, blocked, reason)
260 fba.update_block_reason(reason, blocker, blocked if blocked.count("*") <= 1 else blocked_hash, block_level)
263 except Exception as e:
264 print("error:", e, blocker, software)
265 elif software == "friendica" or software == "misskey":
266 print("INFO: blocker:", blocker)
268 if software == "friendica":
269 json = fba.get_friendica_blocks(blocker)
270 elif software == "misskey":
271 json = fba.get_misskey_blocks(blocker)
273 for block_level, blocks in json.items():
274 # NOISY-DEBUG: print("DEBUG: blocker,block_level,blocks():", blocker, block_level, len(blocks))
275 block_level = fba.tidyup(block_level)
276 # NOISY-DEBUG: print("DEBUG: AFTER-block_level:", block_level)
277 if block_level == "":
278 print("WARNING: block_level is empty, blocker:", blocker)
281 for instance in blocks:
282 blocked, reason = instance.values()
283 # NOISY-DEBUG: print("DEBUG: BEFORE-blocked:", blocked)
284 blocked = fba.tidyup(blocked)
285 # NOISY-DEBUG: print("DEBUG: AFTER-blocked:", blocked)
288 print("WARNING: blocked is empty:", blocker)
290 if blocked.count("*") > 0:
291 # Some friendica servers also obscure domains without hash
293 "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", (blocked.replace("*", "_"),)
295 searchres = fba.c.fetchone()
296 if searchres != None:
297 blocked = searchres[0]
299 if blocked.count("?") > 0:
300 # Some obscure them with question marks, not sure if that's dependent on version or not
302 "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", (blocked.replace("?", "_"),)
304 searchres = fba.c.fetchone()
305 if searchres != None:
306 blocked = searchres[0]
308 # NOISY-DEBUG: print("DEBUG: AFTER-blocked:", blocked)
310 "SELECT domain FROM instances WHERE domain = ?", (blocked,)
313 if fba.c.fetchone() == None:
314 # NOISY-DEBUG: print("DEBUG: Hash wasn't found, adding:", blocked)
315 fba.add_instance(blocked)
317 timestamp = int(time.time())
319 "SELECT * FROM blocks WHERE blocker = ? AND blocked = ?",
322 if fba.c.fetchone() == None:
323 fba.block_instance(blocker, blocked, reason, block_level, timestamp, timestamp)
325 if block_level == "reject":
332 fba.update_last_seen(timestamp, blocker, blocked, block_level)
335 # NOISY-DEBUG: print("DEBUG: Updating block reason:", blocker, blocked, reason)
336 fba.update_block_reason(reason, blocker, blocked, block_level)
339 except Exception as e:
340 print("error:", e, blocker, software)
341 elif software == "gotosocial":
342 print("INFO: blocker:", blocker)
345 federation = reqto.get(
346 f"https://{blocker}/api/v1/instance/peers?filter=suspended", headers=fba.headers, timeout=5
349 if (federation == None):
350 print("WARNING: No valid response:", blocker);
352 for peer in federation:
353 # NOISY-DEBUG: print("DEBUG: peer(),[]:", len(peer), type(peer))
354 if (isinstance(peer, str) and peer == "error"):
355 print("WARNING: Cannot continue, maybe authentication required?", blocker)
358 blocked = peer["domain"].lower()
359 # NOISY-DEBUG: print("DEBUG: BEFORE-blocked:", blocked)
360 blocked = fba.tidyup(blocked)
361 # NOISY-DEBUG: print("DEBUG: AFTER-blocked:", blocked)
364 print("WARNING: blocked is empty:", blocker)
366 if blocked.count("*") > 0:
367 # GTS does not have hashes for obscured domains, so we have to guess it
369 "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", (blocked.replace("*", "_"),)
371 searchres = fba.c.fetchone()
373 if searchres != None:
374 blocked = searchres[0]
377 "SELECT domain FROM instances WHERE domain = ?", (blocked,)
380 if fba.c.fetchone() == None:
381 # NOISY-DEBUG: print("DEBUG: Hash wasn't found, adding:", blocked)
382 fba.add_instance(blocked)
385 "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ?",
392 timestamp = int(time.time())
394 if fba.c.fetchone() == None:
395 fba.block_instance(blocker, blocked, "unknown", "reject", timestamp, timestamp)
403 fba.update_last_seen(timestamp, blocker, blocked, "reject")
405 if "public_comment" in peer:
406 reason = peer["public_comment"]
407 # NOISY-DEBUG: print("DEBUG: Updating block reason:", blocker, blocked, reason)
408 fba.update_block_reason(reason, blocker, blocked, "reject")
410 for entry in blockdict:
411 if entry["blocked"] == blocked:
412 entry["reason"] = reason
415 except Exception as e:
416 print("error:", e, blocker, software)
418 print("WARNING: Unknown software:", software)
420 if fba.config["bot_enabled"] and len(blockdict) > 0:
421 send_bot_post(blocker, blockdict)