8 "SELECT domain, software FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial')"
11 for blocker, software in fba.c.fetchall():
12 print("DEBUG: blocker,software:", blocker, software)
14 blocker = fba.tidyup(blocker)
15 if software == "pleroma":
16 print("DEBUG: blocker:", blocker)
19 federation = reqto.get(
20 f"https://{blocker}/nodeinfo/2.1.json", headers=headers, timeout=5
21 ).json()["metadata"]["federation"]
22 if "mrf_simple" in federation:
23 for block_level, blocks in (
24 {**federation["mrf_simple"],
25 **{"quarantined_instances": federation["quarantined_instances"]}}
27 for blocked in blocks:
28 blocked = fba.tidyup(blocked)
31 print("WARNING: blocked is empty after fba.tidyup():", blocker, block_level)
34 if blocked.count("*") > 1:
35 # -ACK!-oma also started obscuring domains without hash
37 "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", (blocked.replace("*", "_"),)
39 searchres = fba.c.fetchone()
41 blocked = searchres[0]
44 "SELECT domain FROM instances WHERE domain = ?", (blocked)
47 if fba.c.fetchone() == None:
48 print("DEBUG: Hash wasn't found, adding:", blocked)
49 fba.add_instance(blocked)
51 timestamp = int(time.time())
53 "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ?",
60 if fba.c.fetchone() == None:
61 fba.block_instance(blocker, blocked, reason, block_level, timestamp, timestamp)
63 if block_level == "reject":
70 fba.update_last_seen(timestamp, blocker, blocked, block_level)
74 if "mrf_simple_info" in federation:
75 for block_level, info in (
76 {**federation["mrf_simple_info"],
77 **(federation["quarantined_instances_info"]
78 if "quarantined_instances_info" in federation
81 for blocked, reason in info.items():
82 blocked = fba.tidyup(blocked)
85 print("WARNING: blocked is empty after fba.tidyup():", blocker, block_level)
88 if blocked.count("*") > 1:
89 # same domain guess as above, but for reasons field
91 "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", (blocked.replace("*", "_"),)
93 searchres = fba.c.fetchone()
96 blocked = searchres[0]
98 fba.update_block_reason(reason["reason"], blocker, blocked, block_level)
100 for entry in blockdict:
101 if entry["blocked"] == blocked:
102 entry["reason"] = reason["reason"]
105 except Exception as e:
106 print("error:", e, blocker, software)
107 elif software == "mastodon":
108 print("DEBUG: blocker:", blocker)
110 # json endpoint for newer mastodongs
115 "followers_only": [],
119 # handling CSRF, I've saw at least one server requiring it to access the endpoint
120 meta = bs4.BeautifulSoup(
121 reqto.get(f"https://{blocker}/about", headers=headers, timeout=5).text,
125 csrf = meta.find("meta", attrs={"name": "csrf-token"})["content"]
126 reqheaders = {**headers, **{"x-csrf-token": csrf}}
131 f"https://{blocker}/api/v1/instance/domain_blocks", headers=reqheaders, timeout=5
134 print("DEBUG: blocks():", len(blocks))
136 entry = {'domain': block['domain'], 'hash': block['digest'], 'reason': block['comment']}
138 print("DEBUG: severity,domain,hash,comment:", block['severity'], block['domain'], block['digest'], block['comment'])
139 if block['severity'] == 'suspend':
140 json['reject'].append(entry)
141 elif block['severity'] == 'silence':
142 json['followers_only'].append(entry)
143 elif block['severity'] == 'reject_media':
144 json['media_removal'].append(entry)
145 elif block['severity'] == 'reject_reports':
146 json['report_removal'].append(entry)
148 print("WARNING: Unknown severity:", block['severity'], block['domain'])
150 json = fba.get_mastodon_blocks(blocker)
152 for block_level, blocks in json.items():
153 for instance in blocks:
154 blocked, blocked_hash, reason = instance.values()
155 print("DEBUG: blocked,hash,reason:", blocked, blocked_hash, reason)
157 blocked = fba.tidyup(blocked)
158 print("DEBUG: blocked:", blocked)
160 if blocked.count("*") < 1:
161 # No obsfucation for this instance
163 "SELECT hash FROM instances WHERE domain = ? LIMIT 1", (blocked,)
166 if fba.c.fetchone() == None:
167 print("DEBUG: Hash wasn't found, adding:", blocked)
168 fba.add_instance(blocked)
170 # Doing the hash search for instance names as well to tidy up DB
172 "SELECT domain FROM instances WHERE hash = ? LIMIT 1", (blocked_hash,)
174 searchres = fba.c.fetchone()
176 if searchres != None:
177 print("DEBUG: Updating domain: ", searchres[0])
178 blocked = searchres[0]
180 timestamp = int(time.time())
182 "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ?",
185 blocked if blocked.count("*") <= 1 else blocked_hash,
190 if fba.c.fetchone() == None:
191 fba.block_instance(blocker, blocked if blocked.count("*") <= 1 else blocked_hash, reason, block_level, timestamp, timestamp)
193 if block_level == "reject":
200 fba.update_last_seen(timestamp, blocker, blocked if blocked.count("*") <= 1 else blocked_hash, block_level)
203 fba.update_block_reason(reason, blocker, blocked if blocked.count("*") <= 1 else blocked_hash, block_level)
206 except Exception as e:
207 print("error:", e, blocker, software)
208 elif software == "friendica" or software == "misskey":
209 print("DEBUG: blocker:", blocker)
211 if software == "friendica":
212 json = fba.get_friendica_blocks(blocker)
213 elif software == "misskey":
214 json = fba.get_misskey_blocks(blocker)
215 for block_level, blocks in json.items():
216 for instance in blocks:
217 blocked, reason = instance.values()
218 blocked = fba.tidyup(blocked)
220 print("BEFORE-blocked:", blocked)
221 if blocked.count("*") > 0:
222 # Some friendica servers also obscure domains without hash
224 "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", (blocked.replace("*", "_"),)
226 searchres = fba.c.fetchone()
227 if searchres != None:
228 blocked = searchres[0]
230 if blocked.count("?") > 0:
231 # Some obscure them with question marks, not sure if that's dependent on version or not
233 "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", (blocked.replace("?", "_"),)
235 searchres = fba.c.fetchone()
236 if searchres != None:
237 blocked = searchres[0]
239 print("AFTER-blocked:", blocked)
241 "SELECT domain FROM instances WHERE domain = ?", (blocked,)
244 if fba.c.fetchone() == None:
245 print("DEBUG: Hash wasn't found, adding:", blocked)
246 fba.add_instance(blocked)
248 timestamp = int(time.time())
250 "SELECT * FROM blocks WHERE blocker = ? AND blocked = ?",
253 if fba.c.fetchone() == None:
254 fba.block_instance(blocker, blocked, reason, block_level, timestamp, timestamp)
256 if block_level == "reject":
263 fba.update_last_seen(timestamp, blocker, blocked, block_level)
266 fba.update_block_reason(reason, blocker, blocked, block_level)
269 except Exception as e:
270 print("error:", e, blocker, software)
271 elif software == "gotosocial":
272 print("DEBUG: blocker:", blocker)
275 federation = reqto.get(
276 f"https://{blocker}/api/v1/instance/peers?filter=suspended", headers=headers, timeout=5
279 if (federation == None):
280 print("WARNING: No valid response:", blocker);
282 for peer in federation:
283 blocked = peer["domain"].lower()
285 if blocked.count("*") > 0:
286 # GTS does not have hashes for obscured domains, so we have to guess it
288 "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", (blocked.replace("*", "_"),)
290 searchres = fba.c.fetchone()
292 if searchres != None:
293 blocked = searchres[0]
296 "SELECT domain FROM instances WHERE domain = ?", (blocked,)
299 if fba.c.fetchone() == None:
300 print("DEBUG: Hash wasn't found, adding:", blocked)
301 fba.add_instance(blocked)
304 "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ?",
311 timestamp = int(time.time())
313 if fba.c.fetchone() == None:
314 fba.block_instance(blocker, blocked, "", "reject", timestamp, timestamp)
322 fba.update_last_seen(timestamp, blocker, blocked, "reject")
324 if "public_comment" in peer:
325 reason = peer["public_comment"]
326 fba.update_block_reason(reason, blocker, blocked, "reject")
328 for entry in blockdict:
329 if entry["blocked"] == blocked:
330 entry["reason"] = reason
332 except Exception as e:
333 print("error:", e, blocker, software)
335 print("WARNING: Unknown software:", software)
337 if fba.config["bot_enabled"] and len(blockdict) > 0:
338 send_bot_post(blocker, blockdict)