]> git.mxchange.org Git - fba.git/blob - fetch_blocks.py
040eddcea157e656588a582a6b58180f53a45701
[fba.git] / fetch_blocks.py
1 import time
2 import bs4
3 import fba
4 import itertools
5 import re
6
7 fba.c.execute(
8     "SELECT domain, software FROM instances WHERE domain='tooting.intensifi.es'"
9     #"SELECT domain, software FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial')"
10 )
11
12 for blocker, software in fba.c.fetchall():
13     print("DEBUG: blocker,software:", blocker, software)
14     blockdict = []
15     blocker = fba.tidyup(blocker)
16     if software == "pleroma":
17         print("DEBUG: blocker:", blocker)
18         try:
19             # Blocks
20             federation = reqto.get(
21                 f"https://{blocker}/nodeinfo/2.1.json", headers=headers, timeout=5
22             ).json()["metadata"]["federation"]
23             if "mrf_simple" in federation:
24                 for block_level, blocks in (
25                     {**federation["mrf_simple"],
26                     **{"quarantined_instances": federation["quarantined_instances"]}}
27                 ).items():
28                     for blocked in blocks:
29                         blocked = fba.tidyup(blocked)
30
31                         if blocked == "":
32                             print("WARNING: blocked is empty after fba.tidyup():", blocker, block_level)
33                             continue
34
35                         if blocked.count("*") > 1:
36                             # -ACK!-oma also started obscuring domains without hash
37                             fba.c.execute(
38                                 "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", (blocked.replace("*", "_"),)
39                             )
40                             searchres = fba.c.fetchone()
41                             if searchres != None:
42                                 blocked = searchres[0]
43
44                         fba.c.execute(
45                             "SELECT domain FROM instances WHERE domain = ?", (blocked)
46                         )
47
48                         if fba.c.fetchone() == None:
49                             print("DEBUG: Hash wasn't found, adding:", blocked)
50                             fba.add_instance(blocked)
51
52                         timestamp = int(time.time())
53                         fba.c.execute(
54                             "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ?",
55                             (
56                                blocker,
57                                blocked,
58                                block_level
59                            ),
60                         )
61                         if fba.c.fetchone() == None:
62                             fba.block_instance(blocker, blocked, reason, block_level, timestamp, timestamp)
63
64                             if block_level == "reject":
65                                 blockdict.append(
66                                     {
67                                         "blocked": blocked,
68                                         "reason": None
69                                     })
70                         else:
71                             fba.update_last_seen(timestamp, blocker, blocked, block_level)
72
73             fba.conn.commit()
74             # Reasons
75             if "mrf_simple_info" in federation:
76                 for block_level, info in (
77                     {**federation["mrf_simple_info"],
78                     **(federation["quarantined_instances_info"]
79                     if "quarantined_instances_info" in federation
80                     else {})}
81                 ).items():
82                     for blocked, reason in info.items():
83                         blocked = fba.tidyup(blocked)
84
85                         if blocked == "":
86                             print("WARNING: blocked is empty after fba.tidyup():", blocker, block_level)
87                             continue
88
89                         if blocked.count("*") > 1:
90                             # same domain guess as above, but for reasons field
91                             fba.c.execute(
92                                 "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", (blocked.replace("*", "_"),)
93                             )
94                             searchres = fba.c.fetchone()
95
96                             if searchres != None:
97                                 blocked = searchres[0]
98
99                         fba.update_block_reason(reason["reason"], blocker, blocked, block_level)
100
101                         for entry in blockdict:
102                             if entry["blocked"] == blocked:
103                                 entry["reason"] = reason["reason"]
104
105             fba.conn.commit()
106         except Exception as e:
107             print("error:", e, blocker, software)
108     elif software == "mastodon":
109         print("DEBUG: blocker:", blocker)
110         try:
111             # json endpoint for newer mastodongs
112             try:
113                 json = {
114                     "reject": [],
115                     "media_removal": [],
116                     "followers_only": [],
117                     "report_removal": []
118                 }
119
120                 # handling CSRF, I've saw at least one server requiring it to access the endpoint
121                 meta = bs4.BeautifulSoup(
122                     reqto.get(f"https://{blocker}/about", headers=headers, timeout=5).text,
123                     "html.parser",
124                 )
125                 try:
126                     csrf = meta.find("meta", attrs={"name": "csrf-token"})["content"]
127                     reqheaders = {**headers, **{"x-csrf-token": csrf}}
128                 except:
129                     reqheaders = headers
130
131                 blocks = reqto.get(
132                     f"https://{blocker}/api/v1/instance/domain_blocks", headers=reqheaders, timeout=5
133                 ).json()
134
135                 print("DEBUG: blocks():", len(blocks))
136                 for block in blocks:
137                     entry = {'domain': block['domain'], 'hash': block['digest'], 'reason': block['comment']}
138
139                     print("DEBUG: severity,domain,hash,comment:", block['severity'], block['domain'], block['digest'], block['comment'])
140                     if block['severity'] == 'suspend':
141                         json['reject'].append(entry)
142                     elif block['severity'] == 'silence':
143                         json['followers_only'].append(entry)
144                     elif block['severity'] == 'reject_media':
145                         json['media_removal'].append(entry)
146                     elif block['severity'] == 'reject_reports':
147                         json['report_removal'].append(entry)
148                     else:
149                         print("WARNING: Unknown severity:", block['severity'], block['domain'])
150             except:
151                 json = fba.get_mastodon_blocks(blocker)
152
153             for block_level, blocks in json.items():
154                 for instance in blocks:
155                     blocked, blocked_hash, reason = instance.values()
156                     print("DEBUG: blocked,hash,reason:", blocked, blocked_hash, reason)
157
158                     blocked = fba.tidyup(blocked)
159                     print("DEBUG: blocked:", blocked)
160
161                     if blocked.count("*") < 1:
162                         # No obsfucation for this instance
163                         fba.c.execute(
164                             "SELECT hash FROM instances WHERE domain = ? LIMIT 1", (blocked,)
165                         )
166
167                         if fba.c.fetchone() == None:
168                             print("DEBUG: Hash wasn't found, adding:", blocked)
169                             fba.add_instance(blocked)
170                     else:
171                         # Doing the hash search for instance names as well to tidy up DB
172                         fba.c.execute(
173                             "SELECT domain FROM instances WHERE hash = ? LIMIT 1", (blocked_hash,)
174                         )
175                         searchres = fba.c.fetchone()
176
177                         if searchres != None:
178                             print("DEBUG: Updating domain: ", searchres[0])
179                             blocked = searchres[0]
180
181                     timestamp = int(time.time())
182                     fba.c.execute(
183                         "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ?",
184                         (
185                             blocker,
186                             blocked if blocked.count("*") <= 1 else blocked_hash,
187                             block_level
188                         ),
189                     )
190
191                     if fba.c.fetchone() == None:
192                         fba.block_instance(blocker, blocked if blocked.count("*") <= 1 else blocked_hash, reason, block_level, timestamp, timestamp)
193
194                         if block_level == "reject":
195                             blockdict.append(
196                                 {
197                                     "blocked": blocked,
198                                     "reason": reason
199                                 })
200                     else:
201                         fba.update_last_seen(timestamp, blocker, blocked if blocked.count("*") <= 1 else blocked_hash, block_level)
202
203                     if reason != '':
204                         fba.update_block_reason(reason, blocker, blocked if blocked.count("*") <= 1 else blocked_hash, block_level)
205
206             fba.conn.commit()
207         except Exception as e:
208             print("error:", e, blocker, software)
209     elif software == "friendica" or software == "misskey":
210         print("DEBUG: blocker:", blocker)
211         try:
212             if software == "friendica":
213                 json = fba.get_friendica_blocks(blocker)
214             elif software == "misskey":
215                 json = fba.get_misskey_blocks(blocker)
216             for block_level, blocks in json.items():
217                 for instance in blocks:
218                     blocked, reason = instance.values()
219                     blocked = fba.tidyup(blocked)
220
221                     print("BEFORE-blocked:", blocked)
222                     if blocked.count("*") > 0:
223                         # Some friendica servers also obscure domains without hash
224                         fba.c.execute(
225                             "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", (blocked.replace("*", "_"),)
226                         )
227                         searchres = fba.c.fetchone()
228                         if searchres != None:
229                             blocked = searchres[0]
230
231                     if blocked.count("?") > 0:
232                         # Some obscure them with question marks, not sure if that's dependent on version or not
233                         fba.c.execute(
234                             "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", (blocked.replace("?", "_"),)
235                         )
236                         searchres = fba.c.fetchone()
237                         if searchres != None:
238                             blocked = searchres[0]
239
240                     print("AFTER-blocked:", blocked)
241                     fba.c.execute(
242                         "SELECT domain FROM instances WHERE domain = ?", (blocked,)
243                     )
244
245                     if fba.c.fetchone() == None:
246                         print("DEBUG: Hash wasn't found, adding:", blocked)
247                         fba.add_instance(blocked)
248
249                     timestamp = int(time.time())
250                     fba.c.execute(
251                         "SELECT * FROM blocks WHERE blocker = ? AND blocked = ?",
252                         (blocker, blocked),
253                     )
254                     if fba.c.fetchone() == None:
255                         fba.block_instance(blocker, blocked, reason, block_level, timestamp, timestamp)
256
257                         if block_level == "reject":
258                             blockdict.append(
259                                 {
260                                     "blocked": blocked,
261                                     "reason": reason
262                                 })
263                     else:
264                         fba.update_last_seen(timestamp, blocker, blocked, block_level)
265
266                     if reason != '':
267                         fba.update_block_reason(reason, blocker, blocked, block_level)
268
269             fba.conn.commit()
270         except Exception as e:
271             print("error:", e, blocker, software)
272     elif software == "gotosocial":
273         print("DEBUG: blocker:", blocker)
274         try:
275             # Blocks
276             federation = reqto.get(
277                 f"https://{blocker}/api/v1/instance/peers?filter=suspended", headers=headers, timeout=5
278             ).json()
279
280             if (federation == None):
281                 print("WARNING: No valid response:", blocker);
282             else:
283                 for peer in federation:
284                     blocked = peer["domain"].lower()
285
286                     if blocked.count("*") > 0:
287                         # GTS does not have hashes for obscured domains, so we have to guess it
288                         fba.c.execute(
289                             "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", (blocked.replace("*", "_"),)
290                         )
291                         searchres = fba.c.fetchone()
292
293                         if searchres != None:
294                             blocked = searchres[0]
295
296                     fba.c.execute(
297                         "SELECT domain FROM instances WHERE domain = ?", (blocked,)
298                     )
299
300                     if fba.c.fetchone() == None:
301                         print("DEBUG: Hash wasn't found, adding:", blocked)
302                         fba.add_instance(blocked)
303
304                     fba.c.execute(
305                         "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ?",
306                         (
307                             blocker,
308                             blocked,
309                             "reject"
310                         ),
311                     )
312                     timestamp = int(time.time())
313
314                     if fba.c.fetchone() == None:
315                         fba.block_instance(blocker, blocked, "", "reject", timestamp, timestamp)
316
317                         blockdict.append(
318                             {
319                                 "blocked": blocked,
320                                 "reason": None
321                             })
322                     else:
323                         fba.update_last_seen(timestamp, blocker, blocked, "reject")
324
325                     if "public_comment" in peer:
326                         reason = peer["public_comment"]
327                         fba.update_block_reason(reason, blocker, blocked, "reject")
328
329                         for entry in blockdict:
330                             if entry["blocked"] == blocked:
331                                 entry["reason"] = reason
332                 fba.conn.commit()
333         except Exception as e:
334             print("error:", e, blocker, software)
335     else:
336         print("WARNING: Unknown software:", software)
337
338     if fba.config["bot_enabled"] and len(blockdict) > 0:
339         send_bot_post(blocker, blockdict)
340
341     blockdict = []
342
343 fba.conn.close()