fetch_blocks.py

   1 import time
   2 import bs4
   3 import fba
   4 import itertools
   5 import re
   6
   7 fba.c.execute(
   8     "SELECT domain, software FROM instances WHERE domain='tooting.intensifi.es'"
   9     #"SELECT domain, software FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial')"
  10 )
  11
  12 for blocker, software in fba.c.fetchall():
  13     print("DEBUG: blocker,software:", blocker, software)
  14     blockdict = []
  15     blocker = fba.tidyup(blocker)
  16     if software == "pleroma":
  17         print("DEBUG: blocker:", blocker)
  18         try:
  19             # Blocks
  20             federation = reqto.get(
  21                 f"https://{blocker}/nodeinfo/2.1.json", headers=headers, timeout=5
  22             ).json()["metadata"]["federation"]
  23             if "mrf_simple" in federation:
  24                 for block_level, blocks in (
  25                     {**federation["mrf_simple"],
  26                     **{"quarantined_instances": federation["quarantined_instances"]}}
  27                 ).items():
  28                     for blocked in blocks:
  29                         blocked = fba.tidyup(blocked)
  30
  31                         if blocked == "":
  32                             print("WARNING: blocked is empty after fba.tidyup():", blocker, block_level)
  33                             continue
  34
  35                         if blocked.count("*") > 1:
  36                             # -ACK!-oma also started obscuring domains without hash
  37                             fba.c.execute(
  38                                 "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", (blocked.replace("*", "_"),)
  39                             )
  40                             searchres = fba.c.fetchone()
  41                             if searchres != None:
  42                                 blocked = searchres[0]
  43
  44                         fba.c.execute(
  45                             "SELECT domain FROM instances WHERE domain = ?", (blocked)
  46                         )
  47
  48                         if fba.c.fetchone() == None:
  49                             print("DEBUG: Hash wasn't found, adding:", blocked)
  50                             fba.add_instance(blocked)
  51
  52                         timestamp = int(time.time())
  53                         fba.c.execute(
  54                             "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ?",
  55                             (
  56                                blocker,
  57                                blocked,
  58                                block_level
  59                            ),
  60                         )
  61                         if fba.c.fetchone() == None:
  62                             fba.block_instance(blocker, blocked, reason, block_level, timestamp, timestamp)
  63
  64                             if block_level == "reject":
  65                                 blockdict.append(
  66                                     {
  67                                         "blocked": blocked,
  68                                         "reason": None
  69                                     })
  70                         else:
  71                             fba.update_last_seen(timestamp, blocker, blocked, block_level)
  72
  73             fba.conn.commit()
  74             # Reasons
  75             if "mrf_simple_info" in federation:
  76                 for block_level, info in (
  77                     {**federation["mrf_simple_info"],
  78                     **(federation["quarantined_instances_info"]
  79                     if "quarantined_instances_info" in federation
  80                     else {})}
  81                 ).items():
  82                     for blocked, reason in info.items():
  83                         blocked = fba.tidyup(blocked)
  84
  85                         if blocked == "":
  86                             print("WARNING: blocked is empty after fba.tidyup():", blocker, block_level)
  87                             continue
  88
  89                         if blocked.count("*") > 1:
  90                             # same domain guess as above, but for reasons field
  91                             fba.c.execute(
  92                                 "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", (blocked.replace("*", "_"),)
  93                             )
  94                             searchres = fba.c.fetchone()
  95
  96                             if searchres != None:
  97                                 blocked = searchres[0]
  98
  99                         fba.update_block_reason(reason["reason"], blocker, blocked, block_level)
 100
 101                         for entry in blockdict:
 102                             if entry["blocked"] == blocked:
 103                                 entry["reason"] = reason["reason"]
 104
 105             fba.conn.commit()
 106         except Exception as e:
 107             print("error:", e, blocker, software)
 108     elif software == "mastodon":
 109         print("DEBUG: blocker:", blocker)
 110         try:
 111             # json endpoint for newer mastodongs
 112             try:
 113                 json = {
 114                     "reject": [],
 115                     "media_removal": [],
 116                     "followers_only": [],
 117                     "report_removal": []
 118                 }
 119
 120                 # handling CSRF, I've saw at least one server requiring it to access the endpoint
 121                 meta = bs4.BeautifulSoup(
 122                     reqto.get(f"https://{blocker}/about", headers=headers, timeout=5).text,
 123                     "html.parser",
 124                 )
 125                 try:
 126                     csrf = meta.find("meta", attrs={"name": "csrf-token"})["content"]
 127                     reqheaders = {**headers, **{"x-csrf-token": csrf}}
 128                 except:
 129                     reqheaders = headers
 130
 131                 blocks = reqto.get(
 132                     f"https://{blocker}/api/v1/instance/domain_blocks", headers=reqheaders, timeout=5
 133                 ).json()
 134
 135                 print("DEBUG: blocks():", len(blocks))
 136                 for block in blocks:
 137                     entry = {'domain': block['domain'], 'hash': block['digest'], 'reason': block['comment']}
 138
 139                     print("DEBUG: severity,domain,hash,comment:", block['severity'], block['domain'], block['digest'], block['comment'])
 140                     if block['severity'] == 'suspend':
 141                         json['reject'].append(entry)
 142                     elif block['severity'] == 'silence':
 143                         json['followers_only'].append(entry)
 144                     elif block['severity'] == 'reject_media':
 145                         json['media_removal'].append(entry)
 146                     elif block['severity'] == 'reject_reports':
 147                         json['report_removal'].append(entry)
 148                     else:
 149                         print("WARNING: Unknown severity:", block['severity'], block['domain'])
 150             except:
 151                 json = fba.get_mastodon_blocks(blocker)
 152
 153             for block_level, blocks in json.items():
 154                 for instance in blocks:
 155                     blocked, blocked_hash, reason = instance.values()
 156                     print("DEBUG: blocked,hash,reason:", blocked, blocked_hash, reason)
 157
 158                     blocked = fba.tidyup(blocked)
 159                     print("DEBUG: blocked:", blocked)
 160
 161                     if blocked.count("*") < 1:
 162                         # No obsfucation for this instance
 163                         fba.c.execute(
 164                             "SELECT hash FROM instances WHERE domain = ? LIMIT 1", (blocked,)
 165                         )
 166
 167                         if fba.c.fetchone() == None:
 168                             print("DEBUG: Hash wasn't found, adding:", blocked)
 169                             fba.add_instance(blocked)
 170                     else:
 171                         # Doing the hash search for instance names as well to tidy up DB
 172                         fba.c.execute(
 173                             "SELECT domain FROM instances WHERE hash = ? LIMIT 1", (blocked_hash,)
 174                         )
 175                         searchres = fba.c.fetchone()
 176
 177                         if searchres != None:
 178                             print("DEBUG: Updating domain: ", searchres[0])
 179                             blocked = searchres[0]
 180
 181                     timestamp = int(time.time())
 182                     fba.c.execute(
 183                         "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ?",
 184                         (
 185                             blocker,
 186                             blocked if blocked.count("*") <= 1 else blocked_hash,
 187                             block_level
 188                         ),
 189                     )
 190
 191                     if fba.c.fetchone() == None:
 192                         fba.block_instance(blocker, blocked if blocked.count("*") <= 1 else blocked_hash, reason, block_level, timestamp, timestamp)
 193
 194                         if block_level == "reject":
 195                             blockdict.append(
 196                                 {
 197                                     "blocked": blocked,
 198                                     "reason": reason
 199                                 })
 200                     else:
 201                         fba.update_last_seen(timestamp, blocker, blocked if blocked.count("*") <= 1 else blocked_hash, block_level)
 202
 203                     if reason != '':
 204                         fba.update_block_reason(reason, blocker, blocked if blocked.count("*") <= 1 else blocked_hash, block_level)
 205
 206             fba.conn.commit()
 207         except Exception as e:
 208             print("error:", e, blocker, software)
 209     elif software == "friendica" or software == "misskey":
 210         print("DEBUG: blocker:", blocker)
 211         try:
 212             if software == "friendica":
 213                 json = fba.get_friendica_blocks(blocker)
 214             elif software == "misskey":
 215                 json = fba.get_misskey_blocks(blocker)
 216             for block_level, blocks in json.items():
 217                 for instance in blocks:
 218                     blocked, reason = instance.values()
 219                     blocked = fba.tidyup(blocked)
 220
 221                     print("BEFORE-blocked:", blocked)
 222                     if blocked.count("*") > 0:
 223                         # Some friendica servers also obscure domains without hash
 224                         fba.c.execute(
 225                             "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", (blocked.replace("*", "_"),)
 226                         )
 227                         searchres = fba.c.fetchone()
 228                         if searchres != None:
 229                             blocked = searchres[0]
 230
 231                     if blocked.count("?") > 0:
 232                         # Some obscure them with question marks, not sure if that's dependent on version or not
 233                         fba.c.execute(
 234                             "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", (blocked.replace("?", "_"),)
 235                         )
 236                         searchres = fba.c.fetchone()
 237                         if searchres != None:
 238                             blocked = searchres[0]
 239
 240                     print("AFTER-blocked:", blocked)
 241                     fba.c.execute(
 242                         "SELECT domain FROM instances WHERE domain = ?", (blocked,)
 243                     )
 244
 245                     if fba.c.fetchone() == None:
 246                         print("DEBUG: Hash wasn't found, adding:", blocked)
 247                         fba.add_instance(blocked)
 248
 249                     timestamp = int(time.time())
 250                     fba.c.execute(
 251                         "SELECT * FROM blocks WHERE blocker = ? AND blocked = ?",
 252                         (blocker, blocked),
 253                     )
 254                     if fba.c.fetchone() == None:
 255                         fba.block_instance(blocker, blocked, reason, block_level, timestamp, timestamp)
 256
 257                         if block_level == "reject":
 258                             blockdict.append(
 259                                 {
 260                                     "blocked": blocked,
 261                                     "reason": reason
 262                                 })
 263                     else:
 264                         fba.update_last_seen(timestamp, blocker, blocked, block_level)
 265
 266                     if reason != '':
 267                         fba.update_block_reason(reason, blocker, blocked, block_level)
 268
 269             fba.conn.commit()
 270         except Exception as e:
 271             print("error:", e, blocker, software)
 272     elif software == "gotosocial":
 273         print("DEBUG: blocker:", blocker)
 274         try:
 275             # Blocks
 276             federation = reqto.get(
 277                 f"https://{blocker}/api/v1/instance/peers?filter=suspended", headers=headers, timeout=5
 278             ).json()
 279
 280             if (federation == None):
 281                 print("WARNING: No valid response:", blocker);
 282             else:
 283                 for peer in federation:
 284                     blocked = peer["domain"].lower()
 285
 286                     if blocked.count("*") > 0:
 287                         # GTS does not have hashes for obscured domains, so we have to guess it
 288                         fba.c.execute(
 289                             "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", (blocked.replace("*", "_"),)
 290                         )
 291                         searchres = fba.c.fetchone()
 292
 293                         if searchres != None:
 294                             blocked = searchres[0]
 295
 296                     fba.c.execute(
 297                         "SELECT domain FROM instances WHERE domain = ?", (blocked,)
 298                     )
 299
 300                     if fba.c.fetchone() == None:
 301                         print("DEBUG: Hash wasn't found, adding:", blocked)
 302                         fba.add_instance(blocked)
 303
 304                     fba.c.execute(
 305                         "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ?",
 306                         (
 307                             blocker,
 308                             blocked,
 309                             "reject"
 310                         ),
 311                     )
 312                     timestamp = int(time.time())
 313
 314                     if fba.c.fetchone() == None:
 315                         fba.block_instance(blocker, blocked, "", "reject", timestamp, timestamp)
 316
 317                         blockdict.append(
 318                             {
 319                                 "blocked": blocked,
 320                                 "reason": None
 321                             })
 322                     else:
 323                         fba.update_last_seen(timestamp, blocker, blocked, "reject")
 324
 325                     if "public_comment" in peer:
 326                         reason = peer["public_comment"]
 327                         fba.update_block_reason(reason, blocker, blocked, "reject")
 328
 329                         for entry in blockdict:
 330                             if entry["blocked"] == blocked:
 331                                 entry["reason"] = reason
 332                 fba.conn.commit()
 333         except Exception as e:
 334             print("error:", e, blocker, software)
 335     else:
 336         print("WARNING: Unknown software:", software)
 337
 338     if fba.config["bot_enabled"] and len(blockdict) > 0:
 339         send_bot_post(blocker, blockdict)
 340
 341     blockdict = []
 342
 343 fba.conn.close()