fetch_blocks.py

   1 import reqto
   2 import time
   3 import bs4
   4 import fba
   5 import itertools
   6 import re
   7
   8 fba.c.execute(
   9     "SELECT domain, software FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial')"
  10 )
  11
  12 for blocker, software in fba.c.fetchall():
  13     print("DEBUG: blocker,software:", blocker, software)
  14     blockdict = []
  15     blocker = fba.tidyup(blocker)
  16     if software == "pleroma":
  17         print("DEBUG: blocker:", blocker)
  18         try:
  19             # Blocks
  20             federation = reqto.get(
  21                 f"https://{blocker}/nodeinfo/2.1.json", headers=fba.headers, timeout=5
  22             ).json()["metadata"]["federation"]
  23             if "mrf_simple" in federation:
  24                 for block_level, blocks in (
  25                     {**federation["mrf_simple"],
  26                     **{"quarantined_instances": federation["quarantined_instances"]}}
  27                 ).items():
  28                     for blocked in blocks:
  29                         blocked = fba.tidyup(blocked)
  30
  31                         if blocked == "":
  32                             print("WARNING: blocked is empty after fba.tidyup():", blocker, block_level)
  33                             continue
  34
  35                         if blocked.count("*") > 1:
  36                             # -ACK!-oma also started obscuring domains without hash
  37                             fba.c.execute(
  38                                 "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", (blocked.replace("*", "_"),)
  39                             )
  40                             searchres = fba.c.fetchone()
  41                             if searchres != None:
  42                                 blocked = searchres[0]
  43
  44                         fba.c.execute(
  45                             "SELECT domain FROM instances WHERE domain = ?", (blocked,)
  46                         )
  47
  48                         if fba.c.fetchone() == None:
  49                             print("DEBUG: Hash wasn't found, adding:", blocked)
  50                             fba.add_instance(blocked)
  51
  52                         timestamp = int(time.time())
  53                         fba.c.execute(
  54                             "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ?",
  55                             (
  56                                blocker,
  57                                blocked,
  58                                block_level
  59                            ),
  60                         )
  61                         if fba.c.fetchone() == None:
  62                             fba.block_instance(blocker, blocked, reason, block_level, timestamp, timestamp)
  63
  64                             if block_level == "reject":
  65                                 blockdict.append(
  66                                     {
  67                                         "blocked": blocked,
  68                                         "reason": None
  69                                     })
  70                         else:
  71                             fba.update_last_seen(timestamp, blocker, blocked, block_level)
  72
  73             fba.conn.commit()
  74             # Reasons
  75             if "mrf_simple_info" in federation:
  76                 for block_level, info in (
  77                     {**federation["mrf_simple_info"],
  78                     **(federation["quarantined_instances_info"]
  79                     if "quarantined_instances_info" in federation
  80                     else {})}
  81                 ).items():
  82                     for blocked, reason in info.items():
  83                         blocked = fba.tidyup(blocked)
  84
  85                         if blocked == "":
  86                             print("WARNING: blocked is empty after fba.tidyup():", blocker, block_level)
  87                             continue
  88
  89                         if blocked.count("*") > 1:
  90                             # same domain guess as above, but for reasons field
  91                             fba.c.execute(
  92                                 "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", (blocked.replace("*", "_"),)
  93                             )
  94                             searchres = fba.c.fetchone()
  95
  96                             if searchres != None:
  97                                 blocked = searchres[0]
  98
  99                         fba.update_block_reason(reason["reason"], blocker, blocked, block_level)
 100
 101                         for entry in blockdict:
 102                             if entry["blocked"] == blocked:
 103                                 entry["reason"] = reason["reason"]
 104
 105             fba.conn.commit()
 106         except Exception as e:
 107             print("error:", e, blocker, software)
 108     elif software == "mastodon":
 109         print("DEBUG: blocker:", blocker)
 110         try:
 111             # json endpoint for newer mastodongs
 112             try:
 113                 json = {
 114                     "reject": [],
 115                     "media_removal": [],
 116                     "followers_only": [],
 117                     "report_removal": []
 118                 }
 119
 120                 # handling CSRF, I've saw at least one server requiring it to access the endpoint
 121                 print("DEBUG: Fetching meta:", blocker)
 122                 meta = bs4.BeautifulSoup(
 123                     reqto.get(f"https://{blocker}/about", headers=fba.headers, timeout=5).text,
 124                     "html.parser",
 125                 )
 126                 try:
 127                     csrf = meta.find("meta", attrs={"name": "csrf-token"})["content"]
 128                     print("DEBUG: Adding CSRF token:", blocker, csrf)
 129                     reqheaders = {**fba.headers, **{"x-csrf-token": csrf}}
 130                 except:
 131                     print("DEBUG: No CSRF token found, using normal headers:", blocker)
 132                     reqheaders = fba.headers
 133
 134                 print("DEBUG: Quering API domain_blocks:", blocker)
 135                 blocks = reqto.get(
 136                     f"https://{blocker}/api/v1/instance/domain_blocks", headers=reqheaders, timeout=5
 137                 ).json()
 138
 139                 print("DEBUG: blocks():", len(blocks))
 140                 for block in blocks:
 141                     entry = {'domain': block['domain'], 'hash': block['digest'], 'reason': block['comment']}
 142
 143                     print("DEBUG: severity,domain,hash,comment:", block['severity'], block['domain'], block['digest'], block['comment'])
 144                     if block['severity'] == 'suspend':
 145                         json['reject'].append(entry)
 146                     elif block['severity'] == 'silence':
 147                         json['followers_only'].append(entry)
 148                     elif block['severity'] == 'reject_media':
 149                         json['media_removal'].append(entry)
 150                     elif block['severity'] == 'reject_reports':
 151                         json['report_removal'].append(entry)
 152                     else:
 153                         print("WARNING: Unknown severity:", block['severity'], block['domain'])
 154             except:
 155                 print("DEBUG: Failed, Trying mastodon-specific fetches:", blocker)
 156                 json = fba.get_mastodon_blocks(blocker)
 157
 158             print("DEBUG: json.items():", blocker, len(json.items()))
 159             for block_level, blocks in json.items():
 160                 print("DEBUG: blocker,block_level,blocks():", blocker, block_level, len(blocks))
 161                 for instance in blocks:
 162                     blocked, blocked_hash, reason = instance.values()
 163                     print("DEBUG: blocked,hash,reason:", blocked, blocked_hash, reason)
 164
 165                     blocked = fba.tidyup(blocked)
 166                     print("DEBUG: blocked:", blocked)
 167
 168                     if blocked.count("*") < 1:
 169                         # No obsfucation for this instance
 170                         fba.c.execute(
 171                             "SELECT hash FROM instances WHERE domain = ? LIMIT 1", (blocked,)
 172                         )
 173
 174                         if fba.c.fetchone() == None:
 175                             print("DEBUG: Hash wasn't found, adding:", blocked)
 176                             fba.add_instance(blocked)
 177                     else:
 178                         # Doing the hash search for instance names as well to tidy up DB
 179                         fba.c.execute(
 180                             "SELECT domain FROM instances WHERE hash = ? LIMIT 1", (blocked_hash,)
 181                         )
 182                         searchres = fba.c.fetchone()
 183
 184                         if searchres != None:
 185                             print("DEBUG: Updating domain: ", searchres[0])
 186                             blocked = searchres[0]
 187
 188                     timestamp = int(time.time())
 189                     fba.c.execute(
 190                         "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ?",
 191                         (
 192                             blocker,
 193                             blocked if blocked.count("*") <= 1 else blocked_hash,
 194                             block_level
 195                         ),
 196                     )
 197
 198                     if fba.c.fetchone() == None:
 199                         fba.block_instance(blocker, blocked if blocked.count("*") <= 1 else blocked_hash, reason, block_level, timestamp, timestamp)
 200
 201                         if block_level == "reject":
 202                             blockdict.append(
 203                                 {
 204                                     "blocked": blocked,
 205                                     "reason": reason
 206                                 })
 207                     else:
 208                         fba.update_last_seen(timestamp, blocker, blocked if blocked.count("*") <= 1 else blocked_hash, block_level)
 209
 210                     if reason != '':
 211                         fba.update_block_reason(reason, blocker, blocked if blocked.count("*") <= 1 else blocked_hash, block_level)
 212
 213             fba.conn.commit()
 214         except Exception as e:
 215             print("error:", e, blocker, software)
 216     elif software == "friendica" or software == "misskey":
 217         print("DEBUG: blocker:", blocker)
 218         try:
 219             if software == "friendica":
 220                 json = fba.get_friendica_blocks(blocker)
 221             elif software == "misskey":
 222                 json = fba.get_misskey_blocks(blocker)
 223             for block_level, blocks in json.items():
 224                 for instance in blocks:
 225                     blocked, reason = instance.values()
 226                     blocked = fba.tidyup(blocked)
 227
 228                     print("BEFORE-blocked:", blocked)
 229                     if blocked.count("*") > 0:
 230                         # Some friendica servers also obscure domains without hash
 231                         fba.c.execute(
 232                             "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", (blocked.replace("*", "_"),)
 233                         )
 234                         searchres = fba.c.fetchone()
 235                         if searchres != None:
 236                             blocked = searchres[0]
 237
 238                     if blocked.count("?") > 0:
 239                         # Some obscure them with question marks, not sure if that's dependent on version or not
 240                         fba.c.execute(
 241                             "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", (blocked.replace("?", "_"),)
 242                         )
 243                         searchres = fba.c.fetchone()
 244                         if searchres != None:
 245                             blocked = searchres[0]
 246
 247                     print("AFTER-blocked:", blocked)
 248                     fba.c.execute(
 249                         "SELECT domain FROM instances WHERE domain = ?", (blocked,)
 250                     )
 251
 252                     if fba.c.fetchone() == None:
 253                         print("DEBUG: Hash wasn't found, adding:", blocked)
 254                         fba.add_instance(blocked)
 255
 256                     timestamp = int(time.time())
 257                     fba.c.execute(
 258                         "SELECT * FROM blocks WHERE blocker = ? AND blocked = ?",
 259                         (blocker, blocked),
 260                     )
 261                     if fba.c.fetchone() == None:
 262                         fba.block_instance(blocker, blocked, reason, block_level, timestamp, timestamp)
 263
 264                         if block_level == "reject":
 265                             blockdict.append(
 266                                 {
 267                                     "blocked": blocked,
 268                                     "reason": reason
 269                                 })
 270                     else:
 271                         fba.update_last_seen(timestamp, blocker, blocked, block_level)
 272
 273                     if reason != '':
 274                         fba.update_block_reason(reason, blocker, blocked, block_level)
 275
 276             fba.conn.commit()
 277         except Exception as e:
 278             print("error:", e, blocker, software)
 279     elif software == "gotosocial":
 280         print("DEBUG: blocker:", blocker)
 281         try:
 282             # Blocks
 283             federation = reqto.get(
 284                 f"https://{blocker}/api/v1/instance/peers?filter=suspended", headers=fba.headers, timeout=5
 285             ).json()
 286
 287             if (federation == None):
 288                 print("WARNING: No valid response:", blocker);
 289             else:
 290                 for peer in federation:
 291                     print("DEBUG: peer(),[]:", len(peer), type(peer))
 292                     if (isinstance(peer, str) and peer == "error"):
 293                         print("WARNING: Cannot continue, maybe authentication required?", blocker)
 294                         break
 295
 296                     blocked = peer["domain"].lower()
 297                     print("DEBUG: blocked:", blocked)
 298
 299                     if blocked.count("*") > 0:
 300                         # GTS does not have hashes for obscured domains, so we have to guess it
 301                         fba.c.execute(
 302                             "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", (blocked.replace("*", "_"),)
 303                         )
 304                         searchres = fba.c.fetchone()
 305
 306                         if searchres != None:
 307                             blocked = searchres[0]
 308
 309                     fba.c.execute(
 310                         "SELECT domain FROM instances WHERE domain = ?", (blocked,)
 311                     )
 312
 313                     if fba.c.fetchone() == None:
 314                         print("DEBUG: Hash wasn't found, adding:", blocked)
 315                         fba.add_instance(blocked)
 316
 317                     fba.c.execute(
 318                         "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ?",
 319                         (
 320                             blocker,
 321                             blocked,
 322                             "reject"
 323                         ),
 324                     )
 325                     timestamp = int(time.time())
 326
 327                     if fba.c.fetchone() == None:
 328                         fba.block_instance(blocker, blocked, "", "reject", timestamp, timestamp)
 329
 330                         blockdict.append(
 331                             {
 332                                 "blocked": blocked,
 333                                 "reason": None
 334                             })
 335                     else:
 336                         fba.update_last_seen(timestamp, blocker, blocked, "reject")
 337
 338                     if "public_comment" in peer:
 339                         reason = peer["public_comment"]
 340                         fba.update_block_reason(reason, blocker, blocked, "reject")
 341
 342                         for entry in blockdict:
 343                             if entry["blocked"] == blocked:
 344                                 entry["reason"] = reason
 345                 fba.conn.commit()
 346         except Exception as e:
 347             print("error:", e, blocker, software)
 348     else:
 349         print("WARNING: Unknown software:", software)
 350
 351     if fba.config["bot_enabled"] and len(blockdict) > 0:
 352         send_bot_post(blocker, blockdict)
 353
 354     blockdict = []
 355
 356 fba.conn.close()