fetch_blocks.py

   1 from reqto import get
   2 from reqto import post
   3 from hashlib import sha256
   4 from bs4 import BeautifulSoup
   5 from json import dumps
   6 from json import loads
   7 import re
   8 from time import time
   9 import itertools
  10 from fba import c
  11 import fba
  12
  13 c.execute(
  14     "SELECT domain, software FROM instances WHERE domain='tooting.intensifi.es'"
  15     #"SELECT domain, software FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial')"
  16 )
  17
  18 for blocker, software in c.fetchall():
  19     blockdict = []
  20     blocker = fba.tidyup(blocker)
  21     if software == "pleroma":
  22         print(blocker)
  23         try:
  24             # Blocks
  25             federation = get(
  26                 f"https://{blocker}/nodeinfo/2.1.json", headers=headers, timeout=5
  27             ).json()["metadata"]["federation"]
  28             if "mrf_simple" in federation:
  29                 for block_level, blocks in (
  30                     {**federation["mrf_simple"],
  31                     **{"quarantined_instances": federation["quarantined_instances"]}}
  32                 ).items():
  33                     for blocked in blocks:
  34                         blocked = fba.tidyup(blocked)
  35
  36                         if blocked == "":
  37                             print("WARNING: blocked is empty after fba.tidyup():", blocker, block_level)
  38                             continue
  39
  40                         if blocked.count("*") > 1:
  41                             # -ACK!-oma also started obscuring domains without hash
  42                             c.execute(
  43                                 "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", (blocked.replace("*", "_"),)
  44                             )
  45                             searchres = c.fetchone()
  46                             if searchres != None:
  47                                 blocked = searchres[0]
  48
  49                         c.execute(
  50                             "SELECT domain FROM instances WHERE domain = ?", (blocked)
  51                         )
  52
  53                         if c.fetchone() == None:
  54                             add_instance(blocked)
  55
  56                         timestamp = int(time())
  57                         c.execute(
  58                             "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ?",
  59                             (
  60                                blocker,
  61                                blocked,
  62                                block_level
  63                            ),
  64                         )
  65                         if c.fetchone() == None:
  66                             block_instance(blocker, blocked, reason, block_level, timestamp, timestamp)
  67
  68                             if block_level == "reject":
  69                                 blockdict.append(
  70                                     {
  71                                         "blocked": blocked,
  72                                         "reason": None
  73                                     })
  74                         else:
  75                             update_last_seen(timestamp, blocker, blocked, block_level)
  76             fba.conn.commit()
  77             # Reasons
  78             if "mrf_simple_info" in federation:
  79                 for block_level, info in (
  80                     {**federation["mrf_simple_info"],
  81                     **(federation["quarantined_instances_info"]
  82                     if "quarantined_instances_info" in federation
  83                     else {})}
  84                 ).items():
  85                     for blocked, reason in info.items():
  86                         blocked = fba.tidyup(blocked)
  87
  88                         if blocked == "":
  89                             print("WARNING: blocked is empty after fba.tidyup():", blocker, block_level)
  90                             continue
  91
  92                         if blocked.count("*") > 1:
  93                             # same domain guess as above, but for reasons field
  94                             c.execute(
  95                                 "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", (blocked.replace("*", "_"),)
  96                             )
  97                             searchres = c.fetchone()
  98
  99                             if searchres != None:
 100                                 blocked = searchres[0]
 101
 102                         update_block_reason(reason["reason"], blocker, blocked, block_level)
 103
 104                         for entry in blockdict:
 105                             if entry["blocked"] == blocked:
 106                                 entry["reason"] = reason["reason"]
 107
 108             fba.conn.commit()
 109         except Exception as e:
 110             print("error:", e, blocker)
 111     elif software == "mastodon":
 112         print(blocker)
 113         try:
 114             # json endpoint for newer mastodongs
 115             try:
 116                 json = {
 117                     "reject": [],
 118                     "media_removal": [],
 119                     "followers_only": [],
 120                     "report_removal": []
 121                 }
 122
 123                 # handling CSRF, I've saw at least one server requiring it to access the endpoint
 124                 meta = BeautifulSoup(
 125                     get(f"https://{blocker}/about", headers=headers, timeout=5).text,
 126                     "html.parser",
 127                 )
 128                 try:
 129                     csrf = meta.find("meta", attrs={"name": "csrf-token"})["content"]
 130                     reqheaders = {**headers, **{"x-csrf-token": csrf}}
 131                 except:
 132                     reqheaders = headers
 133
 134                 blocks = get(
 135                     f"https://{blocker}/api/v1/instance/domain_blocks", headers=reqheaders, timeout=5
 136                 ).json()
 137
 138                 for block in blocks:
 139                     entry = {'domain': block['domain'], 'hash': block['digest'], 'reason': block['comment']}
 140                     if block['severity'] == 'suspend':
 141                         json['reject'].append(entry)
 142                     elif block['severity'] == 'silence':
 143                         json['followers_only'].append(entry)
 144                     elif block['severity'] == 'reject_media':
 145                         json['media_removal'].append(entry)
 146                     elif block['severity'] == 'reject_reports':
 147                         json['report_removal'].append(entry)
 148                     else:
 149                         print("WARNING: Unknown severity:", block['severity'], block['domain'])
 150             except:
 151                 json = fba.get_mastodon_blocks(blocker)
 152
 153             for block_level, blocks in json.items():
 154                 for instance in blocks:
 155                     blocked, blocked_hash, reason = instance.values()
 156                     blocked = fba.tidyup(blocked)
 157
 158                     if blocked.count("*") <= 1:
 159                         c.execute(
 160                             "SELECT hash FROM instances WHERE hash = ?", (blocked_hash,)
 161                         )
 162
 163                         if c.fetchone() == None:
 164                             add_instance(blocked)
 165                     else:
 166                         # Doing the hash search for instance names as well to tidy up DB
 167                         c.execute(
 168                             "SELECT domain FROM instances WHERE hash = ?", (blocked_hash,)
 169                         )
 170                         searchres = c.fetchone()
 171                         if searchres != None:
 172                             blocked = searchres[0]
 173
 174                     timestamp = int(time())
 175                     c.execute(
 176                         "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ?",
 177                         (
 178                             blocker,
 179                             blocked if blocked.count("*") <= 1 else blocked_hash,
 180                             block_level
 181                         ),
 182                     )
 183                     if c.fetchone() == None:
 184                         block_instance(blocker, blocked if blocked.count("*") <= 1 else blocked_hash, reason, block_level, timestamp, timestamp)
 185
 186                         if block_level == "reject":
 187                             blockdict.append(
 188                                 {
 189                                     "blocked": blocked,
 190                                     "reason": reason
 191                                 })
 192                     else:
 193                         update_last_seen(timestamp, blocker, blocked if blocked.count("*") <= 1 else blocked_hash, block_level)
 194
 195                     if reason != '':
 196                         update_block_reason(reason, blocker, blocked if blocked.count("*") <= 1 else blocked_hash, block_level)
 197
 198             fba.conn.commit()
 199         except Exception as e:
 200             print("error:", e, blocker)
 201     elif software == "friendica" or software == "misskey":
 202         print(blocker)
 203         try:
 204             if software == "friendica":
 205                 json = fba.get_friendica_blocks(blocker)
 206             elif software == "misskey":
 207                 json = fba.get_misskey_blocks(blocker)
 208             for block_level, blocks in json.items():
 209                 for instance in blocks:
 210                     blocked, reason = instance.values()
 211                     blocked = fba.tidyup(blocked)
 212
 213                     print("BEFORE-blocked:", blocked)
 214                     if blocked.count("*") > 0:
 215                         # Some friendica servers also obscure domains without hash
 216                         c.execute(
 217                             "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", (blocked.replace("*", "_"),)
 218                         )
 219                         searchres = c.fetchone()
 220                         if searchres != None:
 221                             blocked = searchres[0]
 222
 223                     if blocked.count("?") > 0:
 224                         # Some obscure them with question marks, not sure if that's dependent on version or not
 225                         c.execute(
 226                             "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", (blocked.replace("?", "_"),)
 227                         )
 228                         searchres = c.fetchone()
 229                         if searchres != None:
 230                             blocked = searchres[0]
 231
 232                     print("AFTER-blocked:", blocked)
 233                     c.execute(
 234                         "SELECT domain FROM instances WHERE domain = ?", (blocked,)
 235                     )
 236
 237                     if c.fetchone() == None:
 238                         add_instance(blocked)
 239
 240                     timestamp = int(time())
 241                     c.execute(
 242                         "SELECT * FROM blocks WHERE blocker = ? AND blocked = ?",
 243                         (blocker, blocked),
 244                     )
 245                     if c.fetchone() == None:
 246                         block_instance(blocker, blocked, reason, block_level, timestamp, timestamp)
 247
 248                         if block_level == "reject":
 249                             blockdict.append(
 250                                 {
 251                                     "blocked": blocked,
 252                                     "reason": reason
 253                                 })
 254                     else:
 255                         update_last_seen(timestamp, blocker, blocked, block_level)
 256
 257                     if reason != '':
 258                         update_block_reason(reason, blocker, blocked, block_level)
 259
 260             fba.conn.commit()
 261         except Exception as e:
 262             print("error:", e, blocker)
 263     elif software == "gotosocial":
 264         print(blocker)
 265         try:
 266             # Blocks
 267             federation = get(
 268                 f"https://{blocker}/api/v1/instance/peers?filter=suspended", headers=headers, timeout=5
 269             ).json()
 270
 271             if (federation == None):
 272                 print("WARNING: No valid response:", blocker);
 273             else:
 274                 for peer in federation:
 275                     blocked = peer["domain"].lower()
 276
 277                     if blocked.count("*") > 0:
 278                         # GTS does not have hashes for obscured domains, so we have to guess it
 279                         c.execute(
 280                             "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", (blocked.replace("*", "_"),)
 281                         )
 282                         searchres = c.fetchone()
 283
 284                         if searchres != None:
 285                             blocked = searchres[0]
 286
 287                     c.execute(
 288                         "SELECT domain FROM instances WHERE domain = ?", (blocked,)
 289                     )
 290
 291                     if c.fetchone() == None:
 292                         add_instance(blocked)
 293
 294                     c.execute(
 295                         "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ?",
 296                         (blocker, blocked, "reject"),
 297                     )
 298                     timestamp = int(time())
 299
 300                     if c.fetchone() == None:
 301                         block_instance(blocker, blocked, "", "reject", timestamp, timestamp)
 302
 303                         blockdict.append(
 304                             {
 305                                 "blocked": blocked,
 306                                 "reason": None
 307                             })
 308                     else:
 309                         update_last_seen(timestamp, blocker, blocked, "reject")
 310
 311                     if "public_comment" in peer:
 312                         reason = peer["public_comment"]
 313                         update_block_reason(reason, blocker, blocked, "reject")
 314
 315                         for entry in blockdict:
 316                             if entry["blocked"] == blocked:
 317                                 entry["reason"] = reason
 318                 fba.conn.commit()
 319         except Exception as e:
 320             print("error:", e, blocker)
 321
 322     if config["bot_enabled"] and len(blockdict) > 0:
 323         send_bot_post(blocker, blockdict)
 324     blockdict = []
 325
 326 fba.conn.close()