fba.py

   1 from bs4 import BeautifulSoup
   2 from hashlib import sha256
   3
   4 import reqto
   5 import re
   6 import sqlite3
   7 import json
   8 import sys
   9 import time
  10
  11 with open("config.json") as f:
  12     config = json.loads(f.read())
  13
  14 blacklist = [
  15     "activitypub-troll.cf",
  16     "gab.best",
  17     "4chan.icu",
  18     "social.shrimpcam.pw",
  19     "mastotroll.netz.org",
  20     "ngrok.io",
  21 ]
  22
  23 headers = {
  24     "user-agent": config["useragent"]
  25 }
  26
  27 connection = sqlite3.connect("blocks.db")
  28 cursor = connection.cursor()
  29
  30 def is_blacklisted(domain: str) -> bool:
  31     blacklisted = False
  32     for peer in blacklist:
  33         if peer in domain:
  34             blacklisted = True
  35
  36     return blacklisted
  37
  38 def get_hash(domain: str) -> str:
  39     return sha256(domain.encode("utf-8")).hexdigest()
  40
  41 def update_last_blocked(domain: str):
  42     # NOISY-DEBUG: print("DEBUG: Updating last_blocked for domain", domain)
  43     try:
  44         cursor.execute("UPDATE instances SET last_blocked = ?, last_updated = ? WHERE domain = ?", [
  45             time.time(),
  46             time.time(),
  47             domain
  48         ])
  49
  50     except:
  51         print("ERROR: failed SQL query:", domain)
  52         sys.exit(255)
  53
  54 def update_last_error(domain: str, res: any):
  55     # NOISY-DEBUG: print("DEBUG: domain,res.status_code:", domain, res.status_code, res.reason)
  56     try:
  57         cursor.execute("UPDATE instances SET last_status_code = ?, last_error_details = ?, last_updated = ? WHERE domain = ?", [
  58             res.status_code,
  59             res.reason,
  60             time.time(),
  61             domain
  62         ])
  63
  64     except:
  65         print("ERROR: failed SQL query:", domain)
  66         sys.exit(255)
  67
  68 def update_last_nodeinfo(domain: str):
  69     # NOISY-DEBUG: print("DEBUG: Updating last_nodeinfo for domain:", domain)
  70     try:
  71         cursor.execute("UPDATE instances SET last_nodeinfo = ?, last_updated = ? WHERE domain = ?", [
  72             time.time(),
  73             time.time(),
  74             domain
  75         ])
  76
  77     except:
  78         print("ERROR: failed SQL query:", domain)
  79         sys.exit(255)
  80
  81     connection.commit()
  82
  83 def get_peers(domain: str) -> list:
  84     # NOISY-DEBUG: print("DEBUG: Getting peers for domain:", domain)
  85     peers = None
  86
  87     try:
  88         res = reqto.get(f"https://{domain}/api/v1/instance/peers", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"]))
  89
  90         if not res.ok or res.status_code >= 400:
  91             print("WARNING: Cannot fetch peers:", domain)
  92             update_last_error(domain, res)
  93         else:
  94             # NOISY-DEBUG: print("DEBUG: Querying API was successful:", domain, len(res.json()))
  95             peers = res.json()
  96
  97     except:
  98         print("WARNING: Some error during get():", domain)
  99
 100     update_last_nodeinfo(domain)
 101
 102     # NOISY-DEBUG: print("DEBUG: Returning peers[]:", type(peers))
 103     return peers
 104
 105 def post_json_api(domain: str, path: str, data: str) -> list:
 106     # NOISY-DEBUG: print("DEBUG: Sending POST to domain,path,data:", domain, path, data)
 107     json = {}
 108     try:
 109         res = reqto.post(f"https://{domain}{path}", data=data, headers=headers, timeout=(config["connection_timeout"], config["read_timeout"]))
 110
 111         if not res.ok or res.status_code >= 400:
 112             print("WARNING: Cannot query JSON API:", domain, path, data, res.status_code)
 113             update_last_error(domain, res)
 114             raise
 115
 116         update_last_nodeinfo(domain)
 117         json = res.json()
 118     except:
 119         print("WARNING: Some error during post():", domain, path, data)
 120
 121     # NOISY-DEBUG: print("DEBUG: Returning json():", len(json))
 122     return json
 123
 124 def fetch_nodeinfo(domain: str) -> list:
 125     # NOISY-DEBUG: print("DEBUG: Fetching nodeinfo from domain:", domain)
 126
 127     requests = [
 128        f"https://{domain}/nodeinfo/2.1.json",
 129        f"https://{domain}/nodeinfo/2.1",
 130        f"https://{domain}/nodeinfo/2.0.json",
 131        f"https://{domain}/nodeinfo/2.0",
 132        f"https://{domain}/nodeinfo/1.0",
 133        f"https://{domain}/api/v1/instance"
 134     ]
 135
 136     json = {}
 137     for request in requests:
 138         try:
 139             # NOISY-DEBUG: print("DEBUG: Fetching request:", request)
 140             res = reqto.get(request, headers=headers, timeout=(config["connection_timeout"], config["read_timeout"]))
 141
 142             # NOISY-DEBUG: print("DEBUG: res.ok,res.json[]:", res.ok, type(res.json()))
 143             if res.ok and res.json() is not None:
 144                 # NOISY-DEBUG: print("DEBUG: Success:", request)
 145                 json = res.json()
 146                 break
 147             elif not res.ok or res.status_code >= 400:
 148                 # NOISY-DEBUG: print("DEBUG: Failed fetching nodeinfo from domain:", domain)
 149                 update_last_error(domain, res)
 150                 continue
 151
 152         except:
 153             # NOISY-DEBUG: print("DEBUG: Cannot fetch API request:", request)
 154             pass
 155
 156     # NOISY-DEBUG: print("DEBUG: json[]:", type(json))
 157     if json is None or len(json) == 0:
 158         print("WARNING: Failed fetching nodeinfo from domain:", domain)
 159
 160     # NOISY-DEBUG: print("DEBUG: Returning json[]:", type(json))
 161     return json
 162
 163 def determine_software(domain: str) -> str:
 164     # NOISY-DEBUG: print("DEBUG: Determining software for domain:", domain)
 165     software = None
 166
 167     json = fetch_nodeinfo(domain)
 168     # NOISY-DEBUG: print("DEBUG: json[]:", type(json))
 169
 170     if json is None or len(json) == 0:
 171         # NOISY-DEBUG: print("DEBUG: Could not determine software type:", domain)
 172         return None
 173
 174     # NOISY-DEBUG: print("DEBUG: json():", len(json), json)
 175     if "software" not in json or "name" not in json["software"]:
 176         print("WARNING: JSON response does not include [software][name], guessing ...")
 177         found = 0
 178         for element in {"uri", "title", "description", "email", "version", "urls", "stats", "thumbnail", "languages", "contact_account"}:
 179             if element in json:
 180                 found = found + 1
 181
 182         # NOISY-DEBUG: print("DEBUG: Found elements:", found)
 183         if found == len(json):
 184             # NOISY-DEBUG: print("DEBUG: Maybe is Mastodon:", domain)
 185             return "mastodon"
 186
 187         print("WARNING: Cannot guess software type:", domain, found, len(json))
 188         return None
 189
 190     software = tidyup(json["software"]["name"])
 191
 192     # NOISY-DEBUG: print("DEBUG: tidyup software:", software)
 193     if software in ["akkoma", "rebased"]:
 194         # NOISY-DEBUG: print("DEBUG: Setting pleroma:", domain, software)
 195         software = "pleroma"
 196     elif software in ["hometown", "ecko"]:
 197         # NOISY-DEBUG: print("DEBUG: Setting mastodon:", domain, software)
 198         software = "mastodon"
 199     elif software in ["calckey", "groundpolis", "foundkey", "cherrypick"]:
 200         # NOISY-DEBUG: print("DEBUG: Setting misskey:", domain, software)
 201         software = "misskey"
 202     elif software.find("/") > 0:
 203         print("WARNING: Spliting of path:", software)
 204         software = software.split("/")[-1];
 205
 206     if software == "":
 207         print("WARNING: tidyup() left no software name behind:", domain)
 208         software = None
 209
 210     # NOISY-DEBUG: print("DEBUG: Returning domain,software:", domain, software)
 211     return software
 212
 213 def update_block_reason(reason: str, blocker: str, blocked: str, block_level: str):
 214     # NOISY: # NOISY-DEBUG: print("DEBUG: Updating block reason:", reason, blocker, blocked, block_level)
 215     try:
 216         cursor.execute(
 217             "UPDATE blocks SET reason = ?, last_seen = ? WHERE blocker = ? AND blocked = ? AND block_level = ? AND reason = ''",
 218             (
 219                 reason,
 220                 time.time(),
 221                 blocker,
 222                 blocked,
 223                 block_level
 224             ),
 225         )
 226
 227     except:
 228         print("ERROR: failed SQL query:", reason, blocker, blocked, block_level)
 229         sys.exit(255)
 230
 231 def update_last_seen(blocker: str, blocked: str, block_level: str):
 232     # NOISY: # NOISY-DEBUG: print("DEBUG: Updating last_seen for:", blocker, blocked, block_level)
 233     try:
 234         cursor.execute(
 235             "UPDATE blocks SET last_seen = ? WHERE blocker = ? AND blocked = ? AND block_level = ?",
 236             (
 237                 time.time(),
 238                 blocker,
 239                 blocked,
 240                 block_level
 241             )
 242         )
 243
 244     except:
 245         print("ERROR: failed SQL query:", last_seen, blocker, blocked, block_level)
 246         sys.exit(255)
 247
 248 def block_instance(blocker: str, blocked: str, reason: str, block_level: str):
 249     # NOISY-DEBUG: print("DEBUG: blocker,blocked,reason,block_level:", blocker, blocked, reason, block_level)
 250     if blocker.find("@") > 0:
 251         print("WARNING: Bad blocker:", blocker)
 252         raise
 253     elif blocked.find("@") > 0:
 254         print("WARNING: Bad blocked:", blocked)
 255         raise
 256
 257     print("INFO: New block:", blocker, blocked, reason, block_level, first_added, last_seen)
 258     try:
 259         cursor.execute(
 260             "INSERT INTO blocks (blocker, blocked, reason, block_level, first_added, last_seen) VALUES(?, ?, ?, ?, ?, ?)",
 261              (
 262                  blocker,
 263                  blocked,
 264                  reason,
 265                  block_level,
 266                  time.time(),
 267                  time.time()
 268              ),
 269         )
 270
 271     except:
 272         print("ERROR: failed SQL query:", blocker, blocked, reason, block_level, first_added, last_seen)
 273         sys.exit(255)
 274
 275 def add_instance(domain: str, origin: str, originator: str):
 276     # NOISY-DEBUG: print("DEBUG: domain,origin:", domain, origin, originator)
 277     if domain.find("@") > 0:
 278         print("WARNING: Bad domain name:", domain)
 279         raise
 280     elif origin is not None and origin.find("@") > 0:
 281         print("WARNING: Bad origin name:", origin)
 282         raise
 283
 284     software = determine_software(domain)
 285     # NOISY-DEBUG: print("DEBUG: Determined software:", software)
 286
 287     print(f"INFO: Adding new instance {domain} (origin: {origin})")
 288     try:
 289         cursor.execute(
 290             "INSERT INTO instances (domain, origin, originator, hash, software, first_seen) VALUES (?, ?, ?, ?, ?, ?)",
 291             (
 292                domain,
 293                origin,
 294                originator,
 295                get_hash(domain),
 296                software,
 297                time.time()
 298             ),
 299         )
 300
 301     except:
 302         print("ERROR: failed SQL query:", domain)
 303         sys.exit(255)
 304     else:
 305         # NOISY-DEBUG: print("DEBUG: Updating nodeinfo for domain:", domain)
 306         update_last_nodeinfo(domain)
 307
 308 def send_bot_post(instance: str, blocks: dict):
 309     message = instance + " has blocked the following instances:\n\n"
 310     truncated = False
 311
 312     if len(blocks) > 20:
 313         truncated = True
 314         blocks = blocks[0 : 19]
 315
 316     for block in blocks:
 317         if block["reason"] == None or block["reason"] == '':
 318             message = message + block["blocked"] + " with unspecified reason\n"
 319         else:
 320             if len(block["reason"]) > 420:
 321                 block["reason"] = block["reason"][0:419] + "[…]"
 322
 323             message = message + block["blocked"] + ' for "' + block["reason"].replace("@", "@\u200b") + '"\n'
 324
 325     if truncated:
 326         message = message + "(the list has been truncated to the first 20 entries)"
 327
 328     botheaders = {**headers, **{"Authorization": "Bearer " + config["bot_token"]}}
 329
 330     req = reqto.post(f"{config['bot_instance']}/api/v1/statuses",
 331         data={"status":message, "visibility":config['bot_visibility'], "content_type":"text/plain"},
 332         headers=botheaders, timeout=10).json()
 333
 334     return True
 335
 336 def get_mastodon_blocks(domain: str) -> dict:
 337     # NOISY-DEBUG: print("DEBUG: Fetching mastodon blocks from domain:", domain)
 338     blocks = {
 339         "Suspended servers": [],
 340         "Filtered media"   : [],
 341         "Limited servers"  : [],
 342         "Silenced servers" : [],
 343     }
 344
 345     translations = {
 346         "Silenced instances": "Silenced servers",
 347         "Suspended instances": "Suspended servers",
 348         "Gesperrte Server": "Suspended servers",
 349         "Gefilterte Medien": "Filtered media",
 350         "Stummgeschaltete Server": "Silenced servers",
 351         "停止済みのサーバー": "Suspended servers",
 352         "メディアを拒否しているサーバー": "Filtered media",
 353         "サイレンス済みのサーバー": "Silenced servers",
 354         "שרתים מושעים": "Suspended servers",
 355         "מדיה מסוננת": "Filtered media",
 356         "שרתים מוגבלים": "Silenced servers",
 357         "Serveurs suspendus": "Suspended servers",
 358         "Médias filtrés": "Filtered media",
 359         "Serveurs limités": "Silenced servers",
 360     }
 361
 362     try:
 363         doc = BeautifulSoup(
 364             reqto.get(f"https://{domain}/about/more", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])).text,
 365             "html.parser",
 366         )
 367     except:
 368         print("ERROR: Cannot fetch from domain:", domain)
 369         return {}
 370
 371     for header in doc.find_all("h3"):
 372         header_text = header.text
 373
 374         if header_text in translations:
 375             header_text = translations[header_text]
 376
 377         if header_text in blocks:
 378             # replaced find_next_siblings with find_all_next to account for instances that e.g. hide lists in dropdown menu
 379             for line in header.find_all_next("table")[0].find_all("tr")[1:]:
 380                 blocks[header_text].append(
 381                     {
 382                         "domain": tidyup(line.find("span").text),
 383                         "hash"  : tidyup(line.find("span")["title"][9:]),
 384                         "reason": tidyup(line.find_all("td")[1].text),
 385                     }
 386                 )
 387
 388     # NOISY-DEBUG: print("DEBUG: Returning blocks for domain:", domain)
 389     return {
 390         "reject"        : blocks["Suspended servers"],
 391         "media_removal" : blocks["Filtered media"],
 392         "followers_only": blocks["Limited servers"] + blocks["Silenced servers"],
 393     }
 394
 395 def get_friendica_blocks(domain: str) -> dict:
 396     # NOISY-DEBUG: print("DEBUG: Fetching friendica blocks from domain:", domain)
 397     blocks = []
 398
 399     try:
 400         doc = BeautifulSoup(
 401             reqto.get(f"https://{domain}/friendica", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])).text,
 402             "html.parser",
 403         )
 404     except:
 405         print("WARNING: Failed to fetch /friendica from domain:", domain)
 406         return {}
 407
 408     blocklist = doc.find(id="about_blocklist")
 409
 410     # Prevents exceptions:
 411     if blocklist is None:
 412         # NOISY-DEBUG: print("DEBUG: Instance has no block list:", domain)
 413         return {}
 414
 415     for line in blocklist.find("table").find_all("tr")[1:]:
 416         blocks.append({
 417             "domain": tidyup(line.find_all("td")[0].text),
 418             "reason": tidyup(line.find_all("td")[1].text)
 419         })
 420
 421     # NOISY-DEBUG: print("DEBUG: Returning blocks() for domain:", domain, len(blocks))
 422     return {
 423         "reject": blocks
 424     }
 425
 426 def get_misskey_blocks(domain: str) -> dict:
 427     # NOISY-DEBUG: print("DEBUG: Fetching misskey blocks from domain:", domain)
 428     blocks = {
 429         "suspended": [],
 430         "blocked": []
 431     }
 432
 433     try:
 434         counter = 0
 435         step = 99
 436         while True:
 437             # iterating through all "suspended" (follow-only in its terminology)
 438             # instances page-by-page, since that troonware doesn't support
 439             # sending them all at once
 440             try:
 441                 if counter == 0:
 442                     # NOISY-DEBUG: print("DEBUG: Sending JSON API request to domain,step,counter:", domain, step, counter)
 443                     doc = post_json_api(domain, "/api/federation/instances/", json.dumps({
 444                         "sort"     : "+caughtAt",
 445                         "host"     : None,
 446                         "suspended": True,
 447                         "limit"    : step
 448                     }))
 449                 else:
 450                     # NOISY-DEBUG: print("DEBUG: Sending JSON API request to domain,step,counter:", domain, step, counter)
 451                     doc = post_json_api(domain, "/api/federation/instances/", json.dumps({
 452                         "sort"     : "+caughtAt",
 453                         "host"     : None,
 454                         "suspended": True,
 455                         "limit"    : step,
 456                         "offset"   : counter-1
 457                     }))
 458
 459                 # NOISY-DEBUG: print("DEBUG: doc():", len(doc))
 460                 if len(doc) == 0:
 461                     # NOISY-DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
 462                     break
 463
 464                 for instance in doc:
 465                     # just in case
 466                     if instance["isSuspended"]:
 467                         blocks["suspended"].append(
 468                             {
 469                                 "domain": tidyup(instance["host"]),
 470                                 # no reason field, nothing
 471                                 "reason": ""
 472                             }
 473                         )
 474
 475                 if len(doc) < step:
 476                     # NOISY-DEBUG: print("DEBUG: End of request:", len(doc), step)
 477                     break
 478
 479                 # NOISY-DEBUG: print("DEBUG: Raising counter by step:", step)
 480                 counter = counter + step
 481
 482             except:
 483                 print("WARNING: Caught error, exiting loop:", domain)
 484                 counter = 0
 485                 break
 486
 487         while True:
 488             # same shit, different asshole ("blocked" aka full suspend)
 489             try:
 490                 if counter == 0:
 491                     # NOISY-DEBUG: print("DEBUG: Sending JSON API request to domain,step,counter:", domain, step, counter)
 492                     doc = post_json_api(domain,"/api/federation/instances", json.dumps({
 493                         "sort"   : "+caughtAt",
 494                         "host"   : None,
 495                         "blocked": True,
 496                         "limit"  : step
 497                     }))
 498                 else:
 499                     # NOISY-DEBUG: print("DEBUG: Sending JSON API request to domain,step,counter:", domain, step, counter)
 500                     doc = post_json_api(domain,"/api/federation/instances", json.dumps({
 501                         "sort"   : "+caughtAt",
 502                         "host"   : None,
 503                         "blocked": True,
 504                         "limit"  : step,
 505                         "offset" : counter-1
 506                     }))
 507
 508                 # NOISY-DEBUG: print("DEBUG: doc():", len(doc))
 509                 if len(doc) == 0:
 510                     # NOISY-DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
 511                     break
 512
 513                 for instance in doc:
 514                     if instance["isBlocked"]:
 515                         blocks["blocked"].append({
 516                             "domain": tidyup(instance["host"]),
 517                             "reason": ""
 518                         })
 519
 520                 if len(doc) < step:
 521                     # NOISY-DEBUG: print("DEBUG: End of request:", len(doc), step)
 522                     break
 523
 524                 # NOISY-DEBUG: print("DEBUG: Raising counter by step:", step)
 525                 counter = counter + step
 526
 527             except:
 528                 counter = 0
 529                 break
 530
 531         # NOISY-DEBUG: print("DEBUG: Returning for domain,blocked(),suspended():", domain, len(blocks["blocked"]), len(blocks["suspended"]))
 532         return {
 533             "reject"        : blocks["blocked"],
 534             "followers_only": blocks["suspended"]
 535         }
 536
 537     except:
 538         print("WARNING: API request failed for domain:", domain)
 539         return {}
 540
 541 def tidyup(string: str) -> str:
 542     # some retards put their blocks in variable case
 543     string = string.lower().strip()
 544
 545     # other retards put the port
 546     string = re.sub("\:\d+$", "", string)
 547
 548     # bigger retards put the schema in their blocklist, sometimes even without slashes
 549     string = re.sub("^https?\:(\/*)", "", string)
 550
 551     # and trailing slash
 552     string = re.sub("\/$", "", string)
 553
 554     # and the @
 555     string = re.sub("^\@", "", string)
 556
 557     # the biggest retards of them all try to block individual users
 558     string = re.sub("(.+)\@", "", string)
 559
 560     return string