fetch_blocks.py

   1 #!/usr/bin/python3
   2 # -*- coding: utf-8 -*-
   3
   4 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
   5 # Copyright (C) 2023 Free Software Foundation
   6 #
   7 # This program is free software: you can redistribute it and/or modify
   8 # it under the terms of the GNU Affero General Public License as published
   9 # by the Free Software Foundation, either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU Affero General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU Affero General Public License
  18 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  19
  20 import reqto
  21 import time
  22 import bs4
  23 import itertools
  24 import re
  25 import fba
  26
  27 fba.cursor.execute(
  28     "SELECT domain, software, origin, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial', 'bookwyrm', 'takahe') AND (last_blocked IS NULL OR last_blocked < ?) ORDER BY rowid DESC", [time.time() - fba.config["recheck_block"]]
  29 )
  30
  31 rows = fba.cursor.fetchall()
  32 print(f"INFO: Checking {len(rows)} entries ...")
  33 for blocker, software, origin, nodeinfo_url in rows:
  34     # NOISY-DEBUG: print("DEBUG: BEFORE blocker,software,origin,nodeinfo_url:", blocker, software, origin, nodeinfo_url)
  35     blockdict = []
  36     blocker = fba.tidyup(blocker)
  37     # NOISY-DEBUG: print("DEBUG: AFTER blocker,software:", blocker, software)
  38
  39     if blocker == "":
  40         print("WARNING: blocker is now empty!")
  41         continue
  42     elif fba.is_blacklisted(blocker):
  43         print(f"WARNING: blocker='{blocker}' is blacklisted now!")
  44         continue
  45
  46     # NOISY-DEBUG: print(f"DEBUG: blocker='{blocker}'")
  47     fba.update_last_blocked(blocker)
  48
  49     if software == "pleroma":
  50         print("INFO: blocker:", blocker)
  51         try:
  52             # Blocks
  53             json = fba.fetch_nodeinfo(blocker, nodeinfo_url)
  54             if json is None:
  55                 print("WARNING: Could not fetch nodeinfo from blocker:", blocker)
  56                 continue
  57
  58             print("DEBUG: Updating nodeinfo:", blocker)
  59             fba.update_last_nodeinfo(blocker)
  60
  61             federation = json["metadata"]["federation"]
  62
  63             if "enabled" in federation:
  64                 # NOISY-DEBUG: print("DEBUG: Instance has no block list to analyze:", blocker)
  65                 continue
  66
  67             if "mrf_simple" in federation:
  68                 for block_level, blocks in (
  69                     {**federation["mrf_simple"],
  70                     **{"quarantined_instances": federation["quarantined_instances"]}}
  71                 ).items():
  72                     # NOISY-DEBUG: print("DEBUG: block_level, blocks():", block_level, len(blocks))
  73                     block_level = fba.tidyup(block_level)
  74                     # NOISY-DEBUG: print("DEBUG: BEFORE block_level:", block_level)
  75
  76                     if block_level == "":
  77                         print("WARNING: block_level is now empty!")
  78                         continue
  79
  80                     for blocked in blocks:
  81                         # NOISY-DEBUG: print("DEBUG: BEFORE blocked:", blocked)
  82                         blocked = fba.tidyup(blocked)
  83                         # NOISY-DEBUG: print("DEBUG: AFTER blocked:", blocked)
  84
  85                         if blocked == "":
  86                             print("WARNING: blocked is empty after fba.tidyup():", blocker, block_level)
  87                             continue
  88
  89                         if blocked.count("*") > 1:
  90                             # -ACK!-oma also started obscuring domains without hash
  91                             fba.cursor.execute(
  92                                 "SELECT domain, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")]
  93                             )
  94                             searchres = fba.cursor.fetchone()
  95                             # NOISY-DEBUG: print("DEBUG: searchres[]:", type(searchres))
  96                             if searchres != None:
  97                                 blocked = searchres[0]
  98                                 nodeinfo_url = searchres[1]
  99                                 # NOISY-DEBUG: print("DEBUG: Looked up domain:", blocked)
 100
 101                         # NOISY-DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
 102                         if not fba.is_instance_registered(blocked):
 103                             # NOISY-DEBUG: print("DEBUG: Domain wasn't found, adding:", blocked, blocker)
 104                             fba.add_instance(blocked, blocker, origin, nodeinfo_url)
 105
 106                         fba.cursor.execute(
 107                             "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ? LIMIT 1",
 108                             (
 109                                blocker,
 110                                blocked,
 111                                block_level
 112                            ),
 113                         )
 114
 115                         if fba.cursor.fetchone() == None:
 116                             # NOISY-DEBUG: print("DEBUG: Blocking:", blocker, blocked, block_level)
 117                             fba.block_instance(blocker, blocked, "unknown", block_level)
 118
 119                             if block_level == "reject":
 120                                 # NOISY-DEBUG: print("DEBUG: Adding to blockdict:", blocked)
 121                                 blockdict.append(
 122                                     {
 123                                         "blocked": blocked,
 124                                         "reason" : None
 125                                     })
 126                         else:
 127                             # NOISY-DEBUG: print("DEBUG: Updating last_seen:", blocker, blocked, block_level)
 128                             fba.update_last_seen(blocker, blocked, block_level)
 129
 130             fba.connection.commit()
 131
 132             # Reasons
 133             if "mrf_simple_info" in federation:
 134                 # NOISY-DEBUG: print("DEBUG: Found mrf_simple_info:", blocker)
 135                 for block_level, info in (
 136                     {**federation["mrf_simple_info"],
 137                     **(federation["quarantined_instances_info"]
 138                     if "quarantined_instances_info" in federation
 139                     else {})}
 140                 ).items():
 141                     # NOISY-DEBUG: print("DEBUG: block_level, info.items():", block_level, len(info.items()))
 142                     block_level = fba.tidyup(block_level)
 143                     # NOISY-DEBUG: print("DEBUG: BEFORE block_level:", block_level)
 144
 145                     if block_level == "":
 146                         print("WARNING: block_level is now empty!")
 147                         continue
 148
 149                     for blocked, reason in info.items():
 150                         # NOISY-DEBUG: print("DEBUG: BEFORE blocked:", blocked)
 151                         blocked = fba.tidyup(blocked)
 152                         # NOISY-DEBUG: print("DEBUG: AFTER blocked:", blocked)
 153
 154                         if blocked == "":
 155                             print("WARNING: blocked is empty after fba.tidyup():", blocker, block_level)
 156                             continue
 157                         elif blocked.count("*") > 1:
 158                             # same domain guess as above, but for reasons field
 159                             fba.cursor.execute(
 160                                 "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")]
 161                             )
 162                             searchres = fba.cursor.fetchone()
 163
 164                             if searchres != None:
 165                                 blocked = searchres[0]
 166
 167                         # NOISY-DEBUG: print("DEBUG: Updating block reason:", blocker, blocked, reason["reason"])
 168                         fba.update_block_reason(reason["reason"], blocker, blocked, block_level)
 169
 170                         for entry in blockdict:
 171                             if entry["blocked"] == blocked:
 172                                 # NOISY-DEBUG: print("DEBUG: Updating entry reason:", blocked)
 173                                 entry["reason"] = reason["reason"]
 174
 175             fba.connection.commit()
 176         except Exception as e:
 177             print(f"ERROR: blocker='{blocker}',software='{software}',exception[{type(e)}]:'{str(e)}'")
 178     elif software == "mastodon":
 179         print("INFO: blocker:", blocker)
 180         try:
 181             # json endpoint for newer mastodongs
 182             try:
 183                 json = {
 184                     "reject"        : [],
 185                     "media_removal" : [],
 186                     "followers_only": [],
 187                     "report_removal": []
 188                 }
 189
 190                 # handling CSRF, I've saw at least one server requiring it to access the endpoint
 191                 # NOISY-DEBUG: print("DEBUG: Fetching meta:", blocker)
 192                 meta = bs4.BeautifulSoup(
 193                     reqto.get(f"https://{blocker}/about", headers=fba.headers, timeout=(fba.config["connection_timeout"], config["read_timeout"])).text,
 194                     "html.parser",
 195                 )
 196                 try:
 197                     csrf = meta.find("meta", attrs={"name": "csrf-token"})["content"]
 198                     # NOISY-DEBUG: print("DEBUG: Adding CSRF token:", blocker, csrf)
 199                     reqheaders = {**fba.api_headers, **{"X-CSRF-Token": csrf}}
 200                 except:
 201                     # NOISY-DEBUG: print("DEBUG: No CSRF token found, using normal headers:", blocker)
 202                     reqheaders = fba.api_headers
 203
 204                 # NOISY-DEBUG: print("DEBUG: Quering API domain_blocks:", blocker)
 205                 blocks = reqto.get(f"https://{blocker}/api/v1/instance/domain_blocks", headers=reqheaders, timeout=(fba.config["connection_timeout"], config["read_timeout"])).json()
 206
 207                 # NOISY-DEBUG: print("DEBUG: blocks():", len(blocks))
 208                 for block in blocks:
 209                     entry = {
 210                         'domain': block['domain'],
 211                         'hash'  : block['digest'],
 212                         'reason': block['comment']
 213                     }
 214
 215                     # NOISY-DEBUG: print("DEBUG: severity,domain,hash,comment:", block['severity'], block['domain'], block['digest'], block['comment'])
 216                     if block['severity'] == 'suspend':
 217                         json['reject'].append(entry)
 218                     elif block['severity'] == 'silence':
 219                         json['followers_only'].append(entry)
 220                     elif block['severity'] == 'reject_media':
 221                         json['media_removal'].append(entry)
 222                     elif block['severity'] == 'reject_reports':
 223                         json['report_removal'].append(entry)
 224                     else:
 225                         print("WARNING: Unknown severity:", block['severity'], block['domain'])
 226             except:
 227                 # NOISY-DEBUG: print("DEBUG: Failed, Trying mastodon-specific fetches:", blocker)
 228                 json = fba.get_mastodon_blocks(blocker)
 229
 230             # NOISY-DEBUG: print("DEBUG: json.items():", blocker, len(json.items()))
 231             for block_level, blocks in json.items():
 232                 # NOISY-DEBUG: print("DEBUG: blocker,block_level,blocks():", blocker, block_level, len(blocks))
 233                 block_level = fba.tidyup(block_level)
 234                 # NOISY-DEBUG: print("DEBUG: AFTER-block_level:", block_level)
 235                 if block_level == "":
 236                     print("WARNING: block_level is empty, blocker:", blocker)
 237                     continue
 238
 239                 for instance in blocks:
 240                     blocked, blocked_hash, reason = instance.values()
 241                     # NOISY-DEBUG: print("DEBUG: blocked,hash,reason:", blocked, blocked_hash, reason)
 242                     blocked = fba.tidyup(blocked)
 243                     # NOISY-DEBUG: print("DEBUG: AFTER-blocked:", blocked)
 244
 245                     if blocked == "":
 246                         print("WARNING: blocked is empty:", blocker)
 247                         continue
 248                     elif blocked.count("*") < 1:
 249                         # No obsfucation for this instance
 250                         fba.cursor.execute(
 251                             "SELECT hash FROM instances WHERE domain = ? LIMIT 1", [blocked]
 252                         )
 253
 254                         if fba.cursor.fetchone() == None:
 255                             # NOISY-DEBUG: print("DEBUG: Hash wasn't found, adding:", blocked, blocker)
 256                             fba.add_instance(blocked, blocker, origin)
 257                     else:
 258                         # Doing the hash search for instance names as well to tidy up DB
 259                         fba.cursor.execute(
 260                             "SELECT domain FROM instances WHERE hash = ? LIMIT 1", [blocked_hash]
 261                         )
 262                         searchres = fba.cursor.fetchone()
 263
 264                         if searchres != None:
 265                             # NOISY-DEBUG: print("DEBUG: Updating domain: ", searchres[0])
 266                             blocked = searchres[0]
 267
 268                     fba.cursor.execute(
 269                         "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ? LIMIT 1",
 270                         (
 271                             blocker,
 272                             blocked if blocked.count("*") <= 1 else blocked_hash,
 273                             block_level
 274                         ),
 275                     )
 276
 277                     if fba.cursor.fetchone() == None:
 278                         fba.block_instance(blocker, blocked if blocked.count("*") <= 1 else blocked_hash, reason, block_level)
 279
 280                         if block_level == "reject":
 281                             blockdict.append(
 282                                 {
 283                                     "blocked": blocked,
 284                                     "reason" : reason
 285                                 })
 286                     else:
 287                         fba.update_last_seen(blocker, blocked if blocked.count("*") <= 1 else blocked_hash, block_level)
 288
 289                     if reason != "":
 290                         # NOISY-DEBUG: print("DEBUG: Updating block reason:", blocker, blocked, reason)
 291                         fba.update_block_reason(reason, blocker, blocked if blocked.count("*") <= 1 else blocked_hash, block_level)
 292
 293             fba.connection.commit()
 294         except Exception as e:
 295             print(f"ERROR: blocker='{blocker}',software='{software}',exception[{type(e)}]:'{str(e)}'")
 296     elif software == "friendica" or software == "misskey" or software == "bookwyrm" or software == "takahe":
 297         print("INFO: blocker:", blocker)
 298         try:
 299             if software == "friendica":
 300                 json = fba.get_friendica_blocks(blocker)
 301             elif software == "misskey":
 302                 json = fba.get_misskey_blocks(blocker)
 303             elif software == "bookwyrm":
 304                 print("WARNING: bookwyrm is not fully supported for fetching blacklist!", blocker)
 305                 #json = fba.get_bookwyrm_blocks(blocker)
 306             elif software == "takahe":
 307                 print("WARNING: takahe is not fully supported for fetching blacklist!", blocker)
 308                 #json = fba.get_takahe_blocks(blocker)
 309
 310             for block_level, blocks in json.items():
 311                 # NOISY-DEBUG: print("DEBUG: blocker,block_level,blocks():", blocker, block_level, len(blocks))
 312                 block_level = fba.tidyup(block_level)
 313                 # NOISY-DEBUG: print("DEBUG: AFTER-block_level:", block_level)
 314                 if block_level == "":
 315                     print("WARNING: block_level is empty, blocker:", blocker)
 316                     continue
 317
 318                 for instance in blocks:
 319                     blocked, reason = instance.values()
 320                     # NOISY-DEBUG: print("DEBUG: BEFORE blocked:", blocked)
 321                     blocked = fba.tidyup(blocked)
 322                     # NOISY-DEBUG: print("DEBUG: AFTER blocked:", blocked)
 323
 324                     if blocked == "":
 325                         print("WARNING: blocked is empty:", blocker)
 326                         continue
 327                     if blocked.count("*") > 0:
 328                         # Some friendica servers also obscure domains without hash
 329                         fba.cursor.execute(
 330                             "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")]
 331                         )
 332                         searchres = fba.cursor.fetchone()
 333                         if searchres != None:
 334                             blocked = searchres[0]
 335
 336                     if blocked.count("?") > 0:
 337                         # Some obscure them with question marks, not sure if that's dependent on version or not
 338                         fba.cursor.execute(
 339                             "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("?", "_")]
 340                         )
 341                         searchres = fba.cursor.fetchone()
 342                         if searchres != None:
 343                             blocked = searchres[0]
 344                             origin = searchres[1]
 345                             nodeinfo_url = searchres[2]
 346
 347                     # NOISY-DEBUG: print("DEBUG: AFTER-blocked:", blocked)
 348                     if not fba.is_instance_registered(blocked):
 349                         # NOISY-DEBUG: print("DEBUG: Hash wasn't found, adding:", blocked, blocker)
 350                         fba.add_instance(blocked, blocker, origin, nodeinfo_url)
 351
 352                     fba.cursor.execute(
 353                         "SELECT * FROM blocks WHERE blocker = ? AND blocked = ?",
 354                         (blocker, blocked),
 355                     )
 356
 357                     if fba.cursor.fetchone() == None:
 358                         fba.block_instance(blocker, blocked, reason, block_level)
 359
 360                         if block_level == "reject":
 361                             blockdict.append(
 362                                 {
 363                                     "blocked": blocked,
 364                                     "reason" : reason
 365                                 })
 366                     else:
 367                         fba.update_last_seen(blocker, blocked, block_level)
 368
 369                     if reason != '':
 370                         # NOISY-DEBUG: print("DEBUG: Updating block reason:", blocker, blocked, reason)
 371                         fba.update_block_reason(reason, blocker, blocked, block_level)
 372
 373             fba.connection.commit()
 374         except Exception as e:
 375             print(f"ERROR: blocker='{blocker}',software='{software}',exception[{type(e)}]:'{str(e)}'")
 376     elif software == "gotosocial":
 377         print("INFO: blocker:", blocker)
 378         try:
 379             # Blocks
 380             federation = reqto.get(f"https://{blocker}{get_peers_url}?filter=suspended", headers=fba.api_headers, timeout=(fba.config["connection_timeout"], config["read_timeout"])).json()
 381
 382             if (federation == None):
 383                 print("WARNING: No valid response:", blocker);
 384             elif "error" in federation:
 385                 print("WARNING: API returned error:", federation["error"])
 386             else:
 387                 # NOISY-DEBUG: print("DEBUG: Checking fenderation():", len(federation))
 388                 for peer in federation:
 389                     blocked = peer["domain"].lower()
 390                     # NOISY-DEBUG: print("DEBUG: BEFORE blocked:", blocked)
 391                     blocked = fba.tidyup(blocked)
 392                     # NOISY-DEBUG: print("DEBUG: AFTER blocked:", blocked)
 393
 394                     if blocked == "":
 395                         print("WARNING: blocked is empty:", blocker)
 396                         continue
 397                     elif blocked.count("*") > 0:
 398                         # GTS does not have hashes for obscured domains, so we have to guess it
 399                         fba.cursor.execute(
 400                             "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")]
 401                         )
 402                         searchres = fba.cursor.fetchone()
 403
 404                         if searchres != None:
 405                             blocked = searchres[0]
 406                             origin = searchres[1]
 407                             nodeinfo_url = searchres[2]
 408
 409                     if not fba.is_instance_registered(blocked):
 410                         # NOISY-DEBUG: print("DEBUG: Domain wasn't found, adding:", blocked, blocker)
 411                         fba.add_instance(blocked, blocker, origin, nodeinfo_url)
 412
 413                     fba.cursor.execute(
 414                         "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ? LIMIT 1",
 415                         (
 416                             blocker,
 417                             blocked,
 418                             "reject"
 419                         ),
 420                     )
 421
 422                     if fba.cursor.fetchone() == None:
 423                         # NOISY-DEBUG: print(f"DEBUG: blocker='{blocker}' is blocking '{blocked}' for unknown reason at this point")
 424                         fba.block_instance(blocker, blocked, "unknown", "reject")
 425
 426                         blockdict.append(
 427                             {
 428                                 "blocked": blocked,
 429                                 "reason" : None
 430                             })
 431                     else:
 432                         fba.update_last_seen(blocker, blocked, "reject")
 433
 434                     if "public_comment" in peer:
 435                         # NOISY-DEBUG: print("DEBUG: Updating block reason:", blocker, blocked, peer["public_comment"])
 436                         fba.update_block_reason(peer["public_comment"], blocker, blocked, "reject")
 437
 438                         for entry in blockdict:
 439                             if entry["blocked"] == blocked:
 440                                 # NOISY-DEBUG: print(f"DEBUG: Setting block reason for blocked='{blocked}':'{peer['public_comment']}'")
 441                                 entry["reason"] = peer["public_comment"]
 442
 443                 fba.connection.commit()
 444         except Exception as e:
 445             print(f"ERROR: blocker='{blocker}',software='{software}',exception[{type(e)}]:'{str(e)}'")
 446     else:
 447         print("WARNING: Unknown software:", blocker, software)
 448
 449     if fba.config["bot_enabled"] and len(blockdict) > 0:
 450         send_bot_post(blocker, blockdict)
 451
 452     blockdict = []
 453
 454 fba.connection.close()