fba/commands.py

   1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
   2 # Copyright (C) 2023 Free Software Foundation
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published
   6 # by the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  16
  17 import argparse
  18 import atoma
  19 import bs4
  20 import itertools
  21 import json
  22 import re
  23 import reqto
  24 import sys
  25 import time
  26 import validators
  27
  28 from fba import boot
  29 from fba import config
  30 from fba import fba
  31
  32 def check_instance(args: argparse.Namespace) -> int:
  33     # DEBUG: print(f"DEBUG: args.domain='{args.domain}' - CALLED!")
  34     status = 0
  35     if not validators.domain(args.domain):
  36         print(f"WARNING: args.domain='{args.domain}' is not valid")
  37         status = 100
  38     elif fba.is_blacklisted(args.domain):
  39         print(f"WARNING: args.domain='{args.domain}' is blacklisted")
  40         status = 101
  41     elif fba.is_instance_registered(args.domain):
  42         print(f"WARNING: args.domain='{args.domain}' is already registered")
  43         staus = 102
  44     else:
  45         print(f"INFO: args.domain='{args.domain}' is not known")
  46
  47     # DEBUG: print(f"DEBUG: status={status} - EXIT!")
  48     return status
  49
  50 def fetch_bkali(args: argparse.Namespace):
  51     # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
  52     domains = list()
  53     try:
  54         fetched = fba.post_json_api("gql.api.bka.li", "/v1/graphql", json.dumps({
  55             "query": "query domainlist {nodeinfo(order_by: {domain: asc}) {domain}}"
  56         }))
  57
  58         # DEBUG: print(f"DEBUG: fetched({len(fetched)})[]='{type(fetched)}'")
  59         if len(fetched) == 0:
  60             raise Exception("WARNING: Returned no records")
  61         elif not "data" in fetched:
  62             raise Exception(f"WARNING: fetched()={len(fetched)} does not contain key 'data'")
  63         elif not "nodeinfo" in fetched["data"]:
  64             raise Exception(f"WARNING: fetched()={len(fetched['data'])} does not contain key 'nodeinfo'")
  65
  66         for entry in fetched["data"]["nodeinfo"]:
  67             # DEBUG: print(f"DEBUG: entry['{type(entry)}']='{entry}'")
  68             if not "domain" in entry:
  69                 print(f"WARNING: entry does not contain 'domain' - SKIPPED!")
  70                 continue
  71             elif not validators.domain(entry["domain"]):
  72                 print(f"WARNING: domain='{entry['domain']}' is not a valid domain - SKIPPED!")
  73                 continue
  74             elif fba.is_blacklisted(entry["domain"]):
  75                 # DEBUG: print(f"DEBUG: domain='{entry['domain']}' is blacklisted - SKIPPED!")
  76                 continue
  77             elif fba.is_instance_registered(entry["domain"]):
  78                 # DEBUG: print(f"DEBUG: domain='{entry['domain']}' is already registered - SKIPPED!")
  79                 continue
  80
  81             # DEBUG: print(f"DEBUG: Adding domain='{entry['domain']}' ...")
  82             domains.append(entry["domain"])
  83
  84     except BaseException as e:
  85         print(f"ERROR: Cannot fetch graphql,exception[{type(e)}]:'{str(e)}'")
  86         sys.exit(255)
  87
  88     # DEBUG: print(f"DEBUG: domains()={len(domains)}")
  89     if len(domains) > 0:
  90         boot.acquire_lock()
  91
  92         print(f"INFO: Adding {len(domains)} new instances ...")
  93         for domain in domains:
  94             print(f"INFO: Fetching instances from domain='{domain}' ...")
  95             fba.fetch_instances(domain, None, None, sys.argv[0])
  96
  97     # DEBUG: print("DEBUG: EXIT!")
  98
  99 def fetch_blocks(args: argparse.Namespace):
 100     # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
 101     if args.domain != None and args.domain != "":
 102         if not validators.domain(args.domain):
 103             print(f"WARNING: domain='{args.domain}' is not valid.")
 104             return
 105         elif fba.is_blacklisted(args.domain):
 106             print(f"WARNING: domain='{args.domain}' is blacklisted, won't check it!")
 107             return
 108         elif not fba.is_instance_registered(args.domain):
 109             print(f"WARNING: domain='{args.domain}' is not registered, please run ./fba.py fetch_instances {args.domain} first.")
 110             return
 111
 112     boot.acquire_lock()
 113
 114     if args.domain != None and args.domain != "":
 115         fba.cursor.execute(
 116             "SELECT domain, software, origin, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial', 'bookwyrm', 'takahe') AND domain = ?", [args.domain]
 117         )
 118     else:
 119         fba.cursor.execute(
 120             "SELECT domain, software, origin, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial', 'bookwyrm', 'takahe') AND (last_blocked IS NULL OR last_blocked < ?) ORDER BY rowid DESC", [time.time() - config.get("recheck_block")]
 121         )
 122
 123     rows = fba.cursor.fetchall()
 124     print(f"INFO: Checking {len(rows)} entries ...")
 125     for blocker, software, origin, nodeinfo_url in rows:
 126         # DEBUG: print("DEBUG: BEFORE blocker,software,origin,nodeinfo_url:", blocker, software, origin, nodeinfo_url)
 127         blockdict = []
 128         blocker = fba.tidyup_domain(blocker)
 129         # DEBUG: print("DEBUG: AFTER blocker,software:", blocker, software)
 130
 131         if blocker == "":
 132             print("WARNING: blocker is now empty!")
 133             continue
 134         elif fba.is_blacklisted(blocker):
 135             print(f"WARNING: blocker='{blocker}' is blacklisted now!")
 136             continue
 137
 138         # DEBUG: print(f"DEBUG: blocker='{blocker}'")
 139         fba.update_last_blocked(blocker)
 140
 141         if software == "pleroma":
 142             print("INFO: blocker:", blocker)
 143             try:
 144                 # Blocks
 145                 json = fba.fetch_nodeinfo(blocker, nodeinfo_url)
 146                 if json is None:
 147                     print("WARNING: Could not fetch nodeinfo from blocker:", blocker)
 148                     continue
 149                 elif not "metadata" in json:
 150                     print(f"WARNING: json()={len(json)} does not have key 'metadata', blocker='{blocker}'")
 151                     continue
 152                 elif not "federation" in json["metadata"]:
 153                     print(f"WARNING: json()={len(json['metadata'])} does not have key 'federation', blocker='{blocker}'")
 154                     continue
 155
 156                 # DEBUG: print("DEBUG: Updating nodeinfo:", blocker)
 157                 fba.update_last_nodeinfo(blocker)
 158
 159                 federation = json["metadata"]["federation"]
 160
 161                 if "enabled" in federation:
 162                     # DEBUG: print("DEBUG: Instance has no block list to analyze:", blocker)
 163                     continue
 164
 165                 if "mrf_simple" in federation:
 166                     for block_level, blocks in (
 167                         {**federation["mrf_simple"],
 168                         **{"quarantined_instances": federation["quarantined_instances"]}}
 169                     ).items():
 170                         # DEBUG: print("DEBUG: block_level, blocks():", block_level, len(blocks))
 171                         block_level = fba.tidyup_domain(block_level)
 172                         # DEBUG: print("DEBUG: BEFORE block_level:", block_level)
 173
 174                         if block_level == "":
 175                             print("WARNING: block_level is now empty!")
 176                             continue
 177
 178                         # DEBUG: print(f"DEBUG: Checking {len(blocks)} entries from blocker='{blocker}',software='{software}',block_level='{block_level}' ...")
 179                         for blocked in blocks:
 180                             # DEBUG: print("DEBUG: BEFORE blocked:", blocked)
 181                             blocked = fba.tidyup_domain(blocked)
 182                             # DEBUG: print("DEBUG: AFTER blocked:", blocked)
 183
 184                             if blocked == "":
 185                                 print("WARNING: blocked is empty after fba.tidyup_domain():", blocker, block_level)
 186                                 continue
 187                             elif fba.is_blacklisted(blocked):
 188                                 # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!")
 189                                 continue
 190                             elif blocked.count("*") > 1:
 191                                 # -ACK!-oma also started obscuring domains without hash
 192                                 fba.cursor.execute(
 193                                     "SELECT domain, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")]
 194                                 )
 195                                 searchres = fba.cursor.fetchone()
 196                                 # DEBUG: print("DEBUG: searchres[]:", type(searchres))
 197
 198                                 if searchres == None:
 199                                     print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
 200                                     continue
 201
 202                                 blocked = searchres[0]
 203                                 nodeinfo_url = searchres[1]
 204                                 # DEBUG: print("DEBUG: Looked up domain:", blocked)
 205                             elif not validators.domain(blocked):
 206                                 print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
 207                                 continue
 208
 209                             # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
 210                             if not validators.domain(blocked):
 211                                 print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
 212                                 continue
 213                             elif not fba.is_instance_registered(blocked):
 214                                 # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., blocker='{blocker}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'")
 215                                 fba.add_instance(blocked, blocker, sys.argv[0], nodeinfo_url)
 216
 217                             if not fba.is_instance_blocked(blocker, blocked, block_level):
 218                                 # DEBUG: print("DEBUG: Blocking:", blocker, blocked, block_level)
 219                                 fba.block_instance(blocker, blocked, "unknown", block_level)
 220
 221                                 if block_level == "reject":
 222                                     # DEBUG: print("DEBUG: Adding to blockdict:", blocked)
 223                                     blockdict.append(
 224                                         {
 225                                             "blocked": blocked,
 226                                             "reason" : None
 227                                         })
 228                             else:
 229                                 # DEBUG: print(f"DEBUG: Updating block last seen for blocker='{blocker}',blocked='{blocked}' ...")
 230                                 fba.update_last_seen(blocker, blocked, block_level)
 231
 232                 # DEBUG: print("DEBUG: Committing changes ...")
 233                 fba.connection.commit()
 234
 235                 # Reasons
 236                 if "mrf_simple_info" in federation:
 237                     # DEBUG: print("DEBUG: Found mrf_simple_info:", blocker)
 238                     for block_level, info in (
 239                         {**federation["mrf_simple_info"],
 240                         **(federation["quarantined_instances_info"]
 241                         if "quarantined_instances_info" in federation
 242                         else {})}
 243                     ).items():
 244                         # DEBUG: print("DEBUG: block_level, info.items():", block_level, len(info.items()))
 245                         block_level = fba.tidyup_domain(block_level)
 246                         # DEBUG: print("DEBUG: BEFORE block_level:", block_level)
 247
 248                         if block_level == "":
 249                             print("WARNING: block_level is now empty!")
 250                             continue
 251
 252                         # DEBUG: print(f"DEBUG: Checking {len(info.items())} entries from blocker='{blocker}',software='{software}',block_level='{block_level}' ...")
 253                         for blocked, reason in info.items():
 254                             # DEBUG: print("DEBUG: BEFORE blocked:", blocked)
 255                             blocked = fba.tidyup_domain(blocked)
 256                             # DEBUG: print("DEBUG: AFTER blocked:", blocked)
 257
 258                             if blocked == "":
 259                                 print("WARNING: blocked is empty after fba.tidyup_domain():", blocker, block_level)
 260                                 continue
 261                             elif fba.is_blacklisted(blocked):
 262                                 # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!")
 263                                 continue
 264                             elif blocked.count("*") > 1:
 265                                 # same domain guess as above, but for reasons field
 266                                 fba.cursor.execute(
 267                                     "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")]
 268                                 )
 269                                 searchres = fba.cursor.fetchone()
 270
 271                                 if searchres == None:
 272                                     print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
 273                                     continue
 274
 275                                 blocked = searchres[0]
 276                                 origin = searchres[1]
 277                                 nodeinfo_url = searchres[2]
 278                             elif not validators.domain(blocked):
 279                                 print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
 280                                 continue
 281
 282                             # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
 283                             if not validators.domain(blocked):
 284                                 print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
 285                                 continue
 286                             elif not fba.is_instance_registered(blocked):
 287                                 # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., blocker='{blocker}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'")
 288                                 fba.add_instance(blocked, blocker, sys.argv[0], nodeinfo_url)
 289
 290                             # DEBUG: print("DEBUG: Updating block reason:", blocker, blocked, reason["reason"])
 291                             fba.update_block_reason(reason["reason"], blocker, blocked, block_level)
 292
 293                             for entry in blockdict:
 294                                 if entry["blocked"] == blocked:
 295                                     # DEBUG: print("DEBUG: Updating entry reason:", blocked)
 296                                     entry["reason"] = reason["reason"]
 297
 298                 fba.connection.commit()
 299             except Exception as e:
 300                 print(f"ERROR: blocker='{blocker}',software='{software}',exception[{type(e)}]:'{str(e)}'")
 301         elif software == "mastodon":
 302             print("INFO: blocker:", blocker)
 303             try:
 304                 # json endpoint for newer mastodongs
 305                 try:
 306                     json = {
 307                         "reject"        : [],
 308                         "media_removal" : [],
 309                         "followers_only": [],
 310                         "report_removal": []
 311                     }
 312
 313                     # handling CSRF, I've saw at least one server requiring it to access the endpoint
 314                     # DEBUG: print("DEBUG: Fetching meta:", blocker)
 315                     meta = bs4.BeautifulSoup(
 316                         fba.get_response(blocker, "/", fba.headers, (config.get("connection_timeout"), config.get("read_timeout"))).text,
 317                         "html.parser",
 318                     )
 319                     try:
 320                         csrf = meta.find("meta", attrs={"name": "csrf-token"})["content"]
 321                         # DEBUG: print("DEBUG: Adding CSRF token:", blocker, csrf)
 322                         reqheaders = {**fba.api_headers, **{"X-CSRF-Token": csrf}}
 323                     except BaseException as e:
 324                         # DEBUG: print("DEBUG: No CSRF token found, using normal headers:", blocker, e)
 325                         reqheaders = fba.api_headers
 326
 327                     # DEBUG: print("DEBUG: Querying API domain_blocks:", blocker)
 328                     blocks = fba.get_response(blocker, "/api/v1/instance/domain_blocks", reqheaders, (config.get("connection_timeout"), config.get("read_timeout"))).json()
 329
 330                     print(f"INFO: Checking {len(blocks)} entries from blocker='{blocker}',software='{software}' ...")
 331                     for block in blocks:
 332                         entry = {
 333                             'domain': block['domain'],
 334                             'hash'  : block['digest'],
 335                             'reason': block['comment']
 336                         }
 337
 338                         # DEBUG: print("DEBUG: severity,domain,hash,comment:", block['severity'], block['domain'], block['digest'], block['comment'])
 339                         if block['severity'] == 'suspend':
 340                             # DEBUG: print(f"DEBUG: Adding entry='{entry}' with severity='{block['severity']}' ...")
 341                             json['reject'].append(entry)
 342                         elif block['severity'] == 'silence':
 343                             # DEBUG: print(f"DEBUG: Adding entry='{entry}' with severity='{block['severity']}' ...")
 344                             json['followers_only'].append(entry)
 345                         elif block['severity'] == 'reject_media':
 346                             # DEBUG: print(f"DEBUG: Adding entry='{entry}' with severity='{block['severity']}' ...")
 347                             json['media_removal'].append(entry)
 348                         elif block['severity'] == 'reject_reports':
 349                             # DEBUG: print(f"DEBUG: Adding entry='{entry}' with severity='{block['severity']}' ...")
 350                             json['report_removal'].append(entry)
 351                         else:
 352                             print("WARNING: Unknown severity:", block['severity'], block['domain'])
 353                 except BaseException as e:
 354                     # DEBUG: print(f"DEBUG: Failed, trying mastodon-specific fetches: blocker='{blocker}',exception[{type(e)}]={str(e)}")
 355                     json = fba.get_mastodon_blocks(blocker)
 356
 357                 print(f"INFO: Checking {len(json.items())} entries from blocker='{blocker}',software='{software}' ...")
 358                 for block_level, blocks in json.items():
 359                     # DEBUG: print("DEBUG: blocker,block_level,blocks():", blocker, block_level, len(blocks))
 360                     block_level = fba.tidyup_domain(block_level)
 361                     # DEBUG: print("DEBUG: AFTER-block_level:", block_level)
 362                     if block_level == "":
 363                         print("WARNING: block_level is empty, blocker:", blocker)
 364                         continue
 365
 366                     # DEBUG: print(f"DEBUG: Checking {len(blocks)} entries from blocker='{blocker}',software='{software}',block_level='{block_level}' ...")
 367                     for block in blocks:
 368                         blocked, blocked_hash, reason = block.values()
 369                         # DEBUG: print("DEBUG: blocked,hash,reason:", blocked, blocked_hash, reason)
 370                         blocked = fba.tidyup_domain(blocked)
 371                         # DEBUG: print("DEBUG: AFTER-blocked:", blocked)
 372
 373                         if blocked == "":
 374                             print("WARNING: blocked is empty:", blocker)
 375                             continue
 376                         elif fba.is_blacklisted(blocked):
 377                             # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!")
 378                             continue
 379                         elif blocked.count("*") > 0:
 380                             # Doing the hash search for instance names as well to tidy up DB
 381                             fba.cursor.execute(
 382                                 "SELECT domain, origin, nodeinfo_url FROM instances WHERE hash = ? LIMIT 1", [blocked_hash]
 383                             )
 384                             searchres = fba.cursor.fetchone()
 385
 386                             if searchres == None:
 387                                 print(f"WARNING: Cannot deobsfucate blocked='{blocked}',blocked_hash='{blocked_hash}' - SKIPPED!")
 388                                 continue
 389
 390                             # DEBUG: print("DEBUG: Updating domain: ", searchres[0])
 391                             blocked = searchres[0]
 392                             origin = searchres[1]
 393                             nodeinfo_url = searchres[2]
 394
 395                             # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
 396                             if not validators.domain(blocked):
 397                                 print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
 398                                 continue
 399                             elif not fba.is_instance_registered(blocked):
 400                                 # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., blocker='{blocker}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'")
 401                                 fba.add_instance(blocked, blocker, sys.argv[0], nodeinfo_url)
 402                         elif not validators.domain(blocked):
 403                             print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
 404                             continue
 405
 406                         # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
 407                         if not validators.domain(blocked):
 408                             print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
 409                             continue
 410                         elif not fba.is_instance_registered(blocked):
 411                             # DEBUG: print("DEBUG: Hash wasn't found, adding:", blocked, blocker)
 412                             fba.add_instance(blocked, blocker, sys.argv[0], nodeinfo_url)
 413
 414                         blocking = blocked if blocked.count("*") <= 1 else blocked_hash
 415                         # DEBUG: print(f"DEBUG: blocking='{blocking}',blocked='{blocked}',blocked_hash='{blocked_hash}'")
 416
 417                         if not fba.is_instance_blocked(blocker, blocked, block_level):
 418                             # DEBUG: print("DEBUG: Blocking:", blocker, blocked, block_level)
 419                             fba.block_instance(blocker, blocking, reason, block_level)
 420
 421                             if block_level == "reject":
 422                                 blockdict.append({
 423                                     "blocked": blocked,
 424                                     "reason" : reason
 425                                 })
 426                         else:
 427                             # DEBUG: print(f"DEBUG: Updating block last seen and reason for blocker='{blocker}',blocking='{blocking}' ...")
 428                             fba.update_last_seen(blocker, blocking, block_level)
 429                             fba.update_block_reason(reason, blocker, blocking, block_level)
 430
 431                 # DEBUG: print("DEBUG: Committing changes ...")
 432                 fba.connection.commit()
 433             except Exception as e:
 434                 print(f"ERROR: blocker='{blocker}',software='{software}',exception[{type(e)}]:'{str(e)}'")
 435         elif software == "friendica" or software == "misskey" or software == "bookwyrm" or software == "takahe":
 436             print("INFO: blocker:", blocker)
 437             try:
 438                 if software == "friendica":
 439                     json = fba.get_friendica_blocks(blocker)
 440                 elif software == "misskey":
 441                     json = fba.get_misskey_blocks(blocker)
 442                 elif software == "bookwyrm":
 443                     print("WARNING: bookwyrm is not fully supported for fetching blacklist!", blocker)
 444                     #json = fba.get_bookwyrm_blocks(blocker)
 445                     continue
 446                 elif software == "takahe":
 447                     print("WARNING: takahe is not fully supported for fetching blacklist!", blocker)
 448                     #json = fba.get_takahe_blocks(blocker)
 449                     continue
 450
 451                 print(f"INFO: Checking {len(json.items())} entries from blocker='{blocker}',software='{software}' ...")
 452                 for block_level, blocks in json.items():
 453                     # DEBUG: print("DEBUG: blocker,block_level,blocks():", blocker, block_level, len(blocks))
 454                     block_level = fba.tidyup_domain(block_level)
 455                     # DEBUG: print("DEBUG: AFTER-block_level:", block_level)
 456                     if block_level == "":
 457                         print("WARNING: block_level is empty, blocker:", blocker)
 458                         continue
 459
 460                     # DEBUG: print(f"DEBUG: Checking {len(blocks)} entries from blocker='{blocker}',software='{software}',block_level='{block_level}' ...")
 461                     for block in blocks:
 462                         blocked, reason = block.values()
 463                         # DEBUG: print("DEBUG: BEFORE blocked:", blocked)
 464                         blocked = fba.tidyup_domain(blocked)
 465                         # DEBUG: print("DEBUG: AFTER blocked:", blocked)
 466
 467                         if blocked == "":
 468                             print("WARNING: blocked is empty:", blocker)
 469                             continue
 470                         elif fba.is_blacklisted(blocked):
 471                             # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!")
 472                             continue
 473                         elif blocked.count("*") > 0:
 474                             # Some friendica servers also obscure domains without hash
 475                             fba.cursor.execute(
 476                                 "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")]
 477                             )
 478
 479                             searchres = fba.cursor.fetchone()
 480
 481                             if searchres == None:
 482                                 print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
 483                                 continue
 484
 485                             blocked = searchres[0]
 486                             origin = searchres[1]
 487                             nodeinfo_url = searchres[2]
 488                         elif blocked.count("?") > 0:
 489                             # Some obscure them with question marks, not sure if that's dependent on version or not
 490                             fba.cursor.execute(
 491                                 "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("?", "_")]
 492                             )
 493
 494                             searchres = fba.cursor.fetchone()
 495
 496                             if searchres == None:
 497                                 print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
 498                                 continue
 499
 500                             blocked = searchres[0]
 501                             origin = searchres[1]
 502                             nodeinfo_url = searchres[2]
 503                         elif not validators.domain(blocked):
 504                             print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
 505                             continue
 506
 507                         # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
 508                         if not validators.domain(blocked):
 509                             print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
 510                             continue
 511                         elif not fba.is_instance_registered(blocked):
 512                             # DEBUG: print("DEBUG: Hash wasn't found, adding:", blocked, blocker)
 513                             fba.add_instance(blocked, blocker, sys.argv[0], nodeinfo_url)
 514
 515                         if not fba.is_instance_blocked(blocker, blocked, block_level):
 516                             fba.block_instance(blocker, blocked, reason, block_level)
 517
 518                             if block_level == "reject":
 519                                 blockdict.append({
 520                                     "blocked": blocked,
 521                                     "reason" : reason
 522                                 })
 523                         else:
 524                             # DEBUG: print(f"DEBUG: Updating block last seen and reason for blocker='{blocker}',blocked='{blocked}' ...")
 525                             fba.update_last_seen(blocker, blocked, block_level)
 526                             fba.update_block_reason(reason, blocker, blocked, block_level)
 527
 528                 # DEBUG: print("DEBUG: Committing changes ...")
 529                 fba.connection.commit()
 530             except Exception as e:
 531                 print(f"ERROR: blocker='{blocker}',software='{software}',exception[{type(e)}]:'{str(e)}'")
 532         elif software == "gotosocial":
 533             print("INFO: blocker:", blocker)
 534             try:
 535                 # Blocks
 536                 federation = fba.get_response(blocker, f"{fba.get_peers_url}?filter=suspended", fba.api_headers, (config.get("connection_timeout"), config.get("read_timeout"))).json()
 537
 538                 if (federation == None):
 539                     print("WARNING: No valid response:", blocker);
 540                 elif "error" in federation:
 541                     print("WARNING: API returned error:", federation["error"])
 542                 else:
 543                     print(f"INFO: Checking {len(federation)} entries from blocker='{blocker}',software='{software}' ...")
 544                     for peer in federation:
 545                         blocked = peer["domain"].lower()
 546                         # DEBUG: print("DEBUG: BEFORE blocked:", blocked)
 547                         blocked = fba.tidyup_domain(blocked)
 548                         # DEBUG: print("DEBUG: AFTER blocked:", blocked)
 549
 550                         if blocked == "":
 551                             print("WARNING: blocked is empty:", blocker)
 552                             continue
 553                         elif fba.is_blacklisted(blocked):
 554                             # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!")
 555                             continue
 556                         elif blocked.count("*") > 0:
 557                             # GTS does not have hashes for obscured domains, so we have to guess it
 558                             fba.cursor.execute(
 559                                 "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")]
 560                             )
 561                             searchres = fba.cursor.fetchone()
 562
 563                             if searchres == None:
 564                                 print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
 565                                 continue
 566
 567                             blocked = searchres[0]
 568                             origin = searchres[1]
 569                             nodeinfo_url = searchres[2]
 570                         elif not validators.domain(blocked):
 571                             print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
 572                             continue
 573
 574                         # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
 575                         if not validators.domain(blocked):
 576                             print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
 577                             continue
 578                         elif not fba.is_instance_registered(blocked):
 579                             # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., blocker='{blocker}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'")
 580                             fba.add_instance(blocked, blocker, sys.argv[0], nodeinfo_url)
 581
 582                         if not fba.is_instance_blocked(blocker, blocked, "reject"):
 583                             # DEBUG: print(f"DEBUG: blocker='{blocker}' is blocking '{blocked}' for unknown reason at this point")
 584                             fba.block_instance(blocker, blocked, "unknown", "reject")
 585
 586                             blockdict.append({
 587                                 "blocked": blocked,
 588                                 "reason" : None
 589                             })
 590                         else:
 591                             # DEBUG: print(f"DEBUG: Updating block last seen for blocker='{blocker}',blocked='{blocked}' ...")
 592                             fba.update_last_seen(blocker, blocked, "reject")
 593
 594                         if "public_comment" in peer:
 595                             # DEBUG: print("DEBUG: Updating block reason:", blocker, blocked, peer["public_comment"])
 596                             fba.update_block_reason(peer["public_comment"], blocker, blocked, "reject")
 597
 598                             for entry in blockdict:
 599                                 if entry["blocked"] == blocked:
 600                                     # DEBUG: print(f"DEBUG: Setting block reason for blocked='{blocked}':'{peer['public_comment']}'")
 601                                     entry["reason"] = peer["public_comment"]
 602
 603                     # DEBUG: print("DEBUG: Committing changes ...")
 604                     fba.connection.commit()
 605             except Exception as e:
 606                 print(f"ERROR: blocker='{blocker}',software='{software}',exception[{type(e)}]:'{str(e)}'")
 607         else:
 608             print("WARNING: Unknown software:", blocker, software)
 609
 610         if config.get("bot_enabled") and len(blockdict) > 0:
 611             send_bot_post(blocker, blockdict)
 612
 613         blockdict = []
 614
 615     # DEBUG: print("DEBUG: EXIT!")
 616
 617 def fetch_cs(args: argparse.Namespace):
 618     # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
 619     domains = {
 620         "silenced": list(),
 621         "blocked": list(),
 622     }
 623
 624     try:
 625         doc = bs4.BeautifulSoup(
 626             fba.get_response("meta.chaos.social", "/federation", fba.headers, (config.get("connection_timeout"), config.get("read_timeout"))).text,
 627             "html.parser",
 628         )
 629         # DEBUG: print(f"DEBUG: doc()={len(doc)}[]={type(doc)}")
 630         silenced = doc.find("h2", {"id": "silenced-instances"}).findNext("table")
 631
 632         # DEBUG: print(f"DEBUG: silenced[]={type(silenced)}")
 633         domains["silenced"] = domains["silenced"] + find_domains(silenced)
 634         blocked = doc.find("h2", {"id": "blocked-instances"}).findNext("table")
 635
 636         # DEBUG: print(f"DEBUG: blocked[]={type(blocked)}")
 637         domains["blocked"] = domains["blocked"] + find_domains(blocked)
 638
 639     except BaseException as e:
 640         print(f"ERROR: Cannot fetch from meta.chaos.social,exception[{type(e)}]:'{str(e)}'")
 641         sys.exit(255)
 642
 643     # DEBUG: print(f"DEBUG: domains()={len(domains)}")
 644     if len(domains) > 0:
 645         boot.acquire_lock()
 646
 647         print(f"INFO: Adding {len(domains)} new instances ...")
 648         for block_level in domains:
 649             # DEBUG: print(f"DEBUG: block_level='{block_level}'")
 650
 651             for row in domains[block_level]:
 652                 # DEBUG: print(f"DEBUG: row='{row}'")
 653                 if not fba.is_instance_registered(row["domain"]):
 654                     print(f"INFO: Fetching instances from domain='{row['domain']}' ...")
 655                     fba.fetch_instances(row["domain"], None, None, sys.argv[0])
 656
 657                 if not fba.is_instance_blocked('chaos.social', row["domain"], block_level):
 658                     # DEBUG: print(f"DEBUG: domain='{row['domain']}',block_level='{block_level}' blocked by chaos.social, adding ...")
 659                     fba.block_instance('chaos.social', row["domain"], row["reason"], block_level)
 660
 661         # DEBUG: print("DEBUG: Committing changes ...")
 662         fba.connection.commit()
 663
 664     # DEBUG: print("DEBUG: EXIT!")
 665
 666 def fetch_fba_rss(args: argparse.Namespace):
 667     # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
 668     domains = list()
 669
 670     try:
 671         print(f"INFO: Fetch FBA-specific RSS args.feed='{args.feed}' ...")
 672         response = fba.get_url(args.feed, fba.headers, (config.get("connection_timeout"), config.get("read_timeout")))
 673
 674         # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code='{response.status_code}',response.text()={len(response.text)}")
 675         if response.ok and response.status_code < 300 and len(response.text) > 0:
 676             # DEBUG: print(f"DEBUG: Parsing RSS feed ...")
 677             rss = atoma.parse_rss_bytes(response.content)
 678
 679             # DEBUG: print(f"DEBUG: rss[]={type(rss)}")
 680             for item in rss.items:
 681                 # DEBUG: print(f"DEBUG: item={item}")
 682                 domain = item.link.split("=")[1]
 683
 684                 if fba.is_blacklisted(domain):
 685                     # DEBUG: print(f"DEBUG: domain='{domain}' is blacklisted - SKIPPED!")
 686                     continue
 687                 elif domain in domains:
 688                     # DEBUG: print(f"DEBUG: domain='{domain}' is already added - SKIPPED!")
 689                     continue
 690                 elif fba.is_instance_registered(domain):
 691                     # DEBUG: print(f"DEBUG: domain='{domain}' is already registered - SKIPPED!")
 692                     continue
 693
 694                 # DEBUG: print(f"DEBUG: Adding domain='{domain}'")
 695                 domains.append(domain)
 696
 697     except BaseException as e:
 698         print(f"ERROR: Cannot fetch feed='{feed}',exception[{type(e)}]:'{str(e)}'")
 699         sys.exit(255)
 700
 701     # DEBUG: print(f"DEBUG: domains()={len(domains)}")
 702     if len(domains) > 0:
 703         boot.acquire_lock()
 704
 705         print(f"INFO: Adding {len(domains)} new instances ...")
 706         for domain in domains:
 707             print(f"INFO: Fetching instances from domain='{domain}' ...")
 708             fba.fetch_instances(domain, None, None, sys.argv[0])
 709
 710     # DEBUG: print("DEBUG: EXIT!")
 711
 712 def fetch_fbabot_atom(args: argparse.Namespace):
 713     # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
 714     feed = "https://ryona.agency/users/fba/feed.atom"
 715
 716     domains = list()
 717     try:
 718         print(f"INFO: Fetching ATOM feed='{feed}' from FBA bot account ...")
 719         response = fba.get_url(feed, fba.headers, (config.get("connection_timeout"), config.get("read_timeout")))
 720
 721         # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code='{response.status_code}',response.text()={len(response.text)}")
 722         if response.ok and response.status_code < 300 and len(response.text) > 0:
 723             # DEBUG: print(f"DEBUG: Parsing ATOM feed ...")
 724             atom = atoma.parse_atom_bytes(response.content)
 725
 726             # DEBUG: print(f"DEBUG: atom[]={type(atom)}")
 727             for entry in atom.entries:
 728                 # DEBUG: print(f"DEBUG: entry[]={type(entry)}")
 729                 doc = bs4.BeautifulSoup(entry.content.value, "html.parser")
 730                 # DEBUG: print(f"DEBUG: doc[]={type(doc)}")
 731                 for element in doc.findAll("a"):
 732                     for href in element["href"].split(","):
 733                         # DEBUG: print(f"DEBUG: href[{type(href)}]={href}")
 734                         domain = fba.tidyup_domain(href)
 735
 736                         # DEBUG: print(f"DEBUG: domain='{domain}'")
 737                         if fba.is_blacklisted(domain):
 738                             # DEBUG: print(f"DEBUG: domain='{domain}' is blacklisted - SKIPPED!")
 739                             continue
 740                         elif domain in domains:
 741                             # DEBUG: print(f"DEBUG: domain='{domain}' is already added - SKIPPED!")
 742                             continue
 743                         elif fba.is_instance_registered(domain):
 744                             # DEBUG: print(f"DEBUG: domain='{domain}' is already registered - SKIPPED!")
 745                             continue
 746
 747                         # DEBUG: print(f"DEBUG: Adding domain='{domain}',domains()={len(domains)}")
 748                         domains.append(domain)
 749
 750     except BaseException as e:
 751         print(f"ERROR: Cannot fetch feed='{feed}',exception[{type(e)}]:'{str(e)}'")
 752         sys.exit(255)
 753
 754     # DEBUG: print(f"DEBUG: domains({len(domains)})={domains}")
 755     if len(domains) > 0:
 756         boot.acquire_lock()
 757
 758         print(f"INFO: Adding {len(domains)} new instances ...")
 759         for domain in domains:
 760             print(f"INFO: Fetching instances from domain='{domain}' ...")
 761             fba.fetch_instances(domain, None, None, sys.argv[0])
 762
 763     # DEBUG: print("DEBUG: EXIT!")
 764
 765 def fetch_instances(args: argparse.Namespace):
 766     # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
 767     boot.acquire_lock()
 768
 769     # Initial fetch
 770     fba.fetch_instances(args.domain, None, None, sys.argv[0])
 771
 772     if args.single:
 773         # DEBUG: print(f"DEBUG: Not fetching more instances - EXIT!")
 774         return
 775
 776     # Loop through some instances
 777     fba.cursor.execute(
 778         "SELECT domain, origin, software, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial', 'bookwyrm', 'takahe', 'lemmy') AND (last_instance_fetch IS NULL OR last_instance_fetch < ?) ORDER BY rowid DESC", [time.time() - config.get("recheck_instance")]
 779     )
 780
 781     rows = fba.cursor.fetchall()
 782     print(f"INFO: Checking {len(rows)} entries ...")
 783     for row in rows:
 784         # DEBUG: print("DEBUG: domain:", row[0])
 785         if fba.is_blacklisted(row[0]):
 786             print("WARNING: domain is blacklisted:", row[0])
 787             continue
 788
 789         print(f"INFO: Fetching instances for instance '{row[0]}' ('{row[2]}') of origin='{row[1]}',nodeinfo_url='{row[3]}'")
 790         fba.fetch_instances(row[0], row[1], row[2], sys.argv[0], row[3])
 791
 792     # DEBUG: print("DEBUG: EXIT!")