fba/commands.py

   1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
   2 # Copyright (C) 2023 Free Software Foundation
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published
   6 # by the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  16
  17 import argparse
  18 import atoma
  19 import bs4
  20 import inspect
  21 import itertools
  22 import json
  23 import re
  24 import reqto
  25 import sys
  26 import time
  27 import validators
  28
  29 from fba import boot
  30 from fba import config
  31 from fba import fba
  32
  33 def check_instance(args: argparse.Namespace) -> int:
  34     # DEBUG: print(f"DEBUG: args.domain='{args.domain}' - CALLED!")
  35     status = 0
  36     if not validators.domain(args.domain):
  37         print(f"WARNING: args.domain='{args.domain}' is not valid")
  38         status = 100
  39     elif fba.is_blacklisted(args.domain):
  40         print(f"WARNING: args.domain='{args.domain}' is blacklisted")
  41         status = 101
  42     elif fba.is_instance_registered(args.domain):
  43         print(f"WARNING: args.domain='{args.domain}' is already registered")
  44         status = 102
  45     else:
  46         print(f"INFO: args.domain='{args.domain}' is not known")
  47
  48     # DEBUG: print(f"DEBUG: status={status} - EXIT!")
  49     return status
  50
  51 def fetch_bkali(args: argparse.Namespace):
  52     # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
  53     domains = list()
  54     try:
  55         fetched = fba.post_json_api("gql.api.bka.li", "/v1/graphql", json.dumps({
  56             "query": "query domainlist {nodeinfo(order_by: {domain: asc}) {domain}}"
  57         }))
  58
  59         # DEBUG: print(f"DEBUG: fetched({len(fetched)})[]='{type(fetched)}'")
  60         if len(fetched) == 0:
  61             raise Exception("WARNING: Returned no records")
  62         elif not "data" in fetched:
  63             raise Exception(f"WARNING: fetched()={len(fetched)} does not contain key 'data'")
  64         elif not "nodeinfo" in fetched["data"]:
  65             raise Exception(f"WARNING: fetched()={len(fetched['data'])} does not contain key 'nodeinfo'")
  66
  67         for entry in fetched["data"]["nodeinfo"]:
  68             # DEBUG: print(f"DEBUG: entry['{type(entry)}']='{entry}'")
  69             if not "domain" in entry:
  70                 print(f"WARNING: entry does not contain 'domain' - SKIPPED!")
  71                 continue
  72             elif not validators.domain(entry["domain"]):
  73                 print(f"WARNING: domain='{entry['domain']}' is not a valid domain - SKIPPED!")
  74                 continue
  75             elif fba.is_blacklisted(entry["domain"]):
  76                 # DEBUG: print(f"DEBUG: domain='{entry['domain']}' is blacklisted - SKIPPED!")
  77                 continue
  78             elif fba.is_instance_registered(entry["domain"]):
  79                 # DEBUG: print(f"DEBUG: domain='{entry['domain']}' is already registered - SKIPPED!")
  80                 continue
  81
  82             # DEBUG: print(f"DEBUG: Adding domain='{entry['domain']}' ...")
  83             domains.append(entry["domain"])
  84
  85     except BaseException as e:
  86         print(f"ERROR: Cannot fetch graphql,exception[{type(e)}]:'{str(e)}'")
  87         sys.exit(255)
  88
  89     # DEBUG: print(f"DEBUG: domains()={len(domains)}")
  90     if len(domains) > 0:
  91         boot.acquire_lock()
  92
  93         print(f"INFO: Adding {len(domains)} new instances ...")
  94         for domain in domains:
  95             print(f"INFO: Fetching instances from domain='{domain}' ...")
  96             fba.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
  97
  98     # DEBUG: print("DEBUG: EXIT!")
  99
 100 def fetch_blocks(args: argparse.Namespace):
 101     # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
 102     if args.domain != None and args.domain != "":
 103         if not validators.domain(args.domain):
 104             print(f"WARNING: domain='{args.domain}' is not valid.")
 105             return
 106         elif fba.is_blacklisted(args.domain):
 107             print(f"WARNING: domain='{args.domain}' is blacklisted, won't check it!")
 108             return
 109         elif not fba.is_instance_registered(args.domain):
 110             print(f"WARNING: domain='{args.domain}' is not registered, please run ./fba.py fetch_instances {args.domain} first.")
 111             return
 112
 113     boot.acquire_lock()
 114
 115     if args.domain != None and args.domain != "":
 116         fba.cursor.execute(
 117             "SELECT domain, software, origin, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial', 'bookwyrm', 'takahe') AND domain = ?", [args.domain]
 118         )
 119     else:
 120         fba.cursor.execute(
 121             "SELECT domain, software, origin, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial', 'bookwyrm', 'takahe') AND (last_blocked IS NULL OR last_blocked < ?) ORDER BY rowid DESC", [time.time() - config.get("recheck_block")]
 122         )
 123
 124     rows = fba.cursor.fetchall()
 125     print(f"INFO: Checking {len(rows)} entries ...")
 126     for blocker, software, origin, nodeinfo_url in rows:
 127         # DEBUG: print("DEBUG: BEFORE blocker,software,origin,nodeinfo_url:", blocker, software, origin, nodeinfo_url)
 128         blockdict = []
 129         blocker = fba.tidyup_domain(blocker)
 130         # DEBUG: print("DEBUG: AFTER blocker,software:", blocker, software)
 131
 132         if blocker == "":
 133             print("WARNING: blocker is now empty!")
 134             continue
 135         elif fba.is_blacklisted(blocker):
 136             print(f"WARNING: blocker='{blocker}' is blacklisted now!")
 137             continue
 138
 139         # DEBUG: print(f"DEBUG: blocker='{blocker}'")
 140         fba.update_last_blocked(blocker)
 141
 142         if software == "pleroma":
 143             print("INFO: blocker:", blocker)
 144             try:
 145                 # Blocks
 146                 json = fba.fetch_nodeinfo(blocker, nodeinfo_url)
 147                 if json is None:
 148                     print("WARNING: Could not fetch nodeinfo from blocker:", blocker)
 149                     continue
 150                 elif not "metadata" in json:
 151                     print(f"WARNING: json()={len(json)} does not have key 'metadata', blocker='{blocker}'")
 152                     continue
 153                 elif not "federation" in json["metadata"]:
 154                     print(f"WARNING: json()={len(json['metadata'])} does not have key 'federation', blocker='{blocker}'")
 155                     continue
 156
 157                 # DEBUG: print("DEBUG: Updating nodeinfo:", blocker)
 158                 fba.update_last_nodeinfo(blocker)
 159
 160                 federation = json["metadata"]["federation"]
 161
 162                 if "enabled" in federation:
 163                     # DEBUG: print("DEBUG: Instance has no block list to analyze:", blocker)
 164                     continue
 165
 166                 if "mrf_simple" in federation:
 167                     for block_level, blocks in (
 168                         {**federation["mrf_simple"],
 169                         **{"quarantined_instances": federation["quarantined_instances"]}}
 170                     ).items():
 171                         # DEBUG: print("DEBUG: block_level, blocks():", block_level, len(blocks))
 172                         block_level = fba.tidyup_domain(block_level)
 173                         # DEBUG: print("DEBUG: BEFORE block_level:", block_level)
 174
 175                         if block_level == "":
 176                             print("WARNING: block_level is now empty!")
 177                             continue
 178
 179                         # DEBUG: print(f"DEBUG: Checking {len(blocks)} entries from blocker='{blocker}',software='{software}',block_level='{block_level}' ...")
 180                         for blocked in blocks:
 181                             # DEBUG: print("DEBUG: BEFORE blocked:", blocked)
 182                             blocked = fba.tidyup_domain(blocked)
 183                             # DEBUG: print("DEBUG: AFTER blocked:", blocked)
 184
 185                             if blocked == "":
 186                                 print("WARNING: blocked is empty after fba.tidyup_domain():", blocker, block_level)
 187                                 continue
 188                             elif fba.is_blacklisted(blocked):
 189                                 # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!")
 190                                 continue
 191                             elif blocked.count("*") > 1:
 192                                 # -ACK!-oma also started obscuring domains without hash
 193                                 fba.cursor.execute(
 194                                     "SELECT domain, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")]
 195                                 )
 196                                 searchres = fba.cursor.fetchone()
 197                                 # DEBUG: print("DEBUG: searchres[]:", type(searchres))
 198
 199                                 if searchres == None:
 200                                     print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
 201                                     continue
 202
 203                                 blocked = searchres[0]
 204                                 nodeinfo_url = searchres[1]
 205                                 # DEBUG: print("DEBUG: Looked up domain:", blocked)
 206                             elif not validators.domain(blocked):
 207                                 print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
 208                                 continue
 209
 210                             # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
 211                             if not validators.domain(blocked):
 212                                 print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
 213                                 continue
 214                             elif not fba.is_instance_registered(blocked):
 215                                 # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., blocker='{blocker}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'")
 216                                 fba.add_instance(blocked, blocker, inspect.currentframe().f_code.co_name, nodeinfo_url)
 217
 218                             if not fba.is_instance_blocked(blocker, blocked, block_level):
 219                                 # DEBUG: print("DEBUG: Blocking:", blocker, blocked, block_level)
 220                                 fba.block_instance(blocker, blocked, "unknown", block_level)
 221
 222                                 if block_level == "reject":
 223                                     # DEBUG: print("DEBUG: Adding to blockdict:", blocked)
 224                                     blockdict.append(
 225                                         {
 226                                             "blocked": blocked,
 227                                             "reason" : None
 228                                         })
 229                             else:
 230                                 # DEBUG: print(f"DEBUG: Updating block last seen for blocker='{blocker}',blocked='{blocked}' ...")
 231                                 fba.update_last_seen(blocker, blocked, block_level)
 232
 233                 # DEBUG: print("DEBUG: Committing changes ...")
 234                 fba.connection.commit()
 235
 236                 # Reasons
 237                 if "mrf_simple_info" in federation:
 238                     # DEBUG: print("DEBUG: Found mrf_simple_info:", blocker)
 239                     for block_level, info in (
 240                         {**federation["mrf_simple_info"],
 241                         **(federation["quarantined_instances_info"]
 242                         if "quarantined_instances_info" in federation
 243                         else {})}
 244                     ).items():
 245                         # DEBUG: print("DEBUG: block_level, info.items():", block_level, len(info.items()))
 246                         block_level = fba.tidyup_domain(block_level)
 247                         # DEBUG: print("DEBUG: BEFORE block_level:", block_level)
 248
 249                         if block_level == "":
 250                             print("WARNING: block_level is now empty!")
 251                             continue
 252
 253                         # DEBUG: print(f"DEBUG: Checking {len(info.items())} entries from blocker='{blocker}',software='{software}',block_level='{block_level}' ...")
 254                         for blocked, reason in info.items():
 255                             # DEBUG: print("DEBUG: BEFORE blocked:", blocked)
 256                             blocked = fba.tidyup_domain(blocked)
 257                             # DEBUG: print("DEBUG: AFTER blocked:", blocked)
 258
 259                             if blocked == "":
 260                                 print("WARNING: blocked is empty after fba.tidyup_domain():", blocker, block_level)
 261                                 continue
 262                             elif fba.is_blacklisted(blocked):
 263                                 # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!")
 264                                 continue
 265                             elif blocked.count("*") > 1:
 266                                 # same domain guess as above, but for reasons field
 267                                 fba.cursor.execute(
 268                                     "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")]
 269                                 )
 270                                 searchres = fba.cursor.fetchone()
 271
 272                                 if searchres == None:
 273                                     print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
 274                                     continue
 275
 276                                 blocked = searchres[0]
 277                                 origin = searchres[1]
 278                                 nodeinfo_url = searchres[2]
 279                             elif not validators.domain(blocked):
 280                                 print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
 281                                 continue
 282
 283                             # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
 284                             if not validators.domain(blocked):
 285                                 print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
 286                                 continue
 287                             elif not fba.is_instance_registered(blocked):
 288                                 # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., blocker='{blocker}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'")
 289                                 fba.add_instance(blocked, blocker, inspect.currentframe().f_code.co_name, nodeinfo_url)
 290
 291                             # DEBUG: print("DEBUG: Updating block reason:", blocker, blocked, reason["reason"])
 292                             fba.update_block_reason(reason["reason"], blocker, blocked, block_level)
 293
 294                             for entry in blockdict:
 295                                 if entry["blocked"] == blocked:
 296                                     # DEBUG: print("DEBUG: Updating entry reason:", blocked)
 297                                     entry["reason"] = reason["reason"]
 298
 299                 fba.connection.commit()
 300             except Exception as e:
 301                 print(f"ERROR: blocker='{blocker}',software='{software}',exception[{type(e)}]:'{str(e)}'")
 302         elif software == "mastodon":
 303             print("INFO: blocker:", blocker)
 304             try:
 305                 # json endpoint for newer mastodongs
 306                 try:
 307                     json = {
 308                         "reject"        : [],
 309                         "media_removal" : [],
 310                         "followers_only": [],
 311                         "report_removal": []
 312                     }
 313
 314                     # handling CSRF, I've saw at least one server requiring it to access the endpoint
 315                     # DEBUG: print("DEBUG: Fetching meta:", blocker)
 316                     meta = bs4.BeautifulSoup(
 317                         fba.get_response(blocker, "/", fba.headers, (config.get("connection_timeout"), config.get("read_timeout"))).text,
 318                         "html.parser",
 319                     )
 320                     try:
 321                         csrf = meta.find("meta", attrs={"name": "csrf-token"})["content"]
 322                         # DEBUG: print("DEBUG: Adding CSRF token:", blocker, csrf)
 323                         reqheaders = {**fba.api_headers, **{"X-CSRF-Token": csrf}}
 324                     except BaseException as e:
 325                         # DEBUG: print("DEBUG: No CSRF token found, using normal headers:", blocker, e)
 326                         reqheaders = fba.api_headers
 327
 328                     # DEBUG: print("DEBUG: Querying API domain_blocks:", blocker)
 329                     blocks = fba.get_response(blocker, "/api/v1/instance/domain_blocks", reqheaders, (config.get("connection_timeout"), config.get("read_timeout"))).json()
 330
 331                     print(f"INFO: Checking {len(blocks)} entries from blocker='{blocker}',software='{software}' ...")
 332                     for block in blocks:
 333                         entry = {
 334                             'domain': block['domain'],
 335                             'hash'  : block['digest'],
 336                             'reason': block['comment']
 337                         }
 338
 339                         # DEBUG: print("DEBUG: severity,domain,hash,comment:", block['severity'], block['domain'], block['digest'], block['comment'])
 340                         if block['severity'] == 'suspend':
 341                             # DEBUG: print(f"DEBUG: Adding entry='{entry}' with severity='{block['severity']}' ...")
 342                             json['reject'].append(entry)
 343                         elif block['severity'] == 'silence':
 344                             # DEBUG: print(f"DEBUG: Adding entry='{entry}' with severity='{block['severity']}' ...")
 345                             json['followers_only'].append(entry)
 346                         elif block['severity'] == 'reject_media':
 347                             # DEBUG: print(f"DEBUG: Adding entry='{entry}' with severity='{block['severity']}' ...")
 348                             json['media_removal'].append(entry)
 349                         elif block['severity'] == 'reject_reports':
 350                             # DEBUG: print(f"DEBUG: Adding entry='{entry}' with severity='{block['severity']}' ...")
 351                             json['report_removal'].append(entry)
 352                         else:
 353                             print("WARNING: Unknown severity:", block['severity'], block['domain'])
 354                 except BaseException as e:
 355                     # DEBUG: print(f"DEBUG: Failed, trying mastodon-specific fetches: blocker='{blocker}',exception[{type(e)}]={str(e)}")
 356                     json = fba.get_mastodon_blocks(blocker)
 357
 358                 print(f"INFO: Checking {len(json.items())} entries from blocker='{blocker}',software='{software}' ...")
 359                 for block_level, blocks in json.items():
 360                     # DEBUG: print("DEBUG: blocker,block_level,blocks():", blocker, block_level, len(blocks))
 361                     block_level = fba.tidyup_domain(block_level)
 362                     # DEBUG: print("DEBUG: AFTER-block_level:", block_level)
 363                     if block_level == "":
 364                         print("WARNING: block_level is empty, blocker:", blocker)
 365                         continue
 366
 367                     # DEBUG: print(f"DEBUG: Checking {len(blocks)} entries from blocker='{blocker}',software='{software}',block_level='{block_level}' ...")
 368                     for block in blocks:
 369                         blocked, blocked_hash, reason = block.values()
 370                         # DEBUG: print("DEBUG: blocked,hash,reason:", blocked, blocked_hash, reason)
 371                         blocked = fba.tidyup_domain(blocked)
 372                         # DEBUG: print("DEBUG: AFTER-blocked:", blocked)
 373
 374                         if blocked == "":
 375                             print("WARNING: blocked is empty:", blocker)
 376                             continue
 377                         elif fba.is_blacklisted(blocked):
 378                             # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!")
 379                             continue
 380                         elif blocked.count("*") > 0:
 381                             # Doing the hash search for instance names as well to tidy up DB
 382                             fba.cursor.execute(
 383                                 "SELECT domain, origin, nodeinfo_url FROM instances WHERE hash = ? LIMIT 1", [blocked_hash]
 384                             )
 385                             searchres = fba.cursor.fetchone()
 386
 387                             if searchres == None:
 388                                 print(f"WARNING: Cannot deobsfucate blocked='{blocked}',blocked_hash='{blocked_hash}' - SKIPPED!")
 389                                 continue
 390
 391                             # DEBUG: print("DEBUG: Updating domain: ", searchres[0])
 392                             blocked = searchres[0]
 393                             origin = searchres[1]
 394                             nodeinfo_url = searchres[2]
 395
 396                             # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
 397                             if not validators.domain(blocked):
 398                                 print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
 399                                 continue
 400                             elif not fba.is_instance_registered(blocked):
 401                                 # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., blocker='{blocker}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'")
 402                                 fba.add_instance(blocked, blocker, inspect.currentframe().f_code.co_name, nodeinfo_url)
 403                         elif not validators.domain(blocked):
 404                             print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
 405                             continue
 406
 407                         # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
 408                         if not validators.domain(blocked):
 409                             print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
 410                             continue
 411                         elif not fba.is_instance_registered(blocked):
 412                             # DEBUG: print("DEBUG: Hash wasn't found, adding:", blocked, blocker)
 413                             fba.add_instance(blocked, blocker, inspect.currentframe().f_code.co_name, nodeinfo_url)
 414
 415                         blocking = blocked if blocked.count("*") <= 1 else blocked_hash
 416                         # DEBUG: print(f"DEBUG: blocking='{blocking}',blocked='{blocked}',blocked_hash='{blocked_hash}'")
 417
 418                         if not fba.is_instance_blocked(blocker, blocked, block_level):
 419                             # DEBUG: print("DEBUG: Blocking:", blocker, blocked, block_level)
 420                             fba.block_instance(blocker, blocking, reason, block_level)
 421
 422                             if block_level == "reject":
 423                                 blockdict.append({
 424                                     "blocked": blocked,
 425                                     "reason" : reason
 426                                 })
 427                         else:
 428                             # DEBUG: print(f"DEBUG: Updating block last seen and reason for blocker='{blocker}',blocking='{blocking}' ...")
 429                             fba.update_last_seen(blocker, blocking, block_level)
 430                             fba.update_block_reason(reason, blocker, blocking, block_level)
 431
 432                 # DEBUG: print("DEBUG: Committing changes ...")
 433                 fba.connection.commit()
 434             except Exception as e:
 435                 print(f"ERROR: blocker='{blocker}',software='{software}',exception[{type(e)}]:'{str(e)}'")
 436         elif software == "friendica" or software == "misskey" or software == "bookwyrm" or software == "takahe":
 437             print("INFO: blocker:", blocker)
 438             try:
 439                 if software == "friendica":
 440                     json = fba.get_friendica_blocks(blocker)
 441                 elif software == "misskey":
 442                     json = fba.get_misskey_blocks(blocker)
 443                 elif software == "bookwyrm":
 444                     print("WARNING: bookwyrm is not fully supported for fetching blacklist!", blocker)
 445                     #json = fba.get_bookwyrm_blocks(blocker)
 446                     continue
 447                 elif software == "takahe":
 448                     print("WARNING: takahe is not fully supported for fetching blacklist!", blocker)
 449                     #json = fba.get_takahe_blocks(blocker)
 450                     continue
 451
 452                 print(f"INFO: Checking {len(json.items())} entries from blocker='{blocker}',software='{software}' ...")
 453                 for block_level, blocks in json.items():
 454                     # DEBUG: print("DEBUG: blocker,block_level,blocks():", blocker, block_level, len(blocks))
 455                     block_level = fba.tidyup_domain(block_level)
 456                     # DEBUG: print("DEBUG: AFTER-block_level:", block_level)
 457                     if block_level == "":
 458                         print("WARNING: block_level is empty, blocker:", blocker)
 459                         continue
 460
 461                     # DEBUG: print(f"DEBUG: Checking {len(blocks)} entries from blocker='{blocker}',software='{software}',block_level='{block_level}' ...")
 462                     for block in blocks:
 463                         blocked, reason = block.values()
 464                         # DEBUG: print("DEBUG: BEFORE blocked:", blocked)
 465                         blocked = fba.tidyup_domain(blocked)
 466                         # DEBUG: print("DEBUG: AFTER blocked:", blocked)
 467
 468                         if blocked == "":
 469                             print("WARNING: blocked is empty:", blocker)
 470                             continue
 471                         elif fba.is_blacklisted(blocked):
 472                             # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!")
 473                             continue
 474                         elif blocked.count("*") > 0:
 475                             # Some friendica servers also obscure domains without hash
 476                             fba.cursor.execute(
 477                                 "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")]
 478                             )
 479
 480                             searchres = fba.cursor.fetchone()
 481
 482                             if searchres == None:
 483                                 print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
 484                                 continue
 485
 486                             blocked = searchres[0]
 487                             origin = searchres[1]
 488                             nodeinfo_url = searchres[2]
 489                         elif blocked.count("?") > 0:
 490                             # Some obscure them with question marks, not sure if that's dependent on version or not
 491                             fba.cursor.execute(
 492                                 "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("?", "_")]
 493                             )
 494
 495                             searchres = fba.cursor.fetchone()
 496
 497                             if searchres == None:
 498                                 print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
 499                                 continue
 500
 501                             blocked = searchres[0]
 502                             origin = searchres[1]
 503                             nodeinfo_url = searchres[2]
 504                         elif not validators.domain(blocked):
 505                             print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
 506                             continue
 507
 508                         # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
 509                         if not validators.domain(blocked):
 510                             print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
 511                             continue
 512                         elif not fba.is_instance_registered(blocked):
 513                             # DEBUG: print("DEBUG: Hash wasn't found, adding:", blocked, blocker)
 514                             fba.add_instance(blocked, blocker, inspect.currentframe().f_code.co_name, nodeinfo_url)
 515
 516                         if not fba.is_instance_blocked(blocker, blocked, block_level):
 517                             fba.block_instance(blocker, blocked, reason, block_level)
 518
 519                             if block_level == "reject":
 520                                 blockdict.append({
 521                                     "blocked": blocked,
 522                                     "reason" : reason
 523                                 })
 524                         else:
 525                             # DEBUG: print(f"DEBUG: Updating block last seen and reason for blocker='{blocker}',blocked='{blocked}' ...")
 526                             fba.update_last_seen(blocker, blocked, block_level)
 527                             fba.update_block_reason(reason, blocker, blocked, block_level)
 528
 529                 # DEBUG: print("DEBUG: Committing changes ...")
 530                 fba.connection.commit()
 531             except Exception as e:
 532                 print(f"ERROR: blocker='{blocker}',software='{software}',exception[{type(e)}]:'{str(e)}'")
 533         elif software == "gotosocial":
 534             print("INFO: blocker:", blocker)
 535             try:
 536                 # Blocks
 537                 federation = fba.get_response(blocker, f"{fba.get_peers_url}?filter=suspended", fba.api_headers, (config.get("connection_timeout"), config.get("read_timeout"))).json()
 538
 539                 if (federation == None):
 540                     print("WARNING: No valid response:", blocker);
 541                 elif "error" in federation:
 542                     print("WARNING: API returned error:", federation["error"])
 543                 else:
 544                     print(f"INFO: Checking {len(federation)} entries from blocker='{blocker}',software='{software}' ...")
 545                     for peer in federation:
 546                         blocked = peer["domain"].lower()
 547                         # DEBUG: print("DEBUG: BEFORE blocked:", blocked)
 548                         blocked = fba.tidyup_domain(blocked)
 549                         # DEBUG: print("DEBUG: AFTER blocked:", blocked)
 550
 551                         if blocked == "":
 552                             print("WARNING: blocked is empty:", blocker)
 553                             continue
 554                         elif fba.is_blacklisted(blocked):
 555                             # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!")
 556                             continue
 557                         elif blocked.count("*") > 0:
 558                             # GTS does not have hashes for obscured domains, so we have to guess it
 559                             fba.cursor.execute(
 560                                 "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")]
 561                             )
 562                             searchres = fba.cursor.fetchone()
 563
 564                             if searchres == None:
 565                                 print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
 566                                 continue
 567
 568                             blocked = searchres[0]
 569                             origin = searchres[1]
 570                             nodeinfo_url = searchres[2]
 571                         elif not validators.domain(blocked):
 572                             print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
 573                             continue
 574
 575                         # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
 576                         if not validators.domain(blocked):
 577                             print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
 578                             continue
 579                         elif not fba.is_instance_registered(blocked):
 580                             # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., blocker='{blocker}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'")
 581                             fba.add_instance(blocked, blocker, inspect.currentframe().f_code.co_name, nodeinfo_url)
 582
 583                         if not fba.is_instance_blocked(blocker, blocked, "reject"):
 584                             # DEBUG: print(f"DEBUG: blocker='{blocker}' is blocking '{blocked}' for unknown reason at this point")
 585                             fba.block_instance(blocker, blocked, "unknown", "reject")
 586
 587                             blockdict.append({
 588                                 "blocked": blocked,
 589                                 "reason" : None
 590                             })
 591                         else:
 592                             # DEBUG: print(f"DEBUG: Updating block last seen for blocker='{blocker}',blocked='{blocked}' ...")
 593                             fba.update_last_seen(blocker, blocked, "reject")
 594
 595                         if "public_comment" in peer:
 596                             # DEBUG: print("DEBUG: Updating block reason:", blocker, blocked, peer["public_comment"])
 597                             fba.update_block_reason(peer["public_comment"], blocker, blocked, "reject")
 598
 599                             for entry in blockdict:
 600                                 if entry["blocked"] == blocked:
 601                                     # DEBUG: print(f"DEBUG: Setting block reason for blocked='{blocked}':'{peer['public_comment']}'")
 602                                     entry["reason"] = peer["public_comment"]
 603
 604                     # DEBUG: print("DEBUG: Committing changes ...")
 605                     fba.connection.commit()
 606             except Exception as e:
 607                 print(f"ERROR: blocker='{blocker}',software='{software}',exception[{type(e)}]:'{str(e)}'")
 608         else:
 609             print("WARNING: Unknown software:", blocker, software)
 610
 611         if config.get("bot_enabled") and len(blockdict) > 0:
 612             send_bot_post(blocker, blockdict)
 613
 614         blockdict = []
 615
 616     # DEBUG: print("DEBUG: EXIT!")
 617
 618 def fetch_cs(args: argparse.Namespace):
 619     # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
 620     domains = {
 621         "silenced": list(),
 622         "blocked": list(),
 623     }
 624
 625     try:
 626         doc = bs4.BeautifulSoup(
 627             fba.get_response("meta.chaos.social", "/federation", fba.headers, (config.get("connection_timeout"), config.get("read_timeout"))).text,
 628             "html.parser",
 629         )
 630         # DEBUG: print(f"DEBUG: doc()={len(doc)}[]={type(doc)}")
 631         silenced = doc.find("h2", {"id": "silenced-instances"}).findNext("table")
 632
 633         # DEBUG: print(f"DEBUG: silenced[]={type(silenced)}")
 634         domains["silenced"] = domains["silenced"] + find_domains(silenced)
 635         blocked = doc.find("h2", {"id": "blocked-instances"}).findNext("table")
 636
 637         # DEBUG: print(f"DEBUG: blocked[]={type(blocked)}")
 638         domains["blocked"] = domains["blocked"] + find_domains(blocked)
 639
 640     except BaseException as e:
 641         print(f"ERROR: Cannot fetch from meta.chaos.social,exception[{type(e)}]:'{str(e)}'")
 642         sys.exit(255)
 643
 644     # DEBUG: print(f"DEBUG: domains()={len(domains)}")
 645     if len(domains) > 0:
 646         boot.acquire_lock()
 647
 648         print(f"INFO: Adding {len(domains)} new instances ...")
 649         for block_level in domains:
 650             # DEBUG: print(f"DEBUG: block_level='{block_level}'")
 651
 652             for row in domains[block_level]:
 653                 # DEBUG: print(f"DEBUG: row='{row}'")
 654                 if not fba.is_instance_registered(row["domain"]):
 655                     print(f"INFO: Fetching instances from domain='{row['domain']}' ...")
 656                     fba.fetch_instances(row["domain"], None, None, inspect.currentframe().f_code.co_name)
 657
 658                 if not fba.is_instance_blocked('chaos.social', row["domain"], block_level):
 659                     # DEBUG: print(f"DEBUG: domain='{row['domain']}',block_level='{block_level}' blocked by chaos.social, adding ...")
 660                     fba.block_instance('chaos.social', row["domain"], row["reason"], block_level)
 661
 662         # DEBUG: print("DEBUG: Committing changes ...")
 663         fba.connection.commit()
 664
 665     # DEBUG: print("DEBUG: EXIT!")
 666
 667 def fetch_fba_rss(args: argparse.Namespace):
 668     # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
 669     domains = list()
 670
 671     try:
 672         print(f"INFO: Fetch FBA-specific RSS args.feed='{args.feed}' ...")
 673         response = fba.get_url(args.feed, fba.headers, (config.get("connection_timeout"), config.get("read_timeout")))
 674
 675         # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code='{response.status_code}',response.text()={len(response.text)}")
 676         if response.ok and response.status_code < 300 and len(response.text) > 0:
 677             # DEBUG: print(f"DEBUG: Parsing RSS feed ...")
 678             rss = atoma.parse_rss_bytes(response.content)
 679
 680             # DEBUG: print(f"DEBUG: rss[]={type(rss)}")
 681             for item in rss.items:
 682                 # DEBUG: print(f"DEBUG: item={item}")
 683                 domain = item.link.split("=")[1]
 684
 685                 if fba.is_blacklisted(domain):
 686                     # DEBUG: print(f"DEBUG: domain='{domain}' is blacklisted - SKIPPED!")
 687                     continue
 688                 elif domain in domains:
 689                     # DEBUG: print(f"DEBUG: domain='{domain}' is already added - SKIPPED!")
 690                     continue
 691                 elif fba.is_instance_registered(domain):
 692                     # DEBUG: print(f"DEBUG: domain='{domain}' is already registered - SKIPPED!")
 693                     continue
 694
 695                 # DEBUG: print(f"DEBUG: Adding domain='{domain}'")
 696                 domains.append(domain)
 697
 698     except BaseException as e:
 699         print(f"ERROR: Cannot fetch feed='{feed}',exception[{type(e)}]:'{str(e)}'")
 700         sys.exit(255)
 701
 702     # DEBUG: print(f"DEBUG: domains()={len(domains)}")
 703     if len(domains) > 0:
 704         boot.acquire_lock()
 705
 706         print(f"INFO: Adding {len(domains)} new instances ...")
 707         for domain in domains:
 708             print(f"INFO: Fetching instances from domain='{domain}' ...")
 709             fba.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
 710
 711     # DEBUG: print("DEBUG: EXIT!")
 712
 713 def fetch_fbabot_atom(args: argparse.Namespace):
 714     # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
 715     feed = "https://ryona.agency/users/fba/feed.atom"
 716
 717     domains = list()
 718     try:
 719         print(f"INFO: Fetching ATOM feed='{feed}' from FBA bot account ...")
 720         response = fba.get_url(feed, fba.headers, (config.get("connection_timeout"), config.get("read_timeout")))
 721
 722         # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code='{response.status_code}',response.text()={len(response.text)}")
 723         if response.ok and response.status_code < 300 and len(response.text) > 0:
 724             # DEBUG: print(f"DEBUG: Parsing ATOM feed ...")
 725             atom = atoma.parse_atom_bytes(response.content)
 726
 727             # DEBUG: print(f"DEBUG: atom[]={type(atom)}")
 728             for entry in atom.entries:
 729                 # DEBUG: print(f"DEBUG: entry[]={type(entry)}")
 730                 doc = bs4.BeautifulSoup(entry.content.value, "html.parser")
 731                 # DEBUG: print(f"DEBUG: doc[]={type(doc)}")
 732                 for element in doc.findAll("a"):
 733                     for href in element["href"].split(","):
 734                         # DEBUG: print(f"DEBUG: href[{type(href)}]={href}")
 735                         domain = fba.tidyup_domain(href)
 736
 737                         # DEBUG: print(f"DEBUG: domain='{domain}'")
 738                         if fba.is_blacklisted(domain):
 739                             # DEBUG: print(f"DEBUG: domain='{domain}' is blacklisted - SKIPPED!")
 740                             continue
 741                         elif domain in domains:
 742                             # DEBUG: print(f"DEBUG: domain='{domain}' is already added - SKIPPED!")
 743                             continue
 744                         elif fba.is_instance_registered(domain):
 745                             # DEBUG: print(f"DEBUG: domain='{domain}' is already registered - SKIPPED!")
 746                             continue
 747
 748                         # DEBUG: print(f"DEBUG: Adding domain='{domain}',domains()={len(domains)}")
 749                         domains.append(domain)
 750
 751     except BaseException as e:
 752         print(f"ERROR: Cannot fetch feed='{feed}',exception[{type(e)}]:'{str(e)}'")
 753         sys.exit(255)
 754
 755     # DEBUG: print(f"DEBUG: domains({len(domains)})={domains}")
 756     if len(domains) > 0:
 757         boot.acquire_lock()
 758
 759         print(f"INFO: Adding {len(domains)} new instances ...")
 760         for domain in domains:
 761             print(f"INFO: Fetching instances from domain='{domain}' ...")
 762             fba.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
 763
 764     # DEBUG: print("DEBUG: EXIT!")
 765
 766 def fetch_instances(args: argparse.Namespace):
 767     # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
 768     boot.acquire_lock()
 769
 770     # Initial fetch
 771     fba.fetch_instances(args.domain, None, None, inspect.currentframe().f_code.co_name)
 772
 773     if args.single:
 774         # DEBUG: print(f"DEBUG: Not fetching more instances - EXIT!")
 775         return
 776
 777     # Loop through some instances
 778     fba.cursor.execute(
 779         "SELECT domain, origin, software, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial', 'bookwyrm', 'takahe', 'lemmy') AND (last_instance_fetch IS NULL OR last_instance_fetch < ?) ORDER BY rowid DESC", [time.time() - config.get("recheck_instance")]
 780     )
 781
 782     rows = fba.cursor.fetchall()
 783     print(f"INFO: Checking {len(rows)} entries ...")
 784     for row in rows:
 785         # DEBUG: print("DEBUG: domain:", row[0])
 786         if fba.is_blacklisted(row[0]):
 787             print("WARNING: domain is blacklisted:", row[0])
 788             continue
 789
 790         print(f"INFO: Fetching instances for instance '{row[0]}' ('{row[2]}') of origin='{row[1]}',nodeinfo_url='{row[3]}'")
 791         fba.fetch_instances(row[0], row[1], row[2], inspect.currentframe().f_code.co_name, row[3])
 792
 793     # DEBUG: print("DEBUG: EXIT!")