fetch_blocks.py

   1 #!/usr/bin/python3
   2 # -*- coding: utf-8 -*-
   3
   4 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
   5 # Copyright (C) 2023 Free Software Foundation
   6 #
   7 # This program is free software: you can redistribute it and/or modify
   8 # it under the terms of the GNU Affero General Public License as published
   9 # by the Free Software Foundation, either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU Affero General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU Affero General Public License
  18 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  19
  20 import reqto
  21 import time
  22 import bs4
  23 import itertools
  24 import re
  25 import validators
  26 from fba import *
  27
  28 boot.acquire_lock()
  29
  30 fba.cursor.execute(
  31     "SELECT domain, software, origin, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial', 'bookwyrm', 'takahe') AND (last_blocked IS NULL OR last_blocked < ?) ORDER BY rowid DESC", [time.time() - fba.config["recheck_block"]]
  32 )
  33
  34 rows = fba.cursor.fetchall()
  35 print(f"INFO: Checking {len(rows)} entries ...")
  36 for blocker, software, origin, nodeinfo_url in rows:
  37     # DEBUG: print("DEBUG: BEFORE blocker,software,origin,nodeinfo_url:", blocker, software, origin, nodeinfo_url)
  38     blockdict = []
  39     blocker = fba.tidyup(blocker)
  40     # DEBUG: print("DEBUG: AFTER blocker,software:", blocker, software)
  41
  42     if blocker == "":
  43         print("WARNING: blocker is now empty!")
  44         continue
  45     elif fba.is_blacklisted(blocker):
  46         print(f"WARNING: blocker='{blocker}' is blacklisted now!")
  47         continue
  48
  49     # DEBUG: print(f"DEBUG: blocker='{blocker}'")
  50     fba.update_last_blocked(blocker)
  51
  52     if software == "pleroma":
  53         print("INFO: blocker:", blocker)
  54         try:
  55             # Blocks
  56             json = fba.fetch_nodeinfo(blocker, nodeinfo_url)
  57             if json is None:
  58                 print("WARNING: Could not fetch nodeinfo from blocker:", blocker)
  59                 continue
  60
  61             # DEBUG: print("DEBUG: Updating nodeinfo:", blocker)
  62             fba.update_last_nodeinfo(blocker)
  63
  64             federation = json["metadata"]["federation"]
  65
  66             if "enabled" in federation:
  67                 # DEBUG: print("DEBUG: Instance has no block list to analyze:", blocker)
  68                 continue
  69
  70             if "mrf_simple" in federation:
  71                 for block_level, blocks in (
  72                     {**federation["mrf_simple"],
  73                     **{"quarantined_instances": federation["quarantined_instances"]}}
  74                 ).items():
  75                     # DEBUG: print("DEBUG: block_level, blocks():", block_level, len(blocks))
  76                     block_level = fba.tidyup(block_level)
  77                     # DEBUG: print("DEBUG: BEFORE block_level:", block_level)
  78
  79                     if block_level == "":
  80                         print("WARNING: block_level is now empty!")
  81                         continue
  82
  83                     # DEBUG: print(f"DEBUG: Checking {len(blocks)} entries from blocker='{blocker}',software='{software}',block_level='{block_level}' ...")
  84                     for blocked in blocks:
  85                         # DEBUG: print("DEBUG: BEFORE blocked:", blocked)
  86                         blocked = fba.tidyup(blocked)
  87                         # DEBUG: print("DEBUG: AFTER blocked:", blocked)
  88
  89                         if blocked == "":
  90                             print("WARNING: blocked is empty after fba.tidyup():", blocker, block_level)
  91                             continue
  92                         elif fba.is_blacklisted(blocked):
  93                             # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!")
  94                             continue
  95                         elif blocked.count("*") > 1:
  96                             # -ACK!-oma also started obscuring domains without hash
  97                             fba.cursor.execute(
  98                                 "SELECT domain, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")]
  99                             )
 100                             searchres = fba.cursor.fetchone()
 101                             # DEBUG: print("DEBUG: searchres[]:", type(searchres))
 102
 103                             if searchres == None:
 104                                 print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
 105                                 continue
 106
 107                             blocked = searchres[0]
 108                             nodeinfo_url = searchres[1]
 109                             # DEBUG: print("DEBUG: Looked up domain:", blocked)
 110                         elif not validators.domain(blocked):
 111                             print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domai name - skipped!")
 112                             continue
 113
 114                         # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
 115                         if not validators.domain(blocked):
 116                             print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
 117                             continue
 118                         elif not fba.is_instance_registered(blocked):
 119                             # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., blocker='{blocker}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'")
 120                             fba.add_instance(blocked, blocker, origin, nodeinfo_url)
 121
 122                         fba.cursor.execute(
 123                             "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ? LIMIT 1",
 124                             (
 125                                blocker,
 126                                blocked,
 127                                block_level
 128                            ),
 129                         )
 130
 131                         if fba.cursor.fetchone() == None:
 132                             # DEBUG: print("DEBUG: Blocking:", blocker, blocked, block_level)
 133                             fba.block_instance(blocker, blocked, "unknown", block_level)
 134
 135                             if block_level == "reject":
 136                                 # DEBUG: print("DEBUG: Adding to blockdict:", blocked)
 137                                 blockdict.append(
 138                                     {
 139                                         "blocked": blocked,
 140                                         "reason" : None
 141                                     })
 142                         else:
 143                             print(f"DEBUG: Updating block last seen for blocker='{blocker}',blocked='{blocked}' ...")
 144                             fba.update_last_seen(blocker, blocked, block_level)
 145
 146             fba.connection.commit()
 147
 148             # Reasons
 149             if "mrf_simple_info" in federation:
 150                 # DEBUG: print("DEBUG: Found mrf_simple_info:", blocker)
 151                 for block_level, info in (
 152                     {**federation["mrf_simple_info"],
 153                     **(federation["quarantined_instances_info"]
 154                     if "quarantined_instances_info" in federation
 155                     else {})}
 156                 ).items():
 157                     # DEBUG: print("DEBUG: block_level, info.items():", block_level, len(info.items()))
 158                     block_level = fba.tidyup(block_level)
 159                     # DEBUG: print("DEBUG: BEFORE block_level:", block_level)
 160
 161                     if block_level == "":
 162                         print("WARNING: block_level is now empty!")
 163                         continue
 164
 165                     # DEBUG: print(f"DEBUG: Checking {len(info.items())} entries from blocker='{blocker}',software='{software}',block_level='{block_level}' ...")
 166                     for blocked, reason in info.items():
 167                         # DEBUG: print("DEBUG: BEFORE blocked:", blocked)
 168                         blocked = fba.tidyup(blocked)
 169                         # DEBUG: print("DEBUG: AFTER blocked:", blocked)
 170
 171                         if blocked == "":
 172                             print("WARNING: blocked is empty after fba.tidyup():", blocker, block_level)
 173                             continue
 174                         elif fba.is_blacklisted(blocked):
 175                             # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!")
 176                             continue
 177                         elif blocked.count("*") > 1:
 178                             # same domain guess as above, but for reasons field
 179                             fba.cursor.execute(
 180                                 "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")]
 181                             )
 182                             searchres = fba.cursor.fetchone()
 183
 184                             if searchres == None:
 185                                 print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
 186                                 continue
 187
 188                             blocked = searchres[0]
 189                             origin = searchres[1]
 190                             nodeinfo_url = searchres[2]
 191                         elif not validators.domain(blocked):
 192                             print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domai name - skipped!")
 193                             continue
 194
 195                         # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
 196                         if not validators.domain(blocked):
 197                             print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
 198                             continue
 199                         elif not fba.is_instance_registered(blocked):
 200                             # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., blocker='{blocker}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'")
 201                             fba.add_instance(blocked, blocker, origin, nodeinfo_url)
 202
 203                         # DEBUG: print("DEBUG: Updating block reason:", blocker, blocked, reason["reason"])
 204                         fba.update_block_reason(reason["reason"], blocker, blocked, block_level)
 205
 206                         for entry in blockdict:
 207                             if entry["blocked"] == blocked:
 208                                 # DEBUG: print("DEBUG: Updating entry reason:", blocked)
 209                                 entry["reason"] = reason["reason"]
 210
 211             fba.connection.commit()
 212         except Exception as e:
 213             print(f"ERROR: blocker='{blocker}',software='{software}',exception[{type(e)}]:'{str(e)}'")
 214     elif software == "mastodon":
 215         print("INFO: blocker:", blocker)
 216         try:
 217             # json endpoint for newer mastodongs
 218             try:
 219                 json = {
 220                     "reject"        : [],
 221                     "media_removal" : [],
 222                     "followers_only": [],
 223                     "report_removal": []
 224                 }
 225
 226                 # handling CSRF, I've saw at least one server requiring it to access the endpoint
 227                 # DEBUG: print("DEBUG: Fetching meta:", blocker)
 228                 meta = bs4.BeautifulSoup(
 229                     reqto.get(f"https://{blocker}/", headers=fba.headers, timeout=(fba.config["connection_timeout"], fba.config["read_timeout"])).text,
 230                     "html.parser",
 231                 )
 232                 try:
 233                     csrf = meta.find("meta", attrs={"name": "csrf-token"})["content"]
 234                     # DEBUG: print("DEBUG: Adding CSRF token:", blocker, csrf)
 235                     reqheaders = {**fba.api_headers, **{"X-CSRF-Token": csrf}}
 236                 except BaseException as e:
 237                     # DEBUG: print("DEBUG: No CSRF token found, using normal headers:", blocker, e)
 238                     reqheaders = fba.api_headers
 239
 240                 # DEBUG: print("DEBUG: Querying API domain_blocks:", blocker)
 241                 blocks = reqto.get(f"https://{blocker}/api/v1/instance/domain_blocks", headers=reqheaders, timeout=(fba.config["connection_timeout"], fba.config["read_timeout"])).json()
 242
 243                 print(f"INFO: Checking {len(blocks)} entries from blocker='{blocker}',software='{software}' ...")
 244                 for block in blocks:
 245                     entry = {
 246                         'domain': block['domain'],
 247                         'hash'  : block['digest'],
 248                         'reason': block['comment']
 249                     }
 250
 251                     # DEBUG: print("DEBUG: severity,domain,hash,comment:", block['severity'], block['domain'], block['digest'], block['comment'])
 252                     if block['severity'] == 'suspend':
 253                         # DEBUG: print(f"DEBUG: Adding entry='{entry}' with severity='{block['severity']}' ...")
 254                         json['reject'].append(entry)
 255                     elif block['severity'] == 'silence':
 256                         # DEBUG: print(f"DEBUG: Adding entry='{entry}' with severity='{block['severity']}' ...")
 257                         json['followers_only'].append(entry)
 258                     elif block['severity'] == 'reject_media':
 259                         # DEBUG: print(f"DEBUG: Adding entry='{entry}' with severity='{block['severity']}' ...")
 260                         json['media_removal'].append(entry)
 261                     elif block['severity'] == 'reject_reports':
 262                         # DEBUG: print(f"DEBUG: Adding entry='{entry}' with severity='{block['severity']}' ...")
 263                         json['report_removal'].append(entry)
 264                     else:
 265                         print("WARNING: Unknown severity:", block['severity'], block['domain'])
 266             except BaseException as e:
 267                 # DEBUG: print(f"DEBUG: Failed, trying mastodon-specific fetches: blocker='{blocker}',exception[{type(e)}]={str(e)}")
 268                 json = fba.get_mastodon_blocks(blocker)
 269
 270             print(f"INFO: Checking {len(json.items())} entries from blocker='{blocker}',software='{software}' ...")
 271             for block_level, blocks in json.items():
 272                 # DEBUG: print("DEBUG: blocker,block_level,blocks():", blocker, block_level, len(blocks))
 273                 block_level = fba.tidyup(block_level)
 274                 # DEBUG: print("DEBUG: AFTER-block_level:", block_level)
 275                 if block_level == "":
 276                     print("WARNING: block_level is empty, blocker:", blocker)
 277                     continue
 278
 279                 # DEBUG: print(f"DEBUG: Checking {len(blocks)} entries from blocker='{blocker}',software='{software}',block_level='{block_level}' ...")
 280                 for block in blocks:
 281                     blocked, blocked_hash, reason = block.values()
 282                     # DEBUG: print("DEBUG: blocked,hash,reason:", blocked, blocked_hash, reason)
 283                     blocked = fba.tidyup(blocked)
 284                     # DEBUG: print("DEBUG: AFTER-blocked:", blocked)
 285
 286                     if blocked == "":
 287                         print("WARNING: blocked is empty:", blocker)
 288                         continue
 289                     elif fba.is_blacklisted(blocked):
 290                         # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!")
 291                         continue
 292                     elif blocked.count("*") > 0:
 293                         # Doing the hash search for instance names as well to tidy up DB
 294                         fba.cursor.execute(
 295                             "SELECT domain, origin, nodeinfo_url FROM instances WHERE hash = ? LIMIT 1", [blocked_hash]
 296                         )
 297                         searchres = fba.cursor.fetchone()
 298
 299                         if searchres == None:
 300                             print(f"WARNING: Cannot deobsfucate blocked='{blocked}',blocked_hash='{blocked_hash}' - SKIPPED!")
 301                             continue
 302
 303                         # DEBUG: print("DEBUG: Updating domain: ", searchres[0])
 304                         blocked = searchres[0]
 305                         origin = searchres[1]
 306                         nodeinfo_url = searchres[2]
 307
 308                         # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
 309                         if not validators.domain(blocked):
 310                             print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
 311                             continue
 312                         elif not fba.is_instance_registered(blocked):
 313                             # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., blocker='{blocker}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'")
 314                             fba.add_instance(blocked, blocker, origin, nodeinfo_url)
 315                     elif not validators.domain(blocked):
 316                         print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domai name - skipped!")
 317                         continue
 318
 319                     # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
 320                     if not validators.domain(blocked):
 321                         print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
 322                         continue
 323                     elif not fba.is_instance_registered(blocked):
 324                         # DEBUG: print("DEBUG: Hash wasn't found, adding:", blocked, blocker)
 325                         fba.add_instance(blocked, blocker, origin, nodeinfo_url)
 326
 327                     blocking = blocked if blocked.count("*") <= 1 else blocked_hash
 328                     # DEBUG: print(f"DEBUG: blocking='{blocking}',blocked='{blocked}',blocked_hash='{blocked_hash}'")
 329
 330                     fba.cursor.execute(
 331                         "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ? LIMIT 1",
 332                         (
 333                             blocker,
 334                             blocking,
 335                             block_level
 336                         ),
 337                     )
 338
 339                     if fba.cursor.fetchone() == None:
 340                         # DEBUG: print("DEBUG: Blocking:", blocker, blocked, block_level)
 341                         fba.block_instance(blocker, blocking, reason, block_level)
 342
 343                         if block_level == "reject":
 344                             blockdict.append(
 345                                 {
 346                                     "blocked": blocked,
 347                                     "reason" : reason
 348                                 })
 349                     else:
 350                         # DEBUG: print(f"DEBUG: Updating block last seen and reason for blocker='{blocker}',blocking='{blocking}' ...")
 351                         fba.update_last_seen(blocker, blocking, block_level)
 352                         fba.update_block_reason(reason, blocker, blocking, block_level)
 353
 354             fba.connection.commit()
 355         except Exception as e:
 356             print(f"ERROR: blocker='{blocker}',software='{software}',exception[{type(e)}]:'{str(e)}'")
 357     elif software == "friendica" or software == "misskey" or software == "bookwyrm" or software == "takahe":
 358         print("INFO: blocker:", blocker)
 359         try:
 360             if software == "friendica":
 361                 json = fba.get_friendica_blocks(blocker)
 362             elif software == "misskey":
 363                 json = fba.get_misskey_blocks(blocker)
 364             elif software == "bookwyrm":
 365                 print("WARNING: bookwyrm is not fully supported for fetching blacklist!", blocker)
 366                 #json = fba.get_bookwyrm_blocks(blocker)
 367                 continue
 368             elif software == "takahe":
 369                 print("WARNING: takahe is not fully supported for fetching blacklist!", blocker)
 370                 #json = fba.get_takahe_blocks(blocker)
 371                 continue
 372
 373             print(f"INFO: Checking {len(json.items())} entries from blocker='{blocker}',software='{software}' ...")
 374             for block_level, blocks in json.items():
 375                 # DEBUG: print("DEBUG: blocker,block_level,blocks():", blocker, block_level, len(blocks))
 376                 block_level = fba.tidyup(block_level)
 377                 # DEBUG: print("DEBUG: AFTER-block_level:", block_level)
 378                 if block_level == "":
 379                     print("WARNING: block_level is empty, blocker:", blocker)
 380                     continue
 381
 382                 # DEBUG: print(f"DEBUG: Checking {len(blocks)} entries from blocker='{blocker}',software='{software}',block_level='{block_level}' ...")
 383                 for block in blocks:
 384                     blocked, reason = block.values()
 385                     # DEBUG: print("DEBUG: BEFORE blocked:", blocked)
 386                     blocked = fba.tidyup(blocked)
 387                     # DEBUG: print("DEBUG: AFTER blocked:", blocked)
 388
 389                     if blocked == "":
 390                         print("WARNING: blocked is empty:", blocker)
 391                         continue
 392                     elif fba.is_blacklisted(blocked):
 393                         # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!")
 394                         continue
 395                     elif blocked.count("*") > 0:
 396                         # Some friendica servers also obscure domains without hash
 397                         fba.cursor.execute(
 398                             "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")]
 399                         )
 400
 401                         searchres = fba.cursor.fetchone()
 402
 403                         if searchres == None:
 404                             print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
 405                             continue
 406
 407                         blocked = searchres[0]
 408                         origin = searchres[1]
 409                         nodeinfo_url = searchres[2]
 410                     elif blocked.count("?") > 0:
 411                         # Some obscure them with question marks, not sure if that's dependent on version or not
 412                         fba.cursor.execute(
 413                             "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("?", "_")]
 414                         )
 415
 416                         searchres = fba.cursor.fetchone()
 417
 418                         if searchres == None:
 419                             print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
 420                             continue
 421
 422                         blocked = searchres[0]
 423                         origin = searchres[1]
 424                         nodeinfo_url = searchres[2]
 425                     elif not validators.domain(blocked):
 426                         print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domai name - skipped!")
 427                         continue
 428
 429                     # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
 430                     if not validators.domain(blocked):
 431                         print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
 432                         continue
 433                     elif not fba.is_instance_registered(blocked):
 434                         # DEBUG: print("DEBUG: Hash wasn't found, adding:", blocked, blocker)
 435                         fba.add_instance(blocked, blocker, origin, nodeinfo_url)
 436
 437                     fba.cursor.execute(
 438                         "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ? LIMIT 1",
 439                         (blocker, blocked, block_level),
 440                     )
 441
 442                     if fba.cursor.fetchone() == None:
 443                         fba.block_instance(blocker, blocked, reason, block_level)
 444
 445                         if block_level == "reject":
 446                             blockdict.append(
 447                                 {
 448                                     "blocked": blocked,
 449                                     "reason" : reason
 450                                 })
 451                     else:
 452                         print(f"DEBUG: Updating block last seen and reason for blocker='{blocker}',blocked='{blocked}' ...")
 453                         fba.update_last_seen(blocker, blocked, block_level)
 454                         fba.update_block_reason(reason, blocker, blocked, block_level)
 455
 456             fba.connection.commit()
 457         except Exception as e:
 458             print(f"ERROR: blocker='{blocker}',software='{software}',exception[{type(e)}]:'{str(e)}'")
 459     elif software == "gotosocial":
 460         print("INFO: blocker:", blocker)
 461         try:
 462             # Blocks
 463             federation = reqto.get(f"https://{blocker}{fba.get_peers_url}?filter=suspended", headers=fba.api_headers, timeout=(fba.config["connection_timeout"], fba.config["read_timeout"])).json()
 464
 465             if (federation == None):
 466                 print("WARNING: No valid response:", blocker);
 467             elif "error" in federation:
 468                 print("WARNING: API returned error:", federation["error"])
 469             else:
 470                 print(f"INFO: Checking {len(federation)} entries from blocker='{blocker}',software='{software}' ...")
 471                 for peer in federation:
 472                     blocked = peer["domain"].lower()
 473                     # DEBUG: print("DEBUG: BEFORE blocked:", blocked)
 474                     blocked = fba.tidyup(blocked)
 475                     # DEBUG: print("DEBUG: AFTER blocked:", blocked)
 476
 477                     if blocked == "":
 478                         print("WARNING: blocked is empty:", blocker)
 479                         continue
 480                     elif fba.is_blacklisted(blocked):
 481                         # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!")
 482                         continue
 483                     elif blocked.count("*") > 0:
 484                         # GTS does not have hashes for obscured domains, so we have to guess it
 485                         fba.cursor.execute(
 486                             "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")]
 487                         )
 488                         searchres = fba.cursor.fetchone()
 489
 490                         if searchres == None:
 491                             print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
 492                             continue
 493
 494                         blocked = searchres[0]
 495                         origin = searchres[1]
 496                         nodeinfo_url = searchres[2]
 497                     elif not validators.domain(blocked):
 498                         print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domai name - skipped!")
 499                         continue
 500
 501                     # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
 502                     if not validators.domain(blocked):
 503                         print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
 504                         continue
 505                     elif not fba.is_instance_registered(blocked):
 506                         # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., blocker='{blocker}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'")
 507                         fba.add_instance(blocked, blocker, origin, nodeinfo_url)
 508
 509                     fba.cursor.execute(
 510                         "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ? LIMIT 1",
 511                         (
 512                             blocker,
 513                             blocked,
 514                             "reject"
 515                         ),
 516                     )
 517
 518                     if fba.cursor.fetchone() == None:
 519                         # DEBUG: print(f"DEBUG: blocker='{blocker}' is blocking '{blocked}' for unknown reason at this point")
 520                         fba.block_instance(blocker, blocked, "unknown", "reject")
 521
 522                         blockdict.append(
 523                             {
 524                                 "blocked": blocked,
 525                                 "reason" : None
 526                             })
 527                     else:
 528                         print(f"DEBUG: Updating block last seen for blocker='{blocker}',blocked='{blocked}' ...")
 529                         fba.update_last_seen(blocker, blocked, "reject")
 530
 531                     if "public_comment" in peer:
 532                         # DEBUG: print("DEBUG: Updating block reason:", blocker, blocked, peer["public_comment"])
 533                         fba.update_block_reason(peer["public_comment"], blocker, blocked, "reject")
 534
 535                         for entry in blockdict:
 536                             if entry["blocked"] == blocked:
 537                                 # DEBUG: print(f"DEBUG: Setting block reason for blocked='{blocked}':'{peer['public_comment']}'")
 538                                 entry["reason"] = peer["public_comment"]
 539
 540                 fba.connection.commit()
 541         except Exception as e:
 542             print(f"ERROR: blocker='{blocker}',software='{software}',exception[{type(e)}]:'{str(e)}'")
 543     else:
 544         print("WARNING: Unknown software:", blocker, software)
 545
 546     if fba.config["bot_enabled"] and len(blockdict) > 0:
 547         send_bot_post(blocker, blockdict)
 548
 549     blockdict = []
 550
 551 boot.shutdown()