fetch_blocks.py

   1 #!/usr/bin/python3
   2 # -*- coding: utf-8 -*-
   3
   4 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
   5 # Copyright (C) 2023 Free Software Foundation
   6 #
   7 # This program is free software: you can redistribute it and/or modify
   8 # it under the terms of the GNU Affero General Public License as published
   9 # by the Free Software Foundation, either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU Affero General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU Affero General Public License
  18 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  19
  20 import reqto
  21 import time
  22 import bs4
  23 import itertools
  24 import re
  25 import validators
  26 import fba
  27
  28 fba.cursor.execute(
  29     "SELECT domain, software, origin, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial', 'bookwyrm', 'takahe') AND (last_blocked IS NULL OR last_blocked < ?) ORDER BY rowid DESC", [time.time() - fba.config["recheck_block"]]
  30 )
  31
  32 rows = fba.cursor.fetchall()
  33 print(f"INFO: Checking {len(rows)} entries ...")
  34 for blocker, software, origin, nodeinfo_url in rows:
  35     # DEBUG: print("DEBUG: BEFORE blocker,software,origin,nodeinfo_url:", blocker, software, origin, nodeinfo_url)
  36     blockdict = []
  37     blocker = fba.tidyup(blocker)
  38     # DEBUG: print("DEBUG: AFTER blocker,software:", blocker, software)
  39
  40     if blocker == "":
  41         print("WARNING: blocker is now empty!")
  42         continue
  43     elif fba.is_blacklisted(blocker):
  44         print(f"WARNING: blocker='{blocker}' is blacklisted now!")
  45         continue
  46
  47     # DEBUG: print(f"DEBUG: blocker='{blocker}'")
  48     fba.update_last_blocked(blocker)
  49
  50     if software == "pleroma":
  51         print("INFO: blocker:", blocker)
  52         try:
  53             # Blocks
  54             json = fba.fetch_nodeinfo(blocker, nodeinfo_url)
  55             if json is None:
  56                 print("WARNING: Could not fetch nodeinfo from blocker:", blocker)
  57                 continue
  58
  59             # DEBUG: print("DEBUG: Updating nodeinfo:", blocker)
  60             fba.update_last_nodeinfo(blocker)
  61
  62             federation = json["metadata"]["federation"]
  63
  64             if "enabled" in federation:
  65                 # DEBUG: print("DEBUG: Instance has no block list to analyze:", blocker)
  66                 continue
  67
  68             if "mrf_simple" in federation:
  69                 for block_level, blocks in (
  70                     {**federation["mrf_simple"],
  71                     **{"quarantined_instances": federation["quarantined_instances"]}}
  72                 ).items():
  73                     # DEBUG: print("DEBUG: block_level, blocks():", block_level, len(blocks))
  74                     block_level = fba.tidyup(block_level)
  75                     # DEBUG: print("DEBUG: BEFORE block_level:", block_level)
  76
  77                     if block_level == "":
  78                         print("WARNING: block_level is now empty!")
  79                         continue
  80
  81                     # DEBUG: print(f"DEBUG: Checking {len(blocks)} entries from blocker='{blocker}',software='{software}',block_level='{block_level}' ...")
  82                     for blocked in blocks:
  83                         # DEBUG: print("DEBUG: BEFORE blocked:", blocked)
  84                         blocked = fba.tidyup(blocked)
  85                         # DEBUG: print("DEBUG: AFTER blocked:", blocked)
  86
  87                         if blocked == "":
  88                             print("WARNING: blocked is empty after fba.tidyup():", blocker, block_level)
  89                             continue
  90                         elif fba.is_blacklisted(blocked):
  91                             # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!")
  92                             continue
  93                         elif blocked.count("*") > 1:
  94                             # -ACK!-oma also started obscuring domains without hash
  95                             fba.cursor.execute(
  96                                 "SELECT domain, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")]
  97                             )
  98                             searchres = fba.cursor.fetchone()
  99                             # DEBUG: print("DEBUG: searchres[]:", type(searchres))
 100
 101                             if searchres == None:
 102                                 print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
 103                                 continue
 104
 105                             blocked = searchres[0]
 106                             nodeinfo_url = searchres[1]
 107                             # DEBUG: print("DEBUG: Looked up domain:", blocked)
 108                         elif not validators.domain(blocked):
 109                             print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domai name - skipped!")
 110                             continue
 111
 112                         # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
 113                         if not validators.domain(blocked):
 114                             print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
 115                             continue
 116                         elif not fba.is_instance_registered(blocked):
 117                             # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., blocker='{blocker}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'")
 118                             fba.add_instance(blocked, blocker, origin, nodeinfo_url)
 119
 120                         fba.cursor.execute(
 121                             "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ? LIMIT 1",
 122                             (
 123                                blocker,
 124                                blocked,
 125                                block_level
 126                            ),
 127                         )
 128
 129                         if fba.cursor.fetchone() == None:
 130                             # DEBUG: print("DEBUG: Blocking:", blocker, blocked, block_level)
 131                             fba.block_instance(blocker, blocked, "unknown", block_level)
 132
 133                             if block_level == "reject":
 134                                 # DEBUG: print("DEBUG: Adding to blockdict:", blocked)
 135                                 blockdict.append(
 136                                     {
 137                                         "blocked": blocked,
 138                                         "reason" : None
 139                                     })
 140                         else:
 141                             print(f"DEBUG: Updating block last seen for blocker='{blocker}',blocked='{blocked}' ...")
 142                             fba.update_last_seen(blocker, blocked, block_level)
 143
 144             fba.connection.commit()
 145
 146             # Reasons
 147             if "mrf_simple_info" in federation:
 148                 # DEBUG: print("DEBUG: Found mrf_simple_info:", blocker)
 149                 for block_level, info in (
 150                     {**federation["mrf_simple_info"],
 151                     **(federation["quarantined_instances_info"]
 152                     if "quarantined_instances_info" in federation
 153                     else {})}
 154                 ).items():
 155                     # DEBUG: print("DEBUG: block_level, info.items():", block_level, len(info.items()))
 156                     block_level = fba.tidyup(block_level)
 157                     # DEBUG: print("DEBUG: BEFORE block_level:", block_level)
 158
 159                     if block_level == "":
 160                         print("WARNING: block_level is now empty!")
 161                         continue
 162
 163                     # DEBUG: print(f"DEBUG: Checking {len(info.items())} entries from blocker='{blocker}',software='{software}',block_level='{block_level}' ...")
 164                     for blocked, reason in info.items():
 165                         # DEBUG: print("DEBUG: BEFORE blocked:", blocked)
 166                         blocked = fba.tidyup(blocked)
 167                         # DEBUG: print("DEBUG: AFTER blocked:", blocked)
 168
 169                         if blocked == "":
 170                             print("WARNING: blocked is empty after fba.tidyup():", blocker, block_level)
 171                             continue
 172                         elif fba.is_blacklisted(blocked):
 173                             # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!")
 174                             continue
 175                         elif blocked.count("*") > 1:
 176                             # same domain guess as above, but for reasons field
 177                             fba.cursor.execute(
 178                                 "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")]
 179                             )
 180                             searchres = fba.cursor.fetchone()
 181
 182                             if searchres == None:
 183                                 print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
 184                                 continue
 185
 186                             blocked = searchres[0]
 187                             origin = searchres[1]
 188                             nodeinfo_url = searchres[2]
 189                         elif not validators.domain(blocked):
 190                             print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domai name - skipped!")
 191                             continue
 192
 193                         # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
 194                         if not validators.domain(blocked):
 195                             print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
 196                             continue
 197                         elif not fba.is_instance_registered(blocked):
 198                             # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., blocker='{blocker}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'")
 199                             fba.add_instance(blocked, blocker, origin, nodeinfo_url)
 200
 201                         # DEBUG: print("DEBUG: Updating block reason:", blocker, blocked, reason["reason"])
 202                         fba.update_block_reason(reason["reason"], blocker, blocked, block_level)
 203
 204                         for entry in blockdict:
 205                             if entry["blocked"] == blocked:
 206                                 # DEBUG: print("DEBUG: Updating entry reason:", blocked)
 207                                 entry["reason"] = reason["reason"]
 208
 209             fba.connection.commit()
 210         except Exception as e:
 211             print(f"ERROR: blocker='{blocker}',software='{software}',exception[{type(e)}]:'{str(e)}'")
 212     elif software == "mastodon":
 213         print("INFO: blocker:", blocker)
 214         try:
 215             # json endpoint for newer mastodongs
 216             try:
 217                 json = {
 218                     "reject"        : [],
 219                     "media_removal" : [],
 220                     "followers_only": [],
 221                     "report_removal": []
 222                 }
 223
 224                 # handling CSRF, I've saw at least one server requiring it to access the endpoint
 225                 # DEBUG: print("DEBUG: Fetching meta:", blocker)
 226                 meta = bs4.BeautifulSoup(
 227                     reqto.get(f"https://{blocker}/", headers=fba.headers, timeout=(fba.config["connection_timeout"], fba.config["read_timeout"])).text,
 228                     "html.parser",
 229                 )
 230                 try:
 231                     csrf = meta.find("meta", attrs={"name": "csrf-token"})["content"]
 232                     # DEBUG: print("DEBUG: Adding CSRF token:", blocker, csrf)
 233                     reqheaders = {**fba.api_headers, **{"X-CSRF-Token": csrf}}
 234                 except BaseException as e:
 235                     # DEBUG: print("DEBUG: No CSRF token found, using normal headers:", blocker, e)
 236                     reqheaders = fba.api_headers
 237
 238                 # DEBUG: print("DEBUG: Querying API domain_blocks:", blocker)
 239                 blocks = reqto.get(f"https://{blocker}/api/v1/instance/domain_blocks", headers=reqheaders, timeout=(fba.config["connection_timeout"], fba.config["read_timeout"])).json()
 240
 241                 print(f"INFO: Checking {len(blocks)} entries from blocker='{blocker}',software='{software}' ...")
 242                 for block in blocks:
 243                     entry = {
 244                         'domain': block['domain'],
 245                         'hash'  : block['digest'],
 246                         'reason': block['comment']
 247                     }
 248
 249                     # DEBUG: print("DEBUG: severity,domain,hash,comment:", block['severity'], block['domain'], block['digest'], block['comment'])
 250                     if block['severity'] == 'suspend':
 251                         # DEBUG: print(f"DEBUG: Adding entry='{entry}' with severity='{block['severity']}' ...")
 252                         json['reject'].append(entry)
 253                     elif block['severity'] == 'silence':
 254                         # DEBUG: print(f"DEBUG: Adding entry='{entry}' with severity='{block['severity']}' ...")
 255                         json['followers_only'].append(entry)
 256                     elif block['severity'] == 'reject_media':
 257                         # DEBUG: print(f"DEBUG: Adding entry='{entry}' with severity='{block['severity']}' ...")
 258                         json['media_removal'].append(entry)
 259                     elif block['severity'] == 'reject_reports':
 260                         # DEBUG: print(f"DEBUG: Adding entry='{entry}' with severity='{block['severity']}' ...")
 261                         json['report_removal'].append(entry)
 262                     else:
 263                         print("WARNING: Unknown severity:", block['severity'], block['domain'])
 264             except BaseException as e:
 265                 # DEBUG: print(f"DEBUG: Failed, trying mastodon-specific fetches: blocker='{blocker}',exception[{type(e)}]={str(e)}")
 266                 json = fba.get_mastodon_blocks(blocker)
 267
 268             print(f"INFO: Checking {len(json.items())} entries from blocker='{blocker}',software='{software}' ...")
 269             for block_level, blocks in json.items():
 270                 # DEBUG: print("DEBUG: blocker,block_level,blocks():", blocker, block_level, len(blocks))
 271                 block_level = fba.tidyup(block_level)
 272                 # DEBUG: print("DEBUG: AFTER-block_level:", block_level)
 273                 if block_level == "":
 274                     print("WARNING: block_level is empty, blocker:", blocker)
 275                     continue
 276
 277                 # DEBUG: print(f"DEBUG: Checking {len(blocks)} entries from blocker='{blocker}',software='{software}',block_level='{block_level}' ...")
 278                 for block in blocks:
 279                     blocked, blocked_hash, reason = block.values()
 280                     # DEBUG: print("DEBUG: blocked,hash,reason:", blocked, blocked_hash, reason)
 281                     blocked = fba.tidyup(blocked)
 282                     # DEBUG: print("DEBUG: AFTER-blocked:", blocked)
 283
 284                     if blocked == "":
 285                         print("WARNING: blocked is empty:", blocker)
 286                         continue
 287                     elif fba.is_blacklisted(blocked):
 288                         # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!")
 289                         continue
 290                     elif blocked.count("*") > 0:
 291                         # Doing the hash search for instance names as well to tidy up DB
 292                         fba.cursor.execute(
 293                             "SELECT domain, origin, nodeinfo_url FROM instances WHERE hash = ? LIMIT 1", [blocked_hash]
 294                         )
 295                         searchres = fba.cursor.fetchone()
 296
 297                         if searchres == None:
 298                             print(f"WARNING: Cannot deobsfucate blocked='{blocked}',blocked_hash='{blocked_hash}' - SKIPPED!")
 299                             continue
 300
 301                         # DEBUG: print("DEBUG: Updating domain: ", searchres[0])
 302                         blocked = searchres[0]
 303                         origin = searchres[1]
 304                         nodeinfo_url = searchres[2]
 305
 306                         # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
 307                         if not validators.domain(blocked):
 308                             print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
 309                             continue
 310                         elif not fba.is_instance_registered(blocked):
 311                             # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., blocker='{blocker}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'")
 312                             fba.add_instance(blocked, blocker, origin, nodeinfo_url)
 313                     elif not validators.domain(blocked):
 314                         print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domai name - skipped!")
 315                         continue
 316
 317                     # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
 318                     if not validators.domain(blocked):
 319                         print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
 320                         continue
 321                     elif not fba.is_instance_registered(blocked):
 322                         # DEBUG: print("DEBUG: Hash wasn't found, adding:", blocked, blocker)
 323                         fba.add_instance(blocked, blocker, origin, nodeinfo_url)
 324
 325                     blocking = blocked if blocked.count("*") <= 1 else blocked_hash
 326                     # DEBUG: print(f"DEBUG: blocking='{blocking}',blocked='{blocked}',blocked_hash='{blocked_hash}'")
 327
 328                     fba.cursor.execute(
 329                         "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ? LIMIT 1",
 330                         (
 331                             blocker,
 332                             blocking,
 333                             block_level
 334                         ),
 335                     )
 336
 337                     if fba.cursor.fetchone() == None:
 338                         # DEBUG: print("DEBUG: Blocking:", blocker, blocked, block_level)
 339                         fba.block_instance(blocker, blocking, reason, block_level)
 340
 341                         if block_level == "reject":
 342                             blockdict.append(
 343                                 {
 344                                     "blocked": blocked,
 345                                     "reason" : reason
 346                                 })
 347                     else:
 348                         # DEBUG: print(f"DEBUG: Updating block last seen and reason for blocker='{blocker}',blocking='{blocking}' ...")
 349                         fba.update_last_seen(blocker, blocking, block_level)
 350                         fba.update_block_reason(reason, blocker, blocking, block_level)
 351
 352             fba.connection.commit()
 353         except Exception as e:
 354             print(f"ERROR: blocker='{blocker}',software='{software}',exception[{type(e)}]:'{str(e)}'")
 355     elif software == "friendica" or software == "misskey" or software == "bookwyrm" or software == "takahe":
 356         print("INFO: blocker:", blocker)
 357         try:
 358             if software == "friendica":
 359                 json = fba.get_friendica_blocks(blocker)
 360             elif software == "misskey":
 361                 json = fba.get_misskey_blocks(blocker)
 362             elif software == "bookwyrm":
 363                 print("WARNING: bookwyrm is not fully supported for fetching blacklist!", blocker)
 364                 #json = fba.get_bookwyrm_blocks(blocker)
 365                 continue
 366             elif software == "takahe":
 367                 print("WARNING: takahe is not fully supported for fetching blacklist!", blocker)
 368                 #json = fba.get_takahe_blocks(blocker)
 369                 continue
 370
 371             print(f"INFO: Checking {len(json.items())} entries from blocker='{blocker}',software='{software}' ...")
 372             for block_level, blocks in json.items():
 373                 # DEBUG: print("DEBUG: blocker,block_level,blocks():", blocker, block_level, len(blocks))
 374                 block_level = fba.tidyup(block_level)
 375                 # DEBUG: print("DEBUG: AFTER-block_level:", block_level)
 376                 if block_level == "":
 377                     print("WARNING: block_level is empty, blocker:", blocker)
 378                     continue
 379
 380                 # DEBUG: print(f"DEBUG: Checking {len(blocks)} entries from blocker='{blocker}',software='{software}',block_level='{block_level}' ...")
 381                 for block in blocks:
 382                     blocked, reason = block.values()
 383                     # DEBUG: print("DEBUG: BEFORE blocked:", blocked)
 384                     blocked = fba.tidyup(blocked)
 385                     # DEBUG: print("DEBUG: AFTER blocked:", blocked)
 386
 387                     if blocked == "":
 388                         print("WARNING: blocked is empty:", blocker)
 389                         continue
 390                     elif fba.is_blacklisted(blocked):
 391                         # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!")
 392                         continue
 393                     elif blocked.count("*") > 0:
 394                         # Some friendica servers also obscure domains without hash
 395                         fba.cursor.execute(
 396                             "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")]
 397                         )
 398
 399                         searchres = fba.cursor.fetchone()
 400
 401                         if searchres == None:
 402                             print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
 403                             continue
 404
 405                         blocked = searchres[0]
 406                         origin = searchres[1]
 407                         nodeinfo_url = searchres[2]
 408                     elif blocked.count("?") > 0:
 409                         # Some obscure them with question marks, not sure if that's dependent on version or not
 410                         fba.cursor.execute(
 411                             "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("?", "_")]
 412                         )
 413
 414                         searchres = fba.cursor.fetchone()
 415
 416                         if searchres == None:
 417                             print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
 418                             continue
 419
 420                         blocked = searchres[0]
 421                         origin = searchres[1]
 422                         nodeinfo_url = searchres[2]
 423                     elif not validators.domain(blocked):
 424                         print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domai name - skipped!")
 425                         continue
 426
 427                     # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
 428                     if not validators.domain(blocked):
 429                         print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
 430                         continue
 431                     elif not fba.is_instance_registered(blocked):
 432                         # DEBUG: print("DEBUG: Hash wasn't found, adding:", blocked, blocker)
 433                         fba.add_instance(blocked, blocker, origin, nodeinfo_url)
 434
 435                     fba.cursor.execute(
 436                         "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ? LIMIT 1",
 437                         (blocker, blocked, block_level),
 438                     )
 439
 440                     if fba.cursor.fetchone() == None:
 441                         fba.block_instance(blocker, blocked, reason, block_level)
 442
 443                         if block_level == "reject":
 444                             blockdict.append(
 445                                 {
 446                                     "blocked": blocked,
 447                                     "reason" : reason
 448                                 })
 449                     else:
 450                         print(f"DEBUG: Updating block last seen and reason for blocker='{blocker}',blocked='{blocked}' ...")
 451                         fba.update_last_seen(blocker, blocked, block_level)
 452                         fba.update_block_reason(reason, blocker, blocked, block_level)
 453
 454             fba.connection.commit()
 455         except Exception as e:
 456             print(f"ERROR: blocker='{blocker}',software='{software}',exception[{type(e)}]:'{str(e)}'")
 457     elif software == "gotosocial":
 458         print("INFO: blocker:", blocker)
 459         try:
 460             # Blocks
 461             federation = reqto.get(f"https://{blocker}{fba.get_peers_url}?filter=suspended", headers=fba.api_headers, timeout=(fba.config["connection_timeout"], fba.config["read_timeout"])).json()
 462
 463             if (federation == None):
 464                 print("WARNING: No valid response:", blocker);
 465             elif "error" in federation:
 466                 print("WARNING: API returned error:", federation["error"])
 467             else:
 468                 print(f"INFO: Checking {len(federation)} entries from blocker='{blocker}',software='{software}' ...")
 469                 for peer in federation:
 470                     blocked = peer["domain"].lower()
 471                     # DEBUG: print("DEBUG: BEFORE blocked:", blocked)
 472                     blocked = fba.tidyup(blocked)
 473                     # DEBUG: print("DEBUG: AFTER blocked:", blocked)
 474
 475                     if blocked == "":
 476                         print("WARNING: blocked is empty:", blocker)
 477                         continue
 478                     elif fba.is_blacklisted(blocked):
 479                         # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!")
 480                         continue
 481                     elif blocked.count("*") > 0:
 482                         # GTS does not have hashes for obscured domains, so we have to guess it
 483                         fba.cursor.execute(
 484                             "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")]
 485                         )
 486                         searchres = fba.cursor.fetchone()
 487
 488                         if searchres == None:
 489                             print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
 490                             continue
 491
 492                         blocked = searchres[0]
 493                         origin = searchres[1]
 494                         nodeinfo_url = searchres[2]
 495                     elif not validators.domain(blocked):
 496                         print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domai name - skipped!")
 497                         continue
 498
 499                     # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
 500                     if not validators.domain(blocked):
 501                         print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
 502                         continue
 503                     elif not fba.is_instance_registered(blocked):
 504                         # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., blocker='{blocker}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'")
 505                         fba.add_instance(blocked, blocker, origin, nodeinfo_url)
 506
 507                     fba.cursor.execute(
 508                         "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ? LIMIT 1",
 509                         (
 510                             blocker,
 511                             blocked,
 512                             "reject"
 513                         ),
 514                     )
 515
 516                     if fba.cursor.fetchone() == None:
 517                         # DEBUG: print(f"DEBUG: blocker='{blocker}' is blocking '{blocked}' for unknown reason at this point")
 518                         fba.block_instance(blocker, blocked, "unknown", "reject")
 519
 520                         blockdict.append(
 521                             {
 522                                 "blocked": blocked,
 523                                 "reason" : None
 524                             })
 525                     else:
 526                         print(f"DEBUG: Updating block last seen for blocker='{blocker}',blocked='{blocked}' ...")
 527                         fba.update_last_seen(blocker, blocked, "reject")
 528
 529                     if "public_comment" in peer:
 530                         # DEBUG: print("DEBUG: Updating block reason:", blocker, blocked, peer["public_comment"])
 531                         fba.update_block_reason(peer["public_comment"], blocker, blocked, "reject")
 532
 533                         for entry in blockdict:
 534                             if entry["blocked"] == blocked:
 535                                 # DEBUG: print(f"DEBUG: Setting block reason for blocked='{blocked}':'{peer['public_comment']}'")
 536                                 entry["reason"] = peer["public_comment"]
 537
 538                 fba.connection.commit()
 539         except Exception as e:
 540             print(f"ERROR: blocker='{blocker}',software='{software}',exception[{type(e)}]:'{str(e)}'")
 541     else:
 542         print("WARNING: Unknown software:", blocker, software)
 543
 544     if fba.config["bot_enabled"] and len(blockdict) > 0:
 545         send_bot_post(blocker, blockdict)
 546
 547     blockdict = []
 548
 549 fba.connection.close()