fba/networks/pleroma.py

   1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
   2 # Copyright (C) 2023 Free Software Foundation
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published
   6 # by the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  16
  17 import inspect
  18
  19 import bs4
  20 import validators
  21
  22 from fba import blacklist
  23 from fba import config
  24 from fba import fba
  25 from fba import federation
  26 from fba import network
  27
  28 from fba.helpers import tidyup
  29
  30 from fba.models import blocks
  31 from fba.models import instances
  32
  33 language_mapping = {
  34     # English -> English
  35     "Reject": "Suspended servers",
  36 }
  37
  38 def fetch_blocks(domain: str, origin: str, nodeinfo_url: str):
  39     # DEBUG: print(f"DEBUG: domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}' - CALLED!")
  40     if not isinstance(domain, str):
  41         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
  42     elif domain == "":
  43         raise ValueError("Parameter 'domain' is empty")
  44     elif not isinstance(origin, str) and origin is not None:
  45         raise ValueError(f"Parameter origin[]='{type(origin)}' is not 'str'")
  46     elif origin == "":
  47         raise ValueError("Parameter 'origin' is empty")
  48     elif not isinstance(nodeinfo_url, str):
  49         raise ValueError(f"Parameter nodeinfo_url[]='{type(nodeinfo_url)}' is not 'str'")
  50     elif nodeinfo_url == "":
  51         raise ValueError("Parameter 'nodeinfo_url' is empty")
  52
  53     # @TODO Unused blockdict
  54     blockdict = list()
  55     rows = None
  56     try:
  57         # DEBUG: print(f"DEBUG: Fetching nodeinfo: domain='{domain}',nodeinfo_url='{nodeinfo_url}'")
  58         rows = federation.fetch_nodeinfo(domain, nodeinfo_url)
  59     except network.exceptions as exception:
  60         print(f"WARNING: Exception '{type(exception)}' during fetching nodeinfo")
  61         instances.set_last_error(domain, exception)
  62
  63     if rows is None:
  64         print("WARNING: Could not fetch nodeinfo from domain:", domain)
  65         return
  66     elif "metadata" not in rows:
  67         print(f"WARNING: rows()={len(rows)} does not have key 'metadata', domain='{domain}'")
  68         return
  69     elif "federation" not in rows["metadata"]:
  70         print(f"WARNING: rows()={len(rows['metadata'])} does not have key 'federation', domain='{domain}'")
  71         return
  72
  73     data = rows["metadata"]["federation"]
  74     found = False
  75
  76     # DEBUG: print(f"DEBUG: data[]='{type(data)}'")
  77     if "mrf_simple" in data:
  78         # DEBUG: print("DEBUG: Found mrf_simple:", domain)
  79         found = True
  80         for block_level, blocklist in (
  81             {
  82                 **data["mrf_simple"],
  83                 **{
  84                     "quarantined_instances": data["quarantined_instances"]
  85                 }
  86             }
  87         ).items():
  88             # DEBUG: print("DEBUG: block_level, blocklist():", block_level, len(blocklist))
  89             block_level = tidyup.domain(block_level)
  90             # DEBUG: print("DEBUG: BEFORE block_level:", block_level)
  91
  92             if block_level == "":
  93                 print("WARNING: block_level is now empty!")
  94                 continue
  95             elif block_level == "accept":
  96                 # DEBUG: print(f"DEBUG: domain='{domain}' skipping block_level='accept'")
  97                 continue
  98
  99             # DEBUG: print(f"DEBUG: Checking {len(blocklist)} entries from domain='{domain}',block_level='{block_level}' ...")
 100             if len(blocklist) > 0:
 101                 for blocked in blocklist:
 102                     # DEBUG: print("DEBUG: BEFORE blocked:", blocked)
 103                     blocked = tidyup.domain(blocked)
 104                     # DEBUG: print("DEBUG: AFTER blocked:", blocked)
 105
 106                     if blocked == "":
 107                         print("WARNING: blocked is empty after tidyup.domain():", domain, block_level)
 108                         continue
 109                     elif blacklist.is_blacklisted(blocked):
 110                         # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!")
 111                         continue
 112                     elif blocked.count("*") > 0:
 113                         # Obscured domain name with no hash
 114                         row = instances.deobscure("*", blocked)
 115
 116                         # DEBUG: print(f"DEBUG: row[]='{type(row)}'")
 117                         if row is None:
 118                             print(f"WARNING: Cannot deobsfucate blocked='{blocked}',domain='{domain}',origin='{origin}' - SKIPPED!")
 119                             continue
 120
 121                         # DEBUG: print(f"DEBUG: blocked='{blocked}' de-obscured to '{row[0]}'")
 122                         blocked      = row[0]
 123                         origin       = row[1]
 124                         nodeinfo_url = row[2]
 125                     elif blocked.count("?") > 0:
 126                         # Obscured domain name with no hash
 127                         row = instances.deobscure("?", blocked)
 128
 129                         # DEBUG: print(f"DEBUG: row[]='{type(row)}'")
 130                         if row is None:
 131                             print(f"WARNING: Cannot deobsfucate blocked='{blocked}',domain='{domain}',origin='{origin}' - SKIPPED!")
 132                             continue
 133
 134                         # DEBUG: print(f"DEBUG: blocked='{blocked}' de-obscured to '{row[0]}'")
 135                         blocked      = row[0]
 136                         origin       = row[1]
 137                         nodeinfo_url = row[2]
 138
 139                     # DEBUG: print(f"DEBUG: blocked='{blocked}'")
 140                     if not validators.domain(blocked):
 141                         print(f"WARNING: blocked='{blocked}',software='pleroma' is not a valid domain name - SKIPPED!")
 142                         continue
 143                     elif blocked.endswith(".arpa"):
 144                         print(f"WARNING: blocked='{blocked}' is a reversed .arpa domain and should not be used generally.")
 145                         continue
 146                     elif blocked.endswith(".tld"):
 147                         print(f"WARNING: blocked='{blocked}' is a fake domain, please don't crawl them!")
 148                         continue
 149                     elif not instances.is_registered(blocked):
 150                         # Commit changes
 151                         fba.connection.commit()
 152
 153                         # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'")
 154                         instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url)
 155
 156                     if not blocks.is_instance_blocked(domain, blocked, block_level):
 157                         # DEBUG: print("DEBUG: Blocking:", domain, blocked, block_level)
 158                         blocks.add_instance(domain, blocked, None, block_level)
 159
 160                         if block_level == "reject":
 161                             # DEBUG: print("DEBUG: Adding to blockdict:", blocked)
 162                             blockdict.append({
 163                                 "blocked": blocked,
 164                                 "reason" : None
 165                             })
 166                     else:
 167                         # DEBUG: print(f"DEBUG: Updating block last seen for domain='{domain}',blocked='{blocked}' ...")
 168                         blocks.update_last_seen(domain, blocked, block_level)
 169     elif "quarantined_instances" in data:
 170         # DEBUG: print(f"DEBUG: Found 'quarantined_instances' in JSON response: domain='{domain}'")
 171         found = True
 172         block_level = "quarantined"
 173
 174         for blocked in data["quarantined_instances"]:
 175             # DEBUG: print("DEBUG: BEFORE blocked:", blocked)
 176             blocked = tidyup.domain(blocked)
 177             # DEBUG: print("DEBUG: AFTER blocked:", blocked)
 178
 179             if blocked == "":
 180                 print("WARNING: blocked is empty after tidyup.domain():", domain, block_level)
 181                 continue
 182             elif blacklist.is_blacklisted(blocked):
 183                 # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!")
 184                 continue
 185             elif blocked.count("*") > 0:
 186                 # Obscured domain name with no hash
 187                 row = instances.deobscure("*", blocked)
 188
 189                 # DEBUG: print(f"DEBUG: row[]='{type(row)}'")
 190                 if row is None:
 191                     print(f"WARNING: Cannot deobsfucate blocked='{blocked}',domain='{domain}',origin='{origin}' - SKIPPED!")
 192                     continue
 193
 194                 # DEBUG: print(f"DEBUG: blocked='{blocked}' de-obscured to '{row[0]}'")
 195                 blocked      = row[0]
 196                 origin       = row[1]
 197                 nodeinfo_url = row[2]
 198             elif blocked.count("?") > 0:
 199                 # Obscured domain name with no hash
 200                 row = instances.deobscure("?", blocked)
 201
 202                 # DEBUG: print(f"DEBUG: row[]='{type(row)}'")
 203                 if row is None:
 204                     print(f"WARNING: Cannot deobsfucate blocked='{blocked}',domain='{domain}',origin='{origin}' - SKIPPED!")
 205                     continue
 206
 207                 # DEBUG: print(f"DEBUG: blocked='{blocked}' de-obscured to '{row[0]}'")
 208                 blocked      = row[0]
 209                 origin       = row[1]
 210                 nodeinfo_url = row[2]
 211
 212             # DEBUG: print(f"DEBUG: blocked='{blocked}'")
 213             if not validators.domain(blocked):
 214                 print(f"WARNING: blocked='{blocked}',software='pleroma' is not a valid domain name - SKIPPED!")
 215                 continue
 216             elif blocked.endswith(".arpa"):
 217                 print(f"WARNING: blocked='{blocked}' is a reversed .arpa domain and should not be used generally.")
 218                 continue
 219             elif blocked.endswith(".tld"):
 220                 print(f"WARNING: blocked='{blocked}' is a fake domain, please don't crawl them!")
 221                 continue
 222             elif not instances.is_registered(blocked):
 223                 # Commit changes
 224                 fba.connection.commit()
 225
 226                 # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'")
 227                 instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url)
 228
 229             if not blocks.is_instance_blocked(domain, blocked, block_level):
 230                 # DEBUG: print("DEBUG: Blocking:", domain, blocked, block_level)
 231                 blocks.add_instance(domain, blocked, None, block_level)
 232
 233                 if block_level == "reject":
 234                     # DEBUG: print("DEBUG: Adding to blockdict:", blocked)
 235                     blockdict.append({
 236                         "blocked": blocked,
 237                         "reason" : None
 238                     })
 239             else:
 240                 # DEBUG: print(f"DEBUG: Updating block last seen for domain='{domain}',blocked='{blocked}' ...")
 241                 blocks.update_last_seen(domain, blocked, block_level)
 242     else:
 243         print(f"WARNING: Cannot find 'mrf_simple' or 'quarantined_instances' in JSON reply: domain='{domain}'")
 244
 245     # DEBUG: print("DEBUG: Committing changes ...")
 246     fba.connection.commit()
 247
 248     # Reasons
 249     if "mrf_simple_info" in data:
 250         # DEBUG: print("DEBUG: Found mrf_simple_info:", domain)
 251         found = True
 252         for block_level, info in (
 253             {
 254                 **data["mrf_simple_info"],
 255                 **(data["quarantined_instances_info"] if "quarantined_instances_info" in data else {})
 256             }
 257         ).items():
 258             # DEBUG: print("DEBUG: block_level, info.items():", block_level, len(info.items()))
 259             block_level = tidyup.domain(block_level)
 260             # DEBUG: print("DEBUG: BEFORE block_level:", block_level)
 261
 262             if block_level == "":
 263                 print("WARNING: block_level is now empty!")
 264                 continue
 265             elif block_level == "accept":
 266                 # DEBUG: print(f"DEBUG: domain='{domain}' skipping block_level='accept'")
 267                 continue
 268
 269             # DEBUG: print(f"DEBUG: Checking {len(info.items())} entries from domain='{domain}',software='pleroma',block_level='{block_level}' ...")
 270             for blocked, reason in info.items():
 271                 # DEBUG: print(f"DEBUG: blocked='{blocked}',reason[{type(reason)}]='{reason}' - BEFORE!")
 272                 blocked = tidyup.domain(blocked)
 273
 274                 if isinstance(reason, str):
 275                     # DEBUG: print("DEBUG: reason[] is a string")
 276                     reason = tidyup.reason(reason)
 277                 elif isinstance(reason, dict) and "reason" in reason:
 278                     # DEBUG: print("DEBUG: reason[] is a dict")
 279                     reason = tidyup.reason(reason["reason"])
 280                 elif reason is not None:
 281                     raise ValueError(f"Cannot handle reason[]='{type(reason)}'")
 282
 283                 # DEBUG: print(f"DEBUG: blocked='{blocked}',reason='{reason}' - AFTER!")
 284
 285                 if blocked == "":
 286                     print("WARNING: blocked is empty after tidyup.domain():", domain, block_level)
 287                     continue
 288                 elif blacklist.is_blacklisted(blocked):
 289                     # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!")
 290                     continue
 291                 elif blocked.count("*") > 0:
 292                     # Obscured domain name with no hash
 293                     row = instances.deobscure("*", blocked)
 294
 295                     # DEBUG: print(f"DEBUG: row[]='{type(row)}'")
 296                     if row is None:
 297                         print(f"WARNING: Cannot deobsfucate blocked='{blocked}',domain='{domain}',origin='{origin}' - SKIPPED!")
 298                         continue
 299
 300                     # DEBUG: print(f"DEBUG: blocked='{blocked}' de-obscured to '{row[0]}'")
 301                     blocked      = row[0]
 302                     origin       = row[1]
 303                     nodeinfo_url = row[2]
 304                 elif blocked.count("?") > 0:
 305                     # Obscured domain name with no hash
 306                     row = instances.deobscure("?", blocked)
 307
 308                     # DEBUG: print(f"DEBUG: row[]='{type(row)}'")
 309                     if row is None:
 310                         print(f"WARNING: Cannot deobsfucate blocked='{blocked}',domain='{domain}',origin='{origin}' - SKIPPED!")
 311                         continue
 312
 313                     # DEBUG: print(f"DEBUG: blocked='{blocked}' de-obscured to '{row[0]}'")
 314                     blocked      = row[0]
 315                     origin       = row[1]
 316                     nodeinfo_url = row[2]
 317
 318                 # DEBUG: print(f"DEBUG: blocked='{blocked}'")
 319                 if not validators.domain(blocked):
 320                     print(f"WARNING: blocked='{blocked}',software='pleroma' is not a valid domain name - SKIPPED!")
 321                     continue
 322                 elif blocked.endswith(".arpa"):
 323                     print(f"WARNING: blocked='{blocked}' is a reversed .arpa domain and should not be used generally.")
 324                     continue
 325                 elif blocked.endswith(".tld"):
 326                     print(f"WARNING: blocked='{blocked}' is a fake domain, please don't crawl them!")
 327                     continue
 328                 elif not instances.is_registered(blocked):
 329                     # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'")
 330                     instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url)
 331
 332                 # DEBUG: print(f"DEBUG: Updating block reason: reason='{reason}',domain='{domain}',blocked='{blocked}',block_level='{block_level}'")
 333                 blocks.update_reason(reason, domain, blocked, block_level)
 334
 335                 # DEBUG: print(f"DEBUG: blockdict()={len(blockdict)}")
 336                 for entry in blockdict:
 337                     if entry["blocked"] == blocked:
 338                         # DEBUG: print(f"DEBUG: Updating entry reason: blocked='{blocked}',reason='{reason}'")
 339                         entry["reason"] = reason
 340
 341     elif "quarantined_instances_info" in data and "quarantined_instances" in data["quarantined_instances_info"]:
 342         # DEBUG: print(f"DEBUG: Found 'quarantined_instances_info' in JSON response: domain='{domain}'")
 343         found = True
 344         block_level = "quarantined"
 345
 346         #print(data["quarantined_instances_info"])
 347         rows = data["quarantined_instances_info"]["quarantined_instances"]
 348         for blocked in rows:
 349             # DEBUG: print("DEBUG: BEFORE blocked:", blocked)
 350             blocked = tidyup.domain(blocked)
 351             # DEBUG: print("DEBUG: AFTER blocked:", blocked)
 352
 353             if blocked not in rows or "reason" not in rows[blocked]:
 354                 print(f"WARNING: Cannot find blocked='{blocked}' in rows()={len(rows)},domain='{domain}'")
 355                 break
 356
 357             reason = rows[blocked]["reason"]
 358             # DEBUG: print(f"DEBUG: reason='{reason}'")
 359
 360             if blocked == "":
 361                 print("WARNING: blocked is empty after tidyup.domain():", domain, block_level)
 362                 continue
 363             elif blacklist.is_blacklisted(blocked):
 364                 # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!")
 365                 continue
 366             elif blocked.count("*") > 0:
 367                 # Obscured domain name with no hash
 368                 row = instances.deobscure("*", blocked)
 369
 370                 # DEBUG: print(f"DEBUG: row[]='{type(row)}'")
 371                 if row is None:
 372                     print(f"WARNING: Cannot deobsfucate blocked='{blocked}',domain='{domain}',origin='{origin}' - SKIPPED!")
 373                     continue
 374
 375                 # DEBUG: print(f"DEBUG: blocked='{blocked}' de-obscured to '{row[0]}'")
 376                 blocked      = row[0]
 377                 origin       = row[1]
 378                 nodeinfo_url = row[2]
 379             elif blocked.count("?") > 0:
 380                 # Obscured domain name with no hash
 381                 row = instances.deobscure("?", blocked)
 382
 383                 # DEBUG: print(f"DEBUG: row[]='{type(row)}'")
 384                 if row is None:
 385                     print(f"WARNING: Cannot deobsfucate blocked='{blocked}',domain='{domain}',origin='{origin}' - SKIPPED!")
 386                     continue
 387
 388                 # DEBUG: print(f"DEBUG: blocked='{blocked}' de-obscured to '{row[0]}'")
 389                 blocked      = row[0]
 390                 origin       = row[1]
 391                 nodeinfo_url = row[2]
 392
 393             # DEBUG: print(f"DEBUG: blocked='{blocked}'")
 394             if not validators.domain(blocked):
 395                 print(f"WARNING: blocked='{blocked}',software='pleroma' is not a valid domain name - SKIPPED!")
 396                 continue
 397             elif blocked.endswith(".arpa"):
 398                 print(f"WARNING: blocked='{blocked}' is a reversed .arpa domain and should not be used generally.")
 399                 continue
 400             elif blocked.endswith(".tld"):
 401                 print(f"WARNING: blocked='{blocked}' is a fake domain, please don't crawl them!")
 402                 continue
 403             elif not instances.is_registered(blocked):
 404                 # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'")
 405                 instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url)
 406
 407             # DEBUG: print(f"DEBUG: Updating block reason: reason='{reason}',domain='{domain}',blocked='{blocked}',block_level='{block_level}'")
 408             blocks.update_reason(reason, domain, blocked, block_level)
 409
 410             # DEBUG: print(f"DEBUG: blockdict()={len(blockdict)}")
 411             for entry in blockdict:
 412                 if entry["blocked"] == blocked:
 413                     # DEBUG: print(f"DEBUG: Updating entry reason: blocked='{blocked}',reason='{reason}'")
 414                     entry["reason"] = reason
 415     else:
 416         print(f"WARNING: Cannot find 'mrf_simple_info' or 'quarantined_instances_info' in JSON reply: domain='{domain}'")
 417
 418     if not found:
 419         # DEBUG: print(f"DEBUG: Did not find any useable JSON elements, domain='{domain}', continuing with /about page ...")
 420         blocklist = fetch_blocks_from_about(domain)
 421
 422         # DEBUG: print(f"DEBUG: blocklist()={len(blocklist)}")
 423         if len(blocklist) > 0:
 424             print(f"INFO: Checking {len(blocklist)} record(s) ...")
 425             for block_level in blocklist:
 426                 # DEBUG: print(f"DEBUG: block_level='{block_level}'")
 427                 rows = blocklist[block_level]
 428                 # DEBUG: print(f"DEBUG: rows['{type(rows)}]()={len(rows)}'")
 429                 for record in rows:
 430                     # DEBUG: print(f"DEBUG: record[]='{type(record)}'")
 431                     blocked = tidyup.domain(record["blocked"])
 432                     reason  = tidyup.reason(record["reason"])
 433                     # DEBUG: print(f"DEBUG: blocked='{blocked}',reason='{reason}' - AFTER!")
 434
 435                     if blocked == "":
 436                         print("WARNING: blocked is empty after tidyup.domain():", domain, block_level)
 437                         continue
 438                     elif blacklist.is_blacklisted(blocked):
 439                         # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!")
 440                         continue
 441                     elif blocked.count("*") > 0:
 442                         # Obscured domain name with no hash
 443                         row = instances.deobscure("*", blocked)
 444
 445                         # DEBUG: print(f"DEBUG: row[]='{type(row)}'")
 446                         if row is None:
 447                             print(f"WARNING: Cannot deobsfucate blocked='{blocked}',domain='{domain}',origin='{origin}' - SKIPPED!")
 448                             continue
 449
 450                         # DEBUG: print(f"DEBUG: blocked='{blocked}' de-obscured to '{row[0]}'")
 451                         blocked      = row[0]
 452                         origin       = row[1]
 453                         nodeinfo_url = row[2]
 454                     elif blocked.count("?") > 0:
 455                         # Obscured domain name with no hash
 456                         row = instances.deobscure("?", blocked)
 457
 458                         # DEBUG: print(f"DEBUG: row[]='{type(row)}'")
 459                         if row is None:
 460                             print(f"WARNING: Cannot deobsfucate blocked='{blocked}',domain='{domain}',origin='{origin}' - SKIPPED!")
 461                             continue
 462
 463                         # DEBUG: print(f"DEBUG: blocked='{blocked}' de-obscured to '{row[0]}'")
 464                         blocked      = row[0]
 465                         origin       = row[1]
 466                         nodeinfo_url = row[2]
 467
 468                     # DEBUG: print(f"DEBUG: blocked='{blocked}'")
 469                     if not validators.domain(blocked):
 470                         print(f"WARNING: blocked='{blocked}',software='pleroma' is not a valid domain name - SKIPPED!")
 471                         continue
 472                     elif blocked.endswith(".arpa"):
 473                         print(f"WARNING: blocked='{blocked}' is a reversed .arpa domain and should not be used generally.")
 474                         continue
 475                     elif blocked.endswith(".tld"):
 476                         print(f"WARNING: blocked='{blocked}' is a fake domain, please don't crawl them!")
 477                         continue
 478                     elif not instances.is_registered(blocked):
 479                         # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'")
 480                         instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url)
 481
 482                     if not blocks.is_instance_blocked(domain, blocked, block_level):
 483                         # DEBUG: print("DEBUG: Blocking:", domain, blocked, block_level)
 484                         blocks.add_instance(domain, blocked, reason, block_level)
 485
 486                         if block_level == "reject":
 487                             # DEBUG: print("DEBUG: Adding to blockdict:", blocked)
 488                             blockdict.append({
 489                                 "blocked": blocked,
 490                                 "reason" : reason
 491                             })
 492                     else:
 493                         # DEBUG: print(f"DEBUG: Updating block last seen for domain='{domain}',blocked='{blocked}' ...")
 494                         blocks.update_reason(reason, domain, blocked, block_level)
 495
 496     fba.connection.commit()
 497     # DEBUG: print("DEBUG: EXIT!")
 498
 499 def fetch_blocks_from_about(domain: str) -> dict:
 500     # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
 501     if not isinstance(domain, str):
 502         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
 503     elif domain == "":
 504         raise ValueError("Parameter 'domain' is empty")
 505
 506     # DEBUG: print(f"DEBUG: Fetching mastodon blocks from domain='{domain}'")
 507     doc = None
 508     for path in ["/instance/about/index.html"]:
 509         try:
 510             # Resetting doc type
 511             doc = None
 512
 513             # DEBUG: print(f"DEBUG: Fetching path='{path}' from domain='{domain}' ...")
 514             response = network.fetch_response(
 515                 domain,
 516                 path,
 517                 network.web_headers,
 518                 (config.get("connection_timeout"), config.get("read_timeout"))
 519             )
 520
 521             # DEBUG: print(f"DEBUG: response.ok='{response.ok}',response.status_code='{response.status_code}',response.text()={len(response.text)}")
 522             if not response.ok or response.text.strip() == "":
 523                 print(f"WARNING: path='{path}' does not exist on domain='{domain}' - SKIPPED!")
 524                 continue
 525
 526             # DEBUG: print(f"DEBUG: Parsing response.text()={len(response.text)} Bytes ...")
 527             doc = bs4.BeautifulSoup(
 528                 response.text,
 529                 "html.parser",
 530             )
 531
 532             # DEBUG: print(f"DEBUG: doc[]='{type(doc)}'")
 533             if doc.find("h2") is not None:
 534                 # DEBUG: print(f"DEBUG: Found 'h2' header in path='{path}' - BREAK!")
 535                 break
 536
 537         except network.exceptions as exception:
 538             print("ERROR: Cannot fetch from domain:", domain, exception)
 539             instances.set_last_error(domain, exception)
 540             break
 541
 542     blocklist = {
 543         "Suspended servers": [],
 544         "Filtered media"   : [],
 545         "Limited servers"  : [],
 546         "Silenced servers" : [],
 547     }
 548
 549     # DEBUG: print(f"DEBUG: doc[]='{type(doc)}'")
 550     if doc is None:
 551         print(f"WARNING: Cannot fetch any /about pages for domain='{domain}' - EXIT!")
 552         return blocklist
 553
 554     for header in doc.find_all("h2"):
 555         header_text = tidyup.reason(header.text)
 556
 557         # DEBUG: print(f"DEBUG: header_text='{header_text}' - BEFORE!")
 558         if header_text in language_mapping:
 559             # DEBUG: print(f"DEBUG: header_text='{header_text}' - FOUND!")
 560             header_text = language_mapping[header_text]
 561         else:
 562             print(f"WARNING: header_text='{header_text}' not found in language mapping table")
 563
 564         # DEBUG: print(f"DEBUG: header_text='{header_text} - AFTER!'")
 565         if header_text in blocklist or header_text.lower() in blocklist:
 566             # replaced find_next_siblings with find_all_next to account for instances that e.g. hide lists in dropdown menu
 567             # DEBUG: print(f"DEBUG: Found header_text='{header_text}', importing domain blocks ...")
 568             for line in header.find_next("table").find_all("tr")[1:]:
 569                 # DEBUG: print(f"DEBUG: line[]='{type(line)}'")
 570                 blocklist[header_text].append({
 571                     "blocked": tidyup.domain(line.find_all("td")[0].text),
 572                     "reason" : tidyup.reason(line.find_all("td")[1].text),
 573                 })
 574         else:
 575             print(f"WARNING: header_text='{header_text}' not found in blocklist()={len(blocklist)}")
 576
 577     # DEBUG: print(f"DEBUG: Returning blocklist for domain='{domain}'")
 578     return {
 579         "reject"        : blocklist["Suspended servers"],
 580         "media_removal" : blocklist["Filtered media"],
 581         "followers_only": blocklist["Limited servers"] + blocklist["Silenced servers"],
 582     }