fba/networks/pleroma.py

   1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
   2 # Copyright (C) 2023 Free Software Foundation
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published
   6 # by the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  16
  17 import inspect
  18 import logging
  19
  20 import bs4
  21 import validators
  22
  23 from fba import database
  24 from fba import utils
  25
  26 from fba.helpers import blacklist
  27 from fba.helpers import config
  28 from fba.helpers import tidyup
  29
  30 from fba.http import federation
  31 from fba.http import network
  32
  33 from fba.models import blocks
  34 from fba.models import instances
  35
  36 logging.basicConfig(level=logging.INFO)
  37 logger = logging.getLogger(__name__)
  38
  39 # Language mapping X -> English
  40 language_mapping = {
  41     # English -> English
  42     "Reject": "Suspended servers",
  43 }
  44
  45 def fetch_blocks(domain: str, origin: str, nodeinfo_url: str):
  46     logger.debug(f"domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}' - CALLED!")
  47     if not isinstance(domain, str):
  48         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
  49     elif domain == "":
  50         raise ValueError("Parameter 'domain' is empty")
  51     elif domain.lower() != domain:
  52         raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
  53     elif not validators.domain(domain.split("/")[0]):
  54         raise ValueError(f"domain='{domain}' is not a valid domain")
  55     elif domain.endswith(".arpa"):
  56         raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!")
  57     elif domain.endswith(".tld"):
  58         raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
  59     elif not isinstance(origin, str) and origin is not None:
  60         raise ValueError(f"Parameter origin[]='{type(origin)}' is not 'str'")
  61     elif origin == "":
  62         raise ValueError("Parameter 'origin' is empty")
  63     elif not isinstance(nodeinfo_url, str):
  64         raise ValueError(f"Parameter nodeinfo_url[]='{type(nodeinfo_url)}' is not 'str'")
  65     elif nodeinfo_url == "":
  66         raise ValueError("Parameter 'nodeinfo_url' is empty")
  67
  68     # @TODO Unused blockdict
  69     blockdict = list()
  70     rows = None
  71     try:
  72         logger.debug(f"Fetching nodeinfo: domain='{domain}',nodeinfo_url='{nodeinfo_url}'")
  73         rows = federation.fetch_nodeinfo(domain, nodeinfo_url)
  74     except network.exceptions as exception:
  75         logger.warning(f"Exception '{type(exception)}' during fetching nodeinfo")
  76         instances.set_last_error(domain, exception)
  77
  78     if rows is None:
  79         logger.warning("Could not fetch nodeinfo from domain:", domain)
  80         return
  81     elif "metadata" not in rows:
  82         logger.warning(f"rows()={len(rows)} does not have key 'metadata', domain='{domain}'")
  83         return
  84     elif "federation" not in rows["metadata"]:
  85         logger.warning(f"rows()={len(rows['metadata'])} does not have key 'federation', domain='{domain}'")
  86         return
  87
  88     data = rows["metadata"]["federation"]
  89     found = False
  90
  91     logger.debug("data[]='%s'", type(data))
  92     if "mrf_simple" in data:
  93         logger.debug("Found mrf_simple:", domain)
  94         found = True
  95         for block_level, blocklist in (
  96             {
  97                 **data["mrf_simple"],
  98                 **{
  99                     "quarantined_instances": data["quarantined_instances"]
 100                 }
 101             }
 102         ).items():
 103             logger.debug("block_level, blocklist():", block_level, len(blocklist))
 104             block_level = tidyup.domain(block_level)
 105             logger.debug("BEFORE block_level:", block_level)
 106
 107             if block_level == "":
 108                 logger.warning("block_level is now empty!")
 109                 continue
 110             elif block_level == "accept":
 111                 logger.debug("domain='%s' skipping block_level='accept'", domain)
 112                 continue
 113
 114             logger.debug(f"Checking {len(blocklist)} entries from domain='{domain}',block_level='{block_level}' ...")
 115             if len(blocklist) > 0:
 116                 for blocked in blocklist:
 117                     logger.debug("BEFORE blocked:", blocked)
 118                     blocked = tidyup.domain(blocked)
 119                     logger.debug("AFTER blocked:", blocked)
 120
 121                     if blocked == "":
 122                         logger.warning("blocked is empty after tidyup.domain():", domain, block_level)
 123                         continue
 124                     elif blacklist.is_blacklisted(blocked):
 125                         logger.debug("blocked='%s' is blacklisted - SKIPPED!", blocked)
 126                         continue
 127                     elif blocked.count("*") > 0:
 128                         # Obscured domain name with no hash
 129                         row = instances.deobscure("*", blocked)
 130
 131                         logger.debug("row[]='%s'", type(row))
 132                         if row is None:
 133                             logger.warning(f"Cannot deobsfucate blocked='{blocked}',domain='{domain}',origin='{origin}' - SKIPPED!")
 134                             continue
 135
 136                         logger.debug(f"blocked='{blocked}' de-obscured to '{row[0]}'")
 137                         blocked      = row[0]
 138                         origin       = row[1]
 139                         nodeinfo_url = row[2]
 140                     elif blocked.count("?") > 0:
 141                         # Obscured domain name with no hash
 142                         row = instances.deobscure("?", blocked)
 143
 144                         logger.debug("row[]='%s'", type(row))
 145                         if row is None:
 146                             logger.warning(f"Cannot deobsfucate blocked='{blocked}',domain='{domain}',origin='{origin}' - SKIPPED!")
 147                             continue
 148
 149                         logger.debug(f"blocked='{blocked}' de-obscured to '{row[0]}'")
 150                         blocked      = row[0]
 151                         origin       = row[1]
 152                         nodeinfo_url = row[2]
 153
 154                     logger.debug(f"blocked='{blocked}'")
 155                     if not utils.is_domain_wanted(blocked):
 156                         logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
 157                         continue
 158                     elif not instances.is_registered(blocked):
 159                         # Commit changes
 160                         logger.debug("Invoking commit() ...")
 161                         database.connection.commit()
 162
 163                         logger.debug(f"Domain blocked='{blocked}' wasn't found, adding ..., domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'")
 164                         instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url)
 165
 166                     if not blocks.is_instance_blocked(domain, blocked, block_level):
 167                         logger.debug("Blocking:", domain, blocked, block_level)
 168                         blocks.add_instance(domain, blocked, None, block_level)
 169
 170                         if block_level == "reject":
 171                             logger.debug("Adding to blockdict:", blocked)
 172                             blockdict.append({
 173                                 "blocked": blocked,
 174                                 "reason" : None
 175                             })
 176                     else:
 177                         logger.debug(f"Updating block last seen for domain='{domain}',blocked='{blocked}' ...")
 178                         blocks.update_last_seen(domain, blocked, block_level)
 179     elif "quarantined_instances" in data:
 180         logger.debug(f"Found 'quarantined_instances' in JSON response: domain='{domain}'")
 181         found = True
 182         block_level = "quarantined"
 183
 184         for blocked in data["quarantined_instances"]:
 185             logger.debug("BEFORE blocked:", blocked)
 186             blocked = tidyup.domain(blocked)
 187             logger.debug("AFTER blocked:", blocked)
 188
 189             if blocked == "":
 190                 logger.warning("blocked is empty after tidyup.domain():", domain, block_level)
 191                 continue
 192             elif blacklist.is_blacklisted(blocked):
 193                 logger.debug("blocked='%s' is blacklisted - SKIPPED!", blocked)
 194                 continue
 195             elif blocked.count("*") > 0:
 196                 # Obscured domain name with no hash
 197                 row = instances.deobscure("*", blocked)
 198
 199                 logger.debug("row[]='%s'", type(row))
 200                 if row is None:
 201                     logger.warning(f"Cannot deobsfucate blocked='{blocked}',domain='{domain}',origin='{origin}' - SKIPPED!")
 202                     continue
 203
 204                 logger.debug(f"blocked='{blocked}' de-obscured to '{row[0]}'")
 205                 blocked      = row[0]
 206                 origin       = row[1]
 207                 nodeinfo_url = row[2]
 208             elif blocked.count("?") > 0:
 209                 # Obscured domain name with no hash
 210                 row = instances.deobscure("?", blocked)
 211
 212                 logger.debug("row[]='%s'", type(row))
 213                 if row is None:
 214                     logger.warning(f"Cannot deobsfucate blocked='{blocked}',domain='{domain}',origin='{origin}' - SKIPPED!")
 215                     continue
 216
 217                 logger.debug(f"blocked='{blocked}' de-obscured to '{row[0]}'")
 218                 blocked      = row[0]
 219                 origin       = row[1]
 220                 nodeinfo_url = row[2]
 221
 222             logger.debug(f"blocked='{blocked}'")
 223             if not utils.is_domain_wanted(blocked):
 224                 logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
 225                 continue
 226             elif not instances.is_registered(blocked):
 227                 # Commit changes
 228                 logger.debug("Invoking commit() ...")
 229                 database.connection.commit()
 230
 231                 logger.debug(f"Domain blocked='{blocked}' wasn't found, adding ..., domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'")
 232                 instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url)
 233
 234             if not blocks.is_instance_blocked(domain, blocked, block_level):
 235                 logger.debug("Blocking:", domain, blocked, block_level)
 236                 blocks.add_instance(domain, blocked, None, block_level)
 237
 238                 if block_level == "reject":
 239                     logger.debug("Adding to blockdict:", blocked)
 240                     blockdict.append({
 241                         "blocked": blocked,
 242                         "reason" : None
 243                     })
 244             else:
 245                 logger.debug(f"Updating block last seen for domain='{domain}',blocked='{blocked}' ...")
 246                 blocks.update_last_seen(domain, blocked, block_level)
 247     else:
 248         logger.warning(f"Cannot find 'mrf_simple' or 'quarantined_instances' in JSON reply: domain='{domain}'")
 249
 250     logger.debug("Invoking commit() ...")
 251     database.connection.commit()
 252
 253     # Reasons
 254     if "mrf_simple_info" in data:
 255         logger.debug("Found mrf_simple_info:", domain)
 256         found = True
 257         for block_level, info in (
 258             {
 259                 **data["mrf_simple_info"],
 260                 **(data["quarantined_instances_info"] if "quarantined_instances_info" in data else {})
 261             }
 262         ).items():
 263             logger.debug("block_level, info.items():", block_level, len(info.items()))
 264             block_level = tidyup.domain(block_level)
 265             logger.debug("BEFORE block_level:", block_level)
 266
 267             if block_level == "":
 268                 logger.warning("block_level is now empty!")
 269                 continue
 270             elif block_level == "accept":
 271                 logger.debug("domain='%s' skipping block_level='accept'", domain)
 272                 continue
 273
 274             logger.debug(f"Checking {len(info.items())} entries from domain='{domain}',block_level='{block_level}' ...")
 275             for blocked, reason in info.items():
 276                 logger.debug(f"blocked='{blocked}',reason[{type(reason)}]='{reason}' - BEFORE!")
 277                 blocked = tidyup.domain(blocked)
 278
 279                 if isinstance(reason, str):
 280                     logger.debug("reason[] is a string")
 281                     reason = tidyup.reason(reason)
 282                 elif isinstance(reason, dict) and "reason" in reason:
 283                     logger.debug("reason[] is a dict")
 284                     reason = tidyup.reason(reason["reason"])
 285                 elif reason is not None:
 286                     raise ValueError(f"Cannot handle reason[]='{type(reason)}'")
 287
 288                 logger.debug("blocked='%s',reason='%s' - AFTER!", blocked, reason)
 289
 290                 if blocked == "":
 291                     logger.warning("blocked is empty after tidyup.domain():", domain, block_level)
 292                     continue
 293                 elif blacklist.is_blacklisted(blocked):
 294                     logger.debug("blocked='%s' is blacklisted - SKIPPED!", blocked)
 295                     continue
 296                 elif blocked.count("*") > 0:
 297                     # Obscured domain name with no hash
 298                     row = instances.deobscure("*", blocked)
 299
 300                     logger.debug("row[]='%s'", type(row))
 301                     if row is None:
 302                         logger.warning(f"Cannot deobsfucate blocked='{blocked}',domain='{domain}',origin='{origin}' - SKIPPED!")
 303                         continue
 304
 305                     logger.debug(f"blocked='{blocked}' de-obscured to '{row[0]}'")
 306                     blocked      = row[0]
 307                     origin       = row[1]
 308                     nodeinfo_url = row[2]
 309                 elif blocked.count("?") > 0:
 310                     # Obscured domain name with no hash
 311                     row = instances.deobscure("?", blocked)
 312
 313                     logger.debug("row[]='%s'", type(row))
 314                     if row is None:
 315                         logger.warning(f"Cannot deobsfucate blocked='{blocked}',domain='{domain}',origin='{origin}' - SKIPPED!")
 316                         continue
 317
 318                     logger.debug(f"blocked='{blocked}' de-obscured to '{row[0]}'")
 319                     blocked      = row[0]
 320                     origin       = row[1]
 321                     nodeinfo_url = row[2]
 322
 323                 logger.debug(f"blocked='{blocked}'")
 324                 if not utils.is_domain_wanted(blocked):
 325                     logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
 326                     continue
 327                 elif not instances.is_registered(blocked):
 328                     logger.debug(f"Domain blocked='{blocked}' wasn't found, adding ..., domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'")
 329                     instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url)
 330
 331                 logger.debug(f"Updating block reason: reason='{reason}',domain='{domain}',blocked='{blocked}',block_level='{block_level}'")
 332                 blocks.update_reason(reason, domain, blocked, block_level)
 333
 334                 logger.debug(f"blockdict()={len(blockdict)}")
 335                 for entry in blockdict:
 336                     if entry["blocked"] == blocked:
 337                         logger.debug(f"Updating entry reason: blocked='{blocked}',reason='{reason}'")
 338                         entry["reason"] = reason
 339
 340     elif "quarantined_instances_info" in data and "quarantined_instances" in data["quarantined_instances_info"]:
 341         logger.debug(f"Found 'quarantined_instances_info' in JSON response: domain='{domain}'")
 342         found = True
 343         block_level = "quarantined"
 344
 345         #print(data["quarantined_instances_info"])
 346         rows = data["quarantined_instances_info"]["quarantined_instances"]
 347         for blocked in rows:
 348             logger.debug("BEFORE blocked:", blocked)
 349             blocked = tidyup.domain(blocked)
 350             logger.debug("AFTER blocked:", blocked)
 351
 352             if blocked not in rows or "reason" not in rows[blocked]:
 353                 logger.warning(f"Cannot find blocked='{blocked}' in rows()={len(rows)},domain='{domain}'")
 354                 break
 355
 356             reason = rows[blocked]["reason"]
 357             logger.debug(f"reason='{reason}'")
 358
 359             if blocked == "":
 360                 logger.warning("blocked is empty after tidyup.domain():", domain, block_level)
 361                 continue
 362             elif blacklist.is_blacklisted(blocked):
 363                 logger.debug("blocked='%s' is blacklisted - SKIPPED!", blocked)
 364                 continue
 365             elif blocked.count("*") > 0:
 366                 # Obscured domain name with no hash
 367                 row = instances.deobscure("*", blocked)
 368
 369                 logger.debug("row[]='%s'", type(row))
 370                 if row is None:
 371                     logger.warning(f"Cannot deobsfucate blocked='{blocked}',domain='{domain}',origin='{origin}' - SKIPPED!")
 372                     continue
 373
 374                 logger.debug(f"blocked='{blocked}' de-obscured to '{row[0]}'")
 375                 blocked      = row[0]
 376                 origin       = row[1]
 377                 nodeinfo_url = row[2]
 378             elif blocked.count("?") > 0:
 379                 # Obscured domain name with no hash
 380                 row = instances.deobscure("?", blocked)
 381
 382                 logger.debug("row[]='%s'", type(row))
 383                 if row is None:
 384                     logger.warning(f"Cannot deobsfucate blocked='{blocked}',domain='{domain}',origin='{origin}' - SKIPPED!")
 385                     continue
 386
 387                 logger.debug(f"blocked='{blocked}' de-obscured to '{row[0]}'")
 388                 blocked      = row[0]
 389                 origin       = row[1]
 390                 nodeinfo_url = row[2]
 391
 392             logger.debug(f"blocked='{blocked}'")
 393             if not utils.is_domain_wanted(blocked):
 394                 logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
 395                 continue
 396             elif not instances.is_registered(blocked):
 397                 logger.debug(f"Domain blocked='{blocked}' wasn't found, adding ..., domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'")
 398                 instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url)
 399
 400             logger.debug(f"Updating block reason: reason='{reason}',domain='{domain}',blocked='{blocked}',block_level='{block_level}'")
 401             blocks.update_reason(reason, domain, blocked, block_level)
 402
 403             logger.debug(f"blockdict()={len(blockdict)}")
 404             for entry in blockdict:
 405                 if entry["blocked"] == blocked:
 406                     logger.debug(f"Updating entry reason: blocked='{blocked}',reason='{reason}'")
 407                     entry["reason"] = reason
 408     else:
 409         logger.warning(f"Cannot find 'mrf_simple_info' or 'quarantined_instances_info' in JSON reply: domain='{domain}'")
 410
 411     if not found:
 412         logger.debug(f"Did not find any useable JSON elements, domain='{domain}', continuing with /about page ...")
 413         blocklist = fetch_blocks_from_about(domain)
 414
 415         logger.debug(f"blocklist()={len(blocklist)}")
 416         if len(blocklist) > 0:
 417             logger.info("Checking %d record(s) ...", len(blocklist))
 418             for block_level in blocklist:
 419                 logger.debug("block_level='%s'", block_level)
 420
 421                 rows = blocklist[block_level]
 422                 logger.debug(f"rows['{type(rows)}]()={len(rows)}'")
 423                 for record in rows:
 424                     logger.debug(f"record[]='{type(record)}'")
 425                     blocked = tidyup.domain(record["blocked"])
 426                     reason  = tidyup.reason(record["reason"])
 427                     logger.debug("blocked='%s',reason='%s' - AFTER!", blocked, reason)
 428
 429                     if blocked == "":
 430                         logger.warning("blocked is empty after tidyup.domain():", domain, block_level)
 431                         continue
 432                     elif blacklist.is_blacklisted(blocked):
 433                         logger.debug("blocked='%s' is blacklisted - SKIPPED!", blocked)
 434                         continue
 435                     elif blocked.count("*") > 0:
 436                         # Obscured domain name with no hash
 437                         row = instances.deobscure("*", blocked)
 438
 439                         logger.debug("row[]='%s'", type(row))
 440                         if row is None:
 441                             logger.warning(f"Cannot deobsfucate blocked='{blocked}',domain='{domain}',origin='{origin}' - SKIPPED!")
 442                             continue
 443
 444                         logger.debug(f"blocked='{blocked}' de-obscured to '{row[0]}'")
 445                         blocked      = row[0]
 446                         origin       = row[1]
 447                         nodeinfo_url = row[2]
 448                     elif blocked.count("?") > 0:
 449                         # Obscured domain name with no hash
 450                         row = instances.deobscure("?", blocked)
 451
 452                         logger.debug("row[]='%s'", type(row))
 453                         if row is None:
 454                             logger.warning(f"Cannot deobsfucate blocked='{blocked}',domain='{domain}',origin='{origin}' - SKIPPED!")
 455                             continue
 456
 457                         logger.debug(f"blocked='{blocked}' de-obscured to '{row[0]}'")
 458                         blocked      = row[0]
 459                         origin       = row[1]
 460                         nodeinfo_url = row[2]
 461
 462                     logger.debug(f"blocked='{blocked}'")
 463                     if not utils.is_domain_wanted(blocked):
 464                         logger.warning("blocked='%s' is not wanted - SKIPPED!", blocked)
 465                         continue
 466                     elif not instances.is_registered(blocked):
 467                         logger.debug(f"Domain blocked='{blocked}' wasn't found, adding ..., domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'")
 468                         instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url)
 469
 470                     if not blocks.is_instance_blocked(domain, blocked, block_level):
 471                         logger.debug("Blocking:", domain, blocked, block_level)
 472                         blocks.add_instance(domain, blocked, reason, block_level)
 473
 474                         if block_level == "reject":
 475                             logger.debug("Adding to blockdict:", blocked)
 476                             blockdict.append({
 477                                 "blocked": blocked,
 478                                 "reason" : reason
 479                             })
 480                     else:
 481                         logger.debug(f"Updating block last seen for domain='{domain}',blocked='{blocked}' ...")
 482                         blocks.update_reason(reason, domain, blocked, block_level)
 483
 484     logger.debug("Invoking commit() ...")
 485     database.connection.commit()
 486
 487     logger.debug("EXIT!")
 488
 489 def fetch_blocks_from_about(domain: str) -> dict:
 490     logger.debug("domain(%d)='%s' - CALLED!", len(domain), domain)
 491     if not isinstance(domain, str):
 492         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
 493     elif domain == "":
 494         raise ValueError("Parameter 'domain' is empty")
 495     elif domain.lower() != domain:
 496         raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
 497     elif not validators.domain(domain.split("/")[0]):
 498         raise ValueError(f"domain='{domain}' is not a valid domain")
 499     elif domain.endswith(".arpa"):
 500         raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!")
 501     elif domain.endswith(".tld"):
 502         raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
 503
 504     logger.debug(f"Fetching mastodon blocks from domain='{domain}'")
 505     doc = None
 506     for path in ["/instance/about/index.html"]:
 507         try:
 508             # Resetting doc type
 509             doc = None
 510
 511             logger.debug(f"Fetching path='{path}' from domain='{domain}' ...")
 512             response = network.fetch_response(
 513                 domain,
 514                 path,
 515                 network.web_headers,
 516                 (config.get("connection_timeout"), config.get("read_timeout"))
 517             )
 518
 519             logger.debug(f"response.ok='{response.ok}',response.status_code='{response.status_code}',response.text()={len(response.text)}")
 520             if not response.ok or response.text.strip() == "":
 521                 logger.warning(f"path='{path}' does not exist on domain='{domain}' - SKIPPED!")
 522                 continue
 523
 524             logger.debug(f"Parsing response.text()={len(response.text)} Bytes ...")
 525             doc = bs4.BeautifulSoup(
 526                 response.text,
 527                 "html.parser",
 528             )
 529
 530             logger.debug("doc[]='%s'", type(doc))
 531             if doc.find("h2") is not None:
 532                 logger.debug(f"Found 'h2' header in path='{path}' - BREAK!")
 533                 break
 534
 535         except network.exceptions as exception:
 536             logger.warning("Cannot fetch from domain:", domain, exception)
 537             instances.set_last_error(domain, exception)
 538             break
 539
 540     blocklist = {
 541         "Suspended servers": [],
 542         "Filtered media"   : [],
 543         "Limited servers"  : [],
 544         "Silenced servers" : [],
 545     }
 546
 547     logger.debug("doc[]='%s'", type(doc))
 548     if doc is None:
 549         logger.warning(f"Cannot fetch any /about pages for domain='{domain}' - EXIT!")
 550         return blocklist
 551
 552     for header in doc.find_all("h2"):
 553         header_text = tidyup.reason(header.text)
 554
 555         logger.debug(f"header_text='{header_text}' - BEFORE!")
 556         if header_text in language_mapping:
 557             logger.debug(f"header_text='{header_text}' - FOUND!")
 558             header_text = language_mapping[header_text]
 559         else:
 560             logger.warning(f"header_text='{header_text}' not found in language mapping table")
 561
 562         logger.debug(f"header_text='{header_text} - AFTER!'")
 563         if header_text in blocklist or header_text.lower() in blocklist:
 564             # replaced find_next_siblings with find_all_next to account for instances that e.g. hide lists in dropdown menu
 565             logger.debug(f"Found header_text='{header_text}', importing domain blocks ...")
 566             for line in header.find_next("table").find_all("tr")[1:]:
 567                 logger.debug(f"line[]='{type(line)}'")
 568                 blocklist[header_text].append({
 569                     "blocked": tidyup.domain(line.find_all("td")[0].text),
 570                     "reason" : tidyup.reason(line.find_all("td")[1].text),
 571                 })
 572         else:
 573             logger.warning(f"header_text='{header_text}' not found in blocklist()={len(blocklist)}")
 574
 575     logger.debug(f"Returning blocklist for domain='{domain}'")
 576     return {
 577         "reject"        : blocklist["Suspended servers"],
 578         "media_removal" : blocklist["Filtered media"],
 579         "followers_only": blocklist["Limited servers"] + blocklist["Silenced servers"],
 580     }