fba/networks/pleroma.py

   1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
   2 # Copyright (C) 2023 Free Software Foundation
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published
   6 # by the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  16
  17 import inspect
  18 import logging
  19
  20 import bs4
  21
  22 from fba import database
  23 from fba import utils
  24
  25 from fba.helpers import blacklist
  26 from fba.helpers import config
  27 from fba.helpers import domain as domain_helper
  28 from fba.helpers import tidyup
  29
  30 from fba.http import federation
  31 from fba.http import network
  32
  33 from fba.models import blocks
  34 from fba.models import instances
  35
  36 logging.basicConfig(level=logging.INFO)
  37 logger = logging.getLogger(__name__)
  38
  39 # Language mapping X -> English
  40 language_mapping = {
  41     # English -> English
  42     "Reject": "Suspended servers",
  43 }
  44
  45 def fetch_blocks(domain: str, origin: str, nodeinfo_url: str):
  46     logger.debug(f"domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}' - CALLED!")
  47     domain_helper.raise_on(domain)
  48     if not isinstance(origin, str) and origin is not None:
  49         raise ValueError(f"Parameter origin[]='{type(origin)}' is not 'str'")
  50     elif origin == "":
  51         raise ValueError("Parameter 'origin' is empty")
  52     elif not isinstance(nodeinfo_url, str):
  53         raise ValueError(f"Parameter nodeinfo_url[]='{type(nodeinfo_url)}' is not 'str'")
  54     elif nodeinfo_url == "":
  55         raise ValueError("Parameter 'nodeinfo_url' is empty")
  56
  57     # @TODO Unused blockdict
  58     blockdict = list()
  59     rows = None
  60     try:
  61         logger.debug(f"Fetching nodeinfo: domain='{domain}',nodeinfo_url='{nodeinfo_url}'")
  62         rows = federation.fetch_nodeinfo(domain, nodeinfo_url)
  63     except network.exceptions as exception:
  64         logger.warning("Exception '%s' during fetching nodeinfo from domain='%s'", type(exception), domain)
  65         instances.set_last_error(domain, exception)
  66
  67     if rows is None:
  68         logger.warning("Could not fetch nodeinfo from domain:", domain)
  69         return
  70     elif "metadata" not in rows:
  71         logger.warning("rows()=%d does not have key 'metadata', domain='%s'", len(rows), domain)
  72         return
  73     elif "federation" not in rows["metadata"]:
  74         logger.warning("rows()=%d does not have key 'federation', domain='%s'", len(rows['metadata']), domain)
  75         return
  76
  77     data = rows["metadata"]["federation"]
  78     found = False
  79
  80     logger.debug("data[]='%s'", type(data))
  81     if "mrf_simple" in data:
  82         logger.debug("Found mrf_simple:", domain)
  83         found = True
  84         for block_level, blocklist in (
  85             {
  86                 **data["mrf_simple"],
  87                 **{
  88                     "quarantined_instances": data["quarantined_instances"]
  89                 }
  90             }
  91         ).items():
  92             logger.debug("block_level='%s', blocklist()=%d", block_level, len(blocklist))
  93             block_level = tidyup.domain(block_level)
  94             logger.debug("block_level='%s' - AFTER!", block_level)
  95
  96             if block_level == "":
  97                 logger.warning("block_level is now empty!")
  98                 continue
  99             elif block_level == "accept":
 100                 logger.debug("domain='%s' skipping block_level='accept'", domain)
 101                 continue
 102
 103             logger.debug("Checking %d entries from domain='%s',block_level='%s' ...", len(blocklist), domain, block_level)
 104             if len(blocklist) > 0:
 105                 for blocked in blocklist:
 106                     logger.debug("blocked='%s' - BEFORE!", blocked)
 107                     blocked = tidyup.domain(blocked)
 108                     logger.debug("blocked='%s' - AFTER!", blocked)
 109
 110                     if blocked == "":
 111                         logger.warning("blocked is empty after tidyup.domain():", domain, block_level)
 112                         continue
 113                     elif blocked.endswith(".arpa"):
 114                         logger.debug("blocked='%s' is a reverse IP address - SKIPPED!", blocked)
 115                         continue
 116                     elif blocked.endswith(".tld"):
 117                         logger.debug("blocked='%s' is a fake domain - SKIPPED!", blocked)
 118                         continue
 119                     elif blacklist.is_blacklisted(blocked):
 120                         logger.debug("blocked='%s' is blacklisted - SKIPPED!", blocked)
 121                         continue
 122                     elif blocked.count("*") > 0:
 123                         # Obscured domain name with no hash
 124                         row = instances.deobscure("*", blocked)
 125
 126                         logger.debug("row[]='%s'", type(row))
 127                         if row is None:
 128                             logger.warning("Cannot deobsfucate blocked='%s',domain='%s',origin='%s' - SKIPPED!", blocked, domain, origin)
 129                             continue
 130
 131                         logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0])
 132                         blocked      = row[0]
 133                         origin       = row[1]
 134                         nodeinfo_url = row[2]
 135                     elif blocked.count("?") > 0:
 136                         # Obscured domain name with no hash
 137                         row = instances.deobscure("?", blocked)
 138
 139                         logger.debug("row[]='%s'", type(row))
 140                         if row is None:
 141                             logger.warning("Cannot deobsfucate blocked='%s',domain='%s',origin='%s' - SKIPPED!", blocked, domain, origin)
 142                             continue
 143
 144                         logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0])
 145                         blocked      = row[0]
 146                         origin       = row[1]
 147                         nodeinfo_url = row[2]
 148
 149                     logger.debug("blocked='%s'", blocked)
 150                     if not utils.is_domain_wanted(blocked):
 151                         logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
 152                         continue
 153                     elif not instances.is_registered(blocked):
 154                         # Commit changes
 155                         logger.debug("Invoking commit() ...")
 156                         database.connection.commit()
 157
 158                         logger.debug("Domain blocked='%s' wasn't found, adding ..., domain='%s',origin='%s',nodeinfo_url='%s'", blocked, domain, origin, nodeinfo_url)
 159                         instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url)
 160
 161                     if not blocks.is_instance_blocked(domain, blocked, block_level):
 162                         logger.debug("Blocking domain='%s',blocked='%s',block_level='%s' ...", domain, blocked, block_level)
 163                         blocks.add_instance(domain, blocked, None, block_level)
 164
 165                         if block_level == "reject":
 166                             logger.debug("Appending blocked='%s' ...", blocked)
 167                             blockdict.append({
 168                                 "blocked": blocked,
 169                                 "reason" : None
 170                             })
 171                     else:
 172                         logger.debug("Updating block last seen for domain='%s',blocked='%s',block_level='%s' ...", domain, blocked, block_level)
 173                         blocks.update_last_seen(domain, blocked, block_level)
 174     elif "quarantined_instances" in data:
 175         logger.debug("Found 'quarantined_instances' in JSON response: domain='%s'", domain)
 176         found = True
 177         block_level = "quarantined"
 178
 179         for blocked in data["quarantined_instances"]:
 180             logger.debug("blocked='%s' - BEFORE!", blocked)
 181             blocked = tidyup.domain(blocked)
 182             logger.debug("blocked='%s' - AFTER!", blocked)
 183
 184             if blocked == "":
 185                 logger.warning("blocked is empty after tidyup.domain():", domain, block_level)
 186                 continue
 187             elif blocked.endswith(".arpa"):
 188                 logger.debug("blocked='%s' is a reverse IP address - SKIPPED!", blocked)
 189                 continue
 190             elif blocked.endswith(".tld"):
 191                 logger.debug("blocked='%s' is a fake domain - SKIPPED!", blocked)
 192                 continue
 193             elif blacklist.is_blacklisted(blocked):
 194                 logger.debug("blocked='%s' is blacklisted - SKIPPED!", blocked)
 195                 continue
 196             elif blocked.count("*") > 0:
 197                 # Obscured domain name with no hash
 198                 row = instances.deobscure("*", blocked)
 199
 200                 logger.debug("row[]='%s'", type(row))
 201                 if row is None:
 202                     logger.warning("Cannot deobsfucate blocked='%s',domain='%s',origin='%s' - SKIPPED!", blocked, domain, origin)
 203                     continue
 204
 205                 logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0])
 206                 blocked      = row[0]
 207                 origin       = row[1]
 208                 nodeinfo_url = row[2]
 209             elif blocked.count("?") > 0:
 210                 # Obscured domain name with no hash
 211                 row = instances.deobscure("?", blocked)
 212
 213                 logger.debug("row[]='%s'", type(row))
 214                 if row is None:
 215                     logger.warning("Cannot deobsfucate blocked='%s',domain='%s',origin='%s' - SKIPPED!", blocked, domain, origin)
 216                     continue
 217
 218                 logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0])
 219                 blocked      = row[0]
 220                 origin       = row[1]
 221                 nodeinfo_url = row[2]
 222
 223             logger.debug("blocked='%s' - DEOBSFUCATED!", blocked)
 224             if not utils.is_domain_wanted(blocked):
 225                 logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
 226                 continue
 227             elif not instances.is_registered(blocked):
 228                 # Commit changes
 229                 logger.debug("Invoking commit() ...")
 230                 database.connection.commit()
 231
 232                 logger.debug("Domain blocked='%s' wasn't found, adding ..., domain='%s',origin='%s',nodeinfo_url='{nodeinfo_url}'", blocked, domain, origin)
 233                 instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url)
 234
 235             if not blocks.is_instance_blocked(domain, blocked, block_level):
 236                 logger.debug("Blocking domain='%s',blocked='%s',block_level='%s' ...", domain, blocked, block_level)
 237                 blocks.add_instance(domain, blocked, None, block_level)
 238
 239                 if block_level == "reject":
 240                     logger.debug("Appending blocked='%s' ...", blocked)
 241                     blockdict.append({
 242                         "blocked": blocked,
 243                         "reason" : None
 244                     })
 245             else:
 246                 logger.debug("Updating block last seen for domain='%s',blocked='%s',block_level='%s' ...", domain, blocked, block_level)
 247                 blocks.update_last_seen(domain, blocked, block_level)
 248     else:
 249         logger.warning("Cannot find 'mrf_simple' or 'quarantined_instances' in JSON reply: domain='%s'", domain)
 250
 251     logger.debug("Invoking commit() ...")
 252     database.connection.commit()
 253
 254     # Reasons
 255     if "mrf_simple_info" in data:
 256         logger.debug("Found mrf_simple_info in API response: domain='%s'", domain)
 257         found = True
 258         for block_level, info in (
 259             {
 260                 **data["mrf_simple_info"],
 261                 **(data["quarantined_instances_info"] if "quarantined_instances_info" in data else {})
 262             }
 263         ).items():
 264             logger.debug("block_level='%s', info.items()=%d", block_level, len(info.items()))
 265             block_level = tidyup.domain(block_level)
 266             logger.debug("block_level='%s' - AFTER!", block_level)
 267
 268             if block_level == "":
 269                 logger.warning("block_level is now empty!")
 270                 continue
 271             elif block_level == "accept":
 272                 logger.debug("domain='%s' skipping block_level='accept'", domain)
 273                 continue
 274
 275             logger.debug("Checking %d entries from domain='%s',block_level='%s' ...", len(info.items()), domain, block_level)
 276             for blocked, reason in info.items():
 277                 logger.debug("blocked='%s',reason[%s]='%s' - BEFORE!", blocked, type(reason), reason)
 278                 blocked = tidyup.domain(blocked)
 279                 logger.debug("blocked='%s' - AFTER!", blocked)
 280
 281                 if isinstance(reason, str):
 282                     logger.debug("reason[] is a string")
 283                     reason = tidyup.reason(reason)
 284                 elif isinstance(reason, dict) and "reason" in reason:
 285                     logger.debug("reason[] is a dict")
 286                     reason = tidyup.reason(reason["reason"])
 287                 elif reason is not None:
 288                     raise ValueError(f"Cannot handle reason[]='{type(reason)}'")
 289
 290                 logger.debug("blocked='%s',reason='%s' - AFTER!", blocked, reason)
 291
 292                 if blocked == "":
 293                     logger.warning("blocked is empty after tidyup.domain():", domain, block_level)
 294                     continue
 295                 elif blacklist.is_blacklisted(blocked):
 296                     logger.debug("blocked='%s' is blacklisted - SKIPPED!", blocked)
 297                     continue
 298                 elif blocked.count("*") > 0:
 299                     # Obscured domain name with no hash
 300                     row = instances.deobscure("*", blocked)
 301
 302                     logger.debug("row[]='%s'", type(row))
 303                     if row is None:
 304                         logger.warning(f"Cannot deobsfucate blocked='{blocked}',domain='{domain}',origin='{origin}' - SKIPPED!")
 305                         continue
 306
 307                     logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0])
 308                     blocked      = row[0]
 309                     origin       = row[1]
 310                     nodeinfo_url = row[2]
 311                 elif blocked.count("?") > 0:
 312                     # Obscured domain name with no hash
 313                     row = instances.deobscure("?", blocked)
 314
 315                     logger.debug("row[]='%s'", type(row))
 316                     if row is None:
 317                         logger.warning(f"Cannot deobsfucate blocked='{blocked}',domain='{domain}',origin='{origin}' - SKIPPED!")
 318                         continue
 319
 320                     logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0])
 321                     blocked      = row[0]
 322                     origin       = row[1]
 323                     nodeinfo_url = row[2]
 324
 325                 logger.debug("blocked='%s' - DEOBSFUCATED!", blocked)
 326                 if not utils.is_domain_wanted(blocked):
 327                     logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
 328                     continue
 329                 elif not instances.is_registered(blocked):
 330                     logger.debug("Domain blocked='%s' wasn't found, adding ..., domain='%s',origin='%s',nodeinfo_url='%s'", blocked, domain, origin, nodeinfo_url)
 331                     instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url)
 332
 333                 logger.debug("Updating block reason: reason='%s',domain='%s',blocked='%s',block_level='%s'", reason, domain, blocked, block_level)
 334                 blocks.update_reason(reason, domain, blocked, block_level)
 335
 336                 logger.debug("Checking %d blockdict records ...", len(blockdict))
 337                 for entry in blockdict:
 338                     if entry["blocked"] == blocked:
 339                         logger.debug("Updating entry reason: blocked='%s',reason='%s'", blocked, reason)
 340                         entry["reason"] = reason
 341
 342     elif "quarantined_instances_info" in data and "quarantined_instances" in data["quarantined_instances_info"]:
 343         logger.debug("Found 'quarantined_instances_info' in JSON response: domain='%s'", domain)
 344         found = True
 345         block_level = "quarantined"
 346
 347         #print(data["quarantined_instances_info"])
 348         rows = data["quarantined_instances_info"]["quarantined_instances"]
 349         for blocked in rows:
 350             logger.debug("blocked='%s' - BEFORE!", blocked)
 351             blocked = tidyup.domain(blocked)
 352             logger.debug("blocked='%s' - AFTER!", blocked)
 353
 354             if blocked not in rows or "reason" not in rows[blocked]:
 355                 logger.warning("Cannot find blocked='%s' in rows()=%d,domain='%s' - BREAK!", blocked, len(rows), domain)
 356                 break
 357
 358             reason = rows[blocked]["reason"]
 359             logger.debug("reason='%s'", reason)
 360
 361             if blocked == "":
 362                 logger.warning("blocked is empty after tidyup.domain():", domain, block_level)
 363                 continue
 364             elif blacklist.is_blacklisted(blocked):
 365                 logger.debug("blocked='%s' is blacklisted - SKIPPED!", blocked)
 366                 continue
 367             elif blocked.count("*") > 0:
 368                 # Obscured domain name with no hash
 369                 row = instances.deobscure("*", blocked)
 370
 371                 logger.debug("row[]='%s'", type(row))
 372                 if row is None:
 373                     logger.warning(f"Cannot deobsfucate blocked='{blocked}',domain='{domain}',origin='{origin}' - SKIPPED!")
 374                     continue
 375
 376                 logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0])
 377                 blocked      = row[0]
 378                 origin       = row[1]
 379                 nodeinfo_url = row[2]
 380             elif blocked.count("?") > 0:
 381                 # Obscured domain name with no hash
 382                 row = instances.deobscure("?", blocked)
 383
 384                 logger.debug("row[]='%s'", type(row))
 385                 if row is None:
 386                     logger.warning(f"Cannot deobsfucate blocked='{blocked}',domain='{domain}',origin='{origin}' - SKIPPED!")
 387                     continue
 388
 389                 logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0])
 390                 blocked      = row[0]
 391                 origin       = row[1]
 392                 nodeinfo_url = row[2]
 393
 394             logger.debug("blocked='%s'", blocked)
 395             if not utils.is_domain_wanted(blocked):
 396                 logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
 397                 continue
 398             elif not instances.is_registered(blocked):
 399                 logger.debug("Domain blocked='%s' wasn't found, adding ..., domain='%s',origin='%s',nodeinfo_url='%s'", blocked, domain, origin, nodeinfo_url)
 400                 instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url)
 401
 402             logger.debug("Updating block reason: reason='%s',domain='%s',blocked='%s',block_level='%s'", reason, domain, blocked, block_level)
 403             blocks.update_reason(reason, domain, blocked, block_level)
 404
 405             logger.debug("Checking %d blockdict records ...", len(blockdict))
 406             for entry in blockdict:
 407                 if entry["blocked"] == blocked:
 408                     logger.debug("Updating entry reason: blocked='%s',reason='%s'", blocked, reason)
 409                     entry["reason"] = reason
 410     else:
 411         logger.warning("Cannot find 'mrf_simple_info' or 'quarantined_instances_info' in JSON reply: domain='%s'", domain)
 412
 413     if not found:
 414         logger.debug("Did not find any useable JSON elements, domain='%s', continuing with /about page ...", domain)
 415         blocklist = fetch_blocks_from_about(domain)
 416
 417         logger.debug("blocklist()=%d", len(blocklist))
 418         if len(blocklist) > 0:
 419             logger.info("Checking %d record(s) ...", len(blocklist))
 420             for block_level in blocklist:
 421                 logger.debug("block_level='%s'", block_level)
 422
 423                 rows = blocklist[block_level]
 424                 logger.debug("rows[%s]()=%d'", type(rows), len(rows))
 425                 for record in rows:
 426                     logger.debug("record[]='%s'", type(record))
 427                     blocked = tidyup.domain(record["blocked"])
 428                     reason  = tidyup.reason(record["reason"])
 429                     logger.debug("blocked='%s',reason='%s' - AFTER!", blocked, reason)
 430
 431                     if blocked == "":
 432                         logger.warning("blocked is empty after tidyup.domain():", domain, block_level)
 433                         continue
 434                     elif blacklist.is_blacklisted(blocked):
 435                         logger.debug("blocked='%s' is blacklisted - SKIPPED!", blocked)
 436                         continue
 437                     elif blocked.count("*") > 0:
 438                         # Obscured domain name with no hash
 439                         row = instances.deobscure("*", blocked)
 440
 441                         logger.debug("row[]='%s'", type(row))
 442                         if row is None:
 443                             logger.warning("Cannot deobsfucate blocked='%s',domain='%s',origin='%s' - SKIPPED!", blocked, domain, origin)
 444                             continue
 445
 446                         logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0])
 447                         blocked      = row[0]
 448                         origin       = row[1]
 449                         nodeinfo_url = row[2]
 450                     elif blocked.count("?") > 0:
 451                         # Obscured domain name with no hash
 452                         row = instances.deobscure("?", blocked)
 453
 454                         logger.debug("row[]='%s'", type(row))
 455                         if row is None:
 456                             logger.warning("Cannot deobsfucate blocked='%s',domain='%s',origin='%s' - SKIPPED!", blocked, domain, origin)
 457                             continue
 458
 459                         logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0])
 460                         blocked      = row[0]
 461                         origin       = row[1]
 462                         nodeinfo_url = row[2]
 463
 464                     logger.debug("blocked='%s' - DEOBSFUCATED!", blocked)
 465                     if not utils.is_domain_wanted(blocked):
 466                         logger.warning("blocked='%s' is not wanted - SKIPPED!", blocked)
 467                         continue
 468                     elif not instances.is_registered(blocked):
 469                         logger.debug("Domain blocked='%s' wasn't found, adding ..., domain='%s',origin='%s',nodeinfo_url='%s'", blocked, domain, origin, nodeinfo_url)
 470                         instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url)
 471
 472                     if not blocks.is_instance_blocked(domain, blocked, block_level):
 473                         logger.debug("Blocking domain='%s',blocked='%s', block_level='%s' ...", domain, blocked, block_level)
 474                         blocks.add_instance(domain, blocked, reason, block_level)
 475
 476                         if block_level == "reject":
 477                             logger.debug("Appending blocked='%s' ...", blocked)
 478                             blockdict.append({
 479                                 "blocked": blocked,
 480                                 "reason" : reason
 481                             })
 482                     else:
 483                         logger.debug("Updating block last seen for domain='%s',blocked='%s',block_level='%s' ...", domain, blocked, block_level)
 484                         blocks.update_reason(reason, domain, blocked, block_level)
 485
 486     logger.debug("Invoking commit() ...")
 487     database.connection.commit()
 488
 489     logger.debug("EXIT!")
 490
 491 def fetch_blocks_from_about(domain: str) -> dict:
 492     logger.debug("domain(%d)='%s' - CALLED!", len(domain), domain)
 493     domain_helper.raise_on(domain)
 494
 495     logger.debug("Fetching mastodon blocks from domain='%s'", domain)
 496     doc = None
 497     for path in ["/instance/about/index.html"]:
 498         try:
 499             # Resetting doc type
 500             doc = None
 501
 502             logger.debug("Fetching path='%s' from domain='%s' ...", path, domain)
 503             response = network.fetch_response(
 504                 domain,
 505                 path,
 506                 network.web_headers,
 507                 (config.get("connection_timeout"), config.get("read_timeout"))
 508             )
 509
 510             logger.debug("response.ok='%s',response.status_code='%d',response.text()=%d", response.ok, response.status_code, len(response.text))
 511             if not response.ok or response.text.strip() == "":
 512                 logger.warning("path='%s' does not exist on domain='%s' - SKIPPED!", path, domain)
 513                 continue
 514
 515             logger.debug("Parsing response.text()=%d Bytes ...", len(response.text))
 516             doc = bs4.BeautifulSoup(
 517                 response.text,
 518                 "html.parser",
 519             )
 520
 521             logger.debug("doc[]='%s'", type(doc))
 522             if doc.find("h2") is not None:
 523                 logger.debug("Found 'h2' header in path='%s' - BREAK!", path)
 524                 break
 525
 526         except network.exceptions as exception:
 527             logger.warning("Cannot fetch from domain:", domain, exception)
 528             instances.set_last_error(domain, exception)
 529             break
 530
 531     blocklist = {
 532         "Suspended servers": [],
 533         "Filtered media"   : [],
 534         "Limited servers"  : [],
 535         "Silenced servers" : [],
 536     }
 537
 538     logger.debug("doc[]='%s'", type(doc))
 539     if doc is None:
 540         logger.warning("Cannot fetch any /about pages for domain='%s' - EXIT!", domain)
 541         return blocklist
 542
 543     for header in doc.find_all("h2"):
 544         header_text = tidyup.reason(header.text)
 545
 546         logger.debug("header_text='%s' - BEFORE!", header_text)
 547         if header_text in language_mapping:
 548             logger.debug("header_text='%s' - FOUND!", header_text)
 549             header_text = language_mapping[header_text]
 550         else:
 551             logger.warning("header_text='%s' not found in language mapping table", header_text)
 552
 553         logger.debug("header_text='%s - AFTER!'", header_text)
 554         if header_text in blocklist or header_text.lower() in blocklist:
 555             # replaced find_next_siblings with find_all_next to account for instances that e.g. hide lists in dropdown menu
 556             logger.debug("Found header_text='%s', importing domain blocks ...", header_text)
 557             for line in header.find_next("table").find_all("tr")[1:]:
 558                 logger.debug("line[]='%s'", type(line))
 559                 blocklist[header_text].append({
 560                     "blocked": tidyup.domain(line.find_all("td")[0].text),
 561                     "reason" : tidyup.reason(line.find_all("td")[1].text),
 562                 })
 563         else:
 564             logger.warning("header_text='%s' not found in blocklist()=%d", header_text, len(blocklist))
 565
 566     logger.debug("Returning blocklist for domain='%s' ...", domain)
 567     return {
 568         "reject"        : blocklist["Suspended servers"],
 569         "media_removal" : blocklist["Filtered media"],
 570         "followers_only": blocklist["Limited servers"] + blocklist["Silenced servers"],
 571     }