fba/networks/pleroma.py

   1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
   2 # Copyright (C) 2023 Free Software Foundation
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published
   6 # by the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  16
  17 import logging
  18
  19 import bs4
  20
  21 from fba import database
  22 from fba import utils
  23
  24 from fba.helpers import config
  25 from fba.helpers import domain as domain_helper
  26 from fba.helpers import tidyup
  27
  28 from fba.http import federation
  29 from fba.http import network
  30
  31 from fba.models import instances
  32
  33 logging.basicConfig(level=logging.INFO)
  34 logger = logging.getLogger(__name__)
  35
  36 # Language mapping X -> English
  37 language_mapping = {
  38     # English -> English
  39     "filtered media"   : "filtered_media",
  40     "limited servers"  : "followers_only",
  41     "followers-only"   : "followers_only",
  42     "media removal"    : "media_removal",
  43     "media_removal"    : "media_removal",
  44     "media force-set as sensitive": "nsfw",
  45     "nsfw"             : "nsfw",
  46     "reject"           : "reject",
  47     "suspended servers": "reject",
  48     "silenced servers" : "silenced",
  49     "removal from \"the whole known network\" timeline": " federated_timeline_removal",
  50 }
  51
  52 def fetch_blocks(domain: str, nodeinfo_url: str) -> list:
  53     logger.debug("domain='%s',nodeinfo_url='%s' - CALLED!", domain, nodeinfo_url)
  54     domain_helper.raise_on(domain)
  55
  56     if not isinstance(nodeinfo_url, str):
  57         raise ValueError(f"Parameter nodeinfo_url[]='{type(nodeinfo_url)}' is not 'str'")
  58     elif nodeinfo_url == "":
  59         raise ValueError("Parameter 'nodeinfo_url' is empty")
  60
  61     blockdict = list()
  62     rows = None
  63     try:
  64         logger.debug("Fetching nodeinfo: domain='%s',nodeinfo_url='%s'", domain, nodeinfo_url)
  65         rows = federation.fetch_nodeinfo(domain, nodeinfo_url)
  66     except network.exceptions as exception:
  67         logger.warning("Exception '%s' during fetching nodeinfo from domain='%s'", type(exception), domain)
  68         instances.set_last_error(domain, exception)
  69
  70     if rows is None:
  71         logger.warning("Could not fetch nodeinfo from domain='%s'", domain)
  72         return list()
  73     elif "metadata" not in rows:
  74         logger.warning("rows()=%d does not have key 'metadata', domain='%s'", len(rows), domain)
  75         return list()
  76     elif "federation" not in rows["metadata"]:
  77         logger.warning("rows()=%d does not have key 'federation', domain='%s'", len(rows["metadata"]), domain)
  78         return list()
  79
  80     data = rows["metadata"]["federation"]
  81     found = False
  82
  83     logger.debug("data[]='%s'", type(data))
  84     if "mrf_simple" in data:
  85         logger.debug("Found mrf_simple in API response from domain='%s'", domain)
  86         found = True
  87         for block_level, blocklist in (
  88             {
  89                 **data["mrf_simple"],
  90                 **{
  91                     "quarantined_instances": data["quarantined_instances"]
  92                 }
  93             }
  94         ).items():
  95             logger.debug("block_level='%s', blocklist()=%d", block_level, len(blocklist))
  96             block_level = tidyup.domain(block_level)
  97             logger.debug("block_level='%s' - AFTER!", block_level)
  98
  99             if block_level == "":
 100                 logger.warning("block_level is now empty!")
 101                 continue
 102             elif block_level == "accept":
 103                 logger.debug("domain='%s' skipping block_level='accept'", domain)
 104                 continue
 105
 106             block_level = utils.alias_block_level(block_level)
 107
 108             logger.debug("Checking %d entries from domain='%s',block_level='%s' ...", len(blocklist), domain, block_level)
 109             if len(blocklist) > 0:
 110                 for blocked in blocklist:
 111                     logger.debug("blocked='%s' - BEFORE!", blocked)
 112                     blocked = tidyup.domain(blocked)
 113                     logger.debug("blocked='%s' - AFTER!", blocked)
 114
 115                     if blocked == "":
 116                         logger.warning("blocked is empty after tidyup.domain(): domain='%s',block_level='%s' - SKIPPED!", domain, block_level)
 117                         continue
 118                     elif not utils.is_domain_wanted(blocked):
 119                         logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
 120                         continue
 121
 122                     logger.debug("Invoking utils.deobfuscate_domain(%s, %s) ...", blocked, domain)
 123                     blocked = utils.deobfuscate_domain(blocked, domain)
 124
 125                     logger.debug("blocked='%s' - DEOBFUSCATED!", blocked)
 126                     if not utils.is_domain_wanted(blocked):
 127                         logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
 128                         continue
 129
 130                     logger.debug("Appending blocker='%s',blocked='%s',block_level='%s' ...", domain, blocked, block_level)
 131                     blockdict.append({
 132                         "blocker"    : domain,
 133                         "blocked"    : blocked,
 134                         "reason"     : None,
 135                         "block_level": block_level,
 136                     })
 137
 138     elif "quarantined_instances" in data:
 139         logger.debug("Found 'quarantined_instances' in JSON response: domain='%s'", domain)
 140         found = True
 141         block_level = "quarantined"
 142
 143         for blocked in data["quarantined_instances"]:
 144             logger.debug("blocked='%s' - BEFORE!", blocked)
 145             blocked = tidyup.domain(blocked)
 146             logger.debug("blocked='%s' - AFTER!", blocked)
 147
 148             if blocked == "":
 149                 logger.warning("blocked is empty after tidyup.domain(): domain='%s',block_level='%s'", domain, block_level)
 150                 continue
 151             elif not utils.is_domain_wanted(blocked):
 152                 logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
 153                 continue
 154
 155             logger.debug("Invoking utils.deobfuscate_domain(%s, %s) ...", blocked, domain)
 156             blocked = utils.deobfuscate_domain(blocked, domain)
 157
 158             logger.debug("blocked='%s' - DEOBFUSCATED!", blocked)
 159             if not utils.is_domain_wanted(blocked):
 160                 logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
 161                 continue
 162
 163             logger.debug("Appending blocker='%s',blocked='%s',block_level='%s' ...", domain, blocked, block_level)
 164             blockdict.append({
 165                 "blocker"    : domain,
 166                 "blocked"    : blocked,
 167                 "reason"     : None,
 168                 "block_level": block_level,
 169             })
 170
 171     else:
 172         logger.warning("Cannot find 'mrf_simple' or 'quarantined_instances' in JSON reply: domain='%s'", domain)
 173
 174     logger.debug("Invoking commit() ...")
 175     database.connection.commit()
 176
 177     # Reasons
 178     if "mrf_simple_info" in data:
 179         logger.debug("Found mrf_simple_info in API response: domain='%s'", domain)
 180         found = True
 181         for block_level, info in (
 182             {
 183                 **data["mrf_simple_info"],
 184                 **(data["quarantined_instances_info"] if "quarantined_instances_info" in data else {})
 185             }
 186         ).items():
 187             logger.debug("block_level='%s', info.items()=%d", block_level, len(info.items()))
 188             block_level = tidyup.domain(block_level)
 189             logger.debug("block_level='%s' - AFTER!", block_level)
 190
 191             if block_level == "":
 192                 logger.warning("block_level is now empty!")
 193                 continue
 194             elif block_level == "accept":
 195                 logger.debug("domain='%s': Skipping block_level='%s' ...", domain, block_level)
 196                 continue
 197
 198             block_level = utils.alias_block_level(block_level)
 199
 200             logger.debug("Checking %d entries from domain='%s',block_level='%s' ...", len(info.items()), domain, block_level)
 201             for blocked, reason in info.items():
 202                 logger.debug("blocked='%s',reason[%s]='%s' - BEFORE!", blocked, type(reason), reason)
 203                 blocked = tidyup.domain(blocked)
 204                 logger.debug("blocked='%s' - AFTER!", blocked)
 205
 206                 if isinstance(reason, str):
 207                     logger.debug("reason[] is a string")
 208                     reason = tidyup.reason(reason)
 209                 elif isinstance(reason, dict) and "reason" in reason:
 210                     logger.debug("reason[] is a dict")
 211                     reason = tidyup.reason(reason["reason"]) if isinstance(reason["reason"], str) else None
 212                 elif reason is not None:
 213                     raise ValueError(f"Cannot handle reason[]='{type(reason)}'")
 214
 215                 logger.debug("blocked='%s',reason='%s' - AFTER!", blocked, reason)
 216
 217                 if blocked == "":
 218                     logger.warning("blocked is empty after tidyup.domain(): domain='%s',block_level='%s'", domain, block_level)
 219                     continue
 220                 elif not utils.is_domain_wanted(blocked):
 221                     logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
 222                     continue
 223
 224                 logger.debug("Invoking utils.deobfuscate_domain(%s, %s) ...", blocked, domain)
 225                 blocked = utils.deobfuscate_domain(blocked, domain)
 226                 logger.debug("blocked='%s' - DEOBFUSCATED!", blocked)
 227
 228                 logger.debug("Checking %d blockdict records ...", len(blockdict))
 229                 for block in blockdict:
 230                     logger.debug("block[blocked]='%s',blocked='%s'", block["blocked"], blocked)
 231                     if block["blocked"] == blocked:
 232                         logger.debug("Updating reason='%s' for blocker='%s'", reason, block["blocked"])
 233                         block["reason"] = reason
 234
 235     elif "quarantined_instances_info" in data and "quarantined_instances" in data["quarantined_instances_info"]:
 236         logger.debug("Found 'quarantined_instances_info' in JSON response: domain='%s'", domain)
 237         found = True
 238         block_level = "quarantined"
 239
 240         #print(data["quarantined_instances_info"])
 241         rows = data["quarantined_instances_info"]["quarantined_instances"]
 242         for blocked in rows:
 243             logger.debug("blocked='%s' - BEFORE!", blocked)
 244             reason = tidyup.reason(rows[blocked]["reason"])
 245             blocked = tidyup.domain(blocked)
 246             logger.debug("blocked='%s',reason='%s' - AFTER!", blocked, reason)
 247
 248             if blocked not in rows or "reason" not in rows[blocked]:
 249                 logger.warning("Cannot find blocked='%s' in rows()=%d,domain='%s' - BREAK!", blocked, len(rows), domain)
 250                 break
 251             elif blocked == "":
 252                 logger.warning("blocked is empty after tidyup.domain(): domain='%s',block_level='%s'", domain, block_level)
 253                 continue
 254             elif not utils.is_domain_wanted(blocked):
 255                 logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
 256                 continue
 257
 258             logger.debug("Invoking utils.deobfuscate_domain(%s, %s) ...", blocked, domain)
 259             blocked = utils.deobfuscate_domain(blocked, domain)
 260
 261             logger.debug("blocked='%s' - DEOBFUSCATED!", blocked)
 262             if not utils.is_domain_wanted(blocked):
 263                 logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
 264                 continue
 265
 266             logger.debug("Checking %d blockdict records ...", len(blockdict))
 267             for block in blockdict:
 268                 logger.debug("block[blocked]='%s',blocked='%s'", block["blocked"], blocked)
 269                 if block["blocked"] == blocked:
 270                     logger.debug("Updating reason='%s' for blocker='%s'", reason, block["blocked"])
 271                     block["reason"] = reason
 272     else:
 273         logger.warning("Cannot find 'mrf_simple_info' or 'quarantined_instances_info' in JSON reply: domain='%s'", domain)
 274
 275     if not found:
 276         logger.debug("Did not find any useable JSON elements, domain='%s', continuing with /about page ...", domain)
 277         blocklist = fetch_blocks_from_about(domain)
 278
 279         logger.debug("blocklist()=%d", len(blocklist))
 280         if len(blocklist) > 0:
 281             logger.info("Checking %d different blocklists ...", len(blocklist))
 282             for block_level in blocklist:
 283                 logger.debug("block_level='%s'", block_level)
 284                 rows = blocklist[block_level]
 285
 286                 logger.debug("rows[%s]()=%d'", type(rows), len(rows))
 287                 for block in rows:
 288                     logger.debug("Invoking utils.deobfuscate_domain(%s, %s) ...", block["blocked"], domain)
 289                     block["blocked"] = utils.deobfuscate_domain(block["blocked"], domain)
 290
 291                     logger.debug("block[blocked]='%s' - DEOBFUSCATED!", block["blocked"])
 292                     if not utils.is_domain_wanted(block["blocked"]):
 293                         logger.debug("block[blocked]='%s' is not wanted - SKIPPED!", block["blocked"])
 294                         continue
 295
 296                     logger.debug("Appending blocker='%s',block[blocked]='%s',block[reason]='%s',block_level='%s' ...",domain, block["blocked"], block["reason"], block_level)
 297                     blockdict.append({
 298                         "blocker"    : domain,
 299                         "blocked"    : block["blocked"],
 300                         "reason"     : block["reason"],
 301                         "block_level": block_level,
 302                     })
 303
 304     logger.debug("blockdict()=%d - EXIT!", len(blockdict))
 305     return blockdict
 306
 307 def fetch_blocks_from_about(domain: str) -> dict:
 308     logger.debug("domain='%s' - CALLED!", domain)
 309     domain_helper.raise_on(domain)
 310
 311     logger.debug("Fetching mastodon blocks from domain='%s'", domain)
 312     doc = None
 313     for path in ["/instance/about/index.html"]:
 314         try:
 315             # Resetting doc type
 316             doc = None
 317
 318             logger.debug("Fetching path='%s' from domain='%s' ...", path, domain)
 319             response = network.fetch_response(
 320                 domain,
 321                 path,
 322                 network.web_headers,
 323                 (config.get("connection_timeout"), config.get("read_timeout"))
 324             )
 325
 326             logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text))
 327             if not response.ok or response.text.strip() == "":
 328                 logger.warning("path='%s' does not exist on domain='%s' - SKIPPED!", path, domain)
 329                 continue
 330
 331             logger.debug("Parsing response.text()=%d Bytes ...", len(response.text))
 332             doc = bs4.BeautifulSoup(
 333                 response.text,
 334                 "html.parser",
 335             )
 336
 337             logger.debug("doc[]='%s'", type(doc))
 338             if doc.find("h2") is not None:
 339                 logger.debug("Found 'h2' header in path='%s' - BREAK!", path)
 340                 break
 341
 342         except network.exceptions as exception:
 343             logger.warning("Cannot fetch from domain='%s',exception[%s]='%s'", domain, type(exception), str(exception))
 344             instances.set_last_error(domain, exception)
 345             break
 346
 347     blocklist = {
 348         "reject"        : [],
 349         "filtered_media": [],
 350         "followers_only": [],
 351         "silenced"      : [],
 352         "nsfw"          : [],
 353         "media_removal" : [],
 354     }
 355
 356     logger.debug("doc[]='%s'", type(doc))
 357     if doc is None:
 358         logger.warning("Cannot fetch any /about pages for domain='%s' - EXIT!", domain)
 359         return list()
 360
 361     headers = doc.find_all("h2")
 362
 363     logger.debug("headers[]='%s'", type(headers))
 364     if headers is None:
 365         logger.warning("Cannot fetch any /about pages for domain='%s' - EXIT!", domain)
 366         return list()
 367
 368     logger.info("Checking %d headers ...", len(headers))
 369     for header in headers:
 370         logger.debug("header[%s]='%s'", type(header), header)
 371         block_level = tidyup.reason(header.text).lower()
 372
 373         logger.debug("block_level='%s' - BEFORE!", block_level)
 374         if block_level in language_mapping:
 375             logger.debug("block_level='%s' - FOUND!", block_level)
 376             block_level = language_mapping[block_level].lower()
 377         else:
 378             logger.warning("block_level='%s' not found in language mapping table", block_level)
 379
 380         logger.debug("block_level='%s - AFTER!'", block_level)
 381         if block_level in blocklist:
 382             # replaced find_next_siblings with find_all_next to account for instances that e.g. hide lists in dropdown menu
 383             logger.debug("Found block_level='%s', importing domain blocks ...", block_level)
 384             for line in header.find_next("table").find_all("tr")[1:]:
 385                 logger.debug("line[]='%s'", type(line))
 386                 blocked = tidyup.domain(line.find_all("td")[0].text)
 387                 reason = tidyup.reason(line.find_all("td")[1].text)
 388
 389                 logger.debug("Appending block_level='%s',blocked='%s',reason='%s' ...", block_level, blocked, reason)
 390                 blocklist[block_level].append({
 391                     "blocked": blocked,
 392                     "reason" : reason,
 393                 })
 394         else:
 395             logger.warning("block_level='%s' not found in blocklist()=%d", block_level, len(blocklist))
 396
 397     logger.debug("Returning blocklist for domain='%s' - EXIT!", domain)
 398     return blocklist