fba/networks/misskey.py

   1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
   2 # Copyright (C) 2023 Free Software Foundation
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published
   6 # by the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  16
  17 import json
  18 import logging
  19
  20 from fba.helpers import config
  21 from fba.helpers import dicts as dict_helper
  22 from fba.helpers import domain as domain_helper
  23 from fba.helpers import tidyup
  24
  25 from fba.http import csrf
  26 from fba.http import network
  27
  28 from fba.models import instances
  29
  30 logging.basicConfig(level=logging.INFO)
  31 logger = logging.getLogger(__name__)
  32
  33 def fetch_peers(domain: str) -> list:
  34     logger.debug("domain='%s' - CALLED!", domain)
  35     domain_helper.raise_on(domain)
  36
  37     logger.debug("domain='%s' is misskey, sending API POST request ...", domain)
  38     peers  = list()
  39     offset = 0
  40     step   = config.get("misskey_limit")
  41
  42     # No CSRF by default, you don't have to add network.api_headers by yourself here
  43     headers = tuple()
  44
  45     try:
  46         logger.debug("Checking CSRF for domain='%s'", domain)
  47         headers = csrf.determine(domain, dict())
  48     except network.exceptions as exception:
  49         logger.warning("Exception '%s' during checking CSRF (fetch_peers,%s)", type(exception), __name__)
  50         instances.set_last_error(domain, exception)
  51
  52         logger.debug("Returning empty list ... - EXIT!")
  53         return list()
  54
  55     # iterating through all "suspended" (follow-only in its terminology)
  56     # instances page-by-page, since that troonware doesn't support
  57     # sending them all at once
  58     while True:
  59         logger.debug("Fetching offset=%d from domain='%s' ...", offset, domain)
  60         if offset == 0:
  61             fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
  62                 "sort" : "+pubAt",
  63                 "host" : None,
  64                 "limit": step
  65             }), headers)
  66         else:
  67             fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
  68                 "sort"  : "+pubAt",
  69                 "host"  : None,
  70                 "limit" : step,
  71                 "offset": offset - 1
  72             }), headers)
  73
  74         # Check records
  75         logger.debug("fetched[]='%s'", type(fetched))
  76         if "error_message" in fetched:
  77             logger.warning("post_json_api() for domain='%s' returned error message: '%s'", domain, fetched['error_message'])
  78             instances.set_last_error(domain, fetched)
  79             break
  80         elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
  81             logger.warning("post_json_api() returned error: '%s'", fetched['error']['message'])
  82             instances.set_last_error(domain, fetched["json"]["error"]["message"])
  83             break
  84
  85         rows = fetched["json"]
  86
  87         logger.debug("rows(%d)[]='%s',step=%d", len(rows), type(rows), step)
  88         if len(rows) == 0:
  89             logger.debug("Returned zero bytes, domain='%s' - BREAK!", domain)
  90             break
  91         elif len(rows) != config.get("misskey_limit"):
  92             logger.debug("Fetched %d row(s) but expected: %d", len(rows), config.get('misskey_limit'))
  93             offset = offset + (config.get("misskey_limit") - len(rows))
  94         else:
  95             logger.debug("Raising offset by step=%d", step)
  96             offset = offset + step
  97
  98         added = 0
  99         logger.debug("rows(%d))[]='%s'", len(rows), type(rows))
 100         for row in rows:
 101             logger.debug("row()=%d", len(row))
 102             if "host" not in row:
 103                 logger.warning("row()=%d does not contain key 'host': row='%s',domain='%s' - SKIPPED!", len(row), row, domain)
 104                 continue
 105             elif not isinstance(row["host"], str):
 106                 logger.warning("row[host][]='%s' is not of type 'str' - SKIPPED!", type(row['host']))
 107                 continue
 108             elif row["host"] in peers:
 109                 logger.debug("Not adding row[host]='%s', already found - SKIPPED!", row['host'])
 110                 continue
 111             elif not domain_helper.is_wanted(row["host"]):
 112                 logger.debug("row[host]='%s' is not wanted - SKIPPED!", row["host"])
 113                 continue
 114
 115             logger.debug("Adding peer: row[host]='%s'", row['host'])
 116             added = added + 1
 117             peers.append(row["host"])
 118
 119         logger.debug("added=%d,rows()=%d", added, len(rows))
 120         if added == 0:
 121             logger.debug("Host returned already added (%d) peers - BREAK!", len(rows))
 122             break
 123
 124     logger.debug("peers()=%d - EXIT!", len(peers))
 125     return peers
 126
 127 def fetch_blocks(domain: str) -> list:
 128     logger.debug("domain='%s' - CALLED!", domain)
 129     domain_helper.raise_on(domain)
 130
 131     if not instances.is_registered(domain):
 132         raise Exception(f"domain='{domain}' is not registered but function is invoked.")
 133
 134     # No CSRF by default, you don't have to add network.api_headers by yourself here
 135     headers = tuple()
 136
 137     try:
 138         logger.debug("Checking CSRF for domain='%s'", domain)
 139         headers = csrf.determine(domain, dict())
 140     except network.exceptions as exception:
 141         logger.warning("Exception '%s' during checking CSRF (fetch_blocks,%s)", type(exception), __name__)
 142         instances.set_last_error(domain, exception)
 143
 144         logger.debug("Returning empty list ... - EXIT!")
 145         return list()
 146
 147     blocklist = list()
 148     offset    = 0
 149     step      = config.get("misskey_limit")
 150
 151     # iterating through all "suspended" (follow-only in its terminology)
 152     # instances page-by-page since it doesn't support sending them all at once
 153     logger.debug("Fetching misskey blocks from domain='%s'", domain)
 154     while True:
 155         try:
 156             logger.debug("Fetching offset=%d from domain='%s' ...", offset, domain)
 157             if offset == 0:
 158                 logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d", domain, step, offset)
 159                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 160                     "sort"     : "+pubAt",
 161                     "host"     : None,
 162                     "suspended": True,
 163                     "limit"    : step
 164                 }), headers)
 165             else:
 166                 logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d", domain, step, offset)
 167                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 168                     "sort"     : "+pubAt",
 169                     "host"     : None,
 170                     "suspended": True,
 171                     "limit"    : step,
 172                     "offset"   : offset - 1
 173                 }), headers)
 174
 175             logger.debug("fetched[]='%s'", type(fetched))
 176             if "error_message" in fetched:
 177                 logger.warning("post_json_api() for domain='%s' returned error message: '%s'", domain, fetched['error_message'])
 178                 instances.set_last_error(domain, fetched)
 179                 break
 180             elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
 181                 logger.warning("post_json_api() returned error: '%s'", fetched['error']['message'])
 182                 instances.set_last_error(domain, fetched["json"]["error"]["message"])
 183                 break
 184
 185             rows = fetched["json"]
 186
 187             logger.debug("rows(%d)[]='%s'", len(rows), type(rows))
 188             if len(rows) == 0:
 189                 logger.debug("Returned zero bytes, domain='%s' - BREAK!", domain)
 190                 break
 191             elif len(rows) != config.get("misskey_limit"):
 192                 logger.debug("Fetched %d row(s) but expected: %d", len(rows), config.get('misskey_limit'))
 193                 offset = offset + (config.get("misskey_limit") - len(rows))
 194             else:
 195                 logger.debug("Raising offset by step=%d", step)
 196                 offset = offset + step
 197
 198             count = 0
 199             for instance in rows:
 200                 # Is it there?
 201                 logger.debug("instance[]='%s'", type(instance))
 202                 if "host" not in instance:
 203                     logger.warning("instance(%d)='%s' has no key 'host' - SKIPPED!", len(instance), instance)
 204                     continue
 205                 elif instance["host"] is None or instance["host"] == "":
 206                     logger.debug("instance[host]='%s' is None or empty - SKIPPED!", instance["host"])
 207                     continue
 208
 209                 logger.debug("instance[host]='%s' - BEFORE!", instance["host"])
 210                 blocked = tidyup.domain(instance["host"])
 211                 logger.debug("blocked[%s]='%s' - AFTER!", type(blocked), blocked)
 212
 213                 if blocked is None or blocked == "":
 214                     logger.warning("instance[host]='%s' is None or empty after tidyup.domain() - SKIPPED!", instance["host"])
 215                     continue
 216                 elif not domain_helper.is_wanted(blocked):
 217                     logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
 218                     continue
 219                 elif "isSuspended" in instance and instance["isSuspended"] and not dict_helper.has_key(blocklist, "blocked", blocked):
 220                     count = count + 1
 221                     logger.debug("Appending blocker='%s',blocked='%s',block_level='suspended'", domain, blocked)
 222                     blocklist.append({
 223                         "blocker"    : domain,
 224                         "blocked"    : blocked,
 225                         "reason"     : None,
 226                         "block_level": "suspended",
 227                     })
 228
 229             logger.debug("count=%d", count)
 230             if count == 0:
 231                 logger.debug("API is no more returning new instances, aborting loop! domain='%s'", domain)
 232                 break
 233
 234         except network.exceptions as exception:
 235             logger.warning("Caught error, exiting loop: domain='%s',exception[%s]='%s'", domain, type(exception), str(exception))
 236             instances.set_last_error(domain, exception)
 237             offset = 0
 238             break
 239
 240     while True:
 241         # Fetch blocked (full suspended) instances
 242         try:
 243             if offset == 0:
 244                 logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d", domain, step, offset)
 245                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 246                     "sort"   : "+pubAt",
 247                     "host"   : None,
 248                     "blocked": True,
 249                     "limit"  : step
 250                 }), headers)
 251             else:
 252                 logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d", domain, step, offset)
 253                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 254                     "sort"   : "+pubAt",
 255                     "host"   : None,
 256                     "blocked": True,
 257                     "limit"  : step,
 258                     "offset" : offset - 1
 259                 }), headers)
 260
 261             logger.debug("fetched[]='%s'", type(fetched))
 262             if "error_message" in fetched:
 263                 logger.warning("post_json_api() for domain='%s' returned error message: '%s'", domain, fetched['error_message'])
 264                 instances.set_last_error(domain, fetched)
 265                 break
 266             elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
 267                 logger.warning("post_json_api() returned error: '%s'", fetched['error']['message'])
 268                 instances.set_last_error(domain, fetched["json"]["error"]["message"])
 269                 break
 270
 271             rows = fetched["json"]
 272
 273             logger.debug("rows(%d)[]='%s'", len(rows), type(rows))
 274             if len(rows) == 0:
 275                 logger.debug("Returned zero bytes, domain='%s' - BREAK!", domain)
 276                 break
 277             elif len(rows) != config.get("misskey_limit"):
 278                 logger.debug("Fetched %d row(s) but expected: %d'", len(rows), config.get('misskey_limit'))
 279                 offset = offset + (config.get("misskey_limit") - len(rows))
 280             else:
 281                 logger.debug("Raising offset by step=%d", step)
 282                 offset = offset + step
 283
 284             count = 0
 285             for instance in rows:
 286                 # Is it there?
 287                 logger.debug("instance[]='%s'", type(instance))
 288                 blocked = tidyup.domain(instance["host"])
 289
 290                 logger.debug("blocked='%s'", blocked)
 291                 if blocked is None or blocked == "":
 292                     logger.warning("instance[host]='%s' is None or empty after tidyup.domain() - SKIPPED!", instance["host"])
 293                     continue
 294                 elif not domain_helper.is_wanted(blocked):
 295                     logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
 296                     continue
 297                 elif "isBlocked" in instance and instance["isBlocked"] and not dict_helper.has_key(blocklist, "blocked", blocked):
 298                     count = count + 1
 299                     logger.debug("Appending blocker='%s',blocked='%s',block_level='reject'", domain, blocked)
 300                     blocklist.append({
 301                         "blocker"    : domain,
 302                         "blocked"    : blocked,
 303                         "reason"     : None,
 304                         "block_level": "reject",
 305                     })
 306
 307             logger.debug("count=%d", count)
 308             if count == 0:
 309                 logger.debug("API is no more returning new instances, aborting loop!")
 310                 break
 311
 312         except network.exceptions as exception:
 313             logger.warning("Caught error, exiting loop: domain='%s',exception[%s]='%s'", domain, type(exception), str(exception))
 314             instances.set_last_error(domain, exception)
 315             offset = 0
 316             break
 317
 318     logger.debug("blocklist()=%d - EXIT!", len(blocklist))
 319     return blocklist