fba/networks/misskey.py

   1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
   2 # Copyright (C) 2023 Free Software Foundation
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published
   6 # by the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  16
  17 import json
  18 import logging
  19
  20 from fba import csrf
  21 from fba import utils
  22
  23 from fba.helpers import config
  24 from fba.helpers import dicts
  25 from fba.helpers import domain as domain_helper
  26 from fba.helpers import tidyup
  27
  28 from fba.http import network
  29
  30 from fba.models import instances
  31
  32 logging.basicConfig(level=logging.INFO)
  33 logger = logging.getLogger(__name__)
  34
  35 def fetch_peers(domain: str) -> list:
  36     logger.debug("domain(%d)='%s' - CALLED!", len(domain), domain)
  37     domain_helper.raise_on(domain)
  38
  39     logger.debug("domain='%s' is misskey, sending API POST request ...", domain)
  40     peers  = list()
  41     offset = 0
  42     step   = config.get("misskey_limit")
  43
  44     # No CSRF by default, you don't have to add network.api_headers by yourself here
  45     headers = tuple()
  46
  47     try:
  48         logger.debug("Checking CSRF for domain='%s'", domain)
  49         headers = csrf.determine(domain, dict())
  50     except network.exceptions as exception:
  51         logger.warning("Exception '%s' during checking CSRF (fetch_peers,%s) - EXIT!", type(exception), __name__)
  52         instances.set_last_error(domain, exception)
  53         return list()
  54
  55     # iterating through all "suspended" (follow-only in its terminology)
  56     # instances page-by-page, since that troonware doesn't support
  57     # sending them all at once
  58     while True:
  59         logger.debug("Fetching offset='%d' from domain='%s' ...", offset, domain)
  60         if offset == 0:
  61             fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
  62                 "sort" : "+pubAt",
  63                 "host" : None,
  64                 "limit": step
  65             }), headers)
  66         else:
  67             fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
  68                 "sort"  : "+pubAt",
  69                 "host"  : None,
  70                 "limit" : step,
  71                 "offset": offset - 1
  72             }), headers)
  73
  74         # Check records
  75         logger.debug("fetched[]='%s'", type(fetched))
  76         if "error_message" in fetched:
  77             logger.warning("post_json_api() for domain='%s' returned error message: '%s'", domain, fetched['error_message'])
  78             instances.set_last_error(domain, fetched)
  79             break
  80         elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
  81             logger.warning("post_json_api() returned error: '%s'", fetched['error']['message'])
  82             instances.set_last_error(domain, fetched["json"]["error"]["message"])
  83             break
  84
  85         rows = fetched["json"]
  86
  87         logger.debug("rows(%d)[]='%s'", len(rows), type(rows))
  88         if len(rows) == 0:
  89             logger.debug("Returned zero bytes, domain='%s' - BREAK!", domain)
  90             break
  91         elif len(rows) != config.get("misskey_limit"):
  92             logger.debug("Fetched %d row(s) but expected: %d", len(rows), config.get('misskey_limit'))
  93             offset = offset + (config.get("misskey_limit") - len(rows))
  94         else:
  95             logger.debug("Raising offset by step='%d'", step)
  96             offset = offset + step
  97
  98         already = 0
  99         logger.debug("rows(%d))[]='%s'", len(rows), type(rows))
 100         for row in rows:
 101             logger.debug(f"row()={len(row)}")
 102             if "host" not in row:
 103                 logger.warning("row()=%d does not contain key 'host': row='%s',domain='%s' - SKIPPED!", len(row), row, domain)
 104                 continue
 105             elif not isinstance(row["host"], str):
 106                 logger.warning("row[host][]='%s' is not 'str' - SKIPPED!", type(row['host']))
 107                 continue
 108             elif not utils.is_domain_wanted(row["host"]):
 109                 logger.debug("row[host]='%s' is not wanted, domain='{domain}' - SKIPPED!", row['host'])
 110                 continue
 111             elif row["host"] in peers:
 112                 logger.debug("Not adding row[host]='%s', already found - SKIPPED!", row['host'])
 113                 already = already + 1
 114                 continue
 115
 116             logger.debug("Adding peer: '%s'", row['host'])
 117             peers.append(row["host"])
 118
 119         if already == len(rows):
 120             logger.debug("Host returned same set of %d instance(s) - BREAK!", already)
 121             break
 122
 123     logger.debug("Adding %d for domain='%s'", len(peers), domain)
 124     instances.set_total_peers(domain, peers)
 125
 126     logger.debug("peers()=%d - EXIT!", len(peers))
 127     return peers
 128
 129 def fetch_blocks(domain: str) -> list:
 130     logger.debug("domain(%d)='%s' - CALLED!", len(domain), domain)
 131     domain_helper.raise_on(domain)
 132
 133     logger.debug("Fetching misskey blocks from domain='%s'", domain)
 134     blocklist = list()
 135
 136     offset  = 0
 137     step    = config.get("misskey_limit")
 138
 139     # No CSRF by default, you don't have to add network.api_headers by yourself here
 140     headers = tuple()
 141
 142     try:
 143         logger.debug("Checking CSRF for domain='%s'", domain)
 144         headers = csrf.determine(domain, dict())
 145     except network.exceptions as exception:
 146         logger.warning("Exception '%s' during checking CSRF (fetch_blocks,%s) - EXIT!", type(exception), __name__)
 147         instances.set_last_error(domain, exception)
 148         return blocklist
 149
 150     # iterating through all "suspended" (follow-only in its terminology)
 151     # instances page-by-page since it doesn't support sending them all at once
 152     while True:
 153         try:
 154             logger.debug("Fetching offset='%d' from domain='%s' ...", offset, domain)
 155             if offset == 0:
 156                 logger.debug("Sending JSON API request to domain='%s',step='%d',offset='%d'", domain, step, offset)
 157                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 158                     "sort"     : "+pubAt",
 159                     "host"     : None,
 160                     "suspended": True,
 161                     "limit"    : step
 162                 }), headers)
 163             else:
 164                 logger.debug("Sending JSON API request to domain='%s',step='%d',offset='%d'", domain, step, offset)
 165                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 166                     "sort"     : "+pubAt",
 167                     "host"     : None,
 168                     "suspended": True,
 169                     "limit"    : step,
 170                     "offset"   : offset - 1
 171                 }), headers)
 172
 173             logger.debug("fetched[]='%s'", type(fetched))
 174             if "error_message" in fetched:
 175                 logger.warning("post_json_api() for domain='%s' returned error message: '%s'", domain, fetched['error_message'])
 176                 instances.set_last_error(domain, fetched)
 177                 break
 178             elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
 179                 logger.warning("post_json_api() returned error: '%s'", fetched['error']['message'])
 180                 instances.set_last_error(domain, fetched["json"]["error"]["message"])
 181                 break
 182
 183             rows = fetched["json"]
 184
 185             logger.debug("rows(%d)[]='%s'", len(rows), type(rows))
 186             if len(rows) == 0:
 187                 logger.debug("Returned zero bytes, domain='%s' - BREAK!", domain)
 188                 break
 189             elif len(rows) != config.get("misskey_limit"):
 190                 logger.debug("Fetched %d row(s) but expected: %d", len(rows), config.get('misskey_limit'))
 191                 offset = offset + (config.get("misskey_limit") - len(rows))
 192             else:
 193                 logger.debug("Raising offset by step='%d'", step)
 194                 offset = offset + step
 195
 196             count = 0
 197             for instance in rows:
 198                 # Is it there?
 199                 logger.debug("instance[%s]='%s'", type(instance), instance)
 200                 if "isSuspended" in instance and instance["isSuspended"]:
 201                     count = count + 1
 202                     blocklist.append({
 203                         "blocker"    : domain,
 204                         "blocked"    : tidyup.domain(instance["host"]),
 205                         "reason"     : None,
 206                         "block_level": "suspended",
 207                     })
 208
 209             logger.debug("count=%d", count)
 210             if count == 0:
 211                 logger.debug("API is no more returning new instances, aborting loop! domain='%s'", domain)
 212                 break
 213
 214         except network.exceptions as exception:
 215             logger.warning("Caught error, exiting loop: domain='%s',exception[%s]='%s'", domain, type(exception), str(exception))
 216             instances.set_last_error(domain, exception)
 217             offset = 0
 218             break
 219
 220     while True:
 221         # Fetch blocked (full suspended) instances
 222         try:
 223             if offset == 0:
 224                 logger.debug("Sending JSON API request to domain='%s',step='%d',offset='%d'", domain, step, offset)
 225                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 226                     "sort"   : "+pubAt",
 227                     "host"   : None,
 228                     "blocked": True,
 229                     "limit"  : step
 230                 }), headers)
 231             else:
 232                 logger.debug("Sending JSON API request to domain='%s',step='%d',offset='%d'", domain, step, offset)
 233                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 234                     "sort"   : "+pubAt",
 235                     "host"   : None,
 236                     "blocked": True,
 237                     "limit"  : step,
 238                     "offset" : offset - 1
 239                 }), headers)
 240
 241             logger.debug("fetched[]='%s'", type(fetched))
 242             if "error_message" in fetched:
 243                 logger.warning("post_json_api() for domain='%s' returned error message: '%s'", domain, fetched['error_message'])
 244                 instances.set_last_error(domain, fetched)
 245                 break
 246             elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
 247                 logger.warning("post_json_api() returned error: '%s'", fetched['error']['message'])
 248                 instances.set_last_error(domain, fetched["json"]["error"]["message"])
 249                 break
 250
 251             rows = fetched["json"]
 252
 253             logger.debug("rows(%d)[]='%s'", len(rows), type(rows))
 254             if len(rows) == 0:
 255                 logger.debug("Returned zero bytes, domain='%s' - BREAK!", domain)
 256                 break
 257             elif len(rows) != config.get("misskey_limit"):
 258                 logger.debug("Fetched %d row(s) but expected: '{config.get('misskey_limit')}'", len(rows))
 259                 offset = offset + (config.get("misskey_limit") - len(rows))
 260             else:
 261                 logger.debug("Raising offset by step='%d'", step)
 262                 offset = offset + step
 263
 264             count = 0
 265             for instance in rows:
 266                 # Is it there?
 267                 logger.debug("instance[%s]='%s'", type(instance), instance)
 268                 if "isBlocked" in instance and instance["isBlocked"]:
 269                     count = count + 1
 270                     blocked = tidyup.domain(instance["host"])
 271                     logger.debug("Appending blocker='%s',blocked='%s',block_level='reject'", domain, blocked)
 272                     blocklist.append({
 273                         "blocker"    : domain,
 274                         "blocked"    : blocked,
 275                         "reason"     : None,
 276                         "block_level": "reject",
 277                     })
 278
 279             logger.debug("count=%d", count)
 280             if count == 0:
 281                 logger.debug("API is no more returning new instances, aborting loop!")
 282                 break
 283
 284         except network.exceptions as exception:
 285             logger.warning("Caught error, exiting loop: domain='%s',exception[%s]='%s'", domain, type(exception), str(exception))
 286             instances.set_last_error(domain, exception)
 287             offset = 0
 288             break
 289
 290     logger.debug("blocklist()=%d - EXIT!", len(blocklist))
 291     return blocklist