fba/networks/misskey.py

   1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
   2 # Copyright (C) 2023 Free Software Foundation
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published
   6 # by the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  16
  17 import json
  18 import logging
  19
  20 from fba import csrf
  21 from fba import utils
  22
  23 from fba.helpers import config
  24 from fba.helpers import domain as domain_helper
  25 from fba.helpers import tidyup
  26
  27 from fba.http import network
  28
  29 from fba.models import instances
  30
  31 logging.basicConfig(level=logging.INFO)
  32 logger = logging.getLogger(__name__)
  33
  34 def fetch_peers(domain: str) -> list:
  35     logger.debug("domain='%s' - CALLED!", domain)
  36     domain_helper.raise_on(domain)
  37
  38     logger.debug("domain='%s' is misskey, sending API POST request ...", domain)
  39     peers  = list()
  40     offset = 0
  41     step   = config.get("misskey_limit")
  42
  43     # No CSRF by default, you don't have to add network.api_headers by yourself here
  44     headers = tuple()
  45
  46     try:
  47         logger.debug("Checking CSRF for domain='%s'", domain)
  48         headers = csrf.determine(domain, dict())
  49     except network.exceptions as exception:
  50         logger.warning("Exception '%s' during checking CSRF (fetch_peers,%s) - EXIT!", type(exception), __name__)
  51         instances.set_last_error(domain, exception)
  52         return list()
  53
  54     # iterating through all "suspended" (follow-only in its terminology)
  55     # instances page-by-page, since that troonware doesn't support
  56     # sending them all at once
  57     while True:
  58         logger.debug("Fetching offset='%d' from domain='%s' ...", offset, domain)
  59         if offset == 0:
  60             fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
  61                 "sort" : "+pubAt",
  62                 "host" : None,
  63                 "limit": step
  64             }), headers)
  65         else:
  66             fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
  67                 "sort"  : "+pubAt",
  68                 "host"  : None,
  69                 "limit" : step,
  70                 "offset": offset - 1
  71             }), headers)
  72
  73         # Check records
  74         logger.debug("fetched[]='%s'", type(fetched))
  75         if "error_message" in fetched:
  76             logger.warning("post_json_api() for domain='%s' returned error message: '%s'", domain, fetched['error_message'])
  77             instances.set_last_error(domain, fetched)
  78             break
  79         elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
  80             logger.warning("post_json_api() returned error: '%s'", fetched['error']['message'])
  81             instances.set_last_error(domain, fetched["json"]["error"]["message"])
  82             break
  83
  84         rows = fetched["json"]
  85
  86         logger.debug("rows(%d)[]='%s'", len(rows), type(rows))
  87         if len(rows) == 0:
  88             logger.debug("Returned zero bytes, domain='%s' - BREAK!", domain)
  89             break
  90         elif len(rows) != config.get("misskey_limit"):
  91             logger.debug("Fetched %d row(s) but expected: %d", len(rows), config.get('misskey_limit'))
  92             offset = offset + (config.get("misskey_limit") - len(rows))
  93         else:
  94             logger.debug("Raising offset by step='%d'", step)
  95             offset = offset + step
  96
  97         already = 0
  98         logger.debug("rows(%d))[]='%s'", len(rows), type(rows))
  99         for row in rows:
 100             logger.debug("row()=%d", len(row))
 101             if "host" not in row:
 102                 logger.warning("row()=%d does not contain key 'host': row='%s',domain='%s' - SKIPPED!", len(row), row, domain)
 103                 continue
 104             elif not isinstance(row["host"], str):
 105                 logger.warning("row[host][]='%s' is not 'str' - SKIPPED!", type(row['host']))
 106                 continue
 107             elif not utils.is_domain_wanted(row["host"]):
 108                 logger.debug("row[host]='%s' is not wanted, domain='%s' - SKIPPED!", row['host'], domain)
 109                 continue
 110             elif row["host"] in peers:
 111                 logger.debug("Not adding row[host]='%s', already found - SKIPPED!", row['host'])
 112                 already = already + 1
 113                 continue
 114
 115             logger.debug("Adding peer: row[host]='%s'", row['host'])
 116             peers.append(row["host"])
 117
 118         if already == len(rows):
 119             logger.debug("Host returned same set of %d instance(s) - BREAK!", already)
 120             break
 121
 122     logger.debug("Adding %d for domain='%s'", len(peers), domain)
 123     instances.set_total_peers(domain, peers)
 124
 125     logger.debug("peers()=%d - EXIT!", len(peers))
 126     return peers
 127
 128 def fetch_blocks(domain: str) -> list:
 129     logger.debug("domain='%s' - CALLED!", domain)
 130     domain_helper.raise_on(domain)
 131
 132     logger.debug("Fetching misskey blocks from domain='%s'", domain)
 133     blocklist = list()
 134
 135     offset  = 0
 136     step    = config.get("misskey_limit")
 137
 138     # No CSRF by default, you don't have to add network.api_headers by yourself here
 139     headers = tuple()
 140
 141     try:
 142         logger.debug("Checking CSRF for domain='%s'", domain)
 143         headers = csrf.determine(domain, dict())
 144     except network.exceptions as exception:
 145         logger.warning("Exception '%s' during checking CSRF (fetch_blocks,%s) - EXIT!", type(exception), __name__)
 146         instances.set_last_error(domain, exception)
 147         return blocklist
 148
 149     # iterating through all "suspended" (follow-only in its terminology)
 150     # instances page-by-page since it doesn't support sending them all at once
 151     while True:
 152         try:
 153             logger.debug("Fetching offset='%d' from domain='%s' ...", offset, domain)
 154             if offset == 0:
 155                 logger.debug("Sending JSON API request to domain='%s',step='%d',offset='%d'", domain, step, offset)
 156                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 157                     "sort"     : "+pubAt",
 158                     "host"     : None,
 159                     "suspended": True,
 160                     "limit"    : step
 161                 }), headers)
 162             else:
 163                 logger.debug("Sending JSON API request to domain='%s',step='%d',offset='%d'", domain, step, offset)
 164                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 165                     "sort"     : "+pubAt",
 166                     "host"     : None,
 167                     "suspended": True,
 168                     "limit"    : step,
 169                     "offset"   : offset - 1
 170                 }), headers)
 171
 172             logger.debug("fetched[]='%s'", type(fetched))
 173             if "error_message" in fetched:
 174                 logger.warning("post_json_api() for domain='%s' returned error message: '%s'", domain, fetched['error_message'])
 175                 instances.set_last_error(domain, fetched)
 176                 break
 177             elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
 178                 logger.warning("post_json_api() returned error: '%s'", fetched['error']['message'])
 179                 instances.set_last_error(domain, fetched["json"]["error"]["message"])
 180                 break
 181
 182             rows = fetched["json"]
 183
 184             logger.debug("rows(%d)[]='%s'", len(rows), type(rows))
 185             if len(rows) == 0:
 186                 logger.debug("Returned zero bytes, domain='%s' - BREAK!", domain)
 187                 break
 188             elif len(rows) != config.get("misskey_limit"):
 189                 logger.debug("Fetched %d row(s) but expected: %d", len(rows), config.get('misskey_limit'))
 190                 offset = offset + (config.get("misskey_limit") - len(rows))
 191             else:
 192                 logger.debug("Raising offset by step='%d'", step)
 193                 offset = offset + step
 194
 195             count = 0
 196             for instance in rows:
 197                 # Is it there?
 198                 logger.debug("instance[%s]='%s'", type(instance), instance)
 199                 if "isSuspended" in instance and instance["isSuspended"]:
 200                     count = count + 1
 201                     blocklist.append({
 202                         "blocker"    : domain,
 203                         "blocked"    : tidyup.domain(instance["host"]),
 204                         "reason"     : None,
 205                         "block_level": "suspended",
 206                     })
 207
 208             logger.debug("count=%d", count)
 209             if count == 0:
 210                 logger.debug("API is no more returning new instances, aborting loop! domain='%s'", domain)
 211                 break
 212
 213         except network.exceptions as exception:
 214             logger.warning("Caught error, exiting loop: domain='%s',exception[%s]='%s'", domain, type(exception), str(exception))
 215             instances.set_last_error(domain, exception)
 216             offset = 0
 217             break
 218
 219     while True:
 220         # Fetch blocked (full suspended) instances
 221         try:
 222             if offset == 0:
 223                 logger.debug("Sending JSON API request to domain='%s',step='%d',offset='%d'", domain, step, offset)
 224                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 225                     "sort"   : "+pubAt",
 226                     "host"   : None,
 227                     "blocked": True,
 228                     "limit"  : step
 229                 }), headers)
 230             else:
 231                 logger.debug("Sending JSON API request to domain='%s',step='%d',offset='%d'", domain, step, offset)
 232                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 233                     "sort"   : "+pubAt",
 234                     "host"   : None,
 235                     "blocked": True,
 236                     "limit"  : step,
 237                     "offset" : offset - 1
 238                 }), headers)
 239
 240             logger.debug("fetched[]='%s'", type(fetched))
 241             if "error_message" in fetched:
 242                 logger.warning("post_json_api() for domain='%s' returned error message: '%s'", domain, fetched['error_message'])
 243                 instances.set_last_error(domain, fetched)
 244                 break
 245             elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
 246                 logger.warning("post_json_api() returned error: '%s'", fetched['error']['message'])
 247                 instances.set_last_error(domain, fetched["json"]["error"]["message"])
 248                 break
 249
 250             rows = fetched["json"]
 251
 252             logger.debug("rows(%d)[]='%s'", len(rows), type(rows))
 253             if len(rows) == 0:
 254                 logger.debug("Returned zero bytes, domain='%s' - BREAK!", domain)
 255                 break
 256             elif len(rows) != config.get("misskey_limit"):
 257                 logger.debug("Fetched %d row(s) but expected: %d'", len(rows), config.get('misskey_limit'))
 258                 offset = offset + (config.get("misskey_limit") - len(rows))
 259             else:
 260                 logger.debug("Raising offset by step='%d'", step)
 261                 offset = offset + step
 262
 263             count = 0
 264             for instance in rows:
 265                 # Is it there?
 266                 logger.debug("instance[%s]='%s'", type(instance), instance)
 267                 if "isBlocked" in instance and instance["isBlocked"]:
 268                     count = count + 1
 269                     blocked = tidyup.domain(instance["host"])
 270                     logger.debug("Appending blocker='%s',blocked='%s',block_level='reject'", domain, blocked)
 271                     blocklist.append({
 272                         "blocker"    : domain,
 273                         "blocked"    : blocked,
 274                         "reason"     : None,
 275                         "block_level": "reject",
 276                     })
 277
 278             logger.debug("count=%d", count)
 279             if count == 0:
 280                 logger.debug("API is no more returning new instances, aborting loop!")
 281                 break
 282
 283         except network.exceptions as exception:
 284             logger.warning("Caught error, exiting loop: domain='%s',exception[%s]='%s'", domain, type(exception), str(exception))
 285             instances.set_last_error(domain, exception)
 286             offset = 0
 287             break
 288
 289     logger.debug("blocklist()=%d - EXIT!", len(blocklist))
 290     return blocklist