fba/networks/misskey.py

   1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
   2 # Copyright (C) 2023 Free Software Foundation
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published
   6 # by the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  16
  17 import json
  18 import logging
  19
  20 from fba import csrf
  21 from fba import utils
  22
  23 from fba.helpers import config
  24 from fba.helpers import dicts as dict_helper
  25 from fba.helpers import domain as domain_helper
  26 from fba.helpers import tidyup
  27
  28 from fba.http import network
  29
  30 from fba.models import instances
  31
  32 logging.basicConfig(level=logging.INFO)
  33 logger = logging.getLogger(__name__)
  34
  35 def fetch_peers(domain: str) -> list:
  36     logger.debug("domain='%s' - CALLED!", domain)
  37     domain_helper.raise_on(domain)
  38
  39     logger.debug("domain='%s' is misskey, sending API POST request ...", domain)
  40     peers  = list()
  41     offset = 0
  42     step   = config.get("misskey_limit")
  43
  44     # No CSRF by default, you don't have to add network.api_headers by yourself here
  45     headers = tuple()
  46
  47     try:
  48         logger.debug("Checking CSRF for domain='%s'", domain)
  49         headers = csrf.determine(domain, dict())
  50     except network.exceptions as exception:
  51         logger.warning("Exception '%s' during checking CSRF (fetch_peers,%s)", type(exception), __name__)
  52         instances.set_last_error(domain, exception)
  53
  54         logger.debug("Returning empty list ... - EXIT!")
  55         return list()
  56
  57     # iterating through all "suspended" (follow-only in its terminology)
  58     # instances page-by-page, since that troonware doesn't support
  59     # sending them all at once
  60     while True:
  61         logger.debug("Fetching offset=%d from domain='%s' ...", offset, domain)
  62         if offset == 0:
  63             fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
  64                 "sort" : "+pubAt",
  65                 "host" : None,
  66                 "limit": step
  67             }), headers)
  68         else:
  69             fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
  70                 "sort"  : "+pubAt",
  71                 "host"  : None,
  72                 "limit" : step,
  73                 "offset": offset - 1
  74             }), headers)
  75
  76         # Check records
  77         logger.debug("fetched[]='%s'", type(fetched))
  78         if "error_message" in fetched:
  79             logger.warning("post_json_api() for domain='%s' returned error message: '%s'", domain, fetched['error_message'])
  80             instances.set_last_error(domain, fetched)
  81             break
  82         elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
  83             logger.warning("post_json_api() returned error: '%s'", fetched['error']['message'])
  84             instances.set_last_error(domain, fetched["json"]["error"]["message"])
  85             break
  86
  87         rows = fetched["json"]
  88
  89         logger.debug("rows(%d)[]='%s',step=%d", len(rows), type(rows), step)
  90         if len(rows) == 0:
  91             logger.debug("Returned zero bytes, domain='%s' - BREAK!", domain)
  92             break
  93         elif len(rows) != config.get("misskey_limit"):
  94             logger.debug("Fetched %d row(s) but expected: %d", len(rows), config.get('misskey_limit'))
  95             offset = offset + (config.get("misskey_limit") - len(rows))
  96         else:
  97             logger.debug("Raising offset by step=%d", step)
  98             offset = offset + step
  99
 100         added = 0
 101         logger.debug("rows(%d))[]='%s'", len(rows), type(rows))
 102         for row in rows:
 103             logger.debug("row()=%d", len(row))
 104             if "host" not in row:
 105                 logger.warning("row()=%d does not contain key 'host': row='%s',domain='%s' - SKIPPED!", len(row), row, domain)
 106                 continue
 107             elif not isinstance(row["host"], str):
 108                 logger.warning("row[host][]='%s' is not of type 'str' - SKIPPED!", type(row['host']))
 109                 continue
 110             elif not utils.is_domain_wanted(row["host"]):
 111                 logger.debug("row[host]='%s' is not wanted, domain='%s' - SKIPPED!", row['host'], domain)
 112                 continue
 113             elif row["host"] in peers:
 114                 logger.debug("Not adding row[host]='%s', already found - SKIPPED!", row['host'])
 115                 continue
 116
 117             logger.debug("Adding peer: row[host]='%s'", row['host'])
 118             added = added + 1
 119             peers.append(row["host"])
 120
 121         logger.debug("added=%d,rows()=%d", added, len(rows))
 122         if added == 0:
 123             logger.debug("Host returned already added (%d) peers - BREAK!", len(rows))
 124             break
 125
 126     logger.debug("peers()=%d - EXIT!", len(peers))
 127     return peers
 128
 129 def fetch_blocks(domain: str) -> list:
 130     logger.debug("domain='%s' - CALLED!", domain)
 131     domain_helper.raise_on(domain)
 132
 133     # No CSRF by default, you don't have to add network.api_headers by yourself here
 134     headers = tuple()
 135
 136     try:
 137         logger.debug("Checking CSRF for domain='%s'", domain)
 138         headers = csrf.determine(domain, dict())
 139     except network.exceptions as exception:
 140         logger.warning("Exception '%s' during checking CSRF (fetch_blocks,%s)", type(exception), __name__)
 141         instances.set_last_error(domain, exception)
 142
 143         logger.debug("Returning empty list ... - EXIT!")
 144         return list()
 145
 146     blocklist = list()
 147     offset    = 0
 148     step      = config.get("misskey_limit")
 149
 150     # iterating through all "suspended" (follow-only in its terminology)
 151     # instances page-by-page since it doesn't support sending them all at once
 152     logger.debug("Fetching misskey blocks from domain='%s'", domain)
 153     while True:
 154         try:
 155             logger.debug("Fetching offset=%d from domain='%s' ...", offset, domain)
 156             if offset == 0:
 157                 logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d", domain, step, offset)
 158                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 159                     "sort"     : "+pubAt",
 160                     "host"     : None,
 161                     "suspended": True,
 162                     "limit"    : step
 163                 }), headers)
 164             else:
 165                 logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d", domain, step, offset)
 166                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 167                     "sort"     : "+pubAt",
 168                     "host"     : None,
 169                     "suspended": True,
 170                     "limit"    : step,
 171                     "offset"   : offset - 1
 172                 }), headers)
 173
 174             logger.debug("fetched[]='%s'", type(fetched))
 175             if "error_message" in fetched:
 176                 logger.warning("post_json_api() for domain='%s' returned error message: '%s'", domain, fetched['error_message'])
 177                 instances.set_last_error(domain, fetched)
 178                 break
 179             elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
 180                 logger.warning("post_json_api() returned error: '%s'", fetched['error']['message'])
 181                 instances.set_last_error(domain, fetched["json"]["error"]["message"])
 182                 break
 183
 184             rows = fetched["json"]
 185
 186             logger.debug("rows(%d)[]='%s'", len(rows), type(rows))
 187             if len(rows) == 0:
 188                 logger.debug("Returned zero bytes, domain='%s' - BREAK!", domain)
 189                 break
 190             elif len(rows) != config.get("misskey_limit"):
 191                 logger.debug("Fetched %d row(s) but expected: %d", len(rows), config.get('misskey_limit'))
 192                 offset = offset + (config.get("misskey_limit") - len(rows))
 193             else:
 194                 logger.debug("Raising offset by step=%d", step)
 195                 offset = offset + step
 196
 197             count = 0
 198             for instance in rows:
 199                 # Is it there?
 200                 logger.debug("instance[]='%s'", type(instance))
 201                 blocked = tidyup.domain(instance["host"])
 202                 if "isSuspended" in instance and instance["isSuspended"] and not dict_helper.has_key(blocklist, "blocked", blocked):
 203                     count = count + 1
 204                     logger.debug("Appending blocker='%s',blocked='%s',block_level='suspended'", domain, blocked)
 205                     blocklist.append({
 206                         "blocker"    : domain,
 207                         "blocked"    : blocked,
 208                         "reason"     : None,
 209                         "block_level": "suspended",
 210                     })
 211
 212             logger.debug("count=%d", count)
 213             if count == 0:
 214                 logger.debug("API is no more returning new instances, aborting loop! domain='%s'", domain)
 215                 break
 216
 217         except network.exceptions as exception:
 218             logger.warning("Caught error, exiting loop: domain='%s',exception[%s]='%s'", domain, type(exception), str(exception))
 219             instances.set_last_error(domain, exception)
 220             offset = 0
 221             break
 222
 223     while True:
 224         # Fetch blocked (full suspended) instances
 225         try:
 226             if offset == 0:
 227                 logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d", domain, step, offset)
 228                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 229                     "sort"   : "+pubAt",
 230                     "host"   : None,
 231                     "blocked": True,
 232                     "limit"  : step
 233                 }), headers)
 234             else:
 235                 logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d", domain, step, offset)
 236                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 237                     "sort"   : "+pubAt",
 238                     "host"   : None,
 239                     "blocked": True,
 240                     "limit"  : step,
 241                     "offset" : offset - 1
 242                 }), headers)
 243
 244             logger.debug("fetched[]='%s'", type(fetched))
 245             if "error_message" in fetched:
 246                 logger.warning("post_json_api() for domain='%s' returned error message: '%s'", domain, fetched['error_message'])
 247                 instances.set_last_error(domain, fetched)
 248                 break
 249             elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
 250                 logger.warning("post_json_api() returned error: '%s'", fetched['error']['message'])
 251                 instances.set_last_error(domain, fetched["json"]["error"]["message"])
 252                 break
 253
 254             rows = fetched["json"]
 255
 256             logger.debug("rows(%d)[]='%s'", len(rows), type(rows))
 257             if len(rows) == 0:
 258                 logger.debug("Returned zero bytes, domain='%s' - BREAK!", domain)
 259                 break
 260             elif len(rows) != config.get("misskey_limit"):
 261                 logger.debug("Fetched %d row(s) but expected: %d'", len(rows), config.get('misskey_limit'))
 262                 offset = offset + (config.get("misskey_limit") - len(rows))
 263             else:
 264                 logger.debug("Raising offset by step=%d", step)
 265                 offset = offset + step
 266
 267             count = 0
 268             for instance in rows:
 269                 # Is it there?
 270                 logger.debug("instance[]='%s'", type(instance))
 271                 blocked = tidyup.domain(instance["host"])
 272                 if "isBlocked" in instance and instance["isBlocked"] and not dict_helper.has_key(blocklist, "blocked", blocked):
 273                     count = count + 1
 274                     logger.debug("Appending blocker='%s',blocked='%s',block_level='reject'", domain, blocked)
 275                     blocklist.append({
 276                         "blocker"    : domain,
 277                         "blocked"    : blocked,
 278                         "reason"     : None,
 279                         "block_level": "reject",
 280                     })
 281
 282             logger.debug("count=%d", count)
 283             if count == 0:
 284                 logger.debug("API is no more returning new instances, aborting loop!")
 285                 break
 286
 287         except network.exceptions as exception:
 288             logger.warning("Caught error, exiting loop: domain='%s',exception[%s]='%s'", domain, type(exception), str(exception))
 289             instances.set_last_error(domain, exception)
 290             offset = 0
 291             break
 292
 293     logger.debug("blocklist()=%d - EXIT!", len(blocklist))
 294     return blocklist