fba/networks/misskey.py

   1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
   2 # Copyright (C) 2023 Free Software Foundation
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published
   6 # by the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  16
  17 import json
  18 import logging
  19
  20 from fba import csrf
  21
  22 from fba.helpers import config
  23 from fba.helpers import dicts as dict_helper
  24 from fba.helpers import domain as domain_helper
  25 from fba.helpers import tidyup
  26
  27 from fba.http import network
  28
  29 from fba.models import instances
  30
  31 logging.basicConfig(level=logging.INFO)
  32 logger = logging.getLogger(__name__)
  33
  34 def fetch_peers(domain: str) -> list:
  35     logger.debug("domain='%s' - CALLED!", domain)
  36     domain_helper.raise_on(domain)
  37
  38     logger.debug("domain='%s' is misskey, sending API POST request ...", domain)
  39     peers  = list()
  40     offset = 0
  41     step   = config.get("misskey_limit")
  42
  43     # No CSRF by default, you don't have to add network.api_headers by yourself here
  44     headers = tuple()
  45
  46     try:
  47         logger.debug("Checking CSRF for domain='%s'", domain)
  48         headers = csrf.determine(domain, dict())
  49     except network.exceptions as exception:
  50         logger.warning("Exception '%s' during checking CSRF (fetch_peers,%s)", type(exception), __name__)
  51         instances.set_last_error(domain, exception)
  52
  53         logger.debug("Returning empty list ... - EXIT!")
  54         return list()
  55
  56     # iterating through all "suspended" (follow-only in its terminology)
  57     # instances page-by-page, since that troonware doesn't support
  58     # sending them all at once
  59     while True:
  60         logger.debug("Fetching offset=%d from domain='%s' ...", offset, domain)
  61         if offset == 0:
  62             fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
  63                 "sort" : "+pubAt",
  64                 "host" : None,
  65                 "limit": step
  66             }), headers)
  67         else:
  68             fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
  69                 "sort"  : "+pubAt",
  70                 "host"  : None,
  71                 "limit" : step,
  72                 "offset": offset - 1
  73             }), headers)
  74
  75         # Check records
  76         logger.debug("fetched[]='%s'", type(fetched))
  77         if "error_message" in fetched:
  78             logger.warning("post_json_api() for domain='%s' returned error message: '%s'", domain, fetched['error_message'])
  79             instances.set_last_error(domain, fetched)
  80             break
  81         elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
  82             logger.warning("post_json_api() returned error: '%s'", fetched['error']['message'])
  83             instances.set_last_error(domain, fetched["json"]["error"]["message"])
  84             break
  85
  86         rows = fetched["json"]
  87
  88         logger.debug("rows(%d)[]='%s',step=%d", len(rows), type(rows), step)
  89         if len(rows) == 0:
  90             logger.debug("Returned zero bytes, domain='%s' - BREAK!", domain)
  91             break
  92         elif len(rows) != config.get("misskey_limit"):
  93             logger.debug("Fetched %d row(s) but expected: %d", len(rows), config.get('misskey_limit'))
  94             offset = offset + (config.get("misskey_limit") - len(rows))
  95         else:
  96             logger.debug("Raising offset by step=%d", step)
  97             offset = offset + step
  98
  99         added = 0
 100         logger.debug("rows(%d))[]='%s'", len(rows), type(rows))
 101         for row in rows:
 102             logger.debug("row()=%d", len(row))
 103             if "host" not in row:
 104                 logger.warning("row()=%d does not contain key 'host': row='%s',domain='%s' - SKIPPED!", len(row), row, domain)
 105                 continue
 106             elif not isinstance(row["host"], str):
 107                 logger.warning("row[host][]='%s' is not of type 'str' - SKIPPED!", type(row['host']))
 108                 continue
 109             elif not domain_helper.is_wanted(row["host"]):
 110                 logger.debug("row[host]='%s' is not wanted, domain='%s' - SKIPPED!", row['host'], domain)
 111                 continue
 112             elif row["host"] in peers:
 113                 logger.debug("Not adding row[host]='%s', already found - SKIPPED!", row['host'])
 114                 continue
 115
 116             logger.debug("Adding peer: row[host]='%s'", row['host'])
 117             added = added + 1
 118             peers.append(row["host"])
 119
 120         logger.debug("added=%d,rows()=%d", added, len(rows))
 121         if added == 0:
 122             logger.debug("Host returned already added (%d) peers - BREAK!", len(rows))
 123             break
 124
 125     logger.debug("peers()=%d - EXIT!", len(peers))
 126     return peers
 127
 128 def fetch_blocks(domain: str) -> list:
 129     logger.debug("domain='%s' - CALLED!", domain)
 130     domain_helper.raise_on(domain)
 131
 132     # No CSRF by default, you don't have to add network.api_headers by yourself here
 133     headers = tuple()
 134
 135     try:
 136         logger.debug("Checking CSRF for domain='%s'", domain)
 137         headers = csrf.determine(domain, dict())
 138     except network.exceptions as exception:
 139         logger.warning("Exception '%s' during checking CSRF (fetch_blocks,%s)", type(exception), __name__)
 140         instances.set_last_error(domain, exception)
 141
 142         logger.debug("Returning empty list ... - EXIT!")
 143         return list()
 144
 145     blocklist = list()
 146     offset    = 0
 147     step      = config.get("misskey_limit")
 148
 149     # iterating through all "suspended" (follow-only in its terminology)
 150     # instances page-by-page since it doesn't support sending them all at once
 151     logger.debug("Fetching misskey blocks from domain='%s'", domain)
 152     while True:
 153         try:
 154             logger.debug("Fetching offset=%d from domain='%s' ...", offset, domain)
 155             if offset == 0:
 156                 logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d", domain, step, offset)
 157                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 158                     "sort"     : "+pubAt",
 159                     "host"     : None,
 160                     "suspended": True,
 161                     "limit"    : step
 162                 }), headers)
 163             else:
 164                 logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d", domain, step, offset)
 165                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 166                     "sort"     : "+pubAt",
 167                     "host"     : None,
 168                     "suspended": True,
 169                     "limit"    : step,
 170                     "offset"   : offset - 1
 171                 }), headers)
 172
 173             logger.debug("fetched[]='%s'", type(fetched))
 174             if "error_message" in fetched:
 175                 logger.warning("post_json_api() for domain='%s' returned error message: '%s'", domain, fetched['error_message'])
 176                 instances.set_last_error(domain, fetched)
 177                 break
 178             elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
 179                 logger.warning("post_json_api() returned error: '%s'", fetched['error']['message'])
 180                 instances.set_last_error(domain, fetched["json"]["error"]["message"])
 181                 break
 182
 183             rows = fetched["json"]
 184
 185             logger.debug("rows(%d)[]='%s'", len(rows), type(rows))
 186             if len(rows) == 0:
 187                 logger.debug("Returned zero bytes, domain='%s' - BREAK!", domain)
 188                 break
 189             elif len(rows) != config.get("misskey_limit"):
 190                 logger.debug("Fetched %d row(s) but expected: %d", len(rows), config.get('misskey_limit'))
 191                 offset = offset + (config.get("misskey_limit") - len(rows))
 192             else:
 193                 logger.debug("Raising offset by step=%d", step)
 194                 offset = offset + step
 195
 196             count = 0
 197             for instance in rows:
 198                 # Is it there?
 199                 logger.debug("instance[]='%s'", type(instance))
 200                 blocked = tidyup.domain(instance["host"])
 201                 if "isSuspended" in instance and instance["isSuspended"] and not dict_helper.has_key(blocklist, "blocked", blocked):
 202                     count = count + 1
 203                     logger.debug("Appending blocker='%s',blocked='%s',block_level='suspended'", domain, blocked)
 204                     blocklist.append({
 205                         "blocker"    : domain,
 206                         "blocked"    : blocked,
 207                         "reason"     : None,
 208                         "block_level": "suspended",
 209                     })
 210
 211             logger.debug("count=%d", count)
 212             if count == 0:
 213                 logger.debug("API is no more returning new instances, aborting loop! domain='%s'", domain)
 214                 break
 215
 216         except network.exceptions as exception:
 217             logger.warning("Caught error, exiting loop: domain='%s',exception[%s]='%s'", domain, type(exception), str(exception))
 218             instances.set_last_error(domain, exception)
 219             offset = 0
 220             break
 221
 222     while True:
 223         # Fetch blocked (full suspended) instances
 224         try:
 225             if offset == 0:
 226                 logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d", domain, step, offset)
 227                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 228                     "sort"   : "+pubAt",
 229                     "host"   : None,
 230                     "blocked": True,
 231                     "limit"  : step
 232                 }), headers)
 233             else:
 234                 logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d", domain, step, offset)
 235                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 236                     "sort"   : "+pubAt",
 237                     "host"   : None,
 238                     "blocked": True,
 239                     "limit"  : step,
 240                     "offset" : offset - 1
 241                 }), headers)
 242
 243             logger.debug("fetched[]='%s'", type(fetched))
 244             if "error_message" in fetched:
 245                 logger.warning("post_json_api() for domain='%s' returned error message: '%s'", domain, fetched['error_message'])
 246                 instances.set_last_error(domain, fetched)
 247                 break
 248             elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
 249                 logger.warning("post_json_api() returned error: '%s'", fetched['error']['message'])
 250                 instances.set_last_error(domain, fetched["json"]["error"]["message"])
 251                 break
 252
 253             rows = fetched["json"]
 254
 255             logger.debug("rows(%d)[]='%s'", len(rows), type(rows))
 256             if len(rows) == 0:
 257                 logger.debug("Returned zero bytes, domain='%s' - BREAK!", domain)
 258                 break
 259             elif len(rows) != config.get("misskey_limit"):
 260                 logger.debug("Fetched %d row(s) but expected: %d'", len(rows), config.get('misskey_limit'))
 261                 offset = offset + (config.get("misskey_limit") - len(rows))
 262             else:
 263                 logger.debug("Raising offset by step=%d", step)
 264                 offset = offset + step
 265
 266             count = 0
 267             for instance in rows:
 268                 # Is it there?
 269                 logger.debug("instance[]='%s'", type(instance))
 270                 blocked = tidyup.domain(instance["host"])
 271                 if "isBlocked" in instance and instance["isBlocked"] and not dict_helper.has_key(blocklist, "blocked", blocked):
 272                     count = count + 1
 273                     logger.debug("Appending blocker='%s',blocked='%s',block_level='reject'", domain, blocked)
 274                     blocklist.append({
 275                         "blocker"    : domain,
 276                         "blocked"    : blocked,
 277                         "reason"     : None,
 278                         "block_level": "reject",
 279                     })
 280
 281             logger.debug("count=%d", count)
 282             if count == 0:
 283                 logger.debug("API is no more returning new instances, aborting loop!")
 284                 break
 285
 286         except network.exceptions as exception:
 287             logger.warning("Caught error, exiting loop: domain='%s',exception[%s]='%s'", domain, type(exception), str(exception))
 288             instances.set_last_error(domain, exception)
 289             offset = 0
 290             break
 291
 292     logger.debug("blocklist()=%d - EXIT!", len(blocklist))
 293     return blocklist