fba/networks/misskey.py

   1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
   2 # Copyright (C) 2023 Free Software Foundation
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published
   6 # by the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  16
  17 import json
  18 import logging
  19
  20 from fba.helpers import blacklist
  21 from fba.helpers import config
  22 from fba.helpers import dicts as dict_helper
  23 from fba.helpers import domain as domain_helper
  24 from fba.helpers import tidyup
  25
  26 from fba.http import csrf
  27 from fba.http import network
  28
  29 from fba.models import instances
  30
  31 logging.basicConfig(level=logging.INFO)
  32 logger = logging.getLogger(__name__)
  33
  34 def fetch_peers(domain: str) -> list:
  35     logger.debug("domain='%s' - CALLED!", domain)
  36     domain_helper.raise_on(domain)
  37
  38     if blacklist.is_blacklisted(domain):
  39         raise Exception(f"domain='{domain}' is blacklisted but function is invoked.")
  40
  41     logger.debug("domain='%s' is misskey, sending API POST request ...", domain)
  42     peers  = list()
  43     offset = 0
  44     step   = config.get("misskey_limit")
  45
  46     # No CSRF by default, you don't have to add network.api_headers by yourself here
  47     headers = tuple()
  48
  49     try:
  50         logger.debug("Checking CSRF for domain='%s'", domain)
  51         headers = csrf.determine(domain, dict())
  52     except network.exceptions as exception:
  53         logger.warning("Exception '%s' during checking CSRF (fetch_peers,%s)", type(exception), __name__)
  54         instances.set_last_error(domain, exception)
  55
  56         logger.debug("Returning empty list ... - EXIT!")
  57         return list()
  58
  59     # iterating through all "suspended" (follow-only in its terminology)
  60     # instances page-by-page, since that troonware doesn't support
  61     # sending them all at once
  62     while True:
  63         logger.debug("Fetching offset=%d from domain='%s' ...", offset, domain)
  64         if offset == 0:
  65             fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
  66                 "sort" : "+pubAt",
  67                 "host" : None,
  68                 "limit": step
  69             }), headers)
  70         else:
  71             fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
  72                 "sort"  : "+pubAt",
  73                 "host"  : None,
  74                 "limit" : step,
  75                 "offset": offset - 1
  76             }), headers)
  77
  78         # Check records
  79         logger.debug("fetched[]='%s'", type(fetched))
  80         if "error_message" in fetched:
  81             logger.warning("post_json_api() for domain='%s' returned error message: '%s'", domain, fetched['error_message'])
  82             instances.set_last_error(domain, fetched)
  83             break
  84         elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
  85             logger.warning("post_json_api() returned error: '%s'", fetched['error']['message'])
  86             instances.set_last_error(domain, fetched["json"]["error"]["message"])
  87             break
  88
  89         rows = fetched["json"]
  90
  91         logger.debug("rows(%d)[]='%s',step=%d", len(rows), type(rows), step)
  92         if len(rows) == 0:
  93             logger.debug("Returned zero bytes, domain='%s' - BREAK!", domain)
  94             break
  95         elif len(rows) != config.get("misskey_limit"):
  96             logger.debug("Fetched %d row(s) but expected: %d", len(rows), config.get('misskey_limit'))
  97             offset = offset + (config.get("misskey_limit") - len(rows))
  98         else:
  99             logger.debug("Raising offset by step=%d", step)
 100             offset = offset + step
 101
 102         added = 0
 103         logger.debug("rows(%d))[]='%s'", len(rows), type(rows))
 104         for row in rows:
 105             logger.debug("row()=%d", len(row))
 106             if "host" not in row:
 107                 logger.warning("row()=%d does not contain key 'host': row='%s',domain='%s' - SKIPPED!", len(row), row, domain)
 108                 continue
 109             elif not isinstance(row["host"], str):
 110                 logger.warning("row[host][]='%s' is not of type 'str' - SKIPPED!", type(row['host']))
 111                 continue
 112             elif row["host"] in peers:
 113                 logger.debug("Not adding row[host]='%s', already found - SKIPPED!", row['host'])
 114                 continue
 115             elif not domain_helper.is_wanted(row["host"]):
 116                 logger.debug("row[host]='%s' is not wanted - SKIPPED!", row["host"])
 117                 continue
 118
 119             logger.debug("Adding peer: row[host]='%s'", row['host'])
 120             added = added + 1
 121             peers.append(row["host"])
 122
 123         logger.debug("added=%d,rows()=%d", added, len(rows))
 124         if added == 0:
 125             logger.debug("Host returned already added (%d) peers - BREAK!", len(rows))
 126             break
 127
 128     logger.debug("peers()=%d - EXIT!", len(peers))
 129     return peers
 130
 131 def fetch_blocks(domain: str) -> list:
 132     logger.debug("domain='%s' - CALLED!", domain)
 133     domain_helper.raise_on(domain)
 134
 135     if blacklist.is_blacklisted(domain):
 136         raise Exception(f"domain='{domain}' is blacklisted but function is invoked.")
 137     elif not instances.is_registered(domain):
 138         raise Exception(f"domain='{domain}' is not registered but function is invoked.")
 139
 140     # No CSRF by default, you don't have to add network.api_headers by yourself here
 141     headers = tuple()
 142
 143     try:
 144         logger.debug("Checking CSRF for domain='%s' ...", domain)
 145         headers = csrf.determine(domain, dict())
 146     except network.exceptions as exception:
 147         logger.warning("Exception '%s' during checking CSRF (fetch_blocks,%s)", type(exception), __name__)
 148         instances.set_last_error(domain, exception)
 149
 150         logger.debug("Returning empty list ... - EXIT!")
 151         return list()
 152
 153     blocklist = list()
 154     offset    = 0
 155     step      = config.get("misskey_limit")
 156
 157     # iterating through all "suspended" (follow-only in its terminology)
 158     # instances page-by-page since it doesn't support sending them all at once
 159     logger.debug("Fetching misskey blocks from domain='%s'", domain)
 160     while True:
 161         logger.debug("offset=%d", offset)
 162         try:
 163             logger.debug("Fetching offset=%d from domain='%s' ...", offset, domain)
 164             if offset == 0:
 165                 logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d", domain, step, offset)
 166                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 167                     "sort"     : "+pubAt",
 168                     "host"     : None,
 169                     "suspended": True,
 170                     "limit"    : step
 171                 }), headers)
 172             else:
 173                 logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d", domain, step, offset)
 174                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 175                     "sort"     : "+pubAt",
 176                     "host"     : None,
 177                     "suspended": True,
 178                     "limit"    : step,
 179                     "offset"   : offset - 1
 180                 }), headers)
 181
 182             logger.debug("fetched[]='%s'", type(fetched))
 183             if "error_message" in fetched:
 184                 logger.warning("post_json_api() for domain='%s' returned error message: '%s'", domain, fetched['error_message'])
 185                 instances.set_last_error(domain, fetched)
 186                 break
 187             elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
 188                 logger.warning("post_json_api() returned error: '%s'", fetched['error']['message'])
 189                 instances.set_last_error(domain, fetched["json"]["error"]["message"])
 190                 break
 191
 192             rows = fetched["json"]
 193
 194             logger.debug("rows(%d)[]='%s'", len(rows), type(rows))
 195             if len(rows) == 0:
 196                 logger.debug("Returned zero bytes, domain='%s' - BREAK!", domain)
 197                 break
 198             elif len(rows) != config.get("misskey_limit"):
 199                 logger.debug("Fetched %d row(s) but expected: %d", len(rows), config.get('misskey_limit'))
 200                 offset = offset + (config.get("misskey_limit") - len(rows))
 201             else:
 202                 logger.debug("Raising offset by step=%d", step)
 203                 offset = offset + step
 204
 205             count = 0
 206             logger.debug("Checking %d row(s) of instances ...", len(rows))
 207             for instance in rows:
 208                 # Is it there?
 209                 logger.debug("instance[]='%s'", type(instance))
 210                 if "host" not in instance:
 211                     logger.warning("instance(%d)='%s' has no key 'host' - SKIPPED!", len(instance), instance)
 212                     continue
 213                 elif instance["host"] is None or instance["host"] == "":
 214                     logger.debug("instance[host]='%s' is None or empty - SKIPPED!", instance["host"])
 215                     continue
 216
 217                 logger.debug("instance[host]='%s' - BEFORE!", instance["host"])
 218                 blocked = tidyup.domain(instance["host"])
 219                 logger.debug("blocked[%s]='%s' - AFTER!", type(blocked), blocked)
 220
 221                 if blocked is None or blocked == "":
 222                     logger.warning("instance[host]='%s' is None or empty after tidyup.domain() - SKIPPED!", instance["host"])
 223                     continue
 224                 elif not domain_helper.is_wanted(blocked):
 225                     logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
 226                     continue
 227                 elif "isSuspended" in instance and instance["isSuspended"] and not dict_helper.has_key(blocklist, "blocked", blocked):
 228                     count = count + 1
 229                     logger.debug("Appending blocker='%s',blocked='%s',block_level='suspended'", domain, blocked)
 230                     blocklist.append({
 231                         "blocker"    : domain,
 232                         "blocked"    : blocked,
 233                         "reason"     : None,
 234                         "block_level": "suspended",
 235                     })
 236
 237             logger.debug("count=%d", count)
 238             if count == 0:
 239                 logger.debug("API is no more returning new instances, aborting loop! domain='%s'", domain)
 240                 break
 241
 242         except network.exceptions as exception:
 243             logger.warning("Caught error, exiting loop: domain='%s',exception[%s]='%s'", domain, type(exception), str(exception))
 244             instances.set_last_error(domain, exception)
 245             offset = 0
 246             break
 247
 248     while True:
 249         # Fetch blocked (full suspended) instances
 250         logger.debug("offset=%d", offset)
 251         try:
 252             if offset == 0:
 253                 logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d", domain, step, offset)
 254                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 255                     "sort"   : "+pubAt",
 256                     "host"   : None,
 257                     "blocked": True,
 258                     "limit"  : step
 259                 }), headers)
 260             else:
 261                 logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d", domain, step, offset)
 262                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 263                     "sort"   : "+pubAt",
 264                     "host"   : None,
 265                     "blocked": True,
 266                     "limit"  : step,
 267                     "offset" : offset - 1
 268                 }), headers)
 269
 270             logger.debug("fetched[]='%s'", type(fetched))
 271             if "error_message" in fetched:
 272                 logger.warning("post_json_api() for domain='%s' returned error message: '%s'", domain, fetched['error_message'])
 273                 instances.set_last_error(domain, fetched)
 274                 break
 275             elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
 276                 logger.warning("post_json_api() returned error: '%s'", fetched['error']['message'])
 277                 instances.set_last_error(domain, fetched["json"]["error"]["message"])
 278                 break
 279
 280             rows = fetched["json"]
 281
 282             logger.debug("rows(%d)[]='%s'", len(rows), type(rows))
 283             if len(rows) == 0:
 284                 logger.debug("Returned zero bytes, domain='%s' - BREAK!", domain)
 285                 break
 286             elif len(rows) != config.get("misskey_limit"):
 287                 logger.debug("Fetched %d row(s) but expected: %d'", len(rows), config.get('misskey_limit'))
 288                 offset = offset + (config.get("misskey_limit") - len(rows))
 289             else:
 290                 logger.debug("Raising offset by step=%d", step)
 291                 offset = offset + step
 292
 293             count = 0
 294             logger.debug("Checking %d row(s) of instances ...", len(rows))
 295             for instance in rows:
 296                 # Is it there?
 297                 logger.debug("instance[]='%s'", type(instance))
 298                 blocked = tidyup.domain(instance["host"])
 299                 logger.debug("blocked='%s'", blocked)
 300
 301                 if blocked is None or blocked == "":
 302                     logger.warning("instance[host]='%s' is None or empty after tidyup.domain() - SKIPPED!", instance["host"])
 303                     continue
 304                 elif not domain_helper.is_wanted(blocked):
 305                     logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
 306                     continue
 307                 elif "isBlocked" in instance and instance["isBlocked"] and not dict_helper.has_key(blocklist, "blocked", blocked):
 308                     count = count + 1
 309                     logger.debug("Appending blocker='%s',blocked='%s',block_level='reject'", domain, blocked)
 310                     blocklist.append({
 311                         "blocker"    : domain,
 312                         "blocked"    : blocked,
 313                         "reason"     : None,
 314                         "block_level": "reject",
 315                     })
 316
 317             logger.debug("count=%d", count)
 318             if count == 0:
 319                 logger.debug("API is no more returning new instances, aborting loop!")
 320                 break
 321
 322         except network.exceptions as exception:
 323             logger.warning("Caught error, exiting loop: domain='%s',exception[%s]='%s'", domain, type(exception), str(exception))
 324             instances.set_last_error(domain, exception)
 325             offset = 0
 326             break
 327
 328     logger.debug("blocklist()=%d - EXIT!", len(blocklist))
 329     return blocklist