fba/networks/misskey.py

   1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
   2 # Copyright (C) 2023 Free Software Foundation
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published
   6 # by the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  16
  17 import json
  18 import logging
  19
  20 from fba.helpers import blacklist
  21 from fba.helpers import config
  22 from fba.helpers import dicts as dict_helper
  23 from fba.helpers import domain as domain_helper
  24 from fba.helpers import tidyup
  25
  26 from fba.http import csrf
  27 from fba.http import network
  28
  29 from fba.models import instances
  30
  31 logging.basicConfig(level=logging.INFO)
  32 logger = logging.getLogger(__name__)
  33 #logger.setLevel(logging.DEBUG)
  34
  35 def fetch_peers(domain: str) -> list:
  36     logger.debug("domain='%s' - CALLED!", domain)
  37     domain_helper.raise_on(domain)
  38
  39     if blacklist.is_blacklisted(domain):
  40         raise RuntimeError(f"domain='{domain}' is blacklisted but function was invoked")
  41     elif not instances.is_registered(domain):
  42         raise RuntimeError(f"domain='{domain}' is not registered but function was invoked")
  43
  44     logger.debug("domain='%s' is misskey, sending API POST request ...", domain)
  45     peers  = []
  46     offset = 0
  47     step   = config.get("misskey_limit")
  48
  49     # No CSRF by default, you don't have to add network.api_headers by yourself here
  50     headers = {}
  51
  52     try:
  53         logger.debug("Checking CSRF for domain='%s'", domain)
  54         headers = csrf.determine(domain, {})
  55     except network.exceptions as exception:
  56         logger.warning("Exception '%s' during checking CSRF (fetch_peers,%s)", type(exception), __name__)
  57         instances.set_last_error(domain, exception)
  58
  59         logger.debug("Returning empty list ... - EXIT!")
  60         return []
  61
  62     # iterating through all "suspended" (follow-only in its terminology)
  63     # instances page-by-page, since that troonware doesn't support
  64     # sending them all at once
  65     while True:
  66         logger.debug("Fetching offset=%d from domain='%s' ...", offset, domain)
  67         if offset == 0:
  68             fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
  69                 "sort" : "+pubSub",
  70                 "host" : None,
  71                 "limit": step
  72             }), headers)
  73         else:
  74             fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
  75                 "sort"  : "+pubSub",
  76                 "host"  : None,
  77                 "limit" : step,
  78                 "offset": offset - 1
  79             }), headers)
  80
  81         # Check records
  82         logger.debug("fetched[]='%s'", type(fetched))
  83         if "error_message" in fetched:
  84             logger.warning("post_json_api() for domain='%s' returned error message: '%s'", domain, fetched['error_message'])
  85             instances.set_last_error(domain, fetched)
  86             break
  87         elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
  88             logger.warning("post_json_api() returned error: '%s'", fetched["json"]["error"]["message"])
  89             instances.set_last_error(domain, fetched["json"]["error"]["message"])
  90             break
  91
  92         rows = fetched["json"]
  93
  94         logger.debug("rows(%d)[]='%s',step=%d", len(rows), type(rows), step)
  95         if len(rows) == 0:
  96             logger.debug("Returned zero bytes, domain='%s' - BREAK!", domain)
  97             break
  98         elif len(rows) != step:
  99             logger.debug("Fetched %d row(s) but expected: %d", len(rows), step)
 100             offset = offset + (step - len(rows))
 101         else:
 102             logger.debug("Raising offset by step=%d", step)
 103             offset = offset + step
 104
 105         added = 0
 106         logger.debug("rows(%d))[]='%s'", len(rows), type(rows))
 107         for row in rows:
 108             logger.debug("row()=%d", len(row))
 109             if "host" not in row:
 110                 logger.warning("row()=%d does not contain key 'host': row='%s',domain='%s' - SKIPPED!", len(row), row, domain)
 111                 continue
 112             elif not isinstance(row["host"], str):
 113                 logger.warning("row[host][]='%s' has not expected type 'str' - SKIPPED!", type(row["host"]))
 114                 continue
 115             elif row["host"] == "":
 116                 logger.warning("row[host] is an empty string,domain='%s' - SKIPPED!", domain)
 117                 continue
 118             elif row["host"] in peers:
 119                 logger.debug("Not adding row[host]='%s', already found - SKIPPED!", row["host"])
 120                 continue
 121             elif not domain_helper.is_wanted(row["host"]):
 122                 logger.debug("row[host]='%s' is not wanted - SKIPPED!", row["host"])
 123                 continue
 124
 125             logger.debug("Adding peer: row[host]='%s'", row["host"])
 126             added = added + 1
 127             peers.append(row["host"])
 128
 129         logger.debug("added=%d,rows()=%d", added, len(rows))
 130         if added == 0:
 131             logger.debug("Host returned already added (%d) peers - BREAK!", len(rows))
 132             break
 133
 134     logger.debug("peers()=%d - EXIT!", len(peers))
 135     return peers
 136
 137 def fetch_blocks(domain: str) -> list:
 138     logger.debug("domain='%s' - CALLED!", domain)
 139     domain_helper.raise_on(domain)
 140
 141     if blacklist.is_blacklisted(domain):
 142         raise RuntimeError(f"domain='{domain}' is blacklisted but function was invoked")
 143     elif not instances.is_registered(domain):
 144         raise RuntimeError(f"domain='{domain}' is not registered but function was invoked")
 145
 146     # No CSRF by default, you don't have to add network.api_headers by yourself here
 147     headers = {}
 148
 149     try:
 150         logger.debug("Checking CSRF for domain='%s' ...", domain)
 151         headers = csrf.determine(domain, {})
 152     except network.exceptions as exception:
 153         logger.warning("Exception '%s' during checking CSRF (fetch_blocks,%s)", type(exception), __name__)
 154         instances.set_last_error(domain, exception)
 155
 156         logger.debug("Returning empty list ... - EXIT!")
 157         return []
 158
 159     blocklist = []
 160     offset    = 0
 161     step      = config.get("misskey_limit")
 162
 163     # iterating through all "suspended" (follow-only in its terminology)
 164     # instances page-by-page since it doesn't support sending them all at once
 165     logger.debug("Fetching misskey blocks from domain='%s' ...", domain)
 166     while True:
 167         logger.debug("offset=%d", offset)
 168         try:
 169             logger.debug("Fetching offset=%d from domain='%s' ...", offset, domain)
 170             if offset == 0:
 171                 logger.debug("Sending JSON API request to domain='%s',step=%d ...", domain, step)
 172                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 173                     "sort"     : "+pubSub",
 174                     "host"     : None,
 175                     "limit"    : step
 176                 }), headers)
 177             else:
 178                 logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d ...", domain, step, offset)
 179                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 180                     "sort"     : "+pubSub",
 181                     "host"     : None,
 182                     "limit"    : step,
 183                     "offset"   : offset - 1
 184                 }), headers)
 185
 186             logger.debug("fetched[]='%s'", type(fetched))
 187             if "error_message" in fetched:
 188                 logger.warning("post_json_api() for domain='%s' returned error message: '%s'", domain, fetched['error_message'])
 189                 instances.set_last_error(domain, fetched)
 190                 break
 191             elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
 192                 logger.warning("post_json_api() returned error: '%s'", fetched["json"]["error"]["message"])
 193                 instances.set_last_error(domain, fetched["json"]["error"]["message"])
 194                 break
 195
 196             rows = fetched["json"]
 197
 198             logger.debug("rows(%d)[]='%s'", len(rows), type(rows))
 199             if len(rows) == 0:
 200                 logger.debug("Returned zero bytes, domain='%s' - BREAK!", domain)
 201                 break
 202             elif len(rows) != step:
 203                 logger.debug("Fetched %d row(s) but expected: %d", len(rows), step)
 204                 offset = offset + (step - len(rows))
 205             else:
 206                 logger.debug("Raising offset by step=%d", step)
 207                 offset = offset + step
 208
 209             count = 0
 210             logger.debug("Checking %d row(s) of instances ...", len(rows))
 211             for instance in rows:
 212                 # Is it there?
 213                 logger.debug("instance[]='%s'", type(instance))
 214                 if "host" not in instance:
 215                     logger.warning("instance(%d)='%s' has no key 'host' - SKIPPED!", len(instance), instance)
 216                     continue
 217                 elif not isinstance(instance["host"], str):
 218                     logger.warning("instance[host][]='%s' has not expected type 'str' - SKIPPED!", type(instance["host"]))
 219                     continue
 220                 elif instance["host"] == "":
 221                     logger.warning("instance[host] is an empty string,domain='%s' - SKIPPED!", domain)
 222                     continue
 223
 224                 logger.debug("instance[host]='%s' - BEFORE!", instance["host"])
 225                 blocked = tidyup.domain(instance["host"])
 226                 logger.debug("blocked[%s]='%s' - AFTER!", type(blocked), blocked)
 227
 228                 if blocked in [None, ""]:
 229                     logger.warning("instance[host]='%s' is None or empty after tidyup.domain() - SKIPPED!", instance["host"])
 230                     continue
 231                 elif not domain_helper.is_wanted(blocked):
 232                     logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
 233                     continue
 234                 elif "isSuspended" in instance and instance["isSuspended"] and not dict_helper.has_key(blocklist, "blocked", blocked):
 235                     count = count + 1
 236                     logger.debug("Appending blocker='%s',blocked='%s',block_level='suspended' ... #1", domain, blocked)
 237                     blocklist.append({
 238                         "blocker"    : domain,
 239                         "blocked"    : blocked,
 240                         "reason"     : None,
 241                         "block_level": "suspended",
 242                     })
 243                 elif "isBlocked" in instance and instance["isBlocked"] and not dict_helper.has_key(blocklist, "blocked", blocked):
 244                     count = count + 1
 245                     logger.debug("Appending blocker='%s',blocked='%s',block_level='suspended' ... #2", domain, blocked)
 246                     blocklist.append({
 247                         "blocker"    : domain,
 248                         "blocked"    : blocked,
 249                         "reason"     : None,
 250                         "block_level": "suspended",
 251                     })
 252                 elif "isSilenced" in instance and instance["isSilenced"] and not dict_helper.has_key(blocklist, "blocked", blocked):
 253                     count = count + 1
 254                     logger.debug("Appending blocker='%s',blocked='%s',block_level='silenced' ...", domain, blocked)
 255                     blocklist.append({
 256                         "blocker"    : domain,
 257                         "blocked"    : blocked,
 258                         "reason"     : None,
 259                         "block_level": "silenced",
 260                     })
 261                 else:
 262                     logger.debug("domain='%s',blocked='%s' is not marked suspended - SKIPPED!", domain, blocked)
 263                     continue
 264
 265             logger.debug("count=%d", count)
 266             if count == 0:
 267                 logger.debug("API is no more returning new instances, aborting loop! domain='%s'", domain)
 268                 break
 269
 270         except network.exceptions as exception:
 271             logger.warning("Caught error, exiting loop: domain='%s',exception[%s]='%s'", domain, type(exception), str(exception))
 272             instances.set_last_error(domain, exception)
 273             offset = 0
 274             break
 275
 276     logger.debug("blocklist()=%d - EXIT!", len(blocklist))
 277     return blocklist