fba/networks/misskey.py

   1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
   2 # Copyright (C) 2023 Free Software Foundation
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published
   6 # by the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  16
  17 import json
  18 import logging
  19
  20 from fba.helpers import blacklist
  21 from fba.helpers import config
  22 from fba.helpers import dicts as dict_helper
  23 from fba.helpers import domain as domain_helper
  24 from fba.helpers import tidyup
  25
  26 from fba.http import csrf
  27 from fba.http import network
  28
  29 from fba.models import instances
  30
  31 logging.basicConfig(level=logging.INFO)
  32 logger = logging.getLogger(__name__)
  33
  34 def fetch_peers(domain: str) -> list:
  35     logger.debug("domain='%s' - CALLED!", domain)
  36     domain_helper.raise_on(domain)
  37
  38     if blacklist.is_blacklisted(domain):
  39         raise Exception(f"domain='{domain}' is blacklisted but function is invoked.")
  40     elif not instances.is_registered(domain):
  41         raise Exception(f"domain='{domain}' is not registered but function is invoked.")
  42
  43     logger.debug("domain='%s' is misskey, sending API POST request ...", domain)
  44     peers  = list()
  45     offset = 0
  46     step   = config.get("misskey_limit")
  47
  48     # No CSRF by default, you don't have to add network.api_headers by yourself here
  49     headers = tuple()
  50
  51     try:
  52         logger.debug("Checking CSRF for domain='%s'", domain)
  53         headers = csrf.determine(domain, dict())
  54     except network.exceptions as exception:
  55         logger.warning("Exception '%s' during checking CSRF (fetch_peers,%s)", type(exception), __name__)
  56         instances.set_last_error(domain, exception)
  57
  58         logger.debug("Returning empty list ... - EXIT!")
  59         return list()
  60
  61     # iterating through all "suspended" (follow-only in its terminology)
  62     # instances page-by-page, since that troonware doesn't support
  63     # sending them all at once
  64     while True:
  65         logger.debug("Fetching offset=%d from domain='%s' ...", offset, domain)
  66         if offset == 0:
  67             fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
  68                 "sort" : "+pubAt",
  69                 "host" : None,
  70                 "limit": step
  71             }), headers)
  72         else:
  73             fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
  74                 "sort"  : "+pubAt",
  75                 "host"  : None,
  76                 "limit" : step,
  77                 "offset": offset - 1
  78             }), headers)
  79
  80         # Check records
  81         logger.debug("fetched[]='%s'", type(fetched))
  82         if "error_message" in fetched:
  83             logger.warning("post_json_api() for domain='%s' returned error message: '%s'", domain, fetched['error_message'])
  84             instances.set_last_error(domain, fetched)
  85             break
  86         elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
  87             logger.warning("post_json_api() returned error: '%s'", fetched["json"]["error"]["message"])
  88             instances.set_last_error(domain, fetched["json"]["error"]["message"])
  89             break
  90
  91         rows = fetched["json"]
  92
  93         logger.debug("rows(%d)[]='%s',step=%d", len(rows), type(rows), step)
  94         if len(rows) == 0:
  95             logger.debug("Returned zero bytes, domain='%s' - BREAK!", domain)
  96             break
  97         elif len(rows) != config.get("misskey_limit"):
  98             logger.debug("Fetched %d row(s) but expected: %d", len(rows), config.get('misskey_limit'))
  99             offset = offset + (config.get("misskey_limit") - len(rows))
 100         else:
 101             logger.debug("Raising offset by step=%d", step)
 102             offset = offset + step
 103
 104         added = 0
 105         logger.debug("rows(%d))[]='%s'", len(rows), type(rows))
 106         for row in rows:
 107             logger.debug("row()=%d", len(row))
 108             if "host" not in row:
 109                 logger.warning("row()=%d does not contain key 'host': row='%s',domain='%s' - SKIPPED!", len(row), row, domain)
 110                 continue
 111             elif not isinstance(row["host"], str):
 112                 logger.warning("row[host][]='%s' is not of type 'str' - SKIPPED!", type(row['host']))
 113                 continue
 114             elif row["host"] in peers:
 115                 logger.debug("Not adding row[host]='%s', already found - SKIPPED!", row['host'])
 116                 continue
 117             elif not domain_helper.is_wanted(row["host"]):
 118                 logger.debug("row[host]='%s' is not wanted - SKIPPED!", row["host"])
 119                 continue
 120
 121             logger.debug("Adding peer: row[host]='%s'", row['host'])
 122             added = added + 1
 123             peers.append(row["host"])
 124
 125         logger.debug("added=%d,rows()=%d", added, len(rows))
 126         if added == 0:
 127             logger.debug("Host returned already added (%d) peers - BREAK!", len(rows))
 128             break
 129
 130     logger.debug("peers()=%d - EXIT!", len(peers))
 131     return peers
 132
 133 def fetch_blocks(domain: str) -> list:
 134     logger.debug("domain='%s' - CALLED!", domain)
 135     domain_helper.raise_on(domain)
 136
 137     if blacklist.is_blacklisted(domain):
 138         raise Exception(f"domain='{domain}' is blacklisted but function is invoked.")
 139     elif not instances.is_registered(domain):
 140         raise Exception(f"domain='{domain}' is not registered but function is invoked.")
 141
 142     # No CSRF by default, you don't have to add network.api_headers by yourself here
 143     headers = tuple()
 144
 145     try:
 146         logger.debug("Checking CSRF for domain='%s' ...", domain)
 147         headers = csrf.determine(domain, dict())
 148     except network.exceptions as exception:
 149         logger.warning("Exception '%s' during checking CSRF (fetch_blocks,%s)", type(exception), __name__)
 150         instances.set_last_error(domain, exception)
 151
 152         logger.debug("Returning empty list ... - EXIT!")
 153         return list()
 154
 155     blocklist = list()
 156     offset    = 0
 157     step      = config.get("misskey_limit")
 158
 159     # iterating through all "suspended" (follow-only in its terminology)
 160     # instances page-by-page since it doesn't support sending them all at once
 161     logger.debug("Fetching misskey blocks from domain='%s'", domain)
 162     while True:
 163         logger.debug("offset=%d", offset)
 164         try:
 165             logger.debug("Fetching offset=%d from domain='%s' ...", offset, domain)
 166             if offset == 0:
 167                 logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d", domain, step, offset)
 168                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 169                     "sort"     : "+pubAt",
 170                     "host"     : None,
 171                     "suspended": True,
 172                     "limit"    : step
 173                 }), headers)
 174             else:
 175                 logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d", domain, step, offset)
 176                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 177                     "sort"     : "+pubAt",
 178                     "host"     : None,
 179                     "suspended": True,
 180                     "limit"    : step,
 181                     "offset"   : offset - 1
 182                 }), headers)
 183
 184             logger.debug("fetched[]='%s'", type(fetched))
 185             if "error_message" in fetched:
 186                 logger.warning("post_json_api() for domain='%s' returned error message: '%s'", domain, fetched['error_message'])
 187                 instances.set_last_error(domain, fetched)
 188                 break
 189             elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
 190                 logger.warning("post_json_api() returned error: '%s'", fetched["json"]["error"]["message"])
 191                 instances.set_last_error(domain, fetched["json"]["error"]["message"])
 192                 break
 193
 194             rows = fetched["json"]
 195
 196             logger.debug("rows(%d)[]='%s'", len(rows), type(rows))
 197             if len(rows) == 0:
 198                 logger.debug("Returned zero bytes, domain='%s' - BREAK!", domain)
 199                 break
 200             elif len(rows) != config.get("misskey_limit"):
 201                 logger.debug("Fetched %d row(s) but expected: %d", len(rows), config.get('misskey_limit'))
 202                 offset = offset + (config.get("misskey_limit") - len(rows))
 203             else:
 204                 logger.debug("Raising offset by step=%d", step)
 205                 offset = offset + step
 206
 207             count = 0
 208             logger.debug("Checking %d row(s) of instances ...", len(rows))
 209             for instance in rows:
 210                 # Is it there?
 211                 logger.debug("instance[]='%s'", type(instance))
 212                 if "host" not in instance:
 213                     logger.warning("instance(%d)='%s' has no key 'host' - SKIPPED!", len(instance), instance)
 214                     continue
 215                 elif instance["host"] in [None, ""]:
 216                     logger.debug("instance[host]='%s' is None or empty - SKIPPED!", instance["host"])
 217                     continue
 218
 219                 logger.debug("instance[host]='%s' - BEFORE!", instance["host"])
 220                 blocked = tidyup.domain(instance["host"])
 221                 logger.debug("blocked[%s]='%s' - AFTER!", type(blocked), blocked)
 222
 223                 if blocked in [None, ""]:
 224                     logger.warning("instance[host]='%s' is None or empty after tidyup.domain() - SKIPPED!", instance["host"])
 225                     continue
 226                 elif not domain_helper.is_wanted(blocked):
 227                     logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
 228                     continue
 229                 elif "isSuspended" in instance and instance["isSuspended"] and not dict_helper.has_key(blocklist, "blocked", blocked):
 230                     count = count + 1
 231                     logger.debug("Appending blocker='%s',blocked='%s',block_level='suspended'", domain, blocked)
 232                     blocklist.append({
 233                         "blocker"    : domain,
 234                         "blocked"    : blocked,
 235                         "reason"     : None,
 236                         "block_level": "suspended",
 237                     })
 238
 239             logger.debug("count=%d", count)
 240             if count == 0:
 241                 logger.debug("API is no more returning new instances, aborting loop! domain='%s'", domain)
 242                 break
 243
 244         except network.exceptions as exception:
 245             logger.warning("Caught error, exiting loop: domain='%s',exception[%s]='%s'", domain, type(exception), str(exception))
 246             instances.set_last_error(domain, exception)
 247             offset = 0
 248             break
 249
 250     while True:
 251         # Fetch blocked (full suspended) instances
 252         logger.debug("offset=%d", offset)
 253         try:
 254             if offset == 0:
 255                 logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d", domain, step, offset)
 256                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 257                     "sort"   : "+pubAt",
 258                     "host"   : None,
 259                     "blocked": True,
 260                     "limit"  : step
 261                 }), headers)
 262             else:
 263                 logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d", domain, step, offset)
 264                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 265                     "sort"   : "+pubAt",
 266                     "host"   : None,
 267                     "blocked": True,
 268                     "limit"  : step,
 269                     "offset" : offset - 1
 270                 }), headers)
 271
 272             logger.debug("fetched[]='%s'", type(fetched))
 273             if "error_message" in fetched:
 274                 logger.warning("post_json_api() for domain='%s' returned error message: '%s'", domain, fetched['error_message'])
 275                 instances.set_last_error(domain, fetched)
 276                 break
 277             elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
 278                 logger.warning("post_json_api() returned error: '%s'", fetched["json"]["error"]["message"])
 279                 instances.set_last_error(domain, fetched["json"]["error"]["message"])
 280                 break
 281
 282             rows = fetched["json"]
 283
 284             logger.debug("rows(%d)[]='%s'", len(rows), type(rows))
 285             if len(rows) == 0:
 286                 logger.debug("Returned zero bytes, domain='%s' - BREAK!", domain)
 287                 break
 288             elif len(rows) != config.get("misskey_limit"):
 289                 logger.debug("Fetched %d row(s) but expected: %d'", len(rows), config.get('misskey_limit'))
 290                 offset = offset + (config.get("misskey_limit") - len(rows))
 291             else:
 292                 logger.debug("Raising offset by step=%d", step)
 293                 offset = offset + step
 294
 295             count = 0
 296             logger.debug("Checking %d row(s) of instances ...", len(rows))
 297             for instance in rows:
 298                 # Is it there?
 299                 logger.debug("instance[]='%s'", type(instance))
 300                 blocked = tidyup.domain(instance["host"]) if instance["host"] != "" else None
 301                 logger.debug("blocked='%s' - AFTER!", blocked)
 302
 303                 if blocked in [None, ""]:
 304                     logger.warning("instance[host]='%s' is None or empty after tidyup.domain() - SKIPPED!", instance["host"])
 305                     continue
 306                 elif not domain_helper.is_wanted(blocked):
 307                     logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
 308                     continue
 309                 elif "isBlocked" in instance and instance["isBlocked"] and not dict_helper.has_key(blocklist, "blocked", blocked):
 310                     count = count + 1
 311                     logger.debug("Appending blocker='%s',blocked='%s',block_level='reject'", domain, blocked)
 312                     blocklist.append({
 313                         "blocker"    : domain,
 314                         "blocked"    : blocked,
 315                         "reason"     : None,
 316                         "block_level": "reject",
 317                     })
 318
 319             logger.debug("count=%d", count)
 320             if count == 0:
 321                 logger.debug("API is no more returning new instances, aborting loop!")
 322                 break
 323
 324         except network.exceptions as exception:
 325             logger.warning("Caught error, exiting loop: domain='%s',exception[%s]='%s'", domain, type(exception), str(exception))
 326             instances.set_last_error(domain, exception)
 327             offset = 0
 328             break
 329
 330     logger.debug("blocklist()=%d - EXIT!", len(blocklist))
 331     return blocklist