fba/networks/misskey.py

   1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
   2 # Copyright (C) 2023 Free Software Foundation
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published
   6 # by the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  16
  17 import json
  18
  19 from fba import blacklist
  20 from fba import config
  21 from fba import csrf
  22 from fba import network
  23
  24 from fba.helpers import dicts
  25 from fba.helpers import tidyup
  26
  27 from fba.models import instances
  28
  29 def fetch_peers(domain: str) -> list:
  30     # DEBUG: print(f"DEBUG: domain({len(domain)})={domain} - CALLED!")
  31     if not isinstance(domain, str):
  32         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
  33     elif domain == "":
  34         raise ValueError("Parameter 'domain' is empty")
  35
  36     # DEBUG: print(f"DEBUG: domain='{domain}' is misskey, sending API POST request ...")
  37     peers   = list()
  38     offset  = 0
  39     step    = config.get("misskey_limit")
  40
  41     # No CSRF by default, you don't have to add network.api_headers by yourself here
  42     headers = tuple()
  43
  44     try:
  45         # DEBUG: print(f"DEBUG: Checking CSRF for domain='{domain}'")
  46         headers = csrf.determine(domain, dict())
  47     except network.exceptions as exception:
  48         print(f"WARNING: Exception '{type(exception)}' during checking CSRF (fetch_peers,{__name__}) - EXIT!")
  49         return peers
  50
  51     # iterating through all "suspended" (follow-only in its terminology)
  52     # instances page-by-page, since that troonware doesn't support
  53     # sending them all at once
  54     while True:
  55         # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...")
  56         if offset == 0:
  57             fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
  58                 "sort" : "+pubAt",
  59                 "host" : None,
  60                 "limit": step
  61             }), headers)
  62         else:
  63             fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
  64                 "sort"  : "+pubAt",
  65                 "host"  : None,
  66                 "limit" : step,
  67                 "offset": offset - 1
  68             }), headers)
  69
  70         # Check records
  71         # DEBUG: print(f"DEBUG: fetched[]='{type(fetched)}'")
  72         if "error_message" in fetched:
  73             print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}")
  74             instances.set_last_error(domain, fetched)
  75             break
  76         elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
  77             print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}")
  78             instances.set_last_error(domain, fetched["json"]["error"]["message"])
  79             break
  80
  81         rows = fetched["json"]
  82
  83         # DEBUG: print(f"DEBUG: rows()={len(rows)}")
  84         if len(rows) == 0:
  85             # DEBUG: print(f"DEBUG: Returned zero bytes, exiting loop, domain='{domain}'")
  86             break
  87         elif len(rows) != config.get("misskey_limit"):
  88             # DEBUG: print(f"DEBUG: Fetched '{len(rows)}' row(s) but expected: '{config.get('misskey_limit')}'")
  89             offset = offset + (config.get("misskey_limit") - len(rows))
  90         else:
  91             # DEBUG: print(f"DEBUG: Raising offset by step={step}")
  92             offset = offset + step
  93
  94         already = 0
  95         # DEBUG: print(f"DEBUG: rows({len(rows)})[]='{type(rows)}'")
  96         for row in rows:
  97             # DEBUG: print(f"DEBUG: row()={len(row)}")
  98             if not "host" in row:
  99                 print(f"WARNING: row()={len(row)} does not contain key 'host': {row},domain='{domain}'")
 100                 continue
 101             elif not isinstance(row["host"], str):
 102                 print(f"WARNING: row[host][]='{type(row['host'])}' is not 'str'")
 103                 continue
 104             elif blacklist.is_blacklisted(row["host"]):
 105                 # DEBUG: print(f"DEBUG: row[host]='{row['host']}' is blacklisted. domain='{domain}'")
 106                 continue
 107             elif row["host"] in peers:
 108                 # DEBUG: print(f"DEBUG: Not adding row[host]='{row['host']}', already found.")
 109                 already = already + 1
 110                 continue
 111
 112             # DEBUG: print(f"DEBUG: Adding peer: '{row['host']}'")
 113             peers.append(row["host"])
 114
 115         if already == len(rows):
 116             # DEBUG: print(f"DEBUG: Host returned same set of '{already}' instances, aborting loop!")
 117             break
 118
 119     # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'")
 120     instances.set_total_peers(domain, peers)
 121
 122     # DEBUG: print(f"DEBUG: Returning peers[]='{type(peers)}'")
 123     return peers
 124
 125 def fetch_blocks(domain: str) -> dict:
 126     # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
 127     if not isinstance(domain, str):
 128         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
 129     elif domain == "":
 130         raise ValueError("Parameter 'domain' is empty")
 131
 132     # DEBUG: print(f"DEBUG: Fetching misskey blocks from domain={domain}")
 133     blocklist = {
 134         "suspended": [],
 135         "blocked"  : []
 136     }
 137
 138     offset  = 0
 139     step    = config.get("misskey_limit")
 140
 141     # No CSRF by default, you don't have to add network.api_headers by yourself here
 142     headers = tuple()
 143
 144     try:
 145         # DEBUG: print(f"DEBUG: Checking CSRF for domain='{domain}'")
 146         headers = csrf.determine(domain, dict())
 147     except network.exceptions as exception:
 148         print(f"WARNING: Exception '{type(exception)}' during checking CSRF (fetch_blocks,{__name__}) - EXIT!")
 149         return blocklist
 150
 151     # iterating through all "suspended" (follow-only in its terminology)
 152     # instances page-by-page since it doesn't support sending them all at once
 153     while True:
 154         try:
 155             # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...")
 156             if offset == 0:
 157                 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
 158                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 159                     "sort"     : "+pubAt",
 160                     "host"     : None,
 161                     "suspended": True,
 162                     "limit"    : step
 163                 }), headers)
 164             else:
 165                 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
 166                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 167                     "sort"     : "+pubAt",
 168                     "host"     : None,
 169                     "suspended": True,
 170                     "limit"    : step,
 171                     "offset"   : offset - 1
 172                 }), headers)
 173
 174             # DEBUG: print(f"DEBUG: fetched[]='{type(fetched)}'")
 175             if "error_message" in fetched:
 176                 print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}")
 177                 instances.set_last_error(domain, fetched)
 178                 break
 179             elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
 180                 print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}")
 181                 instances.set_last_error(domain, fetched["json"]["error"]["message"])
 182                 break
 183
 184             rows = fetched["json"]
 185
 186             # DEBUG: print(f"DEBUG: rows({len(rows)})={rows} - suspend")
 187             if len(rows) == 0:
 188                 # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
 189                 break
 190             elif len(rows) != config.get("misskey_limit"):
 191                 # DEBUG: print(f"DEBUG: Fetched '{len(rows)}' row(s) but expected: '{config.get('misskey_limit')}'")
 192                 offset = offset + (config.get("misskey_limit") - len(rows))
 193             else:
 194                 # DEBUG: print("DEBUG: Raising offset by step:", step)
 195                 offset = offset + step
 196
 197             count = 0
 198             for instance in rows:
 199                 # Is it there?
 200                 # DEBUG: print(f"DEBUG: instance[{type(instance)}]='{instance}' - suspend")
 201                 if "isSuspended" in instance and instance["isSuspended"] and not dicts.has_key(blocklist["suspended"], "domain", instance["host"]):
 202                     count = count + 1
 203                     blocklist["suspended"].append({
 204                         "domain": tidyup.domain(instance["host"]),
 205                         # no reason field, nothing
 206                         "reason": None
 207                     })
 208
 209             # DEBUG: print(f"DEBUG: count={count}")
 210             if count == 0:
 211                 # DEBUG: print("DEBUG: API is no more returning new instances, aborting loop!")
 212                 break
 213
 214         except network.exceptions as exception:
 215             print(f"WARNING: Caught error, exiting loop: domain='{domain}',exception[{type(exception)}]='{str(exception)}'")
 216             instances.set_last_error(domain, exception)
 217             offset = 0
 218             break
 219
 220     while True:
 221         # Fetch blocked (full suspended) instances
 222         try:
 223             if offset == 0:
 224                 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
 225                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 226                     "sort"   : "+pubAt",
 227                     "host"   : None,
 228                     "blocked": True,
 229                     "limit"  : step
 230                 }), headers)
 231             else:
 232                 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
 233                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 234                     "sort"   : "+pubAt",
 235                     "host"   : None,
 236                     "blocked": True,
 237                     "limit"  : step,
 238                     "offset" : offset - 1
 239                 }), headers)
 240
 241             # DEBUG: print(f"DEBUG: fetched[]='{type(fetched)}'")
 242             if "error_message" in fetched:
 243                 print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}")
 244                 instances.set_last_error(domain, fetched)
 245                 break
 246             elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
 247                 print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}")
 248                 instances.set_last_error(domain, fetched["json"]["error"]["message"])
 249                 break
 250
 251             rows = fetched["json"]
 252
 253             # DEBUG: print(f"DEBUG: rows({len(rows)})={rows} - blocked")
 254             if len(rows) == 0:
 255                 # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
 256                 break
 257             elif len(rows) != config.get("misskey_limit"):
 258                 # DEBUG: print(f"DEBUG: Fetched '{len(rows)}' row(s) but expected: '{config.get('misskey_limit')}'")
 259                 offset = offset + (config.get("misskey_limit") - len(rows))
 260             else:
 261                 # DEBUG: print("DEBUG: Raising offset by step:", step)
 262                 offset = offset + step
 263
 264             count = 0
 265             for instance in rows:
 266                 # Is it there?
 267                 # DEBUG: print(f"DEBUG: instance[{type(instance)}]='{instance}' - blocked")
 268                 if "isBlocked" in instance and instance["isBlocked"] and not dicts.has_key(blocklist["blocked"], "domain", instance["host"]):
 269                     count = count + 1
 270                     blocklist["blocked"].append({
 271                         "domain": tidyup.domain(instance["host"]),
 272                         "reason": None
 273                     })
 274
 275             # DEBUG: print(f"DEBUG: count={count}")
 276             if count == 0:
 277                 # DEBUG: print("DEBUG: API is no more returning new instances, aborting loop!")
 278                 break
 279
 280         except network.exceptions as exception:
 281             print(f"WARNING: Caught error, exiting loop: domain='{domain}',exception[{type(exception)}]='{str(exception)}'")
 282             instances.set_last_error(domain, exception)
 283             offset = 0
 284             break
 285
 286     # DEBUG: print(f"DEBUG: Returning for domain='{domain}',blocked()={len(blocklist['blocked'])},suspended()={len(blocklist['suspended'])}")
 287     return {
 288         "reject"        : blocklist["blocked"],
 289         "followers_only": blocklist["suspended"]
 290     }