fba/networks/misskey.py

   1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
   2 # Copyright (C) 2023 Free Software Foundation
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published
   6 # by the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  16
  17 import json
  18
  19 from fba import csrf
  20
  21 from fba.helpers import blacklist
  22 from fba.helpers import config
  23 from fba.helpers import dicts
  24 from fba.helpers import tidyup
  25
  26 from fba.http import network
  27
  28 from fba.models import instances
  29
  30 def fetch_peers(domain: str) -> list:
  31     # DEBUG: print(f"DEBUG: domain({len(domain)})='{domain}' - CALLED!")
  32     if not isinstance(domain, str):
  33         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
  34     elif domain == "":
  35         raise ValueError("Parameter 'domain' is empty")
  36
  37     # DEBUG: print(f"DEBUG: domain='{domain}' is misskey, sending API POST request ...")
  38     peers   = list()
  39     offset  = 0
  40     step    = config.get("misskey_limit")
  41
  42     # No CSRF by default, you don't have to add network.api_headers by yourself here
  43     headers = tuple()
  44
  45     try:
  46         # DEBUG: print(f"DEBUG: Checking CSRF for domain='{domain}'")
  47         headers = csrf.determine(domain, dict())
  48     except network.exceptions as exception:
  49         print(f"WARNING: Exception '{type(exception)}' during checking CSRF (fetch_peers,{__name__}) - EXIT!")
  50         instances.set_last_error(domain, exception)
  51         return peers
  52
  53     # iterating through all "suspended" (follow-only in its terminology)
  54     # instances page-by-page, since that troonware doesn't support
  55     # sending them all at once
  56     while True:
  57         # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...")
  58         if offset == 0:
  59             fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
  60                 "sort" : "+pubAt",
  61                 "host" : None,
  62                 "limit": step
  63             }), headers)
  64         else:
  65             fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
  66                 "sort"  : "+pubAt",
  67                 "host"  : None,
  68                 "limit" : step,
  69                 "offset": offset - 1
  70             }), headers)
  71
  72         # Check records
  73         # DEBUG: print(f"DEBUG: fetched[]='{type(fetched)}'")
  74         if "error_message" in fetched:
  75             print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}")
  76             instances.set_last_error(domain, fetched)
  77             break
  78         elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
  79             print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}")
  80             instances.set_last_error(domain, fetched["json"]["error"]["message"])
  81             break
  82
  83         rows = fetched["json"]
  84
  85         # DEBUG: print(f"DEBUG: rows()={len(rows)}")
  86         if len(rows) == 0:
  87             # DEBUG: print(f"DEBUG: Returned zero bytes, exiting loop, domain='{domain}'")
  88             break
  89         elif len(rows) != config.get("misskey_limit"):
  90             # DEBUG: print(f"DEBUG: Fetched '{len(rows)}' row(s) but expected: '{config.get('misskey_limit')}'")
  91             offset = offset + (config.get("misskey_limit") - len(rows))
  92         else:
  93             # DEBUG: print(f"DEBUG: Raising offset by step={step}")
  94             offset = offset + step
  95
  96         already = 0
  97         # DEBUG: print(f"DEBUG: rows({len(rows)})[]='{type(rows)}'")
  98         for row in rows:
  99             # DEBUG: print(f"DEBUG: row()={len(row)}")
 100             if "host" not in row:
 101                 print(f"WARNING: row()={len(row)} does not contain key 'host': {row},domain='{domain}'")
 102                 continue
 103             elif not isinstance(row["host"], str):
 104                 print(f"WARNING: row[host][]='{type(row['host'])}' is not 'str'")
 105                 continue
 106             elif blacklist.is_blacklisted(row["host"]):
 107                 # DEBUG: print(f"DEBUG: row[host]='{row['host']}' is blacklisted. domain='{domain}'")
 108                 continue
 109             elif row["host"] in peers:
 110                 # DEBUG: print(f"DEBUG: Not adding row[host]='{row['host']}', already found.")
 111                 already = already + 1
 112                 continue
 113
 114             # DEBUG: print(f"DEBUG: Adding peer: '{row['host']}'")
 115             peers.append(row["host"])
 116
 117         if already == len(rows):
 118             # DEBUG: print(f"DEBUG: Host returned same set of '{already}' instances, aborting loop!")
 119             break
 120
 121     # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'")
 122     instances.set_total_peers(domain, peers)
 123
 124     # DEBUG: print(f"DEBUG: Returning peers[]='{type(peers)}'")
 125     return peers
 126
 127 def fetch_blocks(domain: str) -> dict:
 128     # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
 129     if not isinstance(domain, str):
 130         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
 131     elif domain == "":
 132         raise ValueError("Parameter 'domain' is empty")
 133
 134     # DEBUG: print(f"DEBUG: Fetching misskey blocks from domain='{domain}'")
 135     blocklist = {
 136         "suspended": [],
 137         "blocked"  : []
 138     }
 139
 140     offset  = 0
 141     step    = config.get("misskey_limit")
 142
 143     # No CSRF by default, you don't have to add network.api_headers by yourself here
 144     headers = tuple()
 145
 146     try:
 147         # DEBUG: print(f"DEBUG: Checking CSRF for domain='{domain}'")
 148         headers = csrf.determine(domain, dict())
 149     except network.exceptions as exception:
 150         print(f"WARNING: Exception '{type(exception)}' during checking CSRF (fetch_blocks,{__name__}) - EXIT!")
 151         instances.set_last_error(domain, exception)
 152         return blocklist
 153
 154     # iterating through all "suspended" (follow-only in its terminology)
 155     # instances page-by-page since it doesn't support sending them all at once
 156     while True:
 157         try:
 158             # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...")
 159             if offset == 0:
 160                 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
 161                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 162                     "sort"     : "+pubAt",
 163                     "host"     : None,
 164                     "suspended": True,
 165                     "limit"    : step
 166                 }), headers)
 167             else:
 168                 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
 169                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 170                     "sort"     : "+pubAt",
 171                     "host"     : None,
 172                     "suspended": True,
 173                     "limit"    : step,
 174                     "offset"   : offset - 1
 175                 }), headers)
 176
 177             # DEBUG: print(f"DEBUG: fetched[]='{type(fetched)}'")
 178             if "error_message" in fetched:
 179                 print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}")
 180                 instances.set_last_error(domain, fetched)
 181                 break
 182             elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
 183                 print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}")
 184                 instances.set_last_error(domain, fetched["json"]["error"]["message"])
 185                 break
 186
 187             rows = fetched["json"]
 188
 189             # DEBUG: print(f"DEBUG: rows({len(rows)})={rows} - suspend")
 190             if len(rows) == 0:
 191                 # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
 192                 break
 193             elif len(rows) != config.get("misskey_limit"):
 194                 # DEBUG: print(f"DEBUG: Fetched '{len(rows)}' row(s) but expected: '{config.get('misskey_limit')}'")
 195                 offset = offset + (config.get("misskey_limit") - len(rows))
 196             else:
 197                 # DEBUG: print("DEBUG: Raising offset by step:", step)
 198                 offset = offset + step
 199
 200             count = 0
 201             for instance in rows:
 202                 # Is it there?
 203                 # DEBUG: print(f"DEBUG: instance[{type(instance)}]='{instance}' - suspend")
 204                 if "isSuspended" in instance and instance["isSuspended"] and not dicts.has_key(blocklist["suspended"], "domain", instance["host"]):
 205                     count = count + 1
 206                     blocklist["suspended"].append({
 207                         "domain": tidyup.domain(instance["host"]),
 208                         # no reason field, nothing
 209                         "reason": None
 210                     })
 211
 212             # DEBUG: print(f"DEBUG: count={count}")
 213             if count == 0:
 214                 # DEBUG: print("DEBUG: API is no more returning new instances, aborting loop!")
 215                 break
 216
 217         except network.exceptions as exception:
 218             print(f"WARNING: Caught error, exiting loop: domain='{domain}',exception[{type(exception)}]='{str(exception)}'")
 219             instances.set_last_error(domain, exception)
 220             offset = 0
 221             break
 222
 223     while True:
 224         # Fetch blocked (full suspended) instances
 225         try:
 226             if offset == 0:
 227                 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
 228                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 229                     "sort"   : "+pubAt",
 230                     "host"   : None,
 231                     "blocked": True,
 232                     "limit"  : step
 233                 }), headers)
 234             else:
 235                 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
 236                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 237                     "sort"   : "+pubAt",
 238                     "host"   : None,
 239                     "blocked": True,
 240                     "limit"  : step,
 241                     "offset" : offset - 1
 242                 }), headers)
 243
 244             # DEBUG: print(f"DEBUG: fetched[]='{type(fetched)}'")
 245             if "error_message" in fetched:
 246                 print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}")
 247                 instances.set_last_error(domain, fetched)
 248                 break
 249             elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
 250                 print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}")
 251                 instances.set_last_error(domain, fetched["json"]["error"]["message"])
 252                 break
 253
 254             rows = fetched["json"]
 255
 256             # DEBUG: print(f"DEBUG: rows({len(rows)})={rows} - blocked")
 257             if len(rows) == 0:
 258                 # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
 259                 break
 260             elif len(rows) != config.get("misskey_limit"):
 261                 # DEBUG: print(f"DEBUG: Fetched '{len(rows)}' row(s) but expected: '{config.get('misskey_limit')}'")
 262                 offset = offset + (config.get("misskey_limit") - len(rows))
 263             else:
 264                 # DEBUG: print("DEBUG: Raising offset by step:", step)
 265                 offset = offset + step
 266
 267             count = 0
 268             for instance in rows:
 269                 # Is it there?
 270                 # DEBUG: print(f"DEBUG: instance[{type(instance)}]='{instance}' - blocked")
 271                 if "isBlocked" in instance and instance["isBlocked"] and not dicts.has_key(blocklist["blocked"], "domain", instance["host"]):
 272                     count = count + 1
 273                     blocklist["blocked"].append({
 274                         "domain": tidyup.domain(instance["host"]),
 275                         "reason": None
 276                     })
 277
 278             # DEBUG: print(f"DEBUG: count={count}")
 279             if count == 0:
 280                 # DEBUG: print("DEBUG: API is no more returning new instances, aborting loop!")
 281                 break
 282
 283         except network.exceptions as exception:
 284             print(f"WARNING: Caught error, exiting loop: domain='{domain}',exception[{type(exception)}]='{str(exception)}'")
 285             instances.set_last_error(domain, exception)
 286             offset = 0
 287             break
 288
 289     # DEBUG: print(f"DEBUG: Returning for domain='{domain}',blocked()={len(blocklist['blocked'])},suspended()={len(blocklist['suspended'])}")
 290     return {
 291         "reject"        : blocklist["blocked"],
 292         "followers_only": blocklist["suspended"]
 293     }