fba/networks/misskey.py

   1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
   2 # Copyright (C) 2023 Free Software Foundation
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published
   6 # by the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  16
  17 import json
  18
  19 from fba import csrf
  20 from fba import network
  21
  22 from fba.helpers import blacklist
  23 from fba.helpers import config
  24 from fba.helpers import dicts
  25 from fba.helpers import tidyup
  26
  27 from fba.models import instances
  28
  29 def fetch_peers(domain: str) -> list:
  30     # DEBUG: print(f"DEBUG: domain({len(domain)})='{domain}' - CALLED!")
  31     if not isinstance(domain, str):
  32         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
  33     elif domain == "":
  34         raise ValueError("Parameter 'domain' is empty")
  35
  36     # DEBUG: print(f"DEBUG: domain='{domain}' is misskey, sending API POST request ...")
  37     peers   = list()
  38     offset  = 0
  39     step    = config.get("misskey_limit")
  40
  41     # No CSRF by default, you don't have to add network.api_headers by yourself here
  42     headers = tuple()
  43
  44     try:
  45         # DEBUG: print(f"DEBUG: Checking CSRF for domain='{domain}'")
  46         headers = csrf.determine(domain, dict())
  47     except network.exceptions as exception:
  48         print(f"WARNING: Exception '{type(exception)}' during checking CSRF (fetch_peers,{__name__}) - EXIT!")
  49         instances.set_last_error(domain, exception)
  50         return peers
  51
  52     # iterating through all "suspended" (follow-only in its terminology)
  53     # instances page-by-page, since that troonware doesn't support
  54     # sending them all at once
  55     while True:
  56         # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...")
  57         if offset == 0:
  58             fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
  59                 "sort" : "+pubAt",
  60                 "host" : None,
  61                 "limit": step
  62             }), headers)
  63         else:
  64             fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
  65                 "sort"  : "+pubAt",
  66                 "host"  : None,
  67                 "limit" : step,
  68                 "offset": offset - 1
  69             }), headers)
  70
  71         # Check records
  72         # DEBUG: print(f"DEBUG: fetched[]='{type(fetched)}'")
  73         if "error_message" in fetched:
  74             print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}")
  75             instances.set_last_error(domain, fetched)
  76             break
  77         elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
  78             print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}")
  79             instances.set_last_error(domain, fetched["json"]["error"]["message"])
  80             break
  81
  82         rows = fetched["json"]
  83
  84         # DEBUG: print(f"DEBUG: rows()={len(rows)}")
  85         if len(rows) == 0:
  86             # DEBUG: print(f"DEBUG: Returned zero bytes, exiting loop, domain='{domain}'")
  87             break
  88         elif len(rows) != config.get("misskey_limit"):
  89             # DEBUG: print(f"DEBUG: Fetched '{len(rows)}' row(s) but expected: '{config.get('misskey_limit')}'")
  90             offset = offset + (config.get("misskey_limit") - len(rows))
  91         else:
  92             # DEBUG: print(f"DEBUG: Raising offset by step={step}")
  93             offset = offset + step
  94
  95         already = 0
  96         # DEBUG: print(f"DEBUG: rows({len(rows)})[]='{type(rows)}'")
  97         for row in rows:
  98             # DEBUG: print(f"DEBUG: row()={len(row)}")
  99             if "host" not in row:
 100                 print(f"WARNING: row()={len(row)} does not contain key 'host': {row},domain='{domain}'")
 101                 continue
 102             elif not isinstance(row["host"], str):
 103                 print(f"WARNING: row[host][]='{type(row['host'])}' is not 'str'")
 104                 continue
 105             elif blacklist.is_blacklisted(row["host"]):
 106                 # DEBUG: print(f"DEBUG: row[host]='{row['host']}' is blacklisted. domain='{domain}'")
 107                 continue
 108             elif row["host"] in peers:
 109                 # DEBUG: print(f"DEBUG: Not adding row[host]='{row['host']}', already found.")
 110                 already = already + 1
 111                 continue
 112
 113             # DEBUG: print(f"DEBUG: Adding peer: '{row['host']}'")
 114             peers.append(row["host"])
 115
 116         if already == len(rows):
 117             # DEBUG: print(f"DEBUG: Host returned same set of '{already}' instances, aborting loop!")
 118             break
 119
 120     # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'")
 121     instances.set_total_peers(domain, peers)
 122
 123     # DEBUG: print(f"DEBUG: Returning peers[]='{type(peers)}'")
 124     return peers
 125
 126 def fetch_blocks(domain: str) -> dict:
 127     # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
 128     if not isinstance(domain, str):
 129         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
 130     elif domain == "":
 131         raise ValueError("Parameter 'domain' is empty")
 132
 133     # DEBUG: print(f"DEBUG: Fetching misskey blocks from domain='{domain}'")
 134     blocklist = {
 135         "suspended": [],
 136         "blocked"  : []
 137     }
 138
 139     offset  = 0
 140     step    = config.get("misskey_limit")
 141
 142     # No CSRF by default, you don't have to add network.api_headers by yourself here
 143     headers = tuple()
 144
 145     try:
 146         # DEBUG: print(f"DEBUG: Checking CSRF for domain='{domain}'")
 147         headers = csrf.determine(domain, dict())
 148     except network.exceptions as exception:
 149         print(f"WARNING: Exception '{type(exception)}' during checking CSRF (fetch_blocks,{__name__}) - EXIT!")
 150         instances.set_last_error(domain, exception)
 151         return blocklist
 152
 153     # iterating through all "suspended" (follow-only in its terminology)
 154     # instances page-by-page since it doesn't support sending them all at once
 155     while True:
 156         try:
 157             # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...")
 158             if offset == 0:
 159                 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
 160                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 161                     "sort"     : "+pubAt",
 162                     "host"     : None,
 163                     "suspended": True,
 164                     "limit"    : step
 165                 }), headers)
 166             else:
 167                 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
 168                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 169                     "sort"     : "+pubAt",
 170                     "host"     : None,
 171                     "suspended": True,
 172                     "limit"    : step,
 173                     "offset"   : offset - 1
 174                 }), headers)
 175
 176             # DEBUG: print(f"DEBUG: fetched[]='{type(fetched)}'")
 177             if "error_message" in fetched:
 178                 print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}")
 179                 instances.set_last_error(domain, fetched)
 180                 break
 181             elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
 182                 print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}")
 183                 instances.set_last_error(domain, fetched["json"]["error"]["message"])
 184                 break
 185
 186             rows = fetched["json"]
 187
 188             # DEBUG: print(f"DEBUG: rows({len(rows)})={rows} - suspend")
 189             if len(rows) == 0:
 190                 # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
 191                 break
 192             elif len(rows) != config.get("misskey_limit"):
 193                 # DEBUG: print(f"DEBUG: Fetched '{len(rows)}' row(s) but expected: '{config.get('misskey_limit')}'")
 194                 offset = offset + (config.get("misskey_limit") - len(rows))
 195             else:
 196                 # DEBUG: print("DEBUG: Raising offset by step:", step)
 197                 offset = offset + step
 198
 199             count = 0
 200             for instance in rows:
 201                 # Is it there?
 202                 # DEBUG: print(f"DEBUG: instance[{type(instance)}]='{instance}' - suspend")
 203                 if "isSuspended" in instance and instance["isSuspended"] and not dicts.has_key(blocklist["suspended"], "domain", instance["host"]):
 204                     count = count + 1
 205                     blocklist["suspended"].append({
 206                         "domain": tidyup.domain(instance["host"]),
 207                         # no reason field, nothing
 208                         "reason": None
 209                     })
 210
 211             # DEBUG: print(f"DEBUG: count={count}")
 212             if count == 0:
 213                 # DEBUG: print("DEBUG: API is no more returning new instances, aborting loop!")
 214                 break
 215
 216         except network.exceptions as exception:
 217             print(f"WARNING: Caught error, exiting loop: domain='{domain}',exception[{type(exception)}]='{str(exception)}'")
 218             instances.set_last_error(domain, exception)
 219             offset = 0
 220             break
 221
 222     while True:
 223         # Fetch blocked (full suspended) instances
 224         try:
 225             if offset == 0:
 226                 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
 227                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 228                     "sort"   : "+pubAt",
 229                     "host"   : None,
 230                     "blocked": True,
 231                     "limit"  : step
 232                 }), headers)
 233             else:
 234                 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
 235                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 236                     "sort"   : "+pubAt",
 237                     "host"   : None,
 238                     "blocked": True,
 239                     "limit"  : step,
 240                     "offset" : offset - 1
 241                 }), headers)
 242
 243             # DEBUG: print(f"DEBUG: fetched[]='{type(fetched)}'")
 244             if "error_message" in fetched:
 245                 print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}")
 246                 instances.set_last_error(domain, fetched)
 247                 break
 248             elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
 249                 print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}")
 250                 instances.set_last_error(domain, fetched["json"]["error"]["message"])
 251                 break
 252
 253             rows = fetched["json"]
 254
 255             # DEBUG: print(f"DEBUG: rows({len(rows)})={rows} - blocked")
 256             if len(rows) == 0:
 257                 # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
 258                 break
 259             elif len(rows) != config.get("misskey_limit"):
 260                 # DEBUG: print(f"DEBUG: Fetched '{len(rows)}' row(s) but expected: '{config.get('misskey_limit')}'")
 261                 offset = offset + (config.get("misskey_limit") - len(rows))
 262             else:
 263                 # DEBUG: print("DEBUG: Raising offset by step:", step)
 264                 offset = offset + step
 265
 266             count = 0
 267             for instance in rows:
 268                 # Is it there?
 269                 # DEBUG: print(f"DEBUG: instance[{type(instance)}]='{instance}' - blocked")
 270                 if "isBlocked" in instance and instance["isBlocked"] and not dicts.has_key(blocklist["blocked"], "domain", instance["host"]):
 271                     count = count + 1
 272                     blocklist["blocked"].append({
 273                         "domain": tidyup.domain(instance["host"]),
 274                         "reason": None
 275                     })
 276
 277             # DEBUG: print(f"DEBUG: count={count}")
 278             if count == 0:
 279                 # DEBUG: print("DEBUG: API is no more returning new instances, aborting loop!")
 280                 break
 281
 282         except network.exceptions as exception:
 283             print(f"WARNING: Caught error, exiting loop: domain='{domain}',exception[{type(exception)}]='{str(exception)}'")
 284             instances.set_last_error(domain, exception)
 285             offset = 0
 286             break
 287
 288     # DEBUG: print(f"DEBUG: Returning for domain='{domain}',blocked()={len(blocklist['blocked'])},suspended()={len(blocklist['suspended'])}")
 289     return {
 290         "reject"        : blocklist["blocked"],
 291         "followers_only": blocklist["suspended"]
 292     }