fba/networks/misskey.py

   1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
   2 # Copyright (C) 2023 Free Software Foundation
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published
   6 # by the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  16
  17 import json
  18
  19 from fba import blacklist
  20 from fba import config
  21 from fba import csrf
  22 from fba import instances
  23 from fba import network
  24
  25 from fba.helpers import dicts
  26 from fba.helpers import tidyup
  27
  28 def fetch_peers(domain: str) -> list:
  29     # DEBUG: print(f"DEBUG: domain({len(domain)})={domain} - CALLED!")
  30     if not isinstance(domain, str):
  31         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
  32     elif domain == "":
  33         raise ValueError("Parameter 'domain' is empty")
  34
  35     # DEBUG: print(f"DEBUG: domain='{domain}' is misskey, sending API POST request ...")
  36     peers   = list()
  37     offset  = 0
  38     step    = config.get("misskey_limit")
  39
  40     # No CSRF by default, you don't have to add network.api_headers by yourself here
  41     headers = tuple()
  42
  43     try:
  44         # DEBUG: print(f"DEBUG: Checking CSRF for domain='{domain}'")
  45         headers = csrf.determine(domain, dict())
  46     except network.exceptions as exception:
  47         print(f"WARNING: Exception '{type(exception)}' during checking CSRF (fetch_peers,{__name__}) - EXIT!")
  48         return peers
  49
  50     # iterating through all "suspended" (follow-only in its terminology)
  51     # instances page-by-page, since that troonware doesn't support
  52     # sending them all at once
  53     while True:
  54         # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...")
  55         if offset == 0:
  56             fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
  57                 "sort" : "+pubAt",
  58                 "host" : None,
  59                 "limit": step
  60             }), headers)
  61         else:
  62             fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
  63                 "sort"  : "+pubAt",
  64                 "host"  : None,
  65                 "limit" : step,
  66                 "offset": offset - 1
  67             }), headers)
  68
  69         # Check records
  70         # DEBUG: print(f"DEBUG: fetched[]='{type(fetched)}'")
  71         if "error_message" in fetched:
  72             print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}")
  73             instances.update_last_error(domain, fetched)
  74             break
  75         elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
  76             print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}")
  77             instances.update_last_error(domain, fetched["json"]["error"]["message"])
  78             break
  79
  80         rows = fetched["json"]
  81
  82         # DEBUG: print(f"DEBUG: rows()={len(rows)}")
  83         if len(rows) == 0:
  84             # DEBUG: print(f"DEBUG: Returned zero bytes, exiting loop, domain='{domain}'")
  85             break
  86         elif len(rows) != config.get("misskey_limit"):
  87             # DEBUG: print(f"DEBUG: Fetched '{len(rows)}' row(s) but expected: '{config.get('misskey_limit')}'")
  88             offset = offset + (config.get("misskey_limit") - len(rows))
  89         else:
  90             # DEBUG: print(f"DEBUG: Raising offset by step={step}")
  91             offset = offset + step
  92
  93         already = 0
  94         # DEBUG: print(f"DEBUG: rows({len(rows)})[]='{type(rows)}'")
  95         for row in rows:
  96             # DEBUG: print(f"DEBUG: row()={len(row)}")
  97             if not "host" in row:
  98                 print(f"WARNING: row()={len(row)} does not contain key 'host': {row},domain='{domain}'")
  99                 continue
 100             elif not isinstance(row["host"], str):
 101                 print(f"WARNING: row[host][]='{type(row['host'])}' is not 'str'")
 102                 continue
 103             elif blacklist.is_blacklisted(row["host"]):
 104                 # DEBUG: print(f"DEBUG: row[host]='{row['host']}' is blacklisted. domain='{domain}'")
 105                 continue
 106             elif row["host"] in peers:
 107                 # DEBUG: print(f"DEBUG: Not adding row[host]='{row['host']}', already found.")
 108                 already = already + 1
 109                 continue
 110
 111             # DEBUG: print(f"DEBUG: Adding peer: '{row['host']}'")
 112             peers.append(row["host"])
 113
 114         if already == len(rows):
 115             # DEBUG: print(f"DEBUG: Host returned same set of '{already}' instances, aborting loop!")
 116             break
 117
 118     # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'")
 119     instances.set_data("total_peers", domain, len(peers))
 120
 121     # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...")
 122     instances.update_last_instance_fetch(domain)
 123
 124     # DEBUG: print(f"DEBUG: Returning peers[]='{type(peers)}'")
 125     return peers
 126
 127 def fetch_blocks(domain: str) -> dict:
 128     # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
 129     if not isinstance(domain, str):
 130         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
 131     elif domain == "":
 132         raise ValueError("Parameter 'domain' is empty")
 133
 134     # DEBUG: print(f"DEBUG: Fetching misskey blocks from domain={domain}")
 135     blocklist = {
 136         "suspended": [],
 137         "blocked"  : []
 138     }
 139
 140     offset  = 0
 141     step    = config.get("misskey_limit")
 142
 143     # No CSRF by default, you don't have to add network.api_headers by yourself here
 144     headers = tuple()
 145
 146     try:
 147         # DEBUG: print(f"DEBUG: Checking CSRF for domain='{domain}'")
 148         headers = csrf.determine(domain, dict())
 149     except network.exceptions as exception:
 150         print(f"WARNING: Exception '{type(exception)}' during checking CSRF (fetch_blocks,{__name__}) - EXIT!")
 151         return blocklist
 152
 153     # iterating through all "suspended" (follow-only in its terminology)
 154     # instances page-by-page since it doesn't support sending them all at once
 155     while True:
 156         try:
 157             # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...")
 158             if offset == 0:
 159                 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
 160                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 161                     "sort"     : "+pubAt",
 162                     "host"     : None,
 163                     "suspended": True,
 164                     "limit"    : step
 165                 }), headers)
 166             else:
 167                 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
 168                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 169                     "sort"     : "+pubAt",
 170                     "host"     : None,
 171                     "suspended": True,
 172                     "limit"    : step,
 173                     "offset"   : offset - 1
 174                 }), headers)
 175
 176             # DEBUG: print(f"DEBUG: fetched[]='{type(fetched)}'")
 177             if "error_message" in fetched:
 178                 print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}")
 179                 instances.update_last_error(domain, fetched)
 180                 break
 181             elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
 182                 print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}")
 183                 instances.update_last_error(domain, fetched["json"]["error"]["message"])
 184                 break
 185
 186             rows = fetched["json"]
 187
 188             # DEBUG: print(f"DEBUG: rows({len(rows)})={rows} - suspend")
 189             if len(rows) == 0:
 190                 # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
 191                 break
 192             elif len(rows) != config.get("misskey_limit"):
 193                 # DEBUG: print(f"DEBUG: Fetched '{len(rows)}' row(s) but expected: '{config.get('misskey_limit')}'")
 194                 offset = offset + (config.get("misskey_limit") - len(rows))
 195             else:
 196                 # DEBUG: print("DEBUG: Raising offset by step:", step)
 197                 offset = offset + step
 198
 199             count = 0
 200             for instance in rows:
 201                 # Is it there?
 202                 # DEBUG: print(f"DEBUG: instance[{type(instance)}]='{instance}' - suspend")
 203                 if "isSuspended" in instance and instance["isSuspended"] and not dicts.has_key(blocklist["suspended"], "domain", instance["host"]):
 204                     count = count + 1
 205                     blocklist["suspended"].append(
 206                         {
 207                             "domain": tidyup.domain(instance["host"]),
 208                             # no reason field, nothing
 209                             "reason": None
 210                         }
 211                     )
 212
 213             # DEBUG: print(f"DEBUG: count={count}")
 214             if count == 0:
 215                 # DEBUG: print("DEBUG: API is no more returning new instances, aborting loop!")
 216                 break
 217
 218         except network.exceptions as exception:
 219             print(f"WARNING: Caught error, exiting loop: domain='{domain}',exception[{type(exception)}]='{str(exception)}'")
 220             instances.update_last_error(domain, exception)
 221             offset = 0
 222             break
 223
 224     while True:
 225         # Fetch blocked (full suspended) instances
 226         try:
 227             if offset == 0:
 228                 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
 229                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 230                     "sort"   : "+pubAt",
 231                     "host"   : None,
 232                     "blocked": True,
 233                     "limit"  : step
 234                 }), headers)
 235             else:
 236                 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
 237                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 238                     "sort"   : "+pubAt",
 239                     "host"   : None,
 240                     "blocked": True,
 241                     "limit"  : step,
 242                     "offset" : offset - 1
 243                 }), headers)
 244
 245             # DEBUG: print(f"DEBUG: fetched[]='{type(fetched)}'")
 246             if "error_message" in fetched:
 247                 print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}")
 248                 instances.update_last_error(domain, fetched)
 249                 break
 250             elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
 251                 print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}")
 252                 instances.update_last_error(domain, fetched["json"]["error"]["message"])
 253                 break
 254
 255             rows = fetched["json"]
 256
 257             # DEBUG: print(f"DEBUG: rows({len(rows)})={rows} - blocked")
 258             if len(rows) == 0:
 259                 # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
 260                 break
 261             elif len(rows) != config.get("misskey_limit"):
 262                 # DEBUG: print(f"DEBUG: Fetched '{len(rows)}' row(s) but expected: '{config.get('misskey_limit')}'")
 263                 offset = offset + (config.get("misskey_limit") - len(rows))
 264             else:
 265                 # DEBUG: print("DEBUG: Raising offset by step:", step)
 266                 offset = offset + step
 267
 268             count = 0
 269             for instance in rows:
 270                 # Is it there?
 271                 # DEBUG: print(f"DEBUG: instance[{type(instance)}]='{instance}' - blocked")
 272                 if "isBlocked" in instance and instance["isBlocked"] and not dicts.has_key(blocklist["blocked"], "domain", instance["host"]):
 273                     count = count + 1
 274                     blocklist["blocked"].append({
 275                         "domain": tidyup.domain(instance["host"]),
 276                         "reason": None
 277                     })
 278
 279             # DEBUG: print(f"DEBUG: count={count}")
 280             if count == 0:
 281                 # DEBUG: print("DEBUG: API is no more returning new instances, aborting loop!")
 282                 break
 283
 284         except network.exceptions as exception:
 285             print(f"WARNING: Caught error, exiting loop: domain='{domain}',exception[{type(exception)}]='{str(exception)}'")
 286             instances.update_last_error(domain, exception)
 287             offset = 0
 288             break
 289
 290     # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...")
 291     instances.update_last_instance_fetch(domain)
 292
 293     # DEBUG: print(f"DEBUG: Returning for domain='{domain}',blocked()={len(blocklist['blocked'])},suspended()={len(blocklist['suspended'])}")
 294     return {
 295         "reject"        : blocklist["blocked"],
 296         "followers_only": blocklist["suspended"]
 297     }