fba/networks/misskey.py

   1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
   2 # Copyright (C) 2023 Free Software Foundation
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published
   6 # by the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  16
  17 import json
  18
  19 import requests
  20
  21 from fba import blacklist
  22 from fba import config
  23 from fba import csrf
  24 from fba import instances
  25 from fba import network
  26
  27 from fba.helpers import dicts
  28 from fba.helpers import tidyup
  29
  30 def fetch_peers(domain: str) -> list:
  31     # DEBUG: print(f"DEBUG: domain({len(domain)})={domain} - CALLED!")
  32     if not isinstance(domain, str):
  33         raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
  34     elif domain == "":
  35         raise ValueError("Parameter 'domain' is empty")
  36
  37     # DEBUG: print(f"DEBUG: domain='{domain}' is misskey, sending API POST request ...")
  38     peers   = list()
  39     offset  = 0
  40     step    = config.get("misskey_limit")
  41     headers = csrf.determine(domain, {"Origin": domain})
  42
  43     # iterating through all "suspended" (follow-only in its terminology)
  44     # instances page-by-page, since that troonware doesn't support
  45     # sending them all at once
  46     while True:
  47         # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...")
  48         if offset == 0:
  49             fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
  50                 "sort" : "+pubAt",
  51                 "host" : None,
  52                 "limit": step
  53             }), headers)
  54         else:
  55             fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
  56                 "sort"  : "+pubAt",
  57                 "host"  : None,
  58                 "limit" : step,
  59                 "offset": offset - 1
  60             }), headers)
  61
  62         # Check records
  63         # DEBUG: print(f"DEBUG: fetched[]={type(fetched)}")
  64         if "error_message" in fetched:
  65             print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}")
  66             instances.update_last_error(domain, fetched)
  67             break
  68         elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
  69             print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}")
  70             instances.update_last_error(domain, fetched["json"]["error"]["message"])
  71             break
  72
  73         rows = fetched["json"]
  74
  75         # DEBUG: print(f"DEBUG: rows()={len(rows)}")
  76         if len(rows) == 0:
  77             # DEBUG: print(f"DEBUG: Returned zero bytes, exiting loop, domain='{domain}'")
  78             break
  79         elif len(rows) != config.get("misskey_limit"):
  80             # DEBUG: print(f"DEBUG: Fetched '{len(rows)}' row(s) but expected: '{config.get('misskey_limit')}'")
  81             offset = offset + (config.get("misskey_limit") - len(rows))
  82         else:
  83             # DEBUG: print(f"DEBUG: Raising offset by step={step}")
  84             offset = offset + step
  85
  86         already = 0
  87         # DEBUG: print(f"DEBUG: rows({len(rows)})[]={type(rows)}")
  88         for row in rows:
  89             # DEBUG: print(f"DEBUG: row()={len(row)}")
  90             if not "host" in row:
  91                 print(f"WARNING: row()={len(row)} does not contain key 'host': {row},domain='{domain}'")
  92                 continue
  93             elif not isinstance(row["host"], str):
  94                 print(f"WARNING: row[host][]={type(row['host'])} is not 'str'")
  95                 continue
  96             elif blacklist.is_blacklisted(row["host"]):
  97                 # DEBUG: print(f"DEBUG: row[host]='{row['host']}' is blacklisted. domain='{domain}'")
  98                 continue
  99             elif row["host"] in peers:
 100                 # DEBUG: print(f"DEBUG: Not adding row[host]='{row['host']}', already found.")
 101                 already = already + 1
 102                 continue
 103
 104             # DEBUG: print(f"DEBUG: Adding peer: '{row['host']}'")
 105             peers.append(row["host"])
 106
 107         if already == len(rows):
 108             # DEBUG: print(f"DEBUG: Host returned same set of '{already}' instances, aborting loop!")
 109             break
 110
 111     # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'")
 112     instances.set_data("total_peers", domain, len(peers))
 113
 114     # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...")
 115     instances.update_last_instance_fetch(domain)
 116
 117     # DEBUG: print(f"DEBUG: Returning peers[]='{type(peers)}'")
 118     return peers
 119
 120 def fetch_blocks(domain: str) -> dict:
 121     # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
 122     if not isinstance(domain, str):
 123         raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
 124     elif domain == "":
 125         raise ValueError("Parameter 'domain' is empty")
 126
 127     # DEBUG: print(f"DEBUG: Fetching misskey blocks from domain={domain}")
 128     blocklist = {
 129         "suspended": [],
 130         "blocked"  : []
 131     }
 132
 133     offset  = 0
 134     step    = config.get("misskey_limit")
 135     headers = csrf.determine(domain, {"Origin": domain})
 136
 137     # iterating through all "suspended" (follow-only in its terminology)
 138     # instances page-by-page since it doesn't support sending them all at once
 139     while True:
 140         try:
 141             # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...")
 142             if offset == 0:
 143                 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
 144                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 145                     "sort"     : "+pubAt",
 146                     "host"     : None,
 147                     "suspended": True,
 148                     "limit"    : step
 149                 }), headers)
 150             else:
 151                 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
 152                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 153                     "sort"     : "+pubAt",
 154                     "host"     : None,
 155                     "suspended": True,
 156                     "limit"    : step,
 157                     "offset"   : offset - 1
 158                 }), headers)
 159
 160             # DEBUG: print(f"DEBUG: fetched[]={type(fetched)}")
 161             if "error_message" in fetched:
 162                 print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}")
 163                 instances.update_last_error(domain, fetched)
 164                 break
 165             elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
 166                 print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}")
 167                 instances.update_last_error(domain, fetched["json"]["error"]["message"])
 168                 break
 169
 170             rows = fetched["json"]
 171
 172             # DEBUG: print(f"DEBUG: rows({len(rows)})={rows} - suspend")
 173             if len(rows) == 0:
 174                 # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
 175                 break
 176             elif len(rows) != config.get("misskey_limit"):
 177                 # DEBUG: print(f"DEBUG: Fetched '{len(rows)}' row(s) but expected: '{config.get('misskey_limit')}'")
 178                 offset = offset + (config.get("misskey_limit") - len(rows))
 179             else:
 180                 # DEBUG: print("DEBUG: Raising offset by step:", step)
 181                 offset = offset + step
 182
 183             count = 0
 184             for instance in rows:
 185                 # Is it there?
 186                 # DEBUG: print(f"DEBUG: instance[{type(instance)}]='{instance}' - suspend")
 187                 if instance["isSuspended"] and not dicts.has_key(blocklist["suspended"], "domain", instance["host"]):
 188                     count = count + 1
 189                     blocklist["suspended"].append(
 190                         {
 191                             "domain": tidyup.domain(instance["host"]),
 192                             # no reason field, nothing
 193                             "reason": None
 194                         }
 195                     )
 196
 197             # DEBUG: print(f"DEBUG: count={count}")
 198             if count == 0:
 199                 # DEBUG: print("DEBUG: API is no more returning new instances, aborting loop!")
 200                 break
 201
 202         except network.exceptions as exception:
 203             print(f"WARNING: Caught error, exiting loop: domain='{domain}',exception[{type(exception)}]='{str(exception)}'")
 204             instances.update_last_error(domain, exception)
 205             offset = 0
 206             break
 207
 208     while True:
 209         # Fetch blocked (full suspended) instances
 210         try:
 211             if offset == 0:
 212                 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
 213                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 214                     "sort"   : "+pubAt",
 215                     "host"   : None,
 216                     "blocked": True,
 217                     "limit"  : step
 218                 }), headers)
 219             else:
 220                 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
 221                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 222                     "sort"   : "+pubAt",
 223                     "host"   : None,
 224                     "blocked": True,
 225                     "limit"  : step,
 226                     "offset" : offset - 1
 227                 }), headers)
 228
 229             # DEBUG: print(f"DEBUG: fetched[]={type(fetched)}")
 230             if "error_message" in fetched:
 231                 print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}")
 232                 instances.update_last_error(domain, fetched)
 233                 break
 234             elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
 235                 print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}")
 236                 instances.update_last_error(domain, fetched["json"]["error"]["message"])
 237                 break
 238
 239             rows = fetched["json"]
 240
 241             # DEBUG: print(f"DEBUG: rows({len(rows)})={rows} - blocked")
 242             if len(rows) == 0:
 243                 # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
 244                 break
 245             elif len(rows) != config.get("misskey_limit"):
 246                 # DEBUG: print(f"DEBUG: Fetched '{len(rows)}' row(s) but expected: '{config.get('misskey_limit')}'")
 247                 offset = offset + (config.get("misskey_limit") - len(rows))
 248             else:
 249                 # DEBUG: print("DEBUG: Raising offset by step:", step)
 250                 offset = offset + step
 251
 252             count = 0
 253             for instance in rows:
 254                 # Is it there?
 255                 # DEBUG: print(f"DEBUG: instance[{type(instance)}]='{instance}' - blocked")
 256                 if instance["isBlocked"] and not dicts.has_key(blocklist["blocked"], "domain", instance["host"]):
 257                     count = count + 1
 258                     blocklist["blocked"].append({
 259                         "domain": tidyup.domain(instance["host"]),
 260                         "reason": None
 261                     })
 262
 263             # DEBUG: print(f"DEBUG: count={count}")
 264             if count == 0:
 265                 # DEBUG: print("DEBUG: API is no more returning new instances, aborting loop!")
 266                 break
 267
 268         except network.exceptions as exception:
 269             print(f"WARNING: Caught error, exiting loop: domain='{domain}',exception[{type(exception)}]='{str(exception)}'")
 270             instances.update_last_error(domain, exception)
 271             offset = 0
 272             break
 273
 274     # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...")
 275     instances.update_last_instance_fetch(domain)
 276
 277     # DEBUG: print(f"DEBUG: Returning for domain='{domain}',blocked()={len(blocklist['blocked'])},suspended()={len(blocklist['suspended'])}")
 278     return {
 279         "reject"        : blocklist["blocked"],
 280         "followers_only": blocklist["suspended"]
 281     }