fba/federation/misskey.py

   1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
   2 # Copyright (C) 2023 Free Software Foundation
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published
   6 # by the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  16
  17 import json
  18
  19 from fba import blacklist
  20 from fba import config
  21 from fba import fba
  22 from fba import instances
  23 from fba import network
  24
  25 def fetch_peers(domain: str) -> list:
  26     # DEBUG: print(f"DEBUG: domain({len(domain)})={domain} - CALLED!")
  27     if not isinstance(domain, str):
  28         raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
  29     elif domain == "":
  30         raise ValueError("Parameter 'domain' is empty")
  31
  32     # DEBUG: print(f"DEBUG: domain='{domain}' is misskey, sending API POST request ...")
  33     peers = list()
  34     offset = 0
  35     step = config.get("misskey_limit")
  36
  37     # iterating through all "suspended" (follow-only in its terminology)
  38     # instances page-by-page, since that troonware doesn't support
  39     # sending them all at once
  40     while True:
  41         # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...")
  42         if offset == 0:
  43             fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
  44                 "sort" : "+pubAt",
  45                 "host" : None,
  46                 "limit": step
  47             }), {
  48                 "Origin": domain
  49             })
  50         else:
  51             fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
  52                 "sort"  : "+pubAt",
  53                 "host"  : None,
  54                 "limit" : step,
  55                 "offset": offset - 1
  56             }), {
  57                 "Origin": domain
  58             })
  59
  60         # DEBUG: print(f"DEBUG: fetched()={len(fetched)}")
  61         if len(fetched) == 0:
  62             # DEBUG: print(f"DEBUG: Returned zero bytes, exiting loop, domain='{domain}'")
  63             break
  64         elif len(fetched) != config.get("misskey_limit"):
  65             # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'")
  66             offset = offset + (config.get("misskey_limit") - len(fetched))
  67         else:
  68             # DEBUG: print(f"DEBUG: Raising offset by step={step}")
  69             offset = offset + step
  70
  71         # Check records
  72         # DEBUG: print(f"DEBUG: fetched({len(fetched)})[]={type(fetched)}")
  73         if isinstance(fetched, dict) and "error" in fetched and "message" in fetched["error"]:
  74             print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}")
  75             instances.update_last_error(domain, fetched["error"]["message"])
  76             break
  77
  78         already = 0
  79         for row in fetched:
  80             # DEBUG: print(f"DEBUG: row()={len(row)}")
  81             if not "host" in row:
  82                 print(f"WARNING: row()={len(row)} does not contain key 'host': {row},domain='{domain}'")
  83                 continue
  84             elif not isinstance(row["host"], str):
  85                 print(f"WARNING: row[host][]={type(row['host'])} is not 'str'")
  86                 continue
  87             elif blacklist.is_blacklisted(row["host"]):
  88                 # DEBUG: print(f"DEBUG: row[host]='{row['host']}' is blacklisted. domain='{domain}'")
  89                 continue
  90             elif row["host"] in peers:
  91                 # DEBUG: print(f"DEBUG: Not adding row[host]='{row['host']}', already found.")
  92                 already = already + 1
  93                 continue
  94
  95             # DEBUG: print(f"DEBUG: Adding peer: '{row['host']}'")
  96             peers.append(row["host"])
  97
  98         if already == len(fetched):
  99             print(f"WARNING: Host returned same set of '{already}' instances, aborting loop!")
 100             break
 101
 102     # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'")
 103     instances.set("total_peers", domain, len(peers))
 104
 105     # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...")
 106     instances.update_last_instance_fetch(domain)
 107
 108     # DEBUG: print(f"DEBUG: Returning peers[]='{type(peers)}'")
 109     return peers
 110
 111 def fetch_blocks(domain: str) -> dict:
 112     # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
 113     if not isinstance(domain, str):
 114         raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
 115     elif domain == "":
 116         raise ValueError("Parameter 'domain' is empty")
 117
 118     # DEBUG: print("DEBUG: Fetching misskey blocks from domain:", domain)
 119     blocklist = {
 120         "suspended": [],
 121         "blocked"  : []
 122     }
 123
 124     offset = 0
 125     step = config.get("misskey_limit")
 126     while True:
 127         # iterating through all "suspended" (follow-only in its terminology)
 128         # instances page-by-page, since that troonware doesn't support
 129         # sending them all at once
 130         try:
 131             # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...")
 132             if offset == 0:
 133                 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
 134                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 135                     "sort"     : "+pubAt",
 136                     "host"     : None,
 137                     "suspended": True,
 138                     "limit"    : step
 139                 }), {
 140                     "Origin": domain
 141                 })
 142             else:
 143                 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
 144                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 145                     "sort"     : "+pubAt",
 146                     "host"     : None,
 147                     "suspended": True,
 148                     "limit"    : step,
 149                     "offset"   : offset - 1
 150                 }), {
 151                     "Origin": domain
 152                 })
 153
 154             # DEBUG: print("DEBUG: fetched():", len(fetched))
 155             if len(fetched) == 0:
 156                 # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
 157                 break
 158             elif len(fetched) != config.get("misskey_limit"):
 159                 # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'")
 160                 offset = offset + (config.get("misskey_limit") - len(fetched))
 161             else:
 162                 # DEBUG: print("DEBUG: Raising offset by step:", step)
 163                 offset = offset + step
 164
 165             count = 0
 166             for instance in fetched:
 167                 # Is it there?
 168                 if instance["isSuspended"] and not fba.has_key(blocklist["suspended"], "domain", instance):
 169                     count = count + 1
 170                     blocklist["suspended"].append(
 171                         {
 172                             "domain": fba.tidyup_domain(instance["host"]),
 173                             # no reason field, nothing
 174                             "reason": None
 175                         }
 176                     )
 177
 178             # DEBUG: print(f"DEBUG: count={count}")
 179             if count == 0:
 180                 # DEBUG: print(f"DEBUG: API is no more returning new instances, aborting loop!")
 181                 break
 182
 183         except BaseException as exception:
 184             print("WARNING: Caught error, exiting loop:", domain, exception)
 185             instances.update_last_error(domain, exception)
 186             offset = 0
 187             break
 188
 189     while True:
 190         # same shit, different asshole ("blocked" aka full suspend)
 191         try:
 192             if offset == 0:
 193                 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
 194                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 195                     "sort"   : "+pubAt",
 196                     "host"   : None,
 197                     "blocked": True,
 198                     "limit"  : step
 199                 }), {
 200                     "Origin": domain
 201                 })
 202             else:
 203                 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
 204                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 205                     "sort"   : "+pubAt",
 206                     "host"   : None,
 207                     "blocked": True,
 208                     "limit"  : step,
 209                     "offset" : offset - 1
 210                 }), {
 211                     "Origin": domain
 212                 })
 213
 214             # DEBUG: print("DEBUG: fetched():", len(fetched))
 215             if len(fetched) == 0:
 216                 # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
 217                 break
 218             elif len(fetched) != config.get("misskey_limit"):
 219                 # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'")
 220                 offset = offset + (config.get("misskey_limit") - len(fetched))
 221             else:
 222                 # DEBUG: print("DEBUG: Raising offset by step:", step)
 223                 offset = offset + step
 224
 225             count = 0
 226             for instance in fetched:
 227                 # Is it there?
 228                 if instance["isBlocked"] and not fba.has_key(blocklist["blocked"], "domain", instance):
 229                     count = count + 1
 230                     blocklist["blocked"].append({
 231                         "domain": fba.tidyup_domain(instance["host"]),
 232                         "reason": None
 233                     })
 234
 235             # DEBUG: print(f"DEBUG: count={count}")
 236             if count == 0:
 237                 # DEBUG: print(f"DEBUG: API is no more returning new instances, aborting loop!")
 238                 break
 239
 240         except BaseException as exception:
 241             print("ERROR: Exception during POST:", domain, exception)
 242             instances.update_last_error(domain, exception)
 243             offset = 0
 244             break
 245
 246     # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...")
 247     instances.update_last_instance_fetch(domain)
 248
 249     # DEBUG: print(f"DEBUG: Returning for domain='{domain}',blocked()={len(blocklist['blocked'])},suspended()={len(blocklist['suspended'])}")
 250     return {
 251         "reject"        : blocklist["blocked"],
 252         "followers_only": blocklist["suspended"]
 253     }