fba/federation/misskey.py

   1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
   2 # Copyright (C) 2023 Free Software Foundation
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published
   6 # by the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  16
  17 import json
  18
  19 from fba import blacklist
  20 from fba import config
  21 from fba import fba
  22 from fba import instances
  23
  24 def fetch_peers(domain: str) -> list:
  25     # DEBUG: print(f"DEBUG: domain({len(domain)})={domain} - CALLED!")
  26     if type(domain) != str:
  27         raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
  28     elif domain == "":
  29         raise ValueError(f"Parameter 'domain' is empty")
  30
  31     # DEBUG: print(f"DEBUG: domain='{domain}' is misskey, sending API POST request ...")
  32     peers = list()
  33     offset = 0
  34     step = config.get("misskey_limit")
  35
  36     # iterating through all "suspended" (follow-only in its terminology)
  37     # instances page-by-page, since that troonware doesn't support
  38     # sending them all at once
  39     while True:
  40         # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...")
  41         if offset == 0:
  42             fetched = fba.post_json_api(domain, "/api/federation/instances", json.dumps({
  43                 "sort" : "+pubAt",
  44                 "host" : None,
  45                 "limit": step
  46             }), {
  47                 "Origin": domain
  48             })
  49         else:
  50             fetched = fba.post_json_api(domain, "/api/federation/instances", json.dumps({
  51                 "sort"  : "+pubAt",
  52                 "host"  : None,
  53                 "limit" : step,
  54                 "offset": offset - 1
  55             }), {
  56                 "Origin": domain
  57             })
  58
  59         # DEBUG: print(f"DEBUG: fetched()={len(fetched)}")
  60         if len(fetched) == 0:
  61             # DEBUG: print(f"DEBUG: Returned zero bytes, exiting loop, domain='{domain}'")
  62             break
  63         elif len(fetched) != config.get("misskey_limit"):
  64             # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'")
  65             offset = offset + (config.get("misskey_limit") - len(fetched))
  66         else:
  67             # DEBUG: print(f"DEBUG: Raising offset by step={step}")
  68             offset = offset + step
  69
  70         # Check records
  71         # DEBUG: print(f"DEBUG: fetched({len(fetched)})[]={type(fetched)}")
  72         if isinstance(fetched, dict) and "error" in fetched and "message" in fetched["error"]:
  73             print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}")
  74             instances.update_last_error(domain, fetched["error"]["message"])
  75             break
  76
  77         already = 0
  78         for row in fetched:
  79             # DEBUG: print(f"DEBUG: row()={len(row)}")
  80             if not "host" in row:
  81                 print(f"WARNING: row()={len(row)} does not contain key 'host': {row},domain='{domain}'")
  82                 continue
  83             elif type(row["host"]) != str:
  84                 print(f"WARNING: row[host][]={type(row['host'])} is not 'str'")
  85                 continue
  86             elif blacklist.is_blacklisted(row["host"]):
  87                 # DEBUG: print(f"DEBUG: row[host]='{row['host']}' is blacklisted. domain='{domain}'")
  88                 continue
  89             elif row["host"] in peers:
  90                 # DEBUG: print(f"DEBUG: Not adding row[host]='{row['host']}', already found.")
  91                 already = already + 1
  92                 continue
  93
  94             # DEBUG: print(f"DEBUG: Adding peer: '{row['host']}'")
  95             peers.append(row["host"])
  96
  97         if already == len(fetched):
  98             print(f"WARNING: Host returned same set of '{already}' instances, aborting loop!")
  99             break
 100
 101     # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'")
 102     instances.set("total_peers", domain, len(peers))
 103
 104     # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...")
 105     instances.update_last_instance_fetch(domain)
 106
 107     # DEBUG: print(f"DEBUG: Returning peers[]='{type(peers)}'")
 108     return peers
 109
 110 def fetch_blocks(domain: str) -> dict:
 111     # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
 112     if type(domain) != str:
 113         raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
 114     elif domain == "":
 115         raise ValueError(f"Parameter 'domain' is empty")
 116
 117     # DEBUG: print("DEBUG: Fetching misskey blocks from domain:", domain)
 118     blocklist = {
 119         "suspended": [],
 120         "blocked"  : []
 121     }
 122
 123     offset = 0
 124     step = config.get("misskey_limit")
 125     while True:
 126         # iterating through all "suspended" (follow-only in its terminology)
 127         # instances page-by-page, since that troonware doesn't support
 128         # sending them all at once
 129         try:
 130             # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...")
 131             if offset == 0:
 132                 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
 133                 fetched = fba.post_json_api(domain, "/api/federation/instances", json.dumps({
 134                     "sort"     : "+pubAt",
 135                     "host"     : None,
 136                     "suspended": True,
 137                     "limit"    : step
 138                 }), {
 139                     "Origin": domain
 140                 })
 141             else:
 142                 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
 143                 fetched = fba.post_json_api(domain, "/api/federation/instances", json.dumps({
 144                     "sort"     : "+pubAt",
 145                     "host"     : None,
 146                     "suspended": True,
 147                     "limit"    : step,
 148                     "offset"   : offset - 1
 149                 }), {
 150                     "Origin": domain
 151                 })
 152
 153             # DEBUG: print("DEBUG: fetched():", len(fetched))
 154             if len(fetched) == 0:
 155                 # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
 156                 break
 157             elif len(fetched) != config.get("misskey_limit"):
 158                 # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'")
 159                 offset = offset + (config.get("misskey_limit") - len(fetched))
 160             else:
 161                 # DEBUG: print("DEBUG: Raising offset by step:", step)
 162                 offset = offset + step
 163
 164             count = 0
 165             for instance in fetched:
 166                 # Is it there?
 167                 if instance["isSuspended"] and not fba.has_key(blocklist["suspended"], "domain", instance):
 168                     count = count + 1
 169                     blocklist["suspended"].append(
 170                         {
 171                             "domain": fba.tidyup_domain(instance["host"]),
 172                             # no reason field, nothing
 173                             "reason": None
 174                         }
 175                     )
 176
 177             # DEBUG: print(f"DEBUG: count={count}")
 178             if count == 0:
 179                 # DEBUG: print(f"DEBUG: API is no more returning new instances, aborting loop!")
 180                 break
 181
 182         except BaseException as e:
 183             print("WARNING: Caught error, exiting loop:", domain, e)
 184             instances.update_last_error(domain, e)
 185             offset = 0
 186             break
 187
 188     while True:
 189         # same shit, different asshole ("blocked" aka full suspend)
 190         try:
 191             if offset == 0:
 192                 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
 193                 fetched = fba.post_json_api(domain, "/api/federation/instances", json.dumps({
 194                     "sort"   : "+pubAt",
 195                     "host"   : None,
 196                     "blocked": True,
 197                     "limit"  : step
 198                 }), {
 199                     "Origin": domain
 200                 })
 201             else:
 202                 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
 203                 fetched = fba.post_json_api(domain, "/api/federation/instances", json.dumps({
 204                     "sort"   : "+pubAt",
 205                     "host"   : None,
 206                     "blocked": True,
 207                     "limit"  : step,
 208                     "offset" : offset - 1
 209                 }), {
 210                     "Origin": domain
 211                 })
 212
 213             # DEBUG: print("DEBUG: fetched():", len(fetched))
 214             if len(fetched) == 0:
 215                 # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
 216                 break
 217             elif len(fetched) != config.get("misskey_limit"):
 218                 # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'")
 219                 offset = offset + (config.get("misskey_limit") - len(fetched))
 220             else:
 221                 # DEBUG: print("DEBUG: Raising offset by step:", step)
 222                 offset = offset + step
 223
 224             count = 0
 225             for instance in fetched:
 226                 # Is it there?
 227                 if instance["isBlocked"] and not fba.has_key(blocklist["blocked"], "domain", instance):
 228                     count = count + 1
 229                     blocklist["blocked"].append({
 230                         "domain": fba.tidyup_domain(instance["host"]),
 231                         "reason": None
 232                     })
 233
 234             # DEBUG: print(f"DEBUG: count={count}")
 235             if count == 0:
 236                 # DEBUG: print(f"DEBUG: API is no more returning new instances, aborting loop!")
 237                 break
 238
 239         except BaseException as e:
 240             print("ERROR: Exception during POST:", domain, e)
 241             instances.update_last_error(domain, e)
 242             offset = 0
 243             break
 244
 245     # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...")
 246     instances.update_last_instance_fetch(domain)
 247
 248     # DEBUG: print(f"DEBUG: Returning for domain='{domain}',blocked()={len(blocklist['blocked'])},suspended()={len(blocklist['suspended'])}")
 249     return {
 250         "reject"        : blocklist["blocked"],
 251         "followers_only": blocklist["suspended"]
 252     }