fba/networks/misskey.py

   1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
   2 # Copyright (C) 2023 Free Software Foundation
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published
   6 # by the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  16
  17 import json
  18
  19 import requests
  20
  21 from fba import blacklist
  22 from fba import config
  23 from fba import instances
  24 from fba import network
  25
  26 from fba.helpers import dicts
  27 from fba.helpers import tidyup
  28
  29 def fetch_peers(domain: str) -> list:
  30     # DEBUG: print(f"DEBUG: domain({len(domain)})={domain} - CALLED!")
  31     if not isinstance(domain, str):
  32         raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
  33     elif domain == "":
  34         raise ValueError("Parameter 'domain' is empty")
  35
  36     # DEBUG: print(f"DEBUG: domain='{domain}' is misskey, sending API POST request ...")
  37     peers = list()
  38     offset = 0
  39     step = config.get("misskey_limit")
  40
  41     # iterating through all "suspended" (follow-only in its terminology)
  42     # instances page-by-page, since that troonware doesn't support
  43     # sending them all at once
  44     while True:
  45         # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...")
  46         if offset == 0:
  47             fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
  48                 "sort" : "+pubAt",
  49                 "host" : None,
  50                 "limit": step
  51             }), {
  52                 "Origin": domain
  53             })
  54         else:
  55             fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
  56                 "sort"  : "+pubAt",
  57                 "host"  : None,
  58                 "limit" : step,
  59                 "offset": offset - 1
  60             }), {
  61                 "Origin": domain
  62             })
  63
  64         # DEBUG: print(f"DEBUG: fetched()={len(fetched)}")
  65         if len(fetched) == 0:
  66             # DEBUG: print(f"DEBUG: Returned zero bytes, exiting loop, domain='{domain}'")
  67             break
  68         elif len(fetched) != config.get("misskey_limit"):
  69             # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'")
  70             offset = offset + (config.get("misskey_limit") - len(fetched))
  71         else:
  72             # DEBUG: print(f"DEBUG: Raising offset by step={step}")
  73             offset = offset + step
  74
  75         # Check records
  76         # DEBUG: print(f"DEBUG: fetched({len(fetched)})[]={type(fetched)}")
  77         if isinstance(fetched, dict) and "error" in fetched and "message" in fetched["error"]:
  78             print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}")
  79             instances.update_last_error(domain, fetched["error"]["message"])
  80             break
  81         elif "error_message" in fetched:
  82             print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}")
  83             instances.update_last_error(domain, fetched)
  84             break
  85
  86         already = 0
  87         for row in fetched["json"]:
  88             # DEBUG: print(f"DEBUG: row()={len(row)}")
  89             if not "host" in row:
  90                 print(f"WARNING: row()={len(row)} does not contain key 'host': {row},domain='{domain}'")
  91                 continue
  92             elif not isinstance(row["host"], str):
  93                 print(f"WARNING: row[host][]={type(row['host'])} is not 'str'")
  94                 continue
  95             elif blacklist.is_blacklisted(row["host"]):
  96                 # DEBUG: print(f"DEBUG: row[host]='{row['host']}' is blacklisted. domain='{domain}'")
  97                 continue
  98             elif row["host"] in peers:
  99                 # DEBUG: print(f"DEBUG: Not adding row[host]='{row['host']}', already found.")
 100                 already = already + 1
 101                 continue
 102
 103             # DEBUG: print(f"DEBUG: Adding peer: '{row['host']}'")
 104             peers.append(row["host"])
 105
 106         if already == len(fetched):
 107             # DEBUG: print(f"DEBUG: Host returned same set of '{already}' instances, aborting loop!")
 108             break
 109
 110     # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'")
 111     instances.set_data("total_peers", domain, len(peers))
 112
 113     # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...")
 114     instances.update_last_instance_fetch(domain)
 115
 116     # DEBUG: print(f"DEBUG: Returning peers[]='{type(peers)}'")
 117     return peers
 118
 119 def fetch_blocks(domain: str) -> dict:
 120     # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
 121     if not isinstance(domain, str):
 122         raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
 123     elif domain == "":
 124         raise ValueError("Parameter 'domain' is empty")
 125
 126     # DEBUG: print("DEBUG: Fetching misskey blocks from domain:", domain)
 127     blocklist = {
 128         "suspended": [],
 129         "blocked"  : []
 130     }
 131
 132     offset = 0
 133     step = config.get("misskey_limit")
 134     while True:
 135         # iterating through all "suspended" (follow-only in its terminology)
 136         # instances page-by-page, since that troonware doesn't support
 137         # sending them all at once
 138         try:
 139             # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...")
 140             if offset == 0:
 141                 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
 142                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 143                     "sort"     : "+pubAt",
 144                     "host"     : None,
 145                     "suspended": True,
 146                     "limit"    : step
 147                 }), {
 148                     "Origin": domain
 149                 })
 150             else:
 151                 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
 152                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 153                     "sort"     : "+pubAt",
 154                     "host"     : None,
 155                     "suspended": True,
 156                     "limit"    : step,
 157                     "offset"   : offset - 1
 158                 }), {
 159                     "Origin": domain
 160                 })
 161
 162             # DEBUG: print("DEBUG: fetched():", len(fetched))
 163             if len(fetched) == 0:
 164                 # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
 165                 break
 166             elif len(fetched) != config.get("misskey_limit"):
 167                 # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'")
 168                 offset = offset + (config.get("misskey_limit") - len(fetched))
 169             else:
 170                 # DEBUG: print("DEBUG: Raising offset by step:", step)
 171                 offset = offset + step
 172
 173             count = 0
 174             for instance in fetched:
 175                 # Is it there?
 176                 if instance["isSuspended"] and not dicts.has_key(blocklist["suspended"], "domain", instance):
 177                     count = count + 1
 178                     blocklist["suspended"].append(
 179                         {
 180                             "domain": tidyup.domain(instance["host"]),
 181                             # no reason field, nothing
 182                             "reason": None
 183                         }
 184                     )
 185
 186             # DEBUG: print(f"DEBUG: count={count}")
 187             if count == 0:
 188                 # DEBUG: print("DEBUG: API is no more returning new instances, aborting loop!")
 189                 break
 190
 191         except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as exception:
 192             print(f"WARNING: Caught error, exiting loop: domain='{domain}',exception[{type(exception)}]='{str(exception)}'")
 193             instances.update_last_error(domain, exception)
 194             offset = 0
 195             break
 196
 197     while True:
 198         # same shit, different asshole ("blocked" aka full suspend)
 199         try:
 200             if offset == 0:
 201                 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
 202                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 203                     "sort"   : "+pubAt",
 204                     "host"   : None,
 205                     "blocked": True,
 206                     "limit"  : step
 207                 }), {
 208                     "Origin": domain
 209                 })
 210             else:
 211                 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
 212                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 213                     "sort"   : "+pubAt",
 214                     "host"   : None,
 215                     "blocked": True,
 216                     "limit"  : step,
 217                     "offset" : offset - 1
 218                 }), {
 219                     "Origin": domain
 220                 })
 221
 222             # DEBUG: print("DEBUG: fetched():", len(fetched))
 223             if len(fetched) == 0:
 224                 # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
 225                 break
 226             elif len(fetched) != config.get("misskey_limit"):
 227                 # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'")
 228                 offset = offset + (config.get("misskey_limit") - len(fetched))
 229             else:
 230                 # DEBUG: print("DEBUG: Raising offset by step:", step)
 231                 offset = offset + step
 232
 233             count = 0
 234             for instance in fetched:
 235                 # Is it there?
 236                 if instance["isBlocked"] and not dicts.has_key(blocklist["blocked"], "domain", instance):
 237                     count = count + 1
 238                     blocklist["blocked"].append({
 239                         "domain": tidyup.domain(instance["host"]),
 240                         "reason": None
 241                     })
 242
 243             # DEBUG: print(f"DEBUG: count={count}")
 244             if count == 0:
 245                 # DEBUG: print("DEBUG: API is no more returning new instances, aborting loop!")
 246                 break
 247
 248         except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as exception:
 249             print(f"WARNING: Caught error, exiting loop: domain='{domain}',exception[{type(exception)}]='{str(exception)}'")
 250             instances.update_last_error(domain, exception)
 251             offset = 0
 252             break
 253
 254     # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...")
 255     instances.update_last_instance_fetch(domain)
 256
 257     # DEBUG: print(f"DEBUG: Returning for domain='{domain}',blocked()={len(blocklist['blocked'])},suspended()={len(blocklist['suspended'])}")
 258     return {
 259         "reject"        : blocklist["blocked"],
 260         "followers_only": blocklist["suspended"]
 261     }