fba/networks/misskey.py

   1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
   2 # Copyright (C) 2023 Free Software Foundation
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published
   6 # by the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  16
  17 import json
  18
  19 import requests
  20
  21 from fba import blacklist
  22 from fba import config
  23 from fba import csrf
  24 from fba import instances
  25 from fba import network
  26
  27 from fba.helpers import dicts
  28 from fba.helpers import tidyup
  29
  30 def fetch_peers(domain: str) -> list:
  31     # DEBUG: print(f"DEBUG: domain({len(domain)})={domain} - CALLED!")
  32     if not isinstance(domain, str):
  33         raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
  34     elif domain == "":
  35         raise ValueError("Parameter 'domain' is empty")
  36
  37     # DEBUG: print(f"DEBUG: domain='{domain}' is misskey, sending API POST request ...")
  38     peers   = list()
  39     offset  = 0
  40     step    = config.get("misskey_limit")
  41     headers = tuple()
  42
  43     # DEBUG: print(f"DEBUG: Checking CSRF for domain='{domain}'")
  44     try:
  45        headers = csrf.determine(domain, dict())
  46     except network.exceptions as exception:
  47         print(f"WARNING: Exception '{type(exception)}' during checking CSRF - EXIT!")
  48         return
  49
  50     # iterating through all "suspended" (follow-only in its terminology)
  51     # instances page-by-page, since that troonware doesn't support
  52     # sending them all at once
  53     while True:
  54         # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...")
  55         if offset == 0:
  56             fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
  57                 "sort" : "+pubAt",
  58                 "host" : None,
  59                 "limit": step
  60             }), headers)
  61         else:
  62             fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
  63                 "sort"  : "+pubAt",
  64                 "host"  : None,
  65                 "limit" : step,
  66                 "offset": offset - 1
  67             }), headers)
  68
  69         # Check records
  70         # DEBUG: print(f"DEBUG: fetched[]={type(fetched)}")
  71         if "error_message" in fetched:
  72             print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}")
  73             instances.update_last_error(domain, fetched)
  74             break
  75         elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
  76             print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}")
  77             instances.update_last_error(domain, fetched["json"]["error"]["message"])
  78             break
  79
  80         rows = fetched["json"]
  81
  82         # DEBUG: print(f"DEBUG: rows()={len(rows)}")
  83         if len(rows) == 0:
  84             # DEBUG: print(f"DEBUG: Returned zero bytes, exiting loop, domain='{domain}'")
  85             break
  86         elif len(rows) != config.get("misskey_limit"):
  87             # DEBUG: print(f"DEBUG: Fetched '{len(rows)}' row(s) but expected: '{config.get('misskey_limit')}'")
  88             offset = offset + (config.get("misskey_limit") - len(rows))
  89         else:
  90             # DEBUG: print(f"DEBUG: Raising offset by step={step}")
  91             offset = offset + step
  92
  93         already = 0
  94         # DEBUG: print(f"DEBUG: rows({len(rows)})[]={type(rows)}")
  95         for row in rows:
  96             # DEBUG: print(f"DEBUG: row()={len(row)}")
  97             if not "host" in row:
  98                 print(f"WARNING: row()={len(row)} does not contain key 'host': {row},domain='{domain}'")
  99                 continue
 100             elif not isinstance(row["host"], str):
 101                 print(f"WARNING: row[host][]={type(row['host'])} is not 'str'")
 102                 continue
 103             elif blacklist.is_blacklisted(row["host"]):
 104                 # DEBUG: print(f"DEBUG: row[host]='{row['host']}' is blacklisted. domain='{domain}'")
 105                 continue
 106             elif row["host"] in peers:
 107                 # DEBUG: print(f"DEBUG: Not adding row[host]='{row['host']}', already found.")
 108                 already = already + 1
 109                 continue
 110
 111             # DEBUG: print(f"DEBUG: Adding peer: '{row['host']}'")
 112             peers.append(row["host"])
 113
 114         if already == len(rows):
 115             # DEBUG: print(f"DEBUG: Host returned same set of '{already}' instances, aborting loop!")
 116             break
 117
 118     # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'")
 119     instances.set_data("total_peers", domain, len(peers))
 120
 121     # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...")
 122     instances.update_last_instance_fetch(domain)
 123
 124     # DEBUG: print(f"DEBUG: Returning peers[]='{type(peers)}'")
 125     return peers
 126
 127 def fetch_blocks(domain: str) -> dict:
 128     # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
 129     if not isinstance(domain, str):
 130         raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
 131     elif domain == "":
 132         raise ValueError("Parameter 'domain' is empty")
 133
 134     # DEBUG: print(f"DEBUG: Fetching misskey blocks from domain={domain}")
 135     blocklist = {
 136         "suspended": [],
 137         "blocked"  : []
 138     }
 139
 140     offset  = 0
 141     step    = config.get("misskey_limit")
 142     headers = tuple()
 143
 144     # DEBUG: print(f"DEBUG: Checking CSRF for domain='{domain}'")
 145     try:
 146        headers = csrf.determine(domain, dict())
 147     except network.exceptions as exception:
 148         print(f"WARNING: Exception '{type(exception)}' during checking CSRF - EXIT!")
 149         return
 150
 151     # iterating through all "suspended" (follow-only in its terminology)
 152     # instances page-by-page since it doesn't support sending them all at once
 153     while True:
 154         try:
 155             # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...")
 156             if offset == 0:
 157                 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
 158                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 159                     "sort"     : "+pubAt",
 160                     "host"     : None,
 161                     "suspended": True,
 162                     "limit"    : step
 163                 }), headers)
 164             else:
 165                 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
 166                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 167                     "sort"     : "+pubAt",
 168                     "host"     : None,
 169                     "suspended": True,
 170                     "limit"    : step,
 171                     "offset"   : offset - 1
 172                 }), headers)
 173
 174             # DEBUG: print(f"DEBUG: fetched[]={type(fetched)}")
 175             if "error_message" in fetched:
 176                 print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}")
 177                 instances.update_last_error(domain, fetched)
 178                 break
 179             elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
 180                 print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}")
 181                 instances.update_last_error(domain, fetched["json"]["error"]["message"])
 182                 break
 183
 184             rows = fetched["json"]
 185
 186             # DEBUG: print(f"DEBUG: rows({len(rows)})={rows} - suspend")
 187             if len(rows) == 0:
 188                 # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
 189                 break
 190             elif len(rows) != config.get("misskey_limit"):
 191                 # DEBUG: print(f"DEBUG: Fetched '{len(rows)}' row(s) but expected: '{config.get('misskey_limit')}'")
 192                 offset = offset + (config.get("misskey_limit") - len(rows))
 193             else:
 194                 # DEBUG: print("DEBUG: Raising offset by step:", step)
 195                 offset = offset + step
 196
 197             count = 0
 198             for instance in rows:
 199                 # Is it there?
 200                 # DEBUG: print(f"DEBUG: instance[{type(instance)}]='{instance}' - suspend")
 201                 if instance["isSuspended"] and not dicts.has_key(blocklist["suspended"], "domain", instance["host"]):
 202                     count = count + 1
 203                     blocklist["suspended"].append(
 204                         {
 205                             "domain": tidyup.domain(instance["host"]),
 206                             # no reason field, nothing
 207                             "reason": None
 208                         }
 209                     )
 210
 211             # DEBUG: print(f"DEBUG: count={count}")
 212             if count == 0:
 213                 # DEBUG: print("DEBUG: API is no more returning new instances, aborting loop!")
 214                 break
 215
 216         except network.exceptions as exception:
 217             print(f"WARNING: Caught error, exiting loop: domain='{domain}',exception[{type(exception)}]='{str(exception)}'")
 218             instances.update_last_error(domain, exception)
 219             offset = 0
 220             break
 221
 222     while True:
 223         # Fetch blocked (full suspended) instances
 224         try:
 225             if offset == 0:
 226                 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
 227                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 228                     "sort"   : "+pubAt",
 229                     "host"   : None,
 230                     "blocked": True,
 231                     "limit"  : step
 232                 }), headers)
 233             else:
 234                 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
 235                 fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
 236                     "sort"   : "+pubAt",
 237                     "host"   : None,
 238                     "blocked": True,
 239                     "limit"  : step,
 240                     "offset" : offset - 1
 241                 }), headers)
 242
 243             # DEBUG: print(f"DEBUG: fetched[]={type(fetched)}")
 244             if "error_message" in fetched:
 245                 print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}")
 246                 instances.update_last_error(domain, fetched)
 247                 break
 248             elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
 249                 print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}")
 250                 instances.update_last_error(domain, fetched["json"]["error"]["message"])
 251                 break
 252
 253             rows = fetched["json"]
 254
 255             # DEBUG: print(f"DEBUG: rows({len(rows)})={rows} - blocked")
 256             if len(rows) == 0:
 257                 # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
 258                 break
 259             elif len(rows) != config.get("misskey_limit"):
 260                 # DEBUG: print(f"DEBUG: Fetched '{len(rows)}' row(s) but expected: '{config.get('misskey_limit')}'")
 261                 offset = offset + (config.get("misskey_limit") - len(rows))
 262             else:
 263                 # DEBUG: print("DEBUG: Raising offset by step:", step)
 264                 offset = offset + step
 265
 266             count = 0
 267             for instance in rows:
 268                 # Is it there?
 269                 # DEBUG: print(f"DEBUG: instance[{type(instance)}]='{instance}' - blocked")
 270                 if instance["isBlocked"] and not dicts.has_key(blocklist["blocked"], "domain", instance["host"]):
 271                     count = count + 1
 272                     blocklist["blocked"].append({
 273                         "domain": tidyup.domain(instance["host"]),
 274                         "reason": None
 275                     })
 276
 277             # DEBUG: print(f"DEBUG: count={count}")
 278             if count == 0:
 279                 # DEBUG: print("DEBUG: API is no more returning new instances, aborting loop!")
 280                 break
 281
 282         except network.exceptions as exception:
 283             print(f"WARNING: Caught error, exiting loop: domain='{domain}',exception[{type(exception)}]='{str(exception)}'")
 284             instances.update_last_error(domain, exception)
 285             offset = 0
 286             break
 287
 288     # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...")
 289     instances.update_last_instance_fetch(domain)
 290
 291     # DEBUG: print(f"DEBUG: Returning for domain='{domain}',blocked()={len(blocklist['blocked'])},suspended()={len(blocklist['suspended'])}")
 292     return {
 293         "reject"        : blocklist["blocked"],
 294         "followers_only": blocklist["suspended"]
 295     }