fba/networks/pleroma.py

   1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
   2 # Copyright (C) 2023 Free Software Foundation
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published
   6 # by the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  16
  17 import inspect
  18 import validators
  19
  20 import bs4
  21
  22 from fba import blacklist
  23 from fba import blocks
  24 from fba import config
  25 from fba import fba
  26 from fba import federation
  27 from fba import instances
  28 from fba import network
  29
  30 from fba.helpers import tidyup
  31
  32 def fetch_blocks(domain: str, origin: str, nodeinfo_url: str):
  33     # DEBUG: print(f"DEBUG: domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}' - CALLED!")
  34     if not isinstance(domain, str):
  35         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
  36     elif domain == "":
  37         raise ValueError("Parameter 'domain' is empty")
  38     elif not isinstance(origin, str) and origin is not None:
  39         raise ValueError(f"Parameter origin[]='{type(origin)}' is not 'str'")
  40     elif origin == "":
  41         raise ValueError("Parameter 'origin' is empty")
  42     elif not isinstance(nodeinfo_url, str):
  43         raise ValueError(f"Parameter nodeinfo_url[]='{type(nodeinfo_url)}' is not 'str'")
  44     elif nodeinfo_url == "":
  45         raise ValueError("Parameter 'nodeinfo_url' is empty")
  46
  47     # Blocks
  48     blockdict = list()
  49     rows = None
  50     try:
  51         rows = federation.fetch_nodeinfo(domain, nodeinfo_url)
  52     except network.exceptions as exception:
  53         print(f"WARNING: Exception '{type(exception)}' during fetching nodeinfo")
  54
  55     if rows is None:
  56         print("WARNING: Could not fetch nodeinfo from domain:", domain)
  57         return
  58     elif "metadata" not in rows:
  59         print(f"WARNING: rows()={len(rows)} does not have key 'metadata', domain='{domain}'")
  60         return
  61     elif "federation" not in rows["metadata"]:
  62         print(f"WARNING: rows()={len(rows['metadata'])} does not have key 'federation', domain='{domain}'")
  63         return
  64
  65     # DEBUG: print("DEBUG: Updating nodeinfo:", domain)
  66     instances.update_last_nodeinfo(domain)
  67
  68     data = rows["metadata"]["federation"]
  69
  70     if "mrf_simple" in data:
  71         # DEBUG: print("DEBUG: Found mrf_simple:", domain)
  72         for block_level, blocklist in (
  73             {
  74                 **data["mrf_simple"],
  75                 **{
  76                     "quarantined_instances": data["quarantined_instances"]
  77                 }
  78             }
  79         ).items():
  80             # DEBUG: print("DEBUG: block_level, blocklist():", block_level, len(blocklist))
  81             block_level = tidyup.domain(block_level)
  82             # DEBUG: print("DEBUG: BEFORE block_level:", block_level)
  83
  84             if block_level == "":
  85                 print("WARNING: block_level is now empty!")
  86                 continue
  87
  88             # DEBUG: print(f"DEBUG: Checking {len(blocklist)} entries from domain='{domain}',block_level='{block_level}' ...")
  89             if len(blocklist) > 0:
  90                 for blocked in blocklist:
  91                     # DEBUG: print("DEBUG: BEFORE blocked:", blocked)
  92                     blocked = tidyup.domain(blocked)
  93                     # DEBUG: print("DEBUG: AFTER blocked:", blocked)
  94
  95                     if blocked == "":
  96                         print("WARNING: blocked is empty after tidyup.domain():", domain, block_level)
  97                         continue
  98                     elif blacklist.is_blacklisted(blocked):
  99                         # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!")
 100                         continue
 101                     elif blocked.count("*") > 0:
 102                         # Obsured domain name with no hash
 103                         # DEBUG: print(f"DEBUG: Trying to de-obscure blocked='{blocked}' ...")
 104                         fba.cursor.execute(
 105                             "SELECT domain, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")]
 106                         )
 107                         searchres = fba.cursor.fetchone()
 108
 109                         # DEBUG: print(f"DEBUG: searchres[]='{type(searchres)}'")
 110                         if searchres is None:
 111                             print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
 112                             continue
 113
 114                         # DEBUG: print(f"DEBUG: blocked='{blocked}' de-obscured to '{searchres[0]}'")
 115                         blocked      = searchres[0]
 116                         nodeinfo_url = searchres[1]
 117
 118                     # DEBUG: print(f"DEBUG: blocked='{blocked}'")
 119                     if not validators.domain(blocked):
 120                         print(f"WARNING: blocked='{blocked}',software='pleroma' is not a valid domain name - skipped!")
 121                         continue
 122                     elif blocked.split(".")[-1] == "arpa":
 123                         print(f"WARNING: blocked='{blocked}' is a reversed .arpa domain and should not be used generally.")
 124                         continue
 125                     elif not instances.is_registered(blocked):
 126                         # Commit changes
 127                         fba.connection.commit()
 128
 129                         # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'")
 130                         instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url)
 131
 132                     if not blocks.is_instance_blocked(domain, blocked, block_level):
 133                         # DEBUG: print("DEBUG: Blocking:", domain, blocked, block_level)
 134                         blocks.add_instance(domain, blocked, "unknown", block_level)
 135
 136                         if block_level == "reject":
 137                             # DEBUG: print("DEBUG: Adding to blockdict:", blocked)
 138                             blockdict.append({
 139                                     "blocked": blocked,
 140                                     "reason" : None
 141                             })
 142                         else:
 143                             # DEBUG: print(f"DEBUG: Updating block last seen for domain='{domain}',blocked='{blocked}' ...")
 144                             blocks.update_last_seen(domain, blocked, block_level)
 145
 146     # DEBUG: print("DEBUG: Committing changes ...")
 147     fba.connection.commit()
 148
 149     # Reasons
 150     if "mrf_simple_info" in data:
 151         # DEBUG: print("DEBUG: Found mrf_simple_info:", domain)
 152         for block_level, info in (
 153             {
 154                 **data["mrf_simple_info"],
 155                 **(data["quarantined_instances_info"] if "quarantined_instances_info" in data else {})
 156             }
 157         ).items():
 158             # DEBUG: print("DEBUG: block_level, info.items():", block_level, len(info.items()))
 159             block_level = tidyup.domain(block_level)
 160             # DEBUG: print("DEBUG: BEFORE block_level:", block_level)
 161
 162             if block_level == "":
 163                 print("WARNING: block_level is now empty!")
 164                 continue
 165
 166             # DEBUG: print(f"DEBUG: Checking {len(info.items())} entries from domain='{domain}',software='pleroma',block_level='{block_level}' ...")
 167             for blocked, reason in info.items():
 168                 # DEBUG: print(f"DEBUG: blocked='{blocked}',reason[{type(reason)}]='{reason}' - BEFORE!")
 169                 blocked = tidyup.domain(blocked)
 170
 171                 if isinstance(reason, str):
 172                     # DEBUG: print("DEBUG: reason[] is a string")
 173                     reason = tidyup.reason(reason)
 174                 elif isinstance(reason, dict) and "reason" in reason:
 175                     # DEBUG: print("DEBUG: reason[] is a dict")
 176                     reason = tidyup.reason(reason["reason"])
 177                 elif reason is not None:
 178                     raise ValueError(f"Cannot handle reason[]='{type(reason)}'")
 179
 180                 # DEBUG: print(f"DEBUG: blocked='{blocked}',reason='{reason}' - AFTER!")
 181
 182                 if blocked == "":
 183                     print("WARNING: blocked is empty after tidyup.domain():", domain, block_level)
 184                     continue
 185                 elif blacklist.is_blacklisted(blocked):
 186                     # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!")
 187                     continue
 188                 elif blocked.count("*") > 0:
 189                     # Obsured domain with no hash
 190                     # DEBUG: print(f"DEBUG: Trying to de-obscure blocked='{blocked}' ...")
 191                     fba.cursor.execute(
 192                         "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")]
 193                     )
 194                     searchres = fba.cursor.fetchone()
 195
 196                     # DEBUG: print(f"DEBUG: searchres[]='{type(searchres)}'")
 197                     if searchres is None:
 198                         print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
 199                         continue
 200
 201                     # DEBUG: print(f"DEBUG: blocked='{blocked}' de-obscured to '{searchres[0]}'")
 202                     blocked = searchres[0]
 203                     origin = searchres[1]
 204                     nodeinfo_url = searchres[2]
 205                 elif not validators.domain(blocked):
 206                     print(f"WARNING: blocked='{blocked}',software='pleroma' is not a valid domain name - skipped!")
 207                     continue
 208
 209                 # DEBUG: print(f"DEBUG: blocked='{blocked}'")
 210                 if blocked.split(".")[-1] == "arpa":
 211                     print(f"WARNING: blocked='{blocked}' is a reversed .arpa domain and should not be used generally.")
 212                     continue
 213                 elif not instances.is_registered(blocked):
 214                     # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., domain='{domain}',origin='{origin}',nodein
 215                     instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url)
 216
 217                 # DEBUG: print(f"DEBUG: Updating block reason: reason='{reason}',domain='{domain}',blocked='{blocked}',block_level='{block_level}'")
 218                 blocks.update_reason(reason, domain, blocked, block_level)
 219
 220                 # DEBUG: print(f"DEBUG: blockdict()={len(blockdict)}")
 221                 for entry in blockdict:
 222                     if entry["blocked"] == blocked:
 223                         # DEBUG: print(f"DEBUG: Updating entry reason: blocked='{blocked}',reason='{reason}'")
 224                         entry["reason"] = reason
 225
 226     fba.connection.commit()
 227     # DEBUG: print("DEBUG: EXIT!")