fba/networks/pleroma.py

   1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
   2 # Copyright (C) 2023 Free Software Foundation
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published
   6 # by the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  16
  17 import inspect
  18
  19 import validators
  20
  21 from fba import blacklist
  22 from fba import blocks
  23 from fba import fba
  24 from fba import federation
  25 from fba import instances
  26 from fba import network
  27
  28 from fba.helpers import tidyup
  29
  30 def fetch_blocks(domain: str, origin: str, nodeinfo_url: str):
  31     # DEBUG: print(f"DEBUG: domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}' - CALLED!")
  32     if not isinstance(domain, str):
  33         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
  34     elif domain == "":
  35         raise ValueError("Parameter 'domain' is empty")
  36     elif not isinstance(origin, str) and origin is not None:
  37         raise ValueError(f"Parameter origin[]='{type(origin)}' is not 'str'")
  38     elif origin == "":
  39         raise ValueError("Parameter 'origin' is empty")
  40     elif not isinstance(nodeinfo_url, str):
  41         raise ValueError(f"Parameter nodeinfo_url[]='{type(nodeinfo_url)}' is not 'str'")
  42     elif nodeinfo_url == "":
  43         raise ValueError("Parameter 'nodeinfo_url' is empty")
  44
  45     # Blocks
  46     blockdict = list()
  47     rows = None
  48     try:
  49         # DEBUG: print(f"DEBUG: Fetching nodeinfo: domain='{domain}',nodeinfo_url='{nodeinfo_url}'")
  50         rows = federation.fetch_nodeinfo(domain, nodeinfo_url)
  51     except network.exceptions as exception:
  52         print(f"WARNING: Exception '{type(exception)}' during fetching nodeinfo")
  53
  54     if rows is None:
  55         print("WARNING: Could not fetch nodeinfo from domain:", domain)
  56         return
  57     elif "metadata" not in rows:
  58         print(f"WARNING: rows()={len(rows)} does not have key 'metadata', domain='{domain}'")
  59         return
  60     elif "federation" not in rows["metadata"]:
  61         print(f"WARNING: rows()={len(rows['metadata'])} does not have key 'federation', domain='{domain}'")
  62         return
  63
  64     # DEBUG: print("DEBUG: Updating nodeinfo:", domain)
  65     instances.update_last_nodeinfo(domain)
  66
  67     data = rows["metadata"]["federation"]
  68
  69     if "mrf_simple" in data:
  70         # DEBUG: print("DEBUG: Found mrf_simple:", domain)
  71         for block_level, blocklist in (
  72             {
  73                 **data["mrf_simple"],
  74                 **{
  75                     "quarantined_instances": data["quarantined_instances"]
  76                 }
  77             }
  78         ).items():
  79             # DEBUG: print("DEBUG: block_level, blocklist():", block_level, len(blocklist))
  80             block_level = tidyup.domain(block_level)
  81             # DEBUG: print("DEBUG: BEFORE block_level:", block_level)
  82
  83             if block_level == "":
  84                 print("WARNING: block_level is now empty!")
  85                 continue
  86
  87             # DEBUG: print(f"DEBUG: Checking {len(blocklist)} entries from domain='{domain}',block_level='{block_level}' ...")
  88             if len(blocklist) > 0:
  89                 for blocked in blocklist:
  90                     # DEBUG: print("DEBUG: BEFORE blocked:", blocked)
  91                     blocked = tidyup.domain(blocked)
  92                     # DEBUG: print("DEBUG: AFTER blocked:", blocked)
  93
  94                     if blocked == "":
  95                         print("WARNING: blocked is empty after tidyup.domain():", domain, block_level)
  96                         continue
  97                     elif blacklist.is_blacklisted(blocked):
  98                         # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!")
  99                         continue
 100                     elif blocked.count("*") > 0:
 101                         # Obscured domain name with no hash
 102                         row = instances.deobscure("*", blocked)
 103
 104                         # DEBUG: print(f"DEBUG: row[]='{type(row)}'")
 105                         if row is None:
 106                             print(f"WARNING: Cannot deobsfucate blocked='{blocked}',domain='{domain}',origin='{origin}' - SKIPPED!")
 107                             continue
 108
 109                         # DEBUG: print(f"DEBUG: blocked='{blocked}' de-obscured to '{row[0]}'")
 110                         blocked      = row[0]
 111                         origin       = row[1]
 112                         nodeinfo_url = row[2]
 113                     elif blocked.count("?") > 0:
 114                         # Obscured domain name with no hash
 115                         row = instances.deobscure("?", blocked)
 116
 117                         # DEBUG: print(f"DEBUG: row[]='{type(row)}'")
 118                         if row is None:
 119                             print(f"WARNING: Cannot deobsfucate blocked='{blocked}',domain='{domain}',origin='{origin}' - SKIPPED!")
 120                             continue
 121
 122                         # DEBUG: print(f"DEBUG: blocked='{blocked}' de-obscured to '{row[0]}'")
 123                         blocked      = row[0]
 124                         origin       = row[1]
 125                         nodeinfo_url = row[2]
 126
 127                     # DEBUG: print(f"DEBUG: blocked='{blocked}'")
 128                     if not validators.domain(blocked):
 129                         print(f"WARNING: blocked='{blocked}',software='pleroma' is not a valid domain name - SKIPPED!")
 130                         continue
 131                     elif blocked.endswith(".arpa"):
 132                         print(f"WARNING: blocked='{blocked}' is a reversed .arpa domain and should not be used generally.")
 133                         continue
 134                     elif not instances.is_registered(blocked):
 135                         # Commit changes
 136                         fba.connection.commit()
 137
 138                         # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'")
 139                         instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url)
 140
 141                     if not blocks.is_instance_blocked(domain, blocked, block_level):
 142                         # DEBUG: print("DEBUG: Blocking:", domain, blocked, block_level)
 143                         blocks.add_instance(domain, blocked, "unknown", block_level)
 144
 145                         if block_level == "reject":
 146                             # DEBUG: print("DEBUG: Adding to blockdict:", blocked)
 147                             blockdict.append({
 148                                     "blocked": blocked,
 149                                     "reason" : None
 150                             })
 151                         else:
 152                             # DEBUG: print(f"DEBUG: Updating block last seen for domain='{domain}',blocked='{blocked}' ...")
 153                             blocks.update_last_seen(domain, blocked, block_level)
 154     elif "quarantined_instances" in data:
 155         # DEBUG: print(f"DEBUG: Found 'quarantined_instances' in JSON response: domain='{domain}'")
 156         block_level = "quarantined"
 157
 158         for blocked in data["quarantined_instances"]:
 159             # DEBUG: print("DEBUG: BEFORE blocked:", blocked)
 160             blocked = tidyup.domain(blocked)
 161             # DEBUG: print("DEBUG: AFTER blocked:", blocked)
 162
 163             if blocked == "":
 164                 print("WARNING: blocked is empty after tidyup.domain():", domain, block_level)
 165                 continue
 166             elif blacklist.is_blacklisted(blocked):
 167                 # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!")
 168                 continue
 169             elif blocked.count("*") > 0:
 170                 # Obscured domain name with no hash
 171                 row = instances.deobscure("*", blocked)
 172
 173                 # DEBUG: print(f"DEBUG: row[]='{type(row)}'")
 174                 if row is None:
 175                     print(f"WARNING: Cannot deobsfucate blocked='{blocked}',domain='{domain}',origin='{origin}' - SKIPPED!")
 176                     continue
 177
 178                 # DEBUG: print(f"DEBUG: blocked='{blocked}' de-obscured to '{row[0]}'")
 179                 blocked      = row[0]
 180                 origin       = row[1]
 181                 nodeinfo_url = row[2]
 182             elif blocked.count("?") > 0:
 183                 # Obscured domain name with no hash
 184                 row = instances.deobscure("?", blocked)
 185
 186                 # DEBUG: print(f"DEBUG: row[]='{type(row)}'")
 187                 if row is None:
 188                     print(f"WARNING: Cannot deobsfucate blocked='{blocked}',domain='{domain}',origin='{origin}' - SKIPPED!")
 189                     continue
 190
 191                 # DEBUG: print(f"DEBUG: blocked='{blocked}' de-obscured to '{row[0]}'")
 192                 blocked      = row[0]
 193                 origin       = row[1]
 194                 nodeinfo_url = row[2]
 195
 196             # DEBUG: print(f"DEBUG: blocked='{blocked}'")
 197             if not validators.domain(blocked):
 198                 print(f"WARNING: blocked='{blocked}',software='pleroma' is not a valid domain name - SKIPPED!")
 199                 continue
 200             elif blocked.endswith(".arpa"):
 201                 print(f"WARNING: blocked='{blocked}' is a reversed .arpa domain and should not be used generally.")
 202                 continue
 203             elif not instances.is_registered(blocked):
 204                 # Commit changes
 205                 fba.connection.commit()
 206
 207                 # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'")
 208                 instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url)
 209
 210             if not blocks.is_instance_blocked(domain, blocked, block_level):
 211                 # DEBUG: print("DEBUG: Blocking:", domain, blocked, block_level)
 212                 blocks.add_instance(domain, blocked, "unknown", block_level)
 213
 214                 if block_level == "reject":
 215                     # DEBUG: print("DEBUG: Adding to blockdict:", blocked)
 216                     blockdict.append({
 217                             "blocked": blocked,
 218                             "reason" : None
 219                     })
 220                 else:
 221                     # DEBUG: print(f"DEBUG: Updating block last seen for domain='{domain}',blocked='{blocked}' ...")
 222                     blocks.update_last_seen(domain, blocked, block_level)
 223     else:
 224         print(f"WARNING: Cannot find 'mrf_simple' or 'quarantined_instances' in JSON reply: domain='{domain}'")
 225
 226     # DEBUG: print("DEBUG: Committing changes ...")
 227     fba.connection.commit()
 228
 229     # Reasons
 230     if "mrf_simple_info" in data:
 231         # DEBUG: print("DEBUG: Found mrf_simple_info:", domain)
 232         for block_level, info in (
 233             {
 234                 **data["mrf_simple_info"],
 235                 **(data["quarantined_instances_info"] if "quarantined_instances_info" in data else {})
 236             }
 237         ).items():
 238             # DEBUG: print("DEBUG: block_level, info.items():", block_level, len(info.items()))
 239             block_level = tidyup.domain(block_level)
 240             # DEBUG: print("DEBUG: BEFORE block_level:", block_level)
 241
 242             if block_level == "":
 243                 print("WARNING: block_level is now empty!")
 244                 continue
 245
 246             # DEBUG: print(f"DEBUG: Checking {len(info.items())} entries from domain='{domain}',software='pleroma',block_level='{block_level}' ...")
 247             for blocked, reason in info.items():
 248                 # DEBUG: print(f"DEBUG: blocked='{blocked}',reason[{type(reason)}]='{reason}' - BEFORE!")
 249                 blocked = tidyup.domain(blocked)
 250
 251                 if isinstance(reason, str):
 252                     # DEBUG: print("DEBUG: reason[] is a string")
 253                     reason = tidyup.reason(reason)
 254                 elif isinstance(reason, dict) and "reason" in reason:
 255                     # DEBUG: print("DEBUG: reason[] is a dict")
 256                     reason = tidyup.reason(reason["reason"])
 257                 elif reason is not None:
 258                     raise ValueError(f"Cannot handle reason[]='{type(reason)}'")
 259
 260                 # DEBUG: print(f"DEBUG: blocked='{blocked}',reason='{reason}' - AFTER!")
 261
 262                 if blocked == "":
 263                     print("WARNING: blocked is empty after tidyup.domain():", domain, block_level)
 264                     continue
 265                 elif blacklist.is_blacklisted(blocked):
 266                     # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!")
 267                     continue
 268                 elif blocked.count("*") > 0:
 269                     # Obscured domain name with no hash
 270                     row = instances.deobscure("*", blocked)
 271
 272                     # DEBUG: print(f"DEBUG: row[]='{type(row)}'")
 273                     if row is None:
 274                         print(f"WARNING: Cannot deobsfucate blocked='{blocked}',domain='{domain}',origin='{origin}' - SKIPPED!")
 275                         continue
 276
 277                     # DEBUG: print(f"DEBUG: blocked='{blocked}' de-obscured to '{row[0]}'")
 278                     blocked      = row[0]
 279                     origin       = row[1]
 280                     nodeinfo_url = row[2]
 281                 elif blocked.count("?") > 0:
 282                     # Obscured domain name with no hash
 283                     row = instances.deobscure("?", blocked)
 284
 285                     # DEBUG: print(f"DEBUG: row[]='{type(row)}'")
 286                     if row is None:
 287                         print(f"WARNING: Cannot deobsfucate blocked='{blocked}',domain='{domain}',origin='{origin}' - SKIPPED!")
 288                         continue
 289
 290                     # DEBUG: print(f"DEBUG: blocked='{blocked}' de-obscured to '{row[0]}'")
 291                     blocked      = row[0]
 292                     origin       = row[1]
 293                     nodeinfo_url = row[2]
 294
 295                 # DEBUG: print(f"DEBUG: blocked='{blocked}'")
 296                 if not validators.domain(blocked):
 297                     print(f"WARNING: blocked='{blocked}',software='pleroma' is not a valid domain name - SKIPPED!")
 298                     continue
 299                 elif blocked.endswith(".arpa"):
 300                     print(f"WARNING: blocked='{blocked}' is a reversed .arpa domain and should not be used generally.")
 301                     continue
 302                 elif not instances.is_registered(blocked):
 303                     # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'")
 304                     instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url)
 305
 306                 # DEBUG: print(f"DEBUG: Updating block reason: reason='{reason}',domain='{domain}',blocked='{blocked}',block_level='{block_level}'")
 307                 blocks.update_reason(reason, domain, blocked, block_level)
 308
 309                 # DEBUG: print(f"DEBUG: blockdict()={len(blockdict)}")
 310                 for entry in blockdict:
 311                     if entry["blocked"] == blocked:
 312                         # DEBUG: print(f"DEBUG: Updating entry reason: blocked='{blocked}',reason='{reason}'")
 313                         entry["reason"] = reason
 314     elif "quarantined_instances_info" in data and "quarantined_instances" in data["quarantined_instances_info"]:
 315         # DEBUG: print(f"DEBUG: Found 'quarantined_instances_info' in JSON response: domain='{domain}'")
 316         block_level = "quarantined"
 317
 318         #print(data["quarantined_instances_info"])
 319         rows = data["quarantined_instances_info"]["quarantined_instances"]
 320         for blocked in rows:
 321             # DEBUG: print("DEBUG: BEFORE blocked:", blocked)
 322             blocked = tidyup.domain(blocked)
 323             # DEBUG: print("DEBUG: AFTER blocked:", blocked)
 324
 325             if blocked not in rows or "reason" not in rows[blocked]:
 326                 print(f"WARNING: Cannot find blocked='{blocked}' in rows()={len(rows)},domain='{domain}'")
 327                 break
 328
 329             reason = rows[blocked]["reason"]
 330             # DEBUG: print(f"DEBUG: reason='{reason}'")
 331
 332             if blocked == "":
 333                 print("WARNING: blocked is empty after tidyup.domain():", domain, block_level)
 334                 continue
 335             elif blacklist.is_blacklisted(blocked):
 336                 # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!")
 337                 continue
 338             elif blocked.count("*") > 0:
 339                 # Obscured domain name with no hash
 340                 row = instances.deobscure("*", blocked)
 341
 342                 # DEBUG: print(f"DEBUG: row[]='{type(row)}'")
 343                 if row is None:
 344                     print(f"WARNING: Cannot deobsfucate blocked='{blocked}',domain='{domain}',origin='{origin}' - SKIPPED!")
 345                     continue
 346
 347                 # DEBUG: print(f"DEBUG: blocked='{blocked}' de-obscured to '{row[0]}'")
 348                 blocked      = row[0]
 349                 origin       = row[1]
 350                 nodeinfo_url = row[2]
 351             elif blocked.count("?") > 0:
 352                 # Obscured domain name with no hash
 353                 row = instances.deobscure("?", blocked)
 354
 355                 # DEBUG: print(f"DEBUG: row[]='{type(row)}'")
 356                 if row is None:
 357                     print(f"WARNING: Cannot deobsfucate blocked='{blocked}',domain='{domain}',origin='{origin}' - SKIPPED!")
 358                     continue
 359
 360                 # DEBUG: print(f"DEBUG: blocked='{blocked}' de-obscured to '{row[0]}'")
 361                 blocked      = row[0]
 362                 origin       = row[1]
 363                 nodeinfo_url = row[2]
 364
 365             # DEBUG: print(f"DEBUG: blocked='{blocked}'")
 366             if not validators.domain(blocked):
 367                 print(f"WARNING: blocked='{blocked}',software='pleroma' is not a valid domain name - SKIPPED!")
 368                 continue
 369             elif blocked.endswith(".arpa"):
 370                 print(f"WARNING: blocked='{blocked}' is a reversed .arpa domain and should not be used generally.")
 371                 continue
 372             elif not instances.is_registered(blocked):
 373                 # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'")
 374                 instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url)
 375
 376             # DEBUG: print(f"DEBUG: Updating block reason: reason='{reason}',domain='{domain}',blocked='{blocked}',block_level='{block_level}'")
 377             blocks.update_reason(reason, domain, blocked, block_level)
 378
 379             # DEBUG: print(f"DEBUG: blockdict()={len(blockdict)}")
 380             for entry in blockdict:
 381                 if entry["blocked"] == blocked:
 382                     # DEBUG: print(f"DEBUG: Updating entry reason: blocked='{blocked}',reason='{reason}'")
 383                     entry["reason"] = reason
 384     else:
 385         print(f"WARNING: Cannot find 'mrf_simple_info' or 'quarantined_instances_info' in JSON reply: domain='{domain}'")
 386
 387     fba.connection.commit()
 388     # DEBUG: print("DEBUG: EXIT!")