fba/commands.py

   1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
   2 # Copyright (C) 2023 Free Software Foundation
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published
   6 # by the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  16
  17 import csv
  18 import inspect
  19 import json
  20 import logging
  21 import time
  22
  23 import argparse
  24 import atoma
  25 import bs4
  26 import markdown
  27 import reqto
  28 import validators
  29
  30 from fba import csrf
  31 from fba import database
  32 from fba import utils
  33
  34 from fba.helpers import blacklist
  35 from fba.helpers import config
  36 from fba.helpers import cookies
  37 from fba.helpers import locking
  38 from fba.helpers import software as software_helper
  39 from fba.helpers import tidyup
  40
  41 from fba.http import federation
  42 from fba.http import network
  43
  44 from fba.models import blocks
  45 from fba.models import instances
  46
  47 from fba.networks import friendica
  48 from fba.networks import lemmy
  49 from fba.networks import mastodon
  50 from fba.networks import misskey
  51 from fba.networks import pleroma
  52
  53 logging.basicConfig(level=logging.INFO)
  54 logger = logging.getLogger(__name__)
  55 #logger.setLevel(logging.DEBUG)
  56
  57 def check_instance(args: argparse.Namespace) -> int:
  58     logger.debug("args.domain='%s' - CALLED!", args.domain)
  59     status = 0
  60     if not validators.domain(args.domain):
  61         logger.warning("args.domain='%s' is not valid", args.domain)
  62         status = 100
  63     elif blacklist.is_blacklisted(args.domain):
  64         logger.warning("args.domain='%s' is blacklisted", args.domain)
  65         status = 101
  66     elif instances.is_registered(args.domain):
  67         logger.warning("args.domain='%s' is already registered", args.domain)
  68         status = 102
  69     else:
  70         logger.info("args.domain='%s' is not known", args.domain)
  71
  72     logger.debug("status=%d - EXIT!", status)
  73     return status
  74
  75 def fetch_pixelfed_api(args: argparse.Namespace) -> int:
  76     logger.debug("args[]='%s' - CALLED!", type(args))
  77
  78     # No CSRF by default, you don't have to add network.api_headers by yourself here
  79     headers = tuple()
  80
  81     try:
  82         logger.debug("Checking CSRF from pixelfed.org")
  83         headers = csrf.determine("pixelfed.org", dict())
  84     except network.exceptions as exception:
  85         logger.warning("Exception '%s' during checking CSRF (fetch_peers,%s) - EXIT!", type(exception), __name__)
  86         return list()
  87
  88     try:
  89         logger.debug("Fetching JSON from pixelfed.org API, headers()=%d ...", len(headers))
  90         fetched = network.get_json_api(
  91             "pixelfed.org",
  92             "/api/v1/servers/all.json?scope=All&country=all&language=all",
  93             headers,
  94             (config.get("connection_timeout"), config.get("read_timeout"))
  95         )
  96
  97         logger.debug("JSON API returned %d elements", len(fetched))
  98         if "error_message" in fetched:
  99             logger.warning("API returned error_message='%s' - EXIT!", fetched["error_message"])
 100             return 101
 101         elif "data" not in fetched["json"]:
 102             logger.warning("API did not return JSON with 'data' element - EXIT!")
 103             return 102
 104
 105         rows = fetched["json"]["data"]
 106         logger.info("Checking %d fetched rows ...", len(rows))
 107         for row in rows:
 108             logger.debug("row[]='%s'", type(row))
 109             if "domain" not in row:
 110                 logger.warning("row='%s' does not contain element 'domain' - SKIPPED!", row)
 111                 continue
 112             elif not utils.is_domain_wanted(row["domain"]):
 113                 logger.debug("row[domain]='%s' is not wanted - SKIPPED!", row["domain"])
 114                 continue
 115             elif instances.is_registered(row["domain"]):
 116                 logger.debug("row[domain]='%s' is already registered - SKIPPED!", row["domain"])
 117                 continue
 118             elif instances.is_recent(row["domain"]):
 119                 logger.debug("row[domain]='%s' has been recently crawled - SKIPPED!", row["domain"])
 120                 continue
 121
 122             logger.debug("Fetching instances from row[domain]='%s' ...", row["domain"])
 123             federation.fetch_instances(row["domain"], None, None, inspect.currentframe().f_code.co_name)
 124
 125     except network.exceptions as exception:
 126         logger.warning("Cannot fetch graphql,exception[%s]:'%s' - EXIT!", type(exception), str(exception))
 127         return 103
 128
 129     logger.debug("Success! - EXIT!")
 130     return 0
 131
 132 def fetch_bkali(args: argparse.Namespace) -> int:
 133     logger.debug("args[]='%s' - CALLED!", type(args))
 134     domains = list()
 135     try:
 136         fetched = network.post_json_api("gql.api.bka.li", "/v1/graphql", json.dumps({
 137             "query": "query domainlist {nodeinfo(order_by: {domain: asc}) {domain}}"
 138         }))
 139
 140         logger.debug("fetched[]='%s'", type(fetched))
 141         if "error_message" in fetched:
 142             logger.warning("post_json_api() for 'gql.api.bka.li' returned error message='%s", fetched["error_message"])
 143             return 100
 144         elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
 145             logger.warning("post_json_api() returned error: '%s", fetched["error"]["message"])
 146             return 101
 147
 148         rows = fetched["json"]
 149
 150         logger.debug("rows(%d)[]='%s'", len(rows), type(rows))
 151         if len(rows) == 0:
 152             raise Exception("WARNING: Returned no records")
 153         elif "data" not in rows:
 154             raise Exception(f"WARNING: rows()={len(rows)} does not contain key 'data'")
 155         elif "nodeinfo" not in rows["data"]:
 156             raise Exception(f"WARNING: rows()={len(rows['data'])} does not contain key 'nodeinfo'")
 157
 158         for entry in rows["data"]["nodeinfo"]:
 159             logger.debug("entry[%s]='%s'", type(entry), entry)
 160             if "domain" not in entry:
 161                 logger.warning("entry()=%d does not contain 'domain' - SKIPPED!", len(entry))
 162                 continue
 163             elif not utils.is_domain_wanted(entry["domain"]):
 164                 logger.debug("entry[domain]='%s' is not wanted - SKIPPED!", entry["domain"])
 165                 continue
 166             elif instances.is_registered(entry["domain"]):
 167                 logger.debug("entry[domain]='%s' is already registered - SKIPPED!", entry["domain"])
 168                 continue
 169             elif instances.is_recent(entry["domain"]):
 170                 logger.debug("entry[domain]='%s' has been recently crawled - SKIPPED!", entry["domain"])
 171                 continue
 172
 173             logger.debug("Adding domain='%s' ...", entry["domain"])
 174             domains.append(entry["domain"])
 175
 176     except network.exceptions as exception:
 177         logger.warning("Cannot fetch graphql,exception[%s]:'%s' - EXIT!", type(exception), str(exception))
 178         return 102
 179
 180     logger.debug("domains()=%d", len(domains))
 181     if len(domains) > 0:
 182         locking.acquire()
 183
 184         logger.info("Adding %d new instances ...", len(domains))
 185         for domain in domains:
 186             try:
 187                 logger.info("Fetching instances from domain='%s' ...", domain)
 188                 federation.fetch_instances(domain, 'tak.teleyal.blog', None, inspect.currentframe().f_code.co_name)
 189             except network.exceptions as exception:
 190                 logger.warning("Exception '%s' during fetching instances (fetch_bkali) from domain='%s'", type(exception), domain)
 191                 instances.set_last_error(domain, exception)
 192
 193     logger.debug("Success - EXIT!")
 194     return 0
 195
 196 def fetch_blocks(args: argparse.Namespace) -> int:
 197     logger.debug("args[]='%s' - CALLED!", type(args))
 198     if args.domain is not None and args.domain != "":
 199         logger.debug("args.domain='%s' - checking ...", args.domain)
 200         if not validators.domain(args.domain):
 201             logger.warning("args.domain='%s' is not valid.", args.domain)
 202             return 100
 203         elif blacklist.is_blacklisted(args.domain):
 204             logger.warning("args.domain='%s' is blacklisted, won't check it!", args.domain)
 205             return 101
 206         elif not instances.is_registered(args.domain):
 207             logger.warning("args.domain='%s' is not registered, please run ./utils.py fetch_instances '%s' first.", args.domain, args.domain)
 208             return 102
 209
 210     locking.acquire()
 211
 212     if args.domain is not None and args.domain != "":
 213         # Re-check single domain
 214         logger.debug("Querying database for single args.domain='%s' ...", args.domain)
 215         database.cursor.execute(
 216             "SELECT domain, software, origin, nodeinfo_url FROM instances WHERE domain = ?", [args.domain]
 217         )
 218     elif args.software is not None and args.software != "":
 219         # Re-check single software
 220         logger.debug("Querying database for args.software='%s' ...", args.software)
 221         database.cursor.execute(
 222             "SELECT domain, software, origin, nodeinfo_url FROM instances WHERE software = ? AND nodeinfo_url IS NOT NULL", [args.software]
 223         )
 224     else:
 225         # Re-check after "timeout" (aka. minimum interval)
 226         database.cursor.execute(
 227             "SELECT domain, software, origin, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'lemmy', 'friendica', 'misskey', 'peertube') AND (last_blocked IS NULL OR last_blocked < ?) AND nodeinfo_url IS NOT NULL ORDER BY rowid DESC", [time.time() - config.get("recheck_block")]
 228         )
 229
 230     rows = database.cursor.fetchall()
 231     logger.info("Checking %d entries ...", len(rows))
 232     for blocker, software, origin, nodeinfo_url in rows:
 233         logger.debug("blocker='%s',software='%s',origin='%s',nodeinfo_url='%s'", blocker, software, origin, nodeinfo_url)
 234         blocker = tidyup.domain(blocker)
 235         logger.debug("blocker='%s' - AFTER!", blocker)
 236
 237         if blocker == "":
 238             logger.warning("blocker is now empty!")
 239             continue
 240         elif nodeinfo_url is None or nodeinfo_url == "":
 241             logger.debug("blocker='%s',software='%s' has empty nodeinfo_url", blocker, software)
 242             continue
 243         elif not utils.is_domain_wanted(blocker):
 244             logger.warning("blocker='%s' is not wanted - SKIPPED!", blocker)
 245             continue
 246
 247         logger.debug("blocker='%s'", blocker)
 248         instances.set_last_blocked(blocker)
 249         instances.set_has_obfuscation(blocker, False)
 250
 251         blocking = list()
 252         blockdict = list()
 253         if software == "pleroma":
 254             logger.info("blocker='%s',software='%s'", blocker, software)
 255             blocking = pleroma.fetch_blocks(blocker, nodeinfo_url)
 256         elif software == "mastodon":
 257             logger.info("blocker='%s',software='%s'", blocker, software)
 258             blocking = mastodon.fetch_blocks(blocker, nodeinfo_url)
 259         elif software == "lemmy":
 260             logger.info("blocker='%s',software='%s'", blocker, software)
 261             blocking = lemmy.fetch_blocks(blocker, nodeinfo_url)
 262         elif software == "friendica":
 263             logger.info("blocker='%s',software='%s'", blocker, software)
 264             blocking = friendica.fetch_blocks(blocker)
 265         elif software == "misskey":
 266             logger.info("blocker='%s',software='%s'", blocker, software)
 267             blocking = misskey.fetch_blocks(blocker)
 268         else:
 269             logger.warning("Unknown software: blocker='%s',software='%s'", blocker, software)
 270
 271         logger.info("Checking %d entries from blocker='%s',software='%s' ...", len(blocking), blocker, software)
 272         blockdict = list()
 273         for block in blocking:
 274             logger.debug("blocked='%s',block_level='%s',reason='%s'", block["blocked"], block["block_level"], block["reason"])
 275
 276             if block["block_level"] == "":
 277                 logger.warning("block_level is empty, blocker='%s',blocked='%s'", block["blocker"], block["blocked"])
 278                 continue
 279
 280             logger.debug("blocked='%s',reason='%s' - BEFORE!", block["blocked"], block["reason"])
 281             block["blocked"] = tidyup.domain(block["blocked"])
 282             block["reason"]  = tidyup.reason(block["reason"]) if block["reason"] is not None and block["reason"] != "" else None
 283             logger.debug("blocked='%s',reason='%s' - AFTER!", block["blocked"], block["reason"])
 284
 285             if block["blocked"] == "":
 286                 logger.warning("blocked is empty, blocker='%s'", blocker)
 287                 continue
 288             elif block["blocked"].endswith(".onion"):
 289                 logger.debug("blocked='%s' is a TOR .onion domain - SKIPPED", block["blocked"])
 290                 continue
 291             elif block["blocked"].endswith(".arpa"):
 292                 logger.debug("blocked='%s' is a reverse IP address - SKIPPED", block["blocked"])
 293                 continue
 294             elif block["blocked"].endswith(".tld"):
 295                 logger.debug("blocked='%s' is a fake domain - SKIPPED", block["blocked"])
 296                 continue
 297             elif block["blocked"].find("*") >= 0:
 298                 logger.debug("blocker='%s' uses obfuscated domains", blocker)
 299
 300                 # Some friendica servers also obscure domains without hash
 301                 row = instances.deobfuscate("*", block["blocked"], block["hash"] if "hash" in block else None)
 302
 303                 logger.debug("row[]='%s'", type(row))
 304                 if row is None:
 305                     logger.warning("Cannot deobfuscate blocked='%s',blocker='%s',software='%s' - SKIPPED!", block["blocked"], blocker, software)
 306                     instances.set_has_obfuscation(blocker, True)
 307                     continue
 308
 309                 block["blocked"] = row[0]
 310                 origin           = row[1]
 311                 nodeinfo_url     = row[2]
 312             elif block["blocked"].find("?") >= 0:
 313                 logger.debug("blocker='%s' uses obfuscated domains", blocker)
 314
 315                 # Some obscure them with question marks, not sure if that's dependent on version or not
 316                 row = instances.deobfuscate("?", block["blocked"], block["hash"] if "hash" in block else None)
 317
 318                 logger.debug("row[]='%s'", type(row))
 319                 if row is None:
 320                     logger.warning("Cannot deobfuscate blocked='%s',blocker='%s',software='%s' - SKIPPED!", block["blocked"], blocker, software)
 321                     instances.set_has_obfuscation(blocker, True)
 322                     continue
 323
 324                 block["blocked"] = row[0]
 325                 origin           = row[1]
 326                 nodeinfo_url     = row[2]
 327
 328             logger.debug("Looking up instance by domainm, blocked='%s'", block["blocked"])
 329             if not utils.is_domain_wanted(block["blocked"]):
 330                 logger.debug("blocked='%s' is not wanted - SKIPPED!", block["blocked"])
 331                 continue
 332             elif block["block_level"] in ["accept", "accepted"]:
 333                 logger.debug("blocked='%s' is accepted, not wanted here - SKIPPED!", block["blocked"])
 334                 continue
 335             elif not instances.is_registered(block["blocked"]):
 336                 logger.debug("Hash wasn't found, adding: blocked='%s',blocker='%s'", block["blocked"], blocker)
 337                 federation.fetch_instances(block["blocked"], blocker, None, inspect.currentframe().f_code.co_name)
 338
 339             if block["block_level"] == "silence":
 340                 logger.debug("Block level 'silence' has been changed to 'silenced'")
 341                 block["block_level"] = "silenced"
 342             elif block["block_level"] == "suspend":
 343                 logger.debug("Block level 'suspend' has been changed to 'suspended'")
 344                 block["block_level"] = "suspended"
 345
 346             if utils.process_block(blocker, block["blocked"], block["reason"], block["block_level"]) and block["block_level"] == "reject" and config.get("bot_enabled"):
 347                 logger.debug("Appending blocked='%s',reason='%s' for blocker='%s' ...", block["blocked"], block["block_level"], blocker)
 348                 blockdict.append({
 349                     "blocked": block["blocked"],
 350                     "reason" : block["reason"],
 351                 })
 352
 353             logger.debug("Invoking cookies.clear(%s) ...", block["blocked"])
 354             cookies.clear(block["blocked"])
 355
 356         logger.debug("Checking if blocker='%s' has pending updates ...", blocker)
 357         if instances.has_pending(blocker):
 358             logger.debug("Flushing updates for blocker='%s' ...", blocker)
 359             instances.update_data(blocker)
 360
 361         logger.debug("Invoking commit() ...")
 362         database.connection.commit()
 363
 364         logger.debug("Invoking cookies.clear(%s) ...", blocker)
 365         cookies.clear(blocker)
 366
 367         logger.debug("config[bot_enabled]='%s',blockdict()=%d'", config.get("bot_enabled"), len(blockdict))
 368         if config.get("bot_enabled") and len(blockdict) > 0:
 369             logger.info("Sending bot POST for blocker='%s',blockdict()=%d ...", blocker, len(blockdict))
 370             network.send_bot_post(blocker, blockdict)
 371
 372     logger.debug("Success! - EXIT!")
 373     return 0
 374
 375 def fetch_observer(args: argparse.Namespace) -> int:
 376     logger.debug("args[]='%s' - CALLED!", type(args))
 377     types = [
 378         "akkoma",
 379         "birdsitelive",
 380         "bookwyrm",
 381         "calckey",
 382         "diaspora",
 383         "foundkey",
 384         "friendica",
 385         "funkwhale",
 386         "gancio",
 387         "gnusocial",
 388         "gotosocial",
 389         "hometown",
 390         "hubzilla",
 391         "kbin",
 392         "ktistec",
 393         "lemmy",
 394         "mastodon",
 395         "microblogpub",
 396         "misskey",
 397         "mitra",
 398         "mobilizon",
 399         "owncast",
 400         "peertube",
 401         "pixelfed",
 402         "pleroma",
 403         "plume",
 404         "snac",
 405         "takahe",
 406         "wildebeest",
 407         "writefreely"
 408     ]
 409
 410     locking.acquire()
 411
 412     logger.info("Fetching %d different table data ...", len(types))
 413     for software in types:
 414         logger.debug("software='%s' - BEFORE!", software)
 415         if args.software is not None and args.software != software:
 416             logger.debug("args.software='%s' does not match software='%s' - SKIPPED!", args.software, software)
 417             continue
 418
 419         doc = None
 420         try:
 421             logger.debug("Fetching table data for software='%s' ...", software)
 422             raw = utils.fetch_url(
 423                 f"https://fediverse.observer/app/views/tabledata.php?software={software}",
 424                 network.web_headers,
 425                 (config.get("connection_timeout"), config.get("read_timeout"))
 426             ).text
 427             logger.debug("raw[%s]()=%d", type(raw), len(raw))
 428
 429             doc = bs4.BeautifulSoup(raw, features='html.parser')
 430             logger.debug("doc[]='%s'", type(doc))
 431         except network.exceptions as exception:
 432             logger.warning("Cannot fetch software='%s' from fediverse.observer: '%s'", software, type(exception))
 433             continue
 434
 435         items = doc.findAll("a", {"class": "url"})
 436         logger.info("Checking %d items,software='%s' ...", len(items), software)
 437         for item in items:
 438             logger.debug("item[]='%s'", type(item))
 439             domain = item.decode_contents()
 440
 441             logger.debug("domain='%s'", domain)
 442             if not utils.is_domain_wanted(domain):
 443                 logger.debug("domain='%s' is not wanted - SKIPPED!", domain)
 444                 continue
 445             elif instances.is_registered(domain):
 446                 logger.debug("domain='%s' is already registered - SKIPPED!", domain)
 447                 continue
 448             elif instances.is_recent(domain):
 449                 logger.debug("domain='%s' is recently being handled - SKIPPED!", domain)
 450                 continue
 451
 452             software = software_helper.alias(software)
 453             logger.info("Fetching instances for domain='%s'", domain)
 454             federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
 455
 456     logger.debug("Success! - EXIT!")
 457     return 0
 458
 459 def fetch_todon_wiki(args: argparse.Namespace) -> int:
 460     logger.debug("args[]='%s' - CALLED!", type(args))
 461
 462     locking.acquire()
 463     blocklist = {
 464         "silenced": list(),
 465         "reject": list(),
 466     }
 467
 468     raw = utils.fetch_url("https://wiki.todon.eu/todon/domainblocks", network.web_headers, (config.get("connection_timeout"), config.get("read_timeout"))).text
 469     logger.debug("raw()=%d,raw[]='%s'", len(raw), type(raw))
 470
 471     doc = bs4.BeautifulSoup(raw, "html.parser")
 472     logger.debug("doc[]='%s'", type(doc))
 473
 474     silenced = doc.find("h3", {"id": "silencedlimited_servers"}).find_next("ul").findAll("li")
 475     logger.info("Checking %d silenced/limited entries ...", len(silenced))
 476     blocklist["silenced"] = utils.find_domains(silenced, "div")
 477
 478     suspended = doc.find("h3", {"id": "suspended_servers"}).find_next("ul").findAll("li")
 479     logger.info("Checking %d suspended entries ...", len(suspended))
 480     blocklist["reject"] = utils.find_domains(suspended, "div")
 481
 482     blockdict = list()
 483     for block_level in blocklist:
 484         blockers = blocklist[block_level]
 485
 486         logger.debug("block_level='%s',blockers()=%d'", block_level, len(blockers))
 487         for blocked in blockers:
 488             logger.debug("blocked='%s'", blocked)
 489
 490             if not instances.is_registered(blocked):
 491                 try:
 492                     logger.info("Fetching instances from domain='%s' ...", blocked)
 493                     federation.fetch_instances(blocked, 'chaos.social', None, inspect.currentframe().f_code.co_name)
 494                 except network.exceptions as exception:
 495                     logger.warning("Exception '%s' during fetching instances (fetch_cs) from blocked='%s'", type(exception), blocked)
 496                     instances.set_last_error(blocked, exception)
 497
 498             if blocks.is_instance_blocked("todon.eu", blocked, block_level):
 499                 logger.debug("blocked='%s',block_level='%s' is already blocked - SKIPPED!", blocked, block_level)
 500                 continue
 501
 502             logger.info("Adding new block: blocked='%s',block_level='%s'", blocked, block_level)
 503             if utils.process_block("todon.eu", blocked, None, block_level) and block_level == "reject" and config.get("bot_enabled"):
 504                 logger.debug("Appending blocked='%s',reason='%s' for blocker='todon.eu' ...", blocked, block_level)
 505                 blockdict.append({
 506                     "blocked": blocked,
 507                     "reason" : None,
 508                 })
 509
 510         logger.debug("Invoking commit() ...")
 511         database.connection.commit()
 512
 513         if config.get("bot_enabled") and len(blockdict) > 0:
 514             logger.info("Sending bot POST for blocker='todon.eu',blockdict()=%d ...", len(blockdict))
 515             network.send_bot_post("todon.eu", blockdict)
 516
 517     logger.debug("Success! - EXIT!")
 518     return 0
 519
 520 def fetch_cs(args: argparse.Namespace):
 521     logger.debug("args[]='%s' - CALLED!", type(args))
 522     extensions = [
 523         "extra",
 524         "abbr",
 525         "attr_list",
 526         "def_list",
 527         "fenced_code",
 528         "footnotes",
 529         "md_in_html",
 530         "admonition",
 531         "codehilite",
 532         "legacy_attrs",
 533         "legacy_em",
 534         "meta",
 535         "nl2br",
 536         "sane_lists",
 537         "smarty",
 538         "toc",
 539         "wikilinks"
 540     ]
 541
 542     domains = {
 543         "silenced": list(),
 544         "reject"  : list(),
 545     }
 546
 547     raw = utils.fetch_url("https://raw.githubusercontent.com/chaossocial/meta/master/federation.md", network.web_headers, (config.get("connection_timeout"), config.get("read_timeout"))).text
 548     logger.debug("raw()=%d,raw[]='%s'", len(raw), type(raw))
 549
 550     doc = bs4.BeautifulSoup(markdown.markdown(raw, extensions=extensions), features='html.parser')
 551     logger.debug("doc()=%d[]='%s'", len(doc), type(doc))
 552
 553     silenced = doc.find("h2", {"id": "silenced-instances"}).findNext("table").find("tbody")
 554     logger.debug("silenced[%s]()=%d", type(silenced), len(silenced))
 555     domains["silenced"] = federation.find_domains(silenced)
 556
 557     blocked = doc.find("h2", {"id": "blocked-instances"}).findNext("table").find("tbody")
 558     logger.debug("blocked[%s]()=%d", type(blocked), len(blocked))
 559     domains["reject"] = federation.find_domains(blocked)
 560
 561     logger.debug("domains[silenced]()=%d,domains[reject]()=%d", len(domains["silenced"]), len(domains["reject"]))
 562     blockdict = list()
 563     if len(domains) > 0:
 564         locking.acquire()
 565
 566         for block_level in domains:
 567             logger.info("block_level='%s' has %d row(s)", block_level, len(domains[block_level]))
 568
 569             for row in domains[block_level]:
 570                 logger.debug("row[%s]='%s'", type(row), row)
 571                 if instances.is_recent(row["domain"], "last_blocked"):
 572                     logger.debug("row[domain]='%s' has been recently crawled - SKIPPED!", row["domain"])
 573                     continue
 574                 elif not instances.is_registered(row["domain"]):
 575                     try:
 576                         logger.info("Fetching instances from domain='%s' ...", row["domain"])
 577                         federation.fetch_instances(row["domain"], 'chaos.social', None, inspect.currentframe().f_code.co_name)
 578                     except network.exceptions as exception:
 579                         logger.warning("Exception '%s' during fetching instances (fetch_cs) from row[domain]='%s'", type(exception), row["domain"])
 580                         instances.set_last_error(row["domain"], exception)
 581
 582                 if utils.process_block("chaos.social", row["domain"], row["reason"], block_level) and block_level == "reject" and config.get("bot_enabled"):
 583                     logger.debug("Appending blocked='%s',reason='%s' for blocker='chaos.social' ...", row["domain"], block_level)
 584                     blockdict.append({
 585                         "blocked": row["domain"],
 586                         "reason" : row["reason"],
 587                     })
 588
 589         logger.debug("Invoking commit() ...")
 590         database.connection.commit()
 591
 592         if config.get("bot_enabled") and len(blockdict) > 0:
 593             logger.info("Sending bot POST for blocker='chaos.social',blockdict()=%d ...", len(blockdict))
 594             network.send_bot_post("chaos.social", blockdict)
 595
 596     logger.debug("Success! - EXIT!")
 597     return 0
 598
 599 def fetch_fba_rss(args: argparse.Namespace) -> int:
 600     logger.debug("args[]='%s' - CALLED!", type(args))
 601     domains = list()
 602
 603     logger.info("Fetch FBA-specific RSS args.feed='%s' ...", args.feed)
 604     response = utils.fetch_url(args.feed, network.web_headers, (config.get("connection_timeout"), config.get("read_timeout")))
 605
 606     logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text))
 607     if response.ok and response.status_code < 300 and len(response.text) > 0:
 608         logger.debug("Parsing RSS feed (%d Bytes) ...", len(response.text))
 609         rss = atoma.parse_rss_bytes(response.content)
 610
 611         logger.debug("rss[]='%s'", type(rss))
 612         for item in rss.items:
 613             logger.debug("item='%s'", item)
 614             domain = item.link.split("=")[1]
 615
 616             if not utils.is_domain_wanted(domain):
 617                 logger.debug("domain='%s' is not wanted - SKIPPED!", domain)
 618                 continue
 619             elif domain in domains:
 620                 logger.debug("domain='%s' is already added - SKIPPED!", domain)
 621                 continue
 622             elif instances.is_registered(domain):
 623                 logger.debug("domain='%s' is already registered - SKIPPED!", domain)
 624                 continue
 625             elif instances.is_recent(domain):
 626                 logger.debug("domain='%s' has been recently crawled - SKIPPED!", domain)
 627                 continue
 628
 629             logger.debug("Adding domain='%s'", domain)
 630             domains.append(domain)
 631
 632     logger.debug("domains()=%d", len(domains))
 633     if len(domains) > 0:
 634         locking.acquire()
 635
 636         logger.info("Adding %d new instances ...", len(domains))
 637         for domain in domains:
 638             try:
 639                 logger.info("Fetching instances from domain='%s' ...", domain)
 640                 federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
 641             except network.exceptions as exception:
 642                 logger.warning("Exception '%s' during fetching instances (fetch_fba_rss) from domain='%s'", type(exception), domain)
 643                 instances.set_last_error(domain, exception)
 644
 645     logger.debug("Success! - EXIT!")
 646     return 0
 647
 648 def fetch_fbabot_atom(args: argparse.Namespace) -> int:
 649     logger.debug("args[]='%s' - CALLED!", type(args))
 650     feed = "https://ryona.agency/users/fba/feed.atom"
 651
 652     domains = list()
 653
 654     logger.info("Fetching ATOM feed='%s' from FBA bot account ...", feed)
 655     response = utils.fetch_url(feed, network.web_headers, (config.get("connection_timeout"), config.get("read_timeout")))
 656
 657     logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text))
 658     if response.ok and response.status_code < 300 and len(response.text) > 0:
 659         logger.debug("Parsing ATOM feed (%d Bytes) ...", len(response.text))
 660         atom = atoma.parse_atom_bytes(response.content)
 661
 662         logger.debug("atom[]='%s'", type(atom))
 663         for entry in atom.entries:
 664             logger.debug("entry[]='%s'", type(entry))
 665             doc = bs4.BeautifulSoup(entry.content.value, "html.parser")
 666             logger.debug("doc[]='%s'", type(doc))
 667             for element in doc.findAll("a"):
 668                 for href in element["href"].split(","):
 669                     logger.debug("href[%s]='%s", type(href), href)
 670                     domain = tidyup.domain(href)
 671
 672                     logger.debug("domain='%s'", domain)
 673                     if not utils.is_domain_wanted(domain):
 674                         logger.debug("domain='%s' is not wanted - SKIPPED!", domain)
 675                         continue
 676                     elif domain in domains:
 677                         logger.debug("domain='%s' is already added - SKIPPED!", domain)
 678                         continue
 679                     elif instances.is_registered(domain):
 680                         logger.debug("domain='%s' is already registered - SKIPPED!", domain)
 681                         continue
 682                     elif instances.is_recent(domain):
 683                         logger.debug("domain='%s' has been recently crawled - SKIPPED!", domain)
 684                         continue
 685
 686                     logger.debug("Adding domain='%s',domains()=%d", domain, len(domains))
 687                     domains.append(domain)
 688
 689     logger.debug("domains()=%d", len(domains))
 690     if len(domains) > 0:
 691         locking.acquire()
 692
 693         logger.info("Adding %d new instances ...", len(domains))
 694         for domain in domains:
 695             try:
 696                 logger.info("Fetching instances from domain='%s' ...", domain)
 697                 federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
 698             except network.exceptions as exception:
 699                 logger.warning("Exception '%s' during fetching instances (fetch_fbabot_atom) from domain='%s'", type(exception), domain)
 700                 instances.set_last_error(domain, exception)
 701
 702     logger.debug("Success! - EXIT!")
 703     return 0
 704
 705 def fetch_instances(args: argparse.Namespace) -> int:
 706     logger.debug("args[]='%s' - CALLED!", type(args))
 707     locking.acquire()
 708
 709     # Initial fetch
 710     try:
 711         logger.info("Fetching instances from args.domain='%s' ...", args.domain)
 712         federation.fetch_instances(args.domain, None, None, inspect.currentframe().f_code.co_name)
 713     except network.exceptions as exception:
 714         logger.warning("Exception '%s' during fetching instances (fetch_instances) from args.domain='%s'", type(exception), args.domain)
 715         instances.set_last_error(args.domain, exception)
 716         instances.update_data(args.domain)
 717         return 100
 718
 719     if args.single:
 720         logger.debug("Not fetching more instances - EXIT!")
 721         return 0
 722
 723     # Loop through some instances
 724     database.cursor.execute(
 725         "SELECT domain, origin, software, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'lemmy', 'peertube') AND (last_instance_fetch IS NULL OR last_instance_fetch < ?) ORDER BY rowid DESC", [time.time() - config.get("recheck_instance")]
 726     )
 727
 728     rows = database.cursor.fetchall()
 729     logger.info("Checking %d entries ...", len(rows))
 730     for row in rows:
 731         logger.debug("domain='%s'", row[0])
 732         if not utils.is_domain_wanted(row[0]):
 733             logger.debug("Domain row[0]='%s' is not wanted - SKIPPED!", row[0])
 734             continue
 735
 736         try:
 737             logger.info("Fetching instances for domain='%s',origin='%s',software='%s',nodeinfo_url='%s'", row[0], row[1], row[2], row[3])
 738             federation.fetch_instances(row[0], row[1], row[2], inspect.currentframe().f_code.co_name, row[3])
 739         except network.exceptions as exception:
 740             logger.warning("Exception '%s' during fetching instances (fetch_instances) from row[0]='%s'", type(exception), row[0])
 741             instances.set_last_error(row[0], exception)
 742
 743     logger.debug("Success - EXIT!")
 744     return 0
 745
 746 def fetch_oliphant(args: argparse.Namespace) -> int:
 747     logger.debug("args[]='%s' - CALLED!", type(args))
 748     locking.acquire()
 749
 750     # Base URL
 751     base_url = "https://codeberg.org/oliphant/blocklists/raw/branch/main/blocklists"
 752
 753     # URLs to fetch
 754     blocklists = (
 755         {
 756             "blocker": "artisan.chat",
 757             "csv_url": "mastodon/artisan.chat.csv",
 758         },{
 759             "blocker": "mastodon.art",
 760             "csv_url": "mastodon/mastodon.art.csv",
 761         },{
 762             "blocker": "pleroma.envs.net",
 763             "csv_url": "mastodon/pleroma.envs.net.csv",
 764         },{
 765             "blocker": "oliphant.social",
 766             "csv_url": "mastodon/_unified_tier3_blocklist.csv",
 767         },{
 768             "blocker": "mastodon.online",
 769             "csv_url": "mastodon/mastodon.online.csv",
 770         },{
 771             "blocker": "mastodon.social",
 772             "csv_url": "mastodon/mastodon.social.csv",
 773         },{
 774             "blocker": "mastodon.social",
 775             "csv_url": "other/missing-tier0-mastodon.social.csv",
 776         },{
 777             "blocker": "rage.love",
 778             "csv_url": "mastodon/rage.love.csv",
 779         },{
 780             "blocker": "sunny.garden",
 781             "csv_url": "mastodon/sunny.garden.csv",
 782         },{
 783             "blocker": "solarpunk.moe",
 784             "csv_url": "mastodon/solarpunk.moe.csv",
 785         },{
 786             "blocker": "toot.wales",
 787             "csv_url": "mastodon/toot.wales.csv",
 788         },{
 789             "blocker": "union.place",
 790             "csv_url": "mastodon/union.place.csv",
 791         }
 792     )
 793
 794     domains = list()
 795
 796     logger.debug("Downloading %d files ...", len(blocklists))
 797     for block in blocklists:
 798         # Is domain given and not equal blocker?
 799         if isinstance(args.domain, str) and args.domain != block["blocker"]:
 800             logger.debug("Skipping blocker='%s', not matching args.domain='%s'", block["blocker"], args.domain)
 801             continue
 802         elif args.domain in domains:
 803             logger.debug("args.domain='%s' already handled - SKIPPED!", args.domain)
 804             continue
 805         elif instances.is_recent(block["blocker"]):
 806             logger.debug("block[blocker]='%s' has been recently crawled - SKIPPED!", block["blocker"])
 807             continue
 808
 809         # Fetch this URL
 810         logger.info("Fetching csv_url='%s' for blocker='%s' ...", block["csv_url"], block["blocker"])
 811         response = utils.fetch_url(f"{base_url}/{block['csv_url']}", network.web_headers, (config.get("connection_timeout"), config.get("read_timeout")))
 812
 813         logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text))
 814         if not response.ok or response.status_code > 399 or response.content == "":
 815             logger.warning("Could not fetch csv_url='%s' for blocker='%s' - SKIPPED!", block["csv_url"], block["blocker"])
 816             continue
 817
 818         logger.debug("Fetched %d Bytes, parsing CSV ...", len(response.content))
 819         reader = csv.DictReader(response.content.decode('utf-8').splitlines(), dialect="unix")
 820
 821         logger.debug("reader[]='%s'", type(reader))
 822         blockdict = list()
 823         for row in reader:
 824             logger.debug("row[%s]='%s'", type(row), row)
 825             domain = severity = None
 826             reject_media = reject_reports = False
 827             if "#domain" in row:
 828                 domain = row["#domain"]
 829             elif "domain" in row:
 830                 domain = row["domain"]
 831             else:
 832                 logger.debug("row='%s' does not contain domain column", row)
 833                 continue
 834
 835             if "#severity" in row:
 836                 severity = row["#severity"]
 837             elif "severity" in row:
 838                 severity = row["severity"]
 839             else:
 840                 logger.debug("row='%s' does not contain severity column", row)
 841                 continue
 842
 843             if "#reject_media" in row and row["#reject_media"].lower() == "true":
 844                 reject_media = True
 845             elif "reject_media" in row and row["reject_media"].lower() == "true":
 846                 reject_media = True
 847
 848             if "#reject_reports" in row and row["#reject_reports"].lower() == "true":
 849                 reject_reports = True
 850             elif "reject_reports" in row and row["reject_reports"].lower() == "true":
 851                 reject_reports = True
 852
 853             logger.debug("domain='%s',severity='%s',reject_media='%s',reject_reports='%s'", domain, severity, reject_media, reject_reports)
 854             if not utils.is_domain_wanted(domain):
 855                 logger.debug("domain='%s' is not wanted - SKIPPED!", domain)
 856                 continue
 857
 858             logger.debug("Marking domain='%s' as handled", domain)
 859             domains.append(domain)
 860
 861             logger.debug("Processing domain='%s' ...", domain)
 862             processed = utils.process_domain(domain, block["blocker"], inspect.currentframe().f_code.co_name)
 863             logger.debug("processed='%s'", processed)
 864
 865             if utils.process_block(block["blocker"], domain, None, "reject") and config.get("bot_enabled"):
 866                 logger.debug("Appending blocked='%s',reason='%s' for blocker='%s' ...", domain, block["block_level"], block["blocker"])
 867                 blockdict.append({
 868                     "blocked": domain,
 869                     "reason" : block["reason"],
 870                 })
 871
 872             if reject_media:
 873                 utils.process_block(block["blocker"], domain, None, "reject_media")
 874             if reject_reports:
 875                 utils.process_block(block["blocker"], domain, None, "reject_reports")
 876
 877         logger.debug("Invoking commit() ...")
 878         database.connection.commit()
 879
 880         if config.get("bot_enabled") and len(blockdict) > 0:
 881             logger.info("Sending bot POST for blocker='%s',blockdict()=%d ...", block["blocker"], len(blockdict))
 882             network.send_bot_post(block["blocker"], blockdict)
 883
 884     logger.debug("Success! - EXIT!")
 885     return 0
 886
 887 def fetch_txt(args: argparse.Namespace) -> int:
 888     logger.debug("args[]='%s' - CALLED!", type(args))
 889     locking.acquire()
 890
 891     # Static URLs
 892     urls = ({
 893         "blocker": "seirdy.one",
 894         "url"    : "https://seirdy.one/pb/bsl.txt",
 895     },)
 896
 897     logger.info("Checking %d text file(s) ...", len(urls))
 898     for row in urls:
 899         logger.debug("Fetching row[url]='%s' ...", row["url"])
 900         response = utils.fetch_url(row["url"], network.web_headers, (config.get("connection_timeout"), config.get("read_timeout")))
 901
 902         logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text))
 903         if response.ok and response.status_code < 300 and response.text != "":
 904             logger.debug("Returned %d Bytes for processing", len(response.text.strip()))
 905             domains = response.text.split("\n")
 906
 907             logger.info("Processing %d domains ...", len(domains))
 908             for domain in domains:
 909                 logger.debug("domain='%s'", domain)
 910                 if domain == "":
 911                     logger.debug("domain is empty - SKIPPED!")
 912                     continue
 913                 elif not utils.is_domain_wanted(domain):
 914                     logger.debug("domain='%s' is not wanted - SKIPPED!", domain)
 915                     continue
 916                 elif instances.is_recent(domain):
 917                     logger.debug("domain='%s' has been recently crawled - SKIPPED!", domain)
 918                     continue
 919
 920                 logger.debug("Processing domain='%s',row[blocker]='%s'", domain, row["blocker"])
 921                 processed = utils.process_domain(domain, row["blocker"], inspect.currentframe().f_code.co_name)
 922
 923                 logger.debug("processed='%s'", processed)
 924                 if not processed:
 925                     logger.debug("domain='%s' was not generically processed - SKIPPED!", domain)
 926                     continue
 927
 928     logger.debug("Success! - EXIT!")
 929     return 0
 930
 931 def fetch_fedipact(args: argparse.Namespace) -> int:
 932     logger.debug("args[]='%s' - CALLED!", type(args))
 933     locking.acquire()
 934
 935     response = utils.fetch_url("https://fedipact.online", network.web_headers, (config.get("connection_timeout"), config.get("read_timeout")))
 936
 937     logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text))
 938     if response.ok and response.status_code < 300 and response.text != "":
 939         logger.debug("Parsing %d Bytes ...", len(response.text))
 940
 941         doc = bs4.BeautifulSoup(response.text, "html.parser")
 942         logger.debug("doc[]='%s'", type(doc))
 943
 944         rows = doc.findAll("li")
 945         logger.info("Checking %d row(s) ...", len(rows))
 946         for row in rows:
 947             logger.debug("row[]='%s'", type(row))
 948             domain = tidyup.domain(row.contents[0])
 949
 950             logger.debug("domain='%s'", domain)
 951             if domain == "":
 952                 logger.debug("domain is empty - SKIPPED!")
 953                 continue
 954             elif not utils.is_domain_wanted(domain):
 955                 logger.debug("domain='%s' is not wanted - SKIPPED!", domain)
 956                 continue
 957             elif instances.is_registered(domain):
 958                 logger.debug("domain='%s' is already registered - SKIPPED!", domain)
 959                 continue
 960             elif instances.is_recent(domain):
 961                 logger.debug("domain='%s' has been recently crawled - SKIPPED!", domain)
 962                 continue
 963
 964             logger.info("Fetching domain='%s' ...", domain)
 965             federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
 966
 967     logger.debug("Success! - EXIT!")
 968     return 0
 969
 970 def fetch_joinfediverse(args: argparse.Namespace) -> int:
 971     logger.debug("args[]='%s' - CALLED!", type(args))
 972     locking.acquire()
 973
 974     raw = utils.fetch_url("https://joinfediverse.wiki/FediBlock", network.web_headers, (config.get("connection_timeout"), config.get("read_timeout"))).text
 975     logger.debug("raw()=%d,raw[]='%s'", len(raw), type(raw))
 976
 977     doc = bs4.BeautifulSoup(raw, "html.parser")
 978     logger.debug("doc[]='%s'", type(doc))
 979
 980     tables = doc.findAll("table", {"class": "wikitable"})
 981
 982     logger.info("Analyzing %d table(s) ...", len(tables))
 983     blocklist = list()
 984     for table in tables:
 985         logger.debug("table[]='%s'", type(table))
 986
 987         rows = table.findAll("tr")
 988         logger.info("Checking %d row(s) ...", len(rows))
 989         block_headers = dict()
 990         for row in rows:
 991             logger.debug("row[%s]='%s'", type(row), row)
 992
 993             headers = row.findAll("th")
 994             logger.debug("Found headers()=%d header(s)", len(headers))
 995             if len(headers) > 1:
 996                 block_headers = dict()
 997                 cnt = 0
 998                 for header in headers:
 999                     cnt = cnt + 1
1000                     logger.debug("header[]='%s',cnt=%d", type(header), cnt)
1001                     text = header.contents[0]
1002
1003                     logger.debug("text[]='%s'", type(text))
1004                     if not isinstance(text, str):
1005                         logger.debug("text[]='%s' is not 'str' - SKIPPED!", type(text))
1006                         continue
1007                     elif validators.domain(text.strip()):
1008                         logger.debug("text='%s' is a domain - SKIPPED!", text.strip())
1009                         continue
1010
1011                     text = tidyup.domain(text.strip())
1012                     logger.debug("text='%s'", text)
1013                     if text in ["domain", "instance", "subdomain(s)", "block reason(s)"]:
1014                         logger.debug("Found header: '%s'=%d", text, cnt)
1015                         block_headers[cnt] = text
1016
1017             elif len(block_headers) == 0:
1018                 logger.debug("row is not scrapable - SKIPPED!")
1019                 continue
1020             elif len(block_headers) > 0:
1021                 logger.debug("Found a row with %d scrapable headers ...", len(block_headers))
1022                 cnt = 0
1023                 block = dict()
1024
1025                 for element in row.find_all(["th", "td"]):
1026                     cnt = cnt + 1
1027                     logger.debug("element[]='%s',cnt=%d", type(element), cnt)
1028                     if cnt in block_headers:
1029                         logger.debug("block_headers[%d]='%s'", cnt, block_headers[cnt])
1030
1031                         text = element.text.strip()
1032                         key = block_headers[cnt] if block_headers[cnt] not in ["domain", "instance"] else "blocked"
1033
1034                         logger.debug("cnt=%d is wanted: key='%s',text[%s]='%s'", cnt, key, type(text), text)
1035                         if key in ["domain", "instance"]:
1036                             block[key] = text
1037                         elif key == "reason":
1038                             block[key] = tidyup.reason(text)
1039                         elif key == "subdomain(s)":
1040                             block[key] = list()
1041                             if text != "":
1042                                 block[key] = text.split("/")
1043                         else:
1044                             logger.debug("key='%s'", key)
1045                             block[key] = text
1046
1047                 logger.debug("block()=%d ...", len(block))
1048                 if len(block) > 0:
1049                     logger.debug("Appending block()=%d ...", len(block))
1050                     blocklist.append(block)
1051
1052     logger.debug("blocklist()=%d", len(blocklist))
1053
1054     database.cursor.execute("SELECT domain FROM instances WHERE domain LIKE 'climatejustice.%'")
1055     domains = database.cursor.fetchall()
1056
1057     logger.debug("domains(%d)[]='%s'", len(domains), type(domains))
1058     blocking = list()
1059     for block in blocklist:
1060         logger.debug("block='%s'", block)
1061         if "subdomain(s)" in block and len(block["subdomain(s)"]) > 0:
1062             origin = block["blocked"]
1063             for subdomain in block["subdomain(s)"]:
1064                 block["blocked"] = subdomain + "." + origin
1065                 blocking.append(block)
1066         else:
1067             blocking.append(block)
1068
1069     logger.debug("blocking()=%d", blocking)
1070     for block in blocking:
1071         block["blocked"] = tidyup.domain(block["blocked"])
1072
1073         if not utils.is_domain_wanted(block["blocked"]):
1074             logger.debug("blocked='%s' is not wanted - SKIPPED!", block["blocked"])
1075             continue
1076         elif instances.is_recent(block["blocked"]):
1077             logger.debug("blocked='%s' has been recently checked - SKIPPED!", block["blocked"])
1078             continue
1079
1080         logger.info("Proccessing blocked='%s' ...", block["blocked"])
1081         utils.process_domain(block["blocked"], "climatejustice.social", inspect.currentframe().f_code.co_name)
1082
1083     blockdict = list()
1084     for blocker in domains:
1085         blocker = blocker[0]
1086         logger.debug("blocker[%s]='%s'", type(blocker), blocker)
1087
1088         for block in blocking:
1089             block["reason"] = tidyup.reason(block["block reason(s)"]) if "block reason(s)" in block else None
1090
1091             if not utils.is_domain_wanted(block["blocked"]):
1092                 logger.debug("blocked='%s' is not wanted - SKIPPED!", block["blocked"])
1093                 continue
1094
1095             logger.debug("blocked='%s',reason='%s'", block["blocked"], block["reason"])
1096             if utils.process_block(blocker, block["blocked"], block["reason"], "reject") and config.get("bot_enabled"):
1097                 logger.debug("Appending blocked='%s',reason='%s' for blocker='%s' ...", block["blocked"], block["block_level"], blocker)
1098                 blockdict.append({
1099                     "blocked": block["blocked"],
1100                     "reason" : block["reason"],
1101                 })
1102
1103         if instances.has_pending(blocker):
1104             logger.debug("Flushing updates for blocker='%s' ...", blocker)
1105             instances.update_data(blocker)
1106
1107         logger.debug("Invoking commit() ...")
1108         database.connection.commit()
1109
1110         if config.get("bot_enabled") and len(blockdict) > 0:
1111             logger.info("Sending bot POST for blocker='%s,blockdict()=%d ...", blocker, len(blockdict))
1112             network.send_bot_post(blocker, blockdict)
1113
1114     logger.debug("Success! - EXIT!")
1115     return 0
1116
1117 def recheck_obfuscation(args: argparse.Namespace) -> int:
1118     logger.debug("args[]='%s' - CALLED!", type(args))
1119
1120     locking.acquire()
1121
1122     database.cursor.execute("SELECT domain, software, nodeinfo_url FROM instances WHERE has_obfuscation = 1")
1123     rows = database.cursor.fetchall()
1124     logger.info("Checking %d domains ...", len(rows))
1125     for row in rows:
1126         logger.debug("Fetching peers from domain='%s',software='%s',nodeinfo_url='%s' ...", row[0], row[1], row[2])
1127
1128         blocking = list()
1129         if row[1] == "pleroma":
1130             logger.debug("domain='%s',software='%s'", row[0], row[1])
1131             blocking = pleroma.fetch_blocks(row[0], row[2])
1132         elif row[1] == "mastodon":
1133             logger.debug("domain='%s',software='%s'", row[0], row[1])
1134             blocking = mastodon.fetch_blocks(row[0], row[2])
1135         elif row[1] == "lemmy":
1136             logger.debug("domain='%s',software='%s'", row[0], row[1])
1137             blocking = lemmy.fetch_blocks(row[0], row[2])
1138         elif row[1] == "friendica":
1139             logger.debug("domain='%s',software='%s'", row[0], row[1])
1140             blocking = friendica.fetch_blocks(row[0])
1141         elif row[1] == "misskey":
1142             logger.debug("domain='%s',software='%s'", row[0], row[1])
1143             blocking = misskey.fetch_blocks(row[0])
1144         else:
1145             logger.warning("Unknown sofware: domain='%s',software='%s'", row[0], row[1])
1146
1147         logger.info("Checking %d block(s) from domain='%s' ...", len(blocking), row[0])
1148         obfuscated = 0
1149         blockdict = list()
1150         for block in blocking:
1151             logger.debug("blocked='%s'", block["blocked"])
1152             blocked = None
1153
1154             if block["blocked"].endswith(".arpa"):
1155                 logger.debug("blocked='%s' is a reversed IP address - SKIPPED!", block["blocked"])
1156                 continue
1157             elif block["blocked"].endswith(".tld"):
1158                 logger.debug("blocked='%s' is a fake domain name - SKIPPED!", block["blocked"])
1159                 continue
1160             elif block["blocked"].endswith(".onion"):
1161                 logger.debug("blocked='%s' is a TOR onion domain name - SKIPPED!", block["blocked"])
1162                 continue
1163             elif block["blocked"].find("*") >= 0 or block["blocked"].find("?") >= 0:
1164                 logger.debug("block='%s' is obfuscated.", block["blocked"])
1165                 obfuscated = obfuscated + 1
1166                 blocked = utils.deobfuscate_domain(block["blocked"], row[0], block["hash"] if "hash" in block else None)
1167             elif not utils.is_domain_wanted(block["blocked"]):
1168                 logger.debug("blocked='%s' is not wanted - SKIPPED!", block["blocked"])
1169                 continue
1170             elif blocks.is_instance_blocked(row[0], block["blocked"]):
1171                 logger.debug("blocked='%s' is already blocked - SKIPPED!", block["blocked"])
1172                 continue
1173
1174             if blocked is not None and blocked != block["blocked"]:
1175                 logger.debug("blocked='%s' was deobfuscated to blocked='%s'", block["blocked"], blocked)
1176                 obfuscated = obfuscated - 1
1177                 if blocks.is_instance_blocked(row[0], blocked):
1178                     logger.debug("blocked='%s' is already blocked by domain='%s' - SKIPPED!", blocked, row[0])
1179                     continue
1180
1181                 if block["block_level"] == "silence":
1182                     logger.debug("Block level 'silence' has been changed to 'silenced'")
1183                     block["block_level"] = "silenced"
1184                 elif block["block_level"] == "suspend":
1185                     logger.debug("Block level 'suspend' has been changed to 'suspended'")
1186                     block["block_level"] = "suspended"
1187
1188                 logger.info("blocked='%s' has been deobfuscated to blocked='%s', adding ...", block["blocked"], blocked)
1189                 if utils.process_block(row[0], blocked, block["reason"], block["block_level"]) and block["block_level"] == "reject" and config.get("bot_enabled"):
1190                     logger.debug("Appending blocked='%s',reason='%s' for blocker='%s' ...", block["blocked"], block["block_level"], row[0])
1191                     blockdict.append({
1192                         "blocked": blocked,
1193                         "reason" : block["reason"],
1194                     })
1195
1196         logger.info("domain='%s' has %d obfuscated domain(s)", row[0], obfuscated)
1197         if obfuscated == 0 and len(blocking) > 0:
1198             logger.info("Block list from domain='%s' has been fully deobfuscated.", row[0])
1199             instances.set_has_obfuscation(row[0], False)
1200
1201         if instances.has_pending(row[0]):
1202             logger.debug("Flushing updates for blocker='%s' ...", row[0])
1203             instances.update_data(row[0])
1204
1205         logger.debug("Invoking commit() ...")
1206         database.connection.commit()
1207
1208         if config.get("bot_enabled") and len(blockdict) > 0:
1209             logger.info("Sending bot POST for blocker='%s,blockdict()=%d ...", row[0], len(blockdict))
1210             network.send_bot_post(row[0], blockdict)
1211
1212     logger.debug("Success! - EXIT!")
1213     return 0