fba/commands.py

   1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
   2 # Copyright (C) 2023 Free Software Foundation
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published
   6 # by the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  16
  17 import csv
  18 import inspect
  19 import json
  20 import logging
  21 import time
  22
  23 import argparse
  24 import atoma
  25 import bs4
  26 import markdown
  27 import reqto
  28 import validators
  29
  30 from fba import csrf
  31 from fba import database
  32 from fba import utils
  33
  34 from fba.helpers import blacklist
  35 from fba.helpers import config
  36 from fba.helpers import cookies
  37 from fba.helpers import locking
  38 from fba.helpers import software as software_helper
  39 from fba.helpers import tidyup
  40
  41 from fba.http import federation
  42 from fba.http import network
  43
  44 from fba.models import blocks
  45 from fba.models import instances
  46
  47 from fba.networks import friendica
  48 from fba.networks import lemmy
  49 from fba.networks import mastodon
  50 from fba.networks import misskey
  51 from fba.networks import pleroma
  52
  53 logging.basicConfig(level=logging.INFO)
  54 logger = logging.getLogger(__name__)
  55 #logger.setLevel(logging.DEBUG)
  56
  57 def check_instance(args: argparse.Namespace) -> int:
  58     logger.debug("args.domain='%s' - CALLED!", args.domain)
  59     status = 0
  60     if not validators.domain(args.domain):
  61         logger.warning("args.domain='%s' is not valid", args.domain)
  62         status = 100
  63     elif blacklist.is_blacklisted(args.domain):
  64         logger.warning("args.domain='%s' is blacklisted", args.domain)
  65         status = 101
  66     elif instances.is_registered(args.domain):
  67         logger.warning("args.domain='%s' is already registered", args.domain)
  68         status = 102
  69     else:
  70         logger.info("args.domain='%s' is not known", args.domain)
  71
  72     logger.debug("status=%d - EXIT!", status)
  73     return status
  74
  75 def fetch_pixelfed_api(args: argparse.Namespace) -> int:
  76     logger.debug("args[]='%s' - CALLED!", type(args))
  77
  78     # No CSRF by default, you don't have to add network.api_headers by yourself here
  79     headers = tuple()
  80
  81     try:
  82         logger.debug("Checking CSRF from pixelfed.org")
  83         headers = csrf.determine("pixelfed.org", dict())
  84     except network.exceptions as exception:
  85         logger.warning("Exception '%s' during checking CSRF (fetch_peers,%s) - EXIT!", type(exception), __name__)
  86         return list()
  87
  88     try:
  89         logger.debug("Fetching JSON from pixelfed.org API, headers()=%d ...", len(headers))
  90         fetched = network.get_json_api(
  91             "pixelfed.org",
  92             "/api/v1/servers/all.json?scope=All&country=all&language=all",
  93             headers,
  94             (config.get("connection_timeout"), config.get("read_timeout"))
  95         )
  96
  97         logger.debug("JSON API returned %d elements", len(fetched))
  98         if "error_message" in fetched:
  99             logger.warning("API returned error_message='%s' - EXIT!", fetched["error_message"])
 100             return 101
 101         elif "data" not in fetched["json"]:
 102             logger.warning("API did not return JSON with 'data' element - EXIT!")
 103             return 102
 104
 105         rows = fetched["json"]["data"]
 106         logger.info("Checking %d fetched rows ...", len(rows))
 107         for row in rows:
 108             logger.debug("row[]='%s'", type(row))
 109             if "domain" not in row:
 110                 logger.warning("row='%s' does not contain element 'domain' - SKIPPED!", row)
 111                 continue
 112             elif not utils.is_domain_wanted(row["domain"]):
 113                 logger.debug("row[domain]='%s' is not wanted - SKIPPED!", row["domain"])
 114                 continue
 115             elif instances.is_registered(row["domain"]):
 116                 logger.debug("row[domain]='%s' is already registered - SKIPPED!", row["domain"])
 117                 continue
 118             elif instances.is_recent(row["domain"]):
 119                 logger.debug("row[domain]='%s' has been recently crawled - SKIPPED!", row["domain"])
 120                 continue
 121
 122             logger.debug("Fetching instances from row[domain]='%s' ...", row["domain"])
 123             federation.fetch_instances(row["domain"], None, None, inspect.currentframe().f_code.co_name)
 124
 125     except network.exceptions as exception:
 126         logger.warning("Cannot fetch graphql,exception[%s]:'%s' - EXIT!", type(exception), str(exception))
 127         return 103
 128
 129     logger.debug("Success! - EXIT!")
 130     return 0
 131
 132 def fetch_bkali(args: argparse.Namespace) -> int:
 133     logger.debug("args[]='%s' - CALLED!", type(args))
 134     domains = list()
 135     try:
 136         fetched = network.post_json_api("gql.api.bka.li", "/v1/graphql", json.dumps({
 137             "query": "query domainlist {nodeinfo(order_by: {domain: asc}) {domain}}"
 138         }))
 139
 140         logger.debug("fetched[]='%s'", type(fetched))
 141         if "error_message" in fetched:
 142             logger.warning("post_json_api() for 'gql.api.bka.li' returned error message='%s", fetched["error_message"])
 143             return 100
 144         elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
 145             logger.warning("post_json_api() returned error: '%s", fetched["error"]["message"])
 146             return 101
 147
 148         rows = fetched["json"]
 149
 150         logger.debug("rows(%d)[]='%s'", len(rows), type(rows))
 151         if len(rows) == 0:
 152             raise Exception("WARNING: Returned no records")
 153         elif "data" not in rows:
 154             raise Exception(f"WARNING: rows()={len(rows)} does not contain key 'data'")
 155         elif "nodeinfo" not in rows["data"]:
 156             raise Exception(f"WARNING: rows()={len(rows['data'])} does not contain key 'nodeinfo'")
 157
 158         for entry in rows["data"]["nodeinfo"]:
 159             logger.debug("entry[%s]='%s'", type(entry), entry)
 160             if "domain" not in entry:
 161                 logger.warning("entry()=%d does not contain 'domain' - SKIPPED!", len(entry))
 162                 continue
 163             elif not utils.is_domain_wanted(entry["domain"]):
 164                 logger.debug("entry[domain]='%s' is not wanted - SKIPPED!", entry["domain"])
 165                 continue
 166             elif instances.is_registered(entry["domain"]):
 167                 logger.debug("entry[domain]='%s' is already registered - SKIPPED!", entry["domain"])
 168                 continue
 169             elif instances.is_recent(entry["domain"]):
 170                 logger.debug("entry[domain]='%s' has been recently crawled - SKIPPED!", entry["domain"])
 171                 continue
 172
 173             logger.debug("Adding domain='%s' ...", entry["domain"])
 174             domains.append(entry["domain"])
 175
 176     except network.exceptions as exception:
 177         logger.warning("Cannot fetch graphql,exception[%s]:'%s' - EXIT!", type(exception), str(exception))
 178         return 102
 179
 180     logger.debug("domains()=%d", len(domains))
 181     if len(domains) > 0:
 182         locking.acquire()
 183
 184         logger.info("Adding %d new instances ...", len(domains))
 185         for domain in domains:
 186             try:
 187                 logger.info("Fetching instances from domain='%s' ...", domain)
 188                 federation.fetch_instances(domain, 'tak.teleyal.blog', None, inspect.currentframe().f_code.co_name)
 189             except network.exceptions as exception:
 190                 logger.warning("Exception '%s' during fetching instances (fetch_bkali) from domain='%s'", type(exception), domain)
 191                 instances.set_last_error(domain, exception)
 192
 193     logger.debug("Success - EXIT!")
 194     return 0
 195
 196 def fetch_blocks(args: argparse.Namespace) -> int:
 197     logger.debug("args[]='%s' - CALLED!", type(args))
 198     if args.domain is not None and args.domain != "":
 199         logger.debug("args.domain='%s' - checking ...", args.domain)
 200         if not validators.domain(args.domain):
 201             logger.warning("args.domain='%s' is not valid.", args.domain)
 202             return 100
 203         elif blacklist.is_blacklisted(args.domain):
 204             logger.warning("args.domain='%s' is blacklisted, won't check it!", args.domain)
 205             return 101
 206         elif not instances.is_registered(args.domain):
 207             logger.warning("args.domain='%s' is not registered, please run ./utils.py fetch_instances '%s' first.", args.domain, args.domain)
 208             return 102
 209
 210     locking.acquire()
 211
 212     if args.domain is not None and args.domain != "":
 213         # Re-check single domain
 214         logger.debug("Querying database for single args.domain='%s' ...", args.domain)
 215         database.cursor.execute(
 216             "SELECT domain, software, origin, nodeinfo_url FROM instances WHERE domain = ?", [args.domain]
 217         )
 218     elif args.software is not None and args.software != "":
 219         # Re-check single software
 220         logger.debug("Querying database for args.software='%s' ...", args.software)
 221         database.cursor.execute(
 222             "SELECT domain, software, origin, nodeinfo_url FROM instances WHERE software = ? AND nodeinfo_url IS NOT NULL", [args.software]
 223         )
 224     else:
 225         # Re-check after "timeout" (aka. minimum interval)
 226         database.cursor.execute(
 227             "SELECT domain, software, origin, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'lemmy', 'friendica', 'misskey', 'peertube') AND (last_blocked IS NULL OR last_blocked < ?) AND nodeinfo_url IS NOT NULL ORDER BY rowid DESC", [time.time() - config.get("recheck_block")]
 228         )
 229
 230     rows = database.cursor.fetchall()
 231     logger.info("Checking %d entries ...", len(rows))
 232     for blocker, software, origin, nodeinfo_url in rows:
 233         logger.debug("blocker='%s',software='%s',origin='%s',nodeinfo_url='%s'", blocker, software, origin, nodeinfo_url)
 234         blocker = tidyup.domain(blocker)
 235         logger.debug("blocker='%s' - AFTER!", blocker)
 236
 237         if blocker == "":
 238             logger.warning("blocker is now empty!")
 239             continue
 240         elif nodeinfo_url is None or nodeinfo_url == "":
 241             logger.debug("blocker='%s',software='%s' has empty nodeinfo_url", blocker, software)
 242             continue
 243         elif not utils.is_domain_wanted(blocker):
 244             logger.warning("blocker='%s' is not wanted - SKIPPED!", blocker)
 245             continue
 246
 247         logger.debug("blocker='%s'", blocker)
 248         instances.set_last_blocked(blocker)
 249         instances.set_has_obfuscation(blocker, False)
 250
 251         blocking = list()
 252         blockdict = list()
 253         if software == "pleroma":
 254             logger.info("blocker='%s',software='%s'", blocker, software)
 255             blocking = pleroma.fetch_blocks(blocker, nodeinfo_url)
 256         elif software == "mastodon":
 257             logger.info("blocker='%s',software='%s'", blocker, software)
 258             blocking = mastodon.fetch_blocks(blocker, nodeinfo_url)
 259         elif software == "lemmy":
 260             logger.info("blocker='%s',software='%s'", blocker, software)
 261             blocking = lemmy.fetch_blocks(blocker, nodeinfo_url)
 262         elif software == "friendica":
 263             logger.info("blocker='%s',software='%s'", blocker, software)
 264             blocking = friendica.fetch_blocks(blocker)
 265         elif software == "misskey":
 266             logger.info("blocker='%s',software='%s'", blocker, software)
 267             blocking = misskey.fetch_blocks(blocker)
 268         else:
 269             logger.warning("Unknown software: blocker='%s',software='%s'", blocker, software)
 270
 271         logger.info("Checking %d entries from blocker='%s',software='%s' ...", len(blocking), blocker, software)
 272         blockdict = list()
 273         for block in blocking:
 274             logger.debug("blocked='%s',block_level='%s',reason='%s'", block["blocked"], block["block_level"], block["reason"])
 275
 276             if block["block_level"] == "":
 277                 logger.warning("block_level is empty, blocker='%s',blocked='%s'", block["blocker"], block["blocked"])
 278                 continue
 279
 280             logger.debug("blocked='%s',reason='%s' - BEFORE!", block["blocked"], block["reason"])
 281             block["blocked"] = tidyup.domain(block["blocked"])
 282             block["reason"]  = tidyup.reason(block["reason"]) if block["reason"] is not None and block["reason"] != "" else None
 283             logger.debug("blocked='%s',reason='%s' - AFTER!", block["blocked"], block["reason"])
 284
 285             if block["blocked"] == "":
 286                 logger.warning("blocked is empty, blocker='%s'", blocker)
 287                 continue
 288             elif block["blocked"].endswith(".onion"):
 289                 logger.debug("blocked='%s' is a TOR .onion domain - SKIPPED", block["blocked"])
 290                 continue
 291             elif block["blocked"].endswith(".arpa"):
 292                 logger.debug("blocked='%s' is a reverse IP address - SKIPPED", block["blocked"])
 293                 continue
 294             elif block["blocked"].endswith(".tld"):
 295                 logger.debug("blocked='%s' is a fake domain - SKIPPED", block["blocked"])
 296                 continue
 297             elif block["blocked"].find("*") >= 0:
 298                 logger.debug("blocker='%s' uses obfuscated domains, marking ...", blocker)
 299                 instances.set_has_obfuscation(blocker, True)
 300
 301                 # Some friendica servers also obscure domains without hash
 302                 row = instances.deobfuscate("*", block["blocked"], block["hash"] if "hash" in block else None)
 303
 304                 logger.debug("row[]='%s'", type(row))
 305                 if row is None:
 306                     logger.warning("Cannot deobfuscate blocked='%s',blocker='%s',software='%s' - SKIPPED!", block["blocked"], blocker, software)
 307                     continue
 308
 309                 block["blocked"] = row[0]
 310                 origin           = row[1]
 311                 nodeinfo_url     = row[2]
 312             elif block["blocked"].find("?") >= 0:
 313                 logger.debug("blocker='%s' uses obfuscated domains, marking ...", blocker)
 314                 instances.set_has_obfuscation(blocker, True)
 315
 316                 # Some obscure them with question marks, not sure if that's dependent on version or not
 317                 row = instances.deobfuscate("?", block["blocked"], block["hash"] if "hash" in block else None)
 318
 319                 logger.debug("row[]='%s'", type(row))
 320                 if row is None:
 321                     logger.warning("Cannot deobfuscate blocked='%s',blocker='%s',software='%s' - SKIPPED!", block["blocked"], blocker, software)
 322                     continue
 323
 324                 block["blocked"] = row[0]
 325                 origin           = row[1]
 326                 nodeinfo_url     = row[2]
 327
 328             logger.debug("Looking up instance by domainm, blocked='%s'", block["blocked"])
 329             if not utils.is_domain_wanted(block["blocked"]):
 330                 logger.debug("blocked='%s' is not wanted - SKIPPED!", block["blocked"])
 331                 continue
 332             elif block["block_level"] in ["accept", "accepted"]:
 333                 logger.debug("blocked='%s' is accepted, not wanted here - SKIPPED!", block["blocked"])
 334                 continue
 335             elif not instances.is_registered(block["blocked"]):
 336                 logger.debug("Hash wasn't found, adding: blocked='%s',blocker='%s'", block["blocked"], blocker)
 337                 federation.fetch_instances(block["blocked"], blocker, None, inspect.currentframe().f_code.co_name)
 338
 339             if block["block_level"] == "silence":
 340                 logger.debug("Block level 'silence' has been changed to 'silenced'")
 341                 block["block_level"] = "silenced"
 342             elif block["block_level"] == "suspend":
 343                 logger.debug("Block level 'suspend' has been changed to 'suspended'")
 344                 block["block_level"] = "suspended"
 345
 346             if utils.process_block(blocker, block["blocked"], block["reason"], block["block_level"]) and block["block_level"] == "reject" and config.get("bot_enabled"):
 347                 logger.debug("Appending blocked='%s',reason='%s' for blocker='%s' ...", block["blocked"], block["block_level"], blocker)
 348                 blockdict.append({
 349                     "blocked": block["blocked"],
 350                     "reason" : block["reason"],
 351                 })
 352
 353             logger.debug("Invoking cookies.clear(%s) ...", block["blocked"])
 354             cookies.clear(block["blocked"])
 355
 356         logger.debug("Checking if blocker='%s' has pending updates ...", blocker)
 357         if instances.has_pending(blocker):
 358             logger.debug("Flushing updates for blocker='%s' ...", blocker)
 359             instances.update_data(blocker)
 360
 361         logger.debug("Invoking commit() ...")
 362         database.connection.commit()
 363
 364         logger.debug("Invoking cookies.clear(%s) ...", blocker)
 365         cookies.clear(blocker)
 366
 367         logger.debug("config[bot_enabled]='%s',blockdict()=%d'", config.get("bot_enabled"), len(blockdict))
 368         if config.get("bot_enabled") and len(blockdict) > 0:
 369             logger.info("Sending bot POST for blocker='%s',blockdict()=%d ...", blocker, len(blockdict))
 370             network.send_bot_post(blocker, blockdict)
 371
 372     logger.debug("Success! - EXIT!")
 373     return 0
 374
 375 def fetch_observer(args: argparse.Namespace) -> int:
 376     logger.debug("args[]='%s' - CALLED!", type(args))
 377     types = [
 378         "akkoma",
 379         "birdsitelive",
 380         "bookwyrm",
 381         "calckey",
 382         "diaspora",
 383         "foundkey",
 384         "friendica",
 385         "funkwhale",
 386         "gancio",
 387         "gnusocial",
 388         "gotosocial",
 389         "hometown",
 390         "hubzilla",
 391         "kbin",
 392         "ktistec",
 393         "lemmy",
 394         "mastodon",
 395         "microblogpub",
 396         "misskey",
 397         "mitra",
 398         "mobilizon",
 399         "owncast",
 400         "peertube",
 401         "pixelfed",
 402         "pleroma",
 403         "plume",
 404         "snac",
 405         "takahe",
 406         "wildebeest",
 407         "writefreely"
 408     ]
 409
 410     locking.acquire()
 411
 412     logger.info("Fetching %d different table data ...", len(types))
 413     for software in types:
 414         logger.debug("software='%s' - BEFORE!", software)
 415         if args.software is not None and args.software != software:
 416             logger.debug("args.software='%s' does not match software='%s' - SKIPPED!", args.software, software)
 417             continue
 418
 419         doc = None
 420         try:
 421             logger.debug("Fetching table data for software='%s' ...", software)
 422             raw = utils.fetch_url(
 423                 f"https://fediverse.observer/app/views/tabledata.php?software={software}",
 424                 network.web_headers,
 425                 (config.get("connection_timeout"), config.get("read_timeout"))
 426             ).text
 427             logger.debug("raw[%s]()=%d", type(raw), len(raw))
 428
 429             doc = bs4.BeautifulSoup(raw, features='html.parser')
 430             logger.debug("doc[]='%s'", type(doc))
 431         except network.exceptions as exception:
 432             logger.warning("Cannot fetch software='%s' from fediverse.observer: '%s'", software, type(exception))
 433             continue
 434
 435         items = doc.findAll("a", {"class": "url"})
 436         logger.info("Checking %d items,software='%s' ...", len(items), software)
 437         for item in items:
 438             logger.debug("item[]='%s'", type(item))
 439             domain = item.decode_contents()
 440
 441             logger.debug("domain='%s'", domain)
 442             if not utils.is_domain_wanted(domain):
 443                 logger.debug("domain='%s' is not wanted - SKIPPED!", domain)
 444                 continue
 445             elif instances.is_registered(domain):
 446                 logger.debug("domain='%s' is already registered - SKIPPED!", domain)
 447                 continue
 448             elif instances.is_recent(domain):
 449                 logger.debug("domain='%s' is recently being handled - SKIPPED!", domain)
 450                 continue
 451
 452             software = software_helper.alias(software)
 453             logger.info("Fetching instances for domain='%s'", domain)
 454             federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
 455
 456     logger.debug("Success! - EXIT!")
 457     return 0
 458
 459 def fetch_todon_wiki(args: argparse.Namespace) -> int:
 460     logger.debug("args[]='%s' - CALLED!", type(args))
 461
 462     locking.acquire()
 463     blocklist = {
 464         "silenced": list(),
 465         "reject": list(),
 466     }
 467
 468     raw = utils.fetch_url("https://wiki.todon.eu/todon/domainblocks", network.web_headers, (config.get("connection_timeout"), config.get("read_timeout"))).text
 469     logger.debug("raw()=%d,raw[]='%s'", len(raw), type(raw))
 470
 471     doc = bs4.BeautifulSoup(raw, "html.parser")
 472     logger.debug("doc[]='%s'", type(doc))
 473
 474     silenced = doc.find("h3", {"id": "silencedlimited_servers"}).find_next("ul").findAll("li")
 475     logger.info("Checking %d silenced/limited entries ...", len(silenced))
 476     blocklist["silenced"] = utils.find_domains(silenced, "div")
 477
 478     suspended = doc.find("h3", {"id": "suspended_servers"}).find_next("ul").findAll("li")
 479     logger.info("Checking %d suspended entries ...", len(suspended))
 480     blocklist["reject"] = utils.find_domains(suspended, "div")
 481
 482     blockdict = list()
 483     for block_level in blocklist:
 484         blockers = blocklist[block_level]
 485
 486         logger.debug("block_level='%s',blockers()=%d'", block_level, len(blockers))
 487         for blocked in blockers:
 488             logger.debug("blocked='%s'", blocked)
 489
 490             if not instances.is_registered(blocked):
 491                 try:
 492                     logger.info("Fetching instances from domain='%s' ...", blocked)
 493                     federation.fetch_instances(blocked, 'chaos.social', None, inspect.currentframe().f_code.co_name)
 494                 except network.exceptions as exception:
 495                     logger.warning("Exception '%s' during fetching instances (fetch_cs) from blocked='%s'", type(exception), blocked)
 496                     instances.set_last_error(blocked, exception)
 497
 498             if blocks.is_instance_blocked("todon.eu", blocked, block_level):
 499                 logger.debug("blocked='%s',block_level='%s' is already blocked - SKIPPED!", blocked, block_level)
 500                 continue
 501
 502             logger.info("Adding new block: blocked='%s',block_level='%s'", blocked, block_level)
 503             if utils.process_block("todon.eu", blocked, None, block_level) and block_level == "reject" and config.get("bot_enabled"):
 504                 logger.debug("Appending blocked='%s',reason='%s' for blocker='todon.eu' ...", blocked, block_level)
 505                 blockdict.append({
 506                     "blocked": blocked,
 507                     "reason" : None,
 508                 })
 509
 510         logger.debug("Invoking commit() ...")
 511         database.connection.commit()
 512
 513         if config.get("bot_enabled") and len(blockdict) > 0:
 514             logger.info("Sending bot POST for blocker='todon.eu',blockdict()=%d ...", len(blockdict))
 515             network.send_bot_post("todon.eu", blockdict)
 516
 517     logger.debug("Success! - EXIT!")
 518     return 0
 519
 520 def fetch_cs(args: argparse.Namespace):
 521     logger.debug("args[]='%s' - CALLED!", type(args))
 522     extensions = [
 523         "extra",
 524         "abbr",
 525         "attr_list",
 526         "def_list",
 527         "fenced_code",
 528         "footnotes",
 529         "md_in_html",
 530         "admonition",
 531         "codehilite",
 532         "legacy_attrs",
 533         "legacy_em",
 534         "meta",
 535         "nl2br",
 536         "sane_lists",
 537         "smarty",
 538         "toc",
 539         "wikilinks"
 540     ]
 541
 542     domains = {
 543         "silenced": list(),
 544         "reject"  : list(),
 545     }
 546
 547     raw = utils.fetch_url("https://raw.githubusercontent.com/chaossocial/meta/master/federation.md", network.web_headers, (config.get("connection_timeout"), config.get("read_timeout"))).text
 548     logger.debug("raw()=%d,raw[]='%s'", len(raw), type(raw))
 549
 550     doc = bs4.BeautifulSoup(markdown.markdown(raw, extensions=extensions), features='html.parser')
 551     logger.debug("doc()=%d[]='%s'", len(doc), type(doc))
 552
 553     silenced = doc.find("h2", {"id": "silenced-instances"}).findNext("table").find("tbody")
 554     logger.debug("silenced[%s]()=%d", type(silenced), len(silenced))
 555     domains["silenced"] = federation.find_domains(silenced)
 556
 557     blocked = doc.find("h2", {"id": "blocked-instances"}).findNext("table").find("tbody")
 558     logger.debug("blocked[%s]()=%d", type(blocked), len(blocked))
 559     domains["reject"] = federation.find_domains(blocked)
 560
 561     logger.debug("domains[silenced]()=%d,domains[reject]()=%d", len(domains["silenced"]), len(domains["reject"]))
 562     blockdict = list()
 563     if len(domains) > 0:
 564         locking.acquire()
 565
 566         for block_level in domains:
 567             logger.info("block_level='%s' has %d row(s)", block_level, len(domains[block_level]))
 568
 569             for row in domains[block_level]:
 570                 logger.debug("row[%s]='%s'", type(row), row)
 571                 if instances.is_recent(row["domain"], "last_blocked"):
 572                     logger.debug("row[domain]='%s' has been recently crawled - SKIPPED!", row["domain"])
 573                     continue
 574                 elif not instances.is_registered(row["domain"]):
 575                     try:
 576                         logger.info("Fetching instances from domain='%s' ...", row["domain"])
 577                         federation.fetch_instances(row["domain"], 'chaos.social', None, inspect.currentframe().f_code.co_name)
 578                     except network.exceptions as exception:
 579                         logger.warning("Exception '%s' during fetching instances (fetch_cs) from row[domain]='%s'", type(exception), row["domain"])
 580                         instances.set_last_error(row["domain"], exception)
 581
 582                 if utils.process_block("chaos.social", row["domain"], row["reason"], block_level) and block_level == "reject" and config.get("bot_enabled"):
 583                     logger.debug("Appending blocked='%s',reason='%s' for blocker='chaos.social' ...", row["domain"], block_level)
 584                     blockdict.append({
 585                         "blocked": row["domain"],
 586                         "reason" : row["reason"],
 587                     })
 588
 589         logger.debug("Invoking commit() ...")
 590         database.connection.commit()
 591
 592         if config.get("bot_enabled") and len(blockdict) > 0:
 593             logger.info("Sending bot POST for blocker='chaos.social',blockdict()=%d ...", len(blockdict))
 594             network.send_bot_post("chaos.social", blockdict)
 595
 596     logger.debug("Success! - EXIT!")
 597     return 0
 598
 599 def fetch_fba_rss(args: argparse.Namespace) -> int:
 600     logger.debug("args[]='%s' - CALLED!", type(args))
 601     domains = list()
 602
 603     logger.info("Fetch FBA-specific RSS args.feed='%s' ...", args.feed)
 604     response = utils.fetch_url(args.feed, network.web_headers, (config.get("connection_timeout"), config.get("read_timeout")))
 605
 606     logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text))
 607     if response.ok and response.status_code < 300 and len(response.text) > 0:
 608         logger.debug("Parsing RSS feed (%d Bytes) ...", len(response.text))
 609         rss = atoma.parse_rss_bytes(response.content)
 610
 611         logger.debug("rss[]='%s'", type(rss))
 612         for item in rss.items:
 613             logger.debug("item='%s'", item)
 614             domain = item.link.split("=")[1]
 615
 616             if not utils.is_domain_wanted(domain):
 617                 logger.debug("domain='%s' is not wanted - SKIPPED!", domain)
 618                 continue
 619             elif domain in domains:
 620                 logger.debug("domain='%s' is already added - SKIPPED!", domain)
 621                 continue
 622             elif instances.is_registered(domain):
 623                 logger.debug("domain='%s' is already registered - SKIPPED!", domain)
 624                 continue
 625             elif instances.is_recent(domain):
 626                 logger.debug("domain='%s' has been recently crawled - SKIPPED!", domain)
 627                 continue
 628
 629             logger.debug("Adding domain='%s'", domain)
 630             domains.append(domain)
 631
 632     logger.debug("domains()=%d", len(domains))
 633     if len(domains) > 0:
 634         locking.acquire()
 635
 636         logger.info("Adding %d new instances ...", len(domains))
 637         for domain in domains:
 638             try:
 639                 logger.info("Fetching instances from domain='%s' ...", domain)
 640                 federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
 641             except network.exceptions as exception:
 642                 logger.warning("Exception '%s' during fetching instances (fetch_fba_rss) from domain='%s'", type(exception), domain)
 643                 instances.set_last_error(domain, exception)
 644
 645     logger.debug("Success! - EXIT!")
 646     return 0
 647
 648 def fetch_fbabot_atom(args: argparse.Namespace) -> int:
 649     logger.debug("args[]='%s' - CALLED!", type(args))
 650     feed = "https://ryona.agency/users/fba/feed.atom"
 651
 652     domains = list()
 653
 654     logger.info("Fetching ATOM feed='%s' from FBA bot account ...", feed)
 655     response = utils.fetch_url(feed, network.web_headers, (config.get("connection_timeout"), config.get("read_timeout")))
 656
 657     logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text))
 658     if response.ok and response.status_code < 300 and len(response.text) > 0:
 659         logger.debug("Parsing ATOM feed (%d Bytes) ...", len(response.text))
 660         atom = atoma.parse_atom_bytes(response.content)
 661
 662         logger.debug("atom[]='%s'", type(atom))
 663         for entry in atom.entries:
 664             logger.debug("entry[]='%s'", type(entry))
 665             doc = bs4.BeautifulSoup(entry.content.value, "html.parser")
 666             logger.debug("doc[]='%s'", type(doc))
 667             for element in doc.findAll("a"):
 668                 for href in element["href"].split(","):
 669                     logger.debug("href[%s]='%s", type(href), href)
 670                     domain = tidyup.domain(href)
 671
 672                     logger.debug("domain='%s'", domain)
 673                     if not utils.is_domain_wanted(domain):
 674                         logger.debug("domain='%s' is not wanted - SKIPPED!", domain)
 675                         continue
 676                     elif domain in domains:
 677                         logger.debug("domain='%s' is already added - SKIPPED!", domain)
 678                         continue
 679                     elif instances.is_registered(domain):
 680                         logger.debug("domain='%s' is already registered - SKIPPED!", domain)
 681                         continue
 682                     elif instances.is_recent(domain):
 683                         logger.debug("domain='%s' has been recently crawled - SKIPPED!", domain)
 684                         continue
 685
 686                     logger.debug("Adding domain='%s',domains()=%d", domain, len(domains))
 687                     domains.append(domain)
 688
 689     logger.debug("domains()=%d", len(domains))
 690     if len(domains) > 0:
 691         locking.acquire()
 692
 693         logger.info("Adding %d new instances ...", len(domains))
 694         for domain in domains:
 695             try:
 696                 logger.info("Fetching instances from domain='%s' ...", domain)
 697                 federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
 698             except network.exceptions as exception:
 699                 logger.warning("Exception '%s' during fetching instances (fetch_fbabot_atom) from domain='%s'", type(exception), domain)
 700                 instances.set_last_error(domain, exception)
 701
 702     logger.debug("Success! - EXIT!")
 703     return 0
 704
 705 def fetch_instances(args: argparse.Namespace) -> int:
 706     logger.debug("args[]='%s' - CALLED!", type(args))
 707     locking.acquire()
 708
 709     # Initial fetch
 710     try:
 711         logger.info("Fetching instances from args.domain='%s' ...", args.domain)
 712         federation.fetch_instances(args.domain, None, None, inspect.currentframe().f_code.co_name)
 713     except network.exceptions as exception:
 714         logger.warning("Exception '%s' during fetching instances (fetch_instances) from args.domain='%s'", type(exception), args.domain)
 715         instances.set_last_error(args.domain, exception)
 716         return 100
 717
 718     if args.single:
 719         logger.debug("Not fetching more instances - EXIT!")
 720         return 0
 721
 722     # Loop through some instances
 723     database.cursor.execute(
 724         "SELECT domain, origin, software, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'lemmy', 'peertube') AND (last_instance_fetch IS NULL OR last_instance_fetch < ?) ORDER BY rowid DESC", [time.time() - config.get("recheck_instance")]
 725     )
 726
 727     rows = database.cursor.fetchall()
 728     logger.info("Checking %d entries ...", len(rows))
 729     for row in rows:
 730         logger.debug("domain='%s'", row[0])
 731         if not utils.is_domain_wanted(row[0]):
 732             logger.debug("Domain row[0]='%s' is not wanted - SKIPPED!", row[0])
 733             continue
 734
 735         try:
 736             logger.info("Fetching instances for domain='%s',origin='%s',software='%s',nodeinfo_url='%s'", row[0], row[1], row[2], row[3])
 737             federation.fetch_instances(row[0], row[1], row[2], inspect.currentframe().f_code.co_name, row[3])
 738         except network.exceptions as exception:
 739             logger.warning("Exception '%s' during fetching instances (fetch_instances) from row[0]='%s'", type(exception), row[0])
 740             instances.set_last_error(row[0], exception)
 741
 742     logger.debug("Success - EXIT!")
 743     return 0
 744
 745 def fetch_oliphant(args: argparse.Namespace) -> int:
 746     logger.debug("args[]='%s' - CALLED!", type(args))
 747     locking.acquire()
 748
 749     # Base URL
 750     base_url = "https://codeberg.org/oliphant/blocklists/raw/branch/main/blocklists"
 751
 752     # URLs to fetch
 753     blocklists = (
 754         {
 755             "blocker": "artisan.chat",
 756             "csv_url": "mastodon/artisan.chat.csv",
 757         },{
 758             "blocker": "mastodon.art",
 759             "csv_url": "mastodon/mastodon.art.csv",
 760         },{
 761             "blocker": "pleroma.envs.net",
 762             "csv_url": "mastodon/pleroma.envs.net.csv",
 763         },{
 764             "blocker": "oliphant.social",
 765             "csv_url": "mastodon/_unified_tier3_blocklist.csv",
 766         },{
 767             "blocker": "mastodon.online",
 768             "csv_url": "mastodon/mastodon.online.csv",
 769         },{
 770             "blocker": "mastodon.social",
 771             "csv_url": "mastodon/mastodon.social.csv",
 772         },{
 773             "blocker": "mastodon.social",
 774             "csv_url": "other/missing-tier0-mastodon.social.csv",
 775         },{
 776             "blocker": "rage.love",
 777             "csv_url": "mastodon/rage.love.csv",
 778         },{
 779             "blocker": "sunny.garden",
 780             "csv_url": "mastodon/sunny.garden.csv",
 781         },{
 782             "blocker": "solarpunk.moe",
 783             "csv_url": "mastodon/solarpunk.moe.csv",
 784         },{
 785             "blocker": "toot.wales",
 786             "csv_url": "mastodon/toot.wales.csv",
 787         },{
 788             "blocker": "union.place",
 789             "csv_url": "mastodon/union.place.csv",
 790         }
 791     )
 792
 793     domains = list()
 794
 795     logger.debug("Downloading %d files ...", len(blocklists))
 796     for block in blocklists:
 797         # Is domain given and not equal blocker?
 798         if isinstance(args.domain, str) and args.domain != block["blocker"]:
 799             logger.debug("Skipping blocker='%s', not matching args.domain='%s'", block["blocker"], args.domain)
 800             continue
 801         elif args.domain in domains:
 802             logger.debug("args.domain='%s' already handled - SKIPPED!", args.domain)
 803             continue
 804         elif instances.is_recent(block["blocker"]):
 805             logger.debug("block[blocker]='%s' has been recently crawled - SKIPPED!", block["blocker"])
 806             continue
 807
 808         # Fetch this URL
 809         logger.info("Fetching csv_url='%s' for blocker='%s' ...", block["csv_url"], block["blocker"])
 810         response = utils.fetch_url(f"{base_url}/{block['csv_url']}", network.web_headers, (config.get("connection_timeout"), config.get("read_timeout")))
 811
 812         logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text))
 813         if not response.ok or response.status_code > 399 or response.content == "":
 814             logger.warning("Could not fetch csv_url='%s' for blocker='%s' - SKIPPED!", block["csv_url"], block["blocker"])
 815             continue
 816
 817         logger.debug("Fetched %d Bytes, parsing CSV ...", len(response.content))
 818         reader = csv.DictReader(response.content.decode('utf-8').splitlines(), dialect="unix")
 819
 820         logger.debug("reader[]='%s'", type(reader))
 821         blockdict = list()
 822         for row in reader:
 823             logger.debug("row[%s]='%s'", type(row), row)
 824             domain = severity = None
 825             reject_media = reject_reports = False
 826             if "#domain" in row:
 827                 domain = row["#domain"]
 828             elif "domain" in row:
 829                 domain = row["domain"]
 830             else:
 831                 logger.debug("row='%s' does not contain domain column", row)
 832                 continue
 833
 834             if "#severity" in row:
 835                 severity = row["#severity"]
 836             elif "severity" in row:
 837                 severity = row["severity"]
 838             else:
 839                 logger.debug("row='%s' does not contain severity column", row)
 840                 continue
 841
 842             if "#reject_media" in row and row["#reject_media"].lower() == "true":
 843                 reject_media = True
 844             elif "reject_media" in row and row["reject_media"].lower() == "true":
 845                 reject_media = True
 846
 847             if "#reject_reports" in row and row["#reject_reports"].lower() == "true":
 848                 reject_reports = True
 849             elif "reject_reports" in row and row["reject_reports"].lower() == "true":
 850                 reject_reports = True
 851
 852             logger.debug("domain='%s',severity='%s',reject_media='%s',reject_reports='%s'", domain, severity, reject_media, reject_reports)
 853             if not utils.is_domain_wanted(domain):
 854                 logger.debug("domain='%s' is not wanted - SKIPPED!", domain)
 855                 continue
 856
 857             logger.debug("Marking domain='%s' as handled", domain)
 858             domains.append(domain)
 859
 860             logger.debug("Processing domain='%s' ...", domain)
 861             processed = utils.process_domain(domain, block["blocker"], inspect.currentframe().f_code.co_name)
 862             logger.debug("processed='%s'", processed)
 863
 864             if utils.process_block(block["blocker"], domain, None, "reject") and config.get("bot_enabled"):
 865                 logger.debug("Appending blocked='%s',reason='%s' for blocker='%s' ...", domain, block["block_level"], block["blocker"])
 866                 blockdict.append({
 867                     "blocked": domain,
 868                     "reason" : block["reason"],
 869                 })
 870
 871             if reject_media:
 872                 utils.process_block(block["blocker"], domain, None, "reject_media")
 873             if reject_reports:
 874                 utils.process_block(block["blocker"], domain, None, "reject_reports")
 875
 876         logger.debug("Invoking commit() ...")
 877         database.connection.commit()
 878
 879         if config.get("bot_enabled") and len(blockdict) > 0:
 880             logger.info("Sending bot POST for blocker='%s',blockdict()=%d ...", block["blocker"], len(blockdict))
 881             network.send_bot_post(block["blocker"], blockdict)
 882
 883     logger.debug("Success! - EXIT!")
 884     return 0
 885
 886 def fetch_txt(args: argparse.Namespace) -> int:
 887     logger.debug("args[]='%s' - CALLED!", type(args))
 888     locking.acquire()
 889
 890     # Static URLs
 891     urls = ({
 892         "blocker": "seirdy.one",
 893         "url"    : "https://seirdy.one/pb/bsl.txt",
 894     },)
 895
 896     logger.info("Checking %d text file(s) ...", len(urls))
 897     for row in urls:
 898         logger.debug("Fetching row[url]='%s' ...", row["url"])
 899         response = utils.fetch_url(row["url"], network.web_headers, (config.get("connection_timeout"), config.get("read_timeout")))
 900
 901         logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text))
 902         if response.ok and response.status_code < 300 and response.text != "":
 903             logger.debug("Returned %d Bytes for processing", len(response.text.strip()))
 904             domains = response.text.split("\n")
 905
 906             logger.info("Processing %d domains ...", len(domains))
 907             for domain in domains:
 908                 logger.debug("domain='%s'", domain)
 909                 if domain == "":
 910                     logger.debug("domain is empty - SKIPPED!")
 911                     continue
 912                 elif not utils.is_domain_wanted(domain):
 913                     logger.debug("domain='%s' is not wanted - SKIPPED!", domain)
 914                     continue
 915                 elif instances.is_recent(domain):
 916                     logger.debug("domain='%s' has been recently crawled - SKIPPED!", domain)
 917                     continue
 918
 919                 logger.debug("Processing domain='%s',row[blocker]='%s'", domain, row["blocker"])
 920                 processed = utils.process_domain(domain, row["blocker"], inspect.currentframe().f_code.co_name)
 921
 922                 logger.debug("processed='%s'", processed)
 923                 if not processed:
 924                     logger.debug("domain='%s' was not generically processed - SKIPPED!", domain)
 925                     continue
 926
 927     logger.debug("Success! - EXIT!")
 928     return 0
 929
 930 def fetch_fedipact(args: argparse.Namespace) -> int:
 931     logger.debug("args[]='%s' - CALLED!", type(args))
 932     locking.acquire()
 933
 934     response = utils.fetch_url("https://fedipact.online", network.web_headers, (config.get("connection_timeout"), config.get("read_timeout")))
 935
 936     logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text))
 937     if response.ok and response.status_code < 300 and response.text != "":
 938         logger.debug("Parsing %d Bytes ...", len(response.text))
 939
 940         doc = bs4.BeautifulSoup(response.text, "html.parser")
 941         logger.debug("doc[]='%s'", type(doc))
 942
 943         rows = doc.findAll("li")
 944         logger.info("Checking %d row(s) ...", len(rows))
 945         for row in rows:
 946             logger.debug("row[]='%s'", type(row))
 947             domain = tidyup.domain(row.contents[0])
 948
 949             logger.debug("domain='%s'", domain)
 950             if domain == "":
 951                 logger.debug("domain is empty - SKIPPED!")
 952                 continue
 953             elif not utils.is_domain_wanted(domain):
 954                 logger.debug("domain='%s' is not wanted - SKIPPED!", domain)
 955                 continue
 956             elif instances.is_registered(domain):
 957                 logger.debug("domain='%s' is already registered - SKIPPED!", domain)
 958                 continue
 959             elif instances.is_recent(domain):
 960                 logger.debug("domain='%s' has been recently crawled - SKIPPED!", domain)
 961                 continue
 962
 963             logger.info("Fetching domain='%s' ...", domain)
 964             federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
 965
 966     logger.debug("Success! - EXIT!")
 967     return 0
 968
 969 def fetch_joinfediverse(args: argparse.Namespace) -> int:
 970     logger.debug("args[]='%s' - CALLED!", type(args))
 971     locking.acquire()
 972
 973     raw = utils.fetch_url("https://joinfediverse.wiki/FediBlock", network.web_headers, (config.get("connection_timeout"), config.get("read_timeout"))).text
 974     logger.debug("raw()=%d,raw[]='%s'", len(raw), type(raw))
 975
 976     doc = bs4.BeautifulSoup(raw, "html.parser")
 977     logger.debug("doc[]='%s'", type(doc))
 978
 979     tables = doc.findAll("table", {"class": "wikitable"})
 980
 981     logger.info("Analyzing %d table(s) ...", len(tables))
 982     blocklist = list()
 983     for table in tables:
 984         logger.debug("table[]='%s'", type(table))
 985
 986         rows = table.findAll("tr")
 987         logger.info("Checking %d row(s) ...", len(rows))
 988         block_headers = dict()
 989         for row in rows:
 990             logger.debug("row[%s]='%s'", type(row), row)
 991
 992             headers = row.findAll("th")
 993             logger.debug("Found headers()=%d header(s)", len(headers))
 994             if len(headers) > 1:
 995                 block_headers = dict()
 996                 cnt = 0
 997                 for header in headers:
 998                     cnt = cnt + 1
 999                     logger.debug("header[]='%s',cnt=%d", type(header), cnt)
1000                     text = header.contents[0]
1001
1002                     logger.debug("text[]='%s'", type(text))
1003                     if not isinstance(text, str):
1004                         logger.debug("text[]='%s' is not 'str' - SKIPPED!", type(text))
1005                         continue
1006                     elif validators.domain(text.strip()):
1007                         logger.debug("text='%s' is a domain - SKIPPED!", text.strip())
1008                         continue
1009
1010                     text = tidyup.domain(text.strip())
1011                     logger.debug("text='%s'", text)
1012                     if text in ["domain", "instance", "subdomain(s)", "block reason(s)"]:
1013                         logger.debug("Found header: '%s'=%d", text, cnt)
1014                         block_headers[cnt] = text
1015
1016             elif len(block_headers) == 0:
1017                 logger.debug("row is not scrapable - SKIPPED!")
1018                 continue
1019             elif len(block_headers) > 0:
1020                 logger.debug("Found a row with %d scrapable headers ...", len(block_headers))
1021                 cnt = 0
1022                 block = dict()
1023
1024                 for element in row.find_all(["th", "td"]):
1025                     cnt = cnt + 1
1026                     logger.debug("element[]='%s',cnt=%d", type(element), cnt)
1027                     if cnt in block_headers:
1028                         logger.debug("block_headers[%d]='%s'", cnt, block_headers[cnt])
1029
1030                         text = element.text.strip()
1031                         key = block_headers[cnt] if block_headers[cnt] not in ["domain", "instance"] else "blocked"
1032
1033                         logger.debug("cnt=%d is wanted: key='%s',text[%s]='%s'", cnt, key, type(text), text)
1034                         if key in ["domain", "instance"]:
1035                             block[key] = text
1036                         elif key == "reason":
1037                             block[key] = tidyup.reason(text)
1038                         elif key == "subdomain(s)":
1039                             block[key] = list()
1040                             if text != "":
1041                                 block[key] = text.split("/")
1042                         else:
1043                             logger.debug("key='%s'", key)
1044                             block[key] = text
1045
1046                 logger.debug("block()=%d ...", len(block))
1047                 if len(block) > 0:
1048                     logger.debug("Appending block()=%d ...", len(block))
1049                     blocklist.append(block)
1050
1051     logger.debug("blocklist()=%d", len(blocklist))
1052
1053     database.cursor.execute("SELECT domain FROM instances WHERE domain LIKE 'climatejustice.%'")
1054     domains = database.cursor.fetchall()
1055
1056     logger.debug("domains(%d)[]='%s'", len(domains), type(domains))
1057     blocking = list()
1058     for block in blocklist:
1059         logger.debug("block='%s'", block)
1060         if "subdomain(s)" in block and len(block["subdomain(s)"]) > 0:
1061             origin = block["blocked"]
1062             for subdomain in block["subdomain(s)"]:
1063                 block["blocked"] = subdomain + "." + origin
1064                 blocking.append(block)
1065         else:
1066             blocking.append(block)
1067
1068     logger.debug("blocking()=%d", blocking)
1069     for block in blocking:
1070         block["blocked"] = tidyup.domain(block["blocked"])
1071
1072         if not utils.is_domain_wanted(block["blocked"]):
1073             logger.debug("blocked='%s' is not wanted - SKIPPED!", block["blocked"])
1074             continue
1075         elif instances.is_recent(block["blocked"]):
1076             logger.debug("blocked='%s' has been recently checked - SKIPPED!", block["blocked"])
1077             continue
1078
1079         logger.info("Proccessing blocked='%s' ...", block["blocked"])
1080         utils.process_domain(block["blocked"], "climatejustice.social", inspect.currentframe().f_code.co_name)
1081
1082     blockdict = list()
1083     for blocker in domains:
1084         blocker = blocker[0]
1085         logger.debug("blocker[%s]='%s'", type(blocker), blocker)
1086
1087         for block in blocking:
1088             block["reason"] = tidyup.reason(block["block reason(s)"]) if "block reason(s)" in block else None
1089
1090             if not utils.is_domain_wanted(block["blocked"]):
1091                 logger.debug("blocked='%s' is not wanted - SKIPPED!", block["blocked"])
1092                 continue
1093
1094             logger.debug("blocked='%s',reason='%s'", block["blocked"], block["reason"])
1095             if utils.process_block(blocker, block["blocked"], block["reason"], "reject") and config.get("bot_enabled"):
1096                 logger.debug("Appending blocked='%s',reason='%s' for blocker='%s' ...", block["blocked"], block["block_level"], blocker)
1097                 blockdict.append({
1098                     "blocked": block["blocked"],
1099                     "reason" : block["reason"],
1100                 })
1101
1102         if instances.has_pending(blocker):
1103             logger.debug("Flushing updates for blocker='%s' ...", blocker)
1104             instances.update_data(blocker)
1105
1106         logger.debug("Invoking commit() ...")
1107         database.connection.commit()
1108
1109         if config.get("bot_enabled") and len(blockdict) > 0:
1110             logger.info("Sending bot POST for blocker='%s,blockdict()=%d ...", blocker, len(blockdict))
1111             network.send_bot_post(blocker, blockdict)
1112
1113     logger.debug("Success! - EXIT!")
1114     return 0
1115
1116 def recheck_obfuscation(args: argparse.Namespace) -> int:
1117     logger.debug("args[]='%s' - CALLED!", type(args))
1118
1119     locking.acquire()
1120
1121     database.cursor.execute("SELECT domain, software, nodeinfo_url FROM instances WHERE has_obfuscation = 1")
1122     rows = database.cursor.fetchall()
1123     logger.info("Checking %d domains ...", len(rows))
1124     for row in rows:
1125         logger.debug("Fetching peers from domain='%s',software='%s',nodeinfo_url='%s' ...", row[0], row[1], row[2])
1126
1127         blocking = list()
1128         if row[1] == "pleroma":
1129             logger.debug("domain='%s',software='%s'", row[0], row[1])
1130             blocking = pleroma.fetch_blocks(row[0], row[2])
1131         elif row[1] == "mastodon":
1132             logger.debug("domain='%s',software='%s'", row[0], row[1])
1133             blocking = mastodon.fetch_blocks(row[0], row[2])
1134         elif row[1] == "lemmy":
1135             logger.debug("domain='%s',software='%s'", row[0], row[1])
1136             blocking = lemmy.fetch_blocks(row[0], row[2])
1137         elif row[1] == "friendica":
1138             logger.debug("domain='%s',software='%s'", row[0], row[1])
1139             blocking = friendica.fetch_blocks(row[0])
1140         elif row[1] == "misskey":
1141             logger.debug("domain='%s',software='%s'", row[0], row[1])
1142             blocking = misskey.fetch_blocks(row[0])
1143         else:
1144             logger.warning("Unknown sofware: domain='%s',software='%s'", row[0], row[1])
1145
1146         logger.info("Checking %d block(s) from domain='%s' ...", len(blocking), row[0])
1147         obfuscated = 0
1148         blockdict = list()
1149         for block in blocking:
1150             logger.debug("blocked='%s'", block["blocked"])
1151             blocked = None
1152
1153             if block["blocked"].endswith(".arpa"):
1154                 logger.debug("blocked='%s' is a reversed IP address - SKIPPED!", block["blocked"])
1155                 continue
1156             elif block["blocked"].endswith(".tld"):
1157                 logger.debug("blocked='%s' is a fake domain name - SKIPPED!", block["blocked"])
1158                 continue
1159             elif block["blocked"].endswith(".onion"):
1160                 logger.debug("blocked='%s' is a TOR onion domain name - SKIPPED!", block["blocked"])
1161                 continue
1162             elif block["blocked"].find("*") >= 0 or block["blocked"].find("?") >= 0:
1163                 logger.debug("block='%s' is obfuscated.", block["blocked"])
1164                 obfuscated = obfuscated + 1
1165                 blocked = utils.deobfuscate_domain(block["blocked"], row[0], block["hash"] if "hash" in block else None)
1166             elif not utils.is_domain_wanted(block["blocked"]):
1167                 logger.debug("blocked='%s' is not wanted - SKIPPED!", block["blocked"])
1168                 continue
1169             elif blocks.is_instance_blocked(row[0], block["blocked"]):
1170                 logger.debug("blocked='%s' is already blocked - SKIPPED!", block["blocked"])
1171                 continue
1172
1173             if blocked is not None and blocked != block["blocked"]:
1174                 logger.debug("blocked='%s' was deobfuscated to blocked='%s'", block["blocked"], blocked)
1175                 obfuscated = obfuscated - 1
1176                 if blocks.is_instance_blocked(row[0], blocked):
1177                     logger.debug("blocked='%s' is already blocked by domain='%s' - SKIPPED!", blocked, row[0])
1178                     continue
1179
1180                 if block["block_level"] == "silence":
1181                     logger.debug("Block level 'silence' has been changed to 'silenced'")
1182                     block["block_level"] = "silenced"
1183                 elif block["block_level"] == "suspend":
1184                     logger.debug("Block level 'suspend' has been changed to 'suspended'")
1185                     block["block_level"] = "suspended"
1186
1187                 logger.info("blocked='%s' has been deobfuscated to blocked='%s', adding ...", block["blocked"], blocked)
1188                 if utils.process_block(row[0], blocked, block["reason"], block["block_level"]) and block["block_level"] == "reject" and config.get("bot_enabled"):
1189                     logger.debug("Appending blocked='%s',reason='%s' for blocker='%s' ...", block["blocked"], block["block_level"], row[0])
1190                     blockdict.append({
1191                         "blocked": blocked,
1192                         "reason" : block["reason"],
1193                     })
1194
1195         logger.info("domain='%s' has %d obfuscated domain(s)", row[0], obfuscated)
1196         if obfuscated == 0 and len(blocking) > 0:
1197             logger.info("Block list from domain='%s' has been fully deobfuscated.", row[0])
1198             instances.set_has_obfuscation(row[0], False)
1199
1200         if instances.has_pending(row[0]):
1201             logger.debug("Flushing updates for blocker='%s' ...", row[0])
1202             instances.update_data(row[0])
1203
1204         logger.debug("Invoking commit() ...")
1205         database.connection.commit()
1206
1207         if config.get("bot_enabled") and len(blockdict) > 0:
1208             logger.info("Sending bot POST for blocker='%s,blockdict()=%d ...", row[0], len(blockdict))
1209             network.send_bot_post(row[0], blockdict)
1210
1211     logger.debug("Success! - EXIT!")
1212     return 0