From ddde2bfdb322f913aab2a7b145d5fc6215f94e00 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Sat, 31 May 2025 20:40:23 +0200 Subject: [PATCH] Continued: - also need to set +pubSub for fetch_instances() command - remove duplicate code, no need to fetch same for POST field "blocked" again - --force-all is generically the norm --- fba/networks/misskey.py | 97 ++--------------------------------------- recheck.sh | 2 +- 2 files changed, 4 insertions(+), 95 deletions(-) diff --git a/fba/networks/misskey.py b/fba/networks/misskey.py index daf2a1a..33563c0 100644 --- a/fba/networks/misskey.py +++ b/fba/networks/misskey.py @@ -66,13 +66,13 @@ def fetch_peers(domain: str) -> list: logger.debug("Fetching offset=%d from domain='%s' ...", offset, domain) if offset == 0: fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({ - "sort" : "+pubAt", + "sort" : "+pubSub", "host" : None, "limit": step }), headers) else: fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({ - "sort" : "+pubAt", + "sort" : "+pubSub", "host" : None, "limit" : step, "offset": offset - 1 @@ -159,7 +159,7 @@ def fetch_blocks(domain: str) -> list: # iterating through all "suspended" (follow-only in its terminology) # instances page-by-page since it doesn't support sending them all at once - logger.debug("Fetching misskey blocks from domain='%s'", domain) + logger.debug("Fetching misskey blocks from domain='%s' ...", domain) while True: logger.debug("offset=%d", offset) try: @@ -267,96 +267,5 @@ def fetch_blocks(domain: str) -> list: offset = 0 break - while True: - # Fetch blocked (full suspended) instances - logger.debug("offset=%d", offset) - try: - if offset == 0: - logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d", domain, step, offset) - fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({ - "sort" : "+pubAt", - "host" : None, - "blocked": True, - "limit" : step - }), headers) - else: - logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d", domain, step, offset) - fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({ - "sort" : "+pubAt", - "host" : None, - "blocked": True, - "limit" : step, - "offset" : offset - 1 - }), headers) - - logger.debug("fetched[]='%s'", type(fetched)) - if "error_message" in fetched: - logger.warning("post_json_api() for domain='%s' returned error message: '%s'", domain, fetched['error_message']) - instances.set_last_error(domain, fetched) - break - elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]: - logger.warning("post_json_api() returned error: '%s'", fetched["json"]["error"]["message"]) - instances.set_last_error(domain, fetched["json"]["error"]["message"]) - break - - rows = fetched["json"] - - logger.debug("rows(%d)[]='%s'", len(rows), type(rows)) - if len(rows) == 0: - logger.debug("Returned zero bytes, domain='%s' - BREAK!", domain) - break - elif len(rows) != step: - logger.debug("Fetched %d row(s) but expected: %d'", len(rows), step) - offset = offset + (step - len(rows)) - else: - logger.debug("Raising offset by step=%d", step) - offset = offset + step - - count = 0 - logger.debug("Checking %d row(s) of instances ...", len(rows)) - for instance in rows: - # Is it there? - logger.debug("instance[]='%s'", type(instance)) - if not isinstance(instance, dict): - logger.warning("instance[]='%s' has not expected type 'dict' - SKIPPED!", type(instance)) - continue - elif "host" not in instance: - logger.warning("instance()=%d has no element 'host' - SKIPPED!", len(instance)) - continue - elif not isinstance(instance["host"], str): - logger.warning("instance[host][]='%s' has not expected type 'str' - SKIPPED!", type(instance["host"])) - continue - - logger.debug("instance[host]='%s' - BEFORE!", instance["host"]) - blocked = tidyup.domain(instance["host"]) if instance["host"] != "" else None - logger.debug("blocked='%s' - AFTER!", blocked) - - if blocked in [None, ""]: - logger.warning("instance[host]='%s' is None or empty after tidyup.domain() - SKIPPED!", instance["host"]) - continue - elif not domain_helper.is_wanted(blocked): - logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked) - continue - elif "isBlocked" in instance and instance["isBlocked"] and not dict_helper.has_key(blocklist, "blocked", blocked): - count = count + 1 - logger.debug("Appending blocker='%s',blocked='%s',block_level='reject'", domain, blocked) - blocklist.append({ - "blocker" : domain, - "blocked" : blocked, - "reason" : None, - "block_level": "rejected", - }) - - logger.debug("count=%d", count) - if count == 0: - logger.debug("API is no more returning new instances, aborting loop!") - break - - except network.exceptions as exception: - logger.warning("Caught error, exiting loop: domain='%s',exception[%s]='%s'", domain, type(exception), str(exception)) - instances.set_last_error(domain, exception) - offset = 0 - break - logger.debug("blocklist()=%d - EXIT!", len(blocklist)) return blocklist diff --git a/recheck.sh b/recheck.sh index 7dc8380..5fc3243 100755 --- a/recheck.sh +++ b/recheck.sh @@ -21,7 +21,7 @@ fi for DOMAIN in ${DOMAIN_LIST}; do echo "$0: DOMAIN='${DOMAIN}'" - ./fba.py recheck_obfuscation --domain="${DOMAIN}" --force + ./fba.py recheck_obfuscation --domain="${DOMAIN}" --force-all STATUS="$?" if [ "${STATUS}" != "0" ] then -- 2.39.5