Continued:

- also need to set +pubSub for fetch_instances() command
- remove duplicate code, no need to fetch same for POST field "blocked" again
- --force-all is generically the norm

diff --git a/fba/networks/misskey.py b/fba/networks/misskey.py
index daf2a1a5360b93646b713416bae52d760e2c1e07..33563c05851274b22c7576cbebae0ad465eabfd4 100644 (file)
--- a/fba/networks/misskey.py
+++ b/fba/networks/misskey.py
@@ -66,13 +66,13 @@ def fetch_peers(domain: str) -> list:
         logger.debug("Fetching offset=%d from domain='%s' ...", offset, domain)
         if offset == 0:
             fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
-                "sort" : "+pubAt",
+                "sort" : "+pubSub",
                 "host" : None,
                 "limit": step
             }), headers)
         else:
             fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
-                "sort"  : "+pubAt",
+                "sort"  : "+pubSub",
                 "host"  : None,
                 "limit" : step,
                 "offset": offset - 1
@@ -159,7 +159,7 @@ def fetch_blocks(domain: str) -> list:
 
     # iterating through all "suspended" (follow-only in its terminology)
     # instances page-by-page since it doesn't support sending them all at once
-    logger.debug("Fetching misskey blocks from domain='%s'", domain)
+    logger.debug("Fetching misskey blocks from domain='%s' ...", domain)
     while True:
         logger.debug("offset=%d", offset)
         try:
@@ -267,96 +267,5 @@ def fetch_blocks(domain: str) -> list:
             offset = 0
             break
 
-    while True:
-        # Fetch blocked (full suspended) instances
-        logger.debug("offset=%d", offset)
-        try:
-            if offset == 0:
-                logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d", domain, step, offset)
-                fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
-                    "sort"   : "+pubAt",
-                    "host"   : None,
-                    "blocked": True,
-                    "limit"  : step
-                }), headers)
-            else:
-                logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d", domain, step, offset)
-                fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
-                    "sort"   : "+pubAt",
-                    "host"   : None,
-                    "blocked": True,
-                    "limit"  : step,
-                    "offset" : offset - 1
-                }), headers)
-
-            logger.debug("fetched[]='%s'", type(fetched))
-            if "error_message" in fetched:
-                logger.warning("post_json_api() for domain='%s' returned error message: '%s'", domain, fetched['error_message'])
-                instances.set_last_error(domain, fetched)
-                break
-            elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
-                logger.warning("post_json_api() returned error: '%s'", fetched["json"]["error"]["message"])
-                instances.set_last_error(domain, fetched["json"]["error"]["message"])
-                break
-
-            rows = fetched["json"]
-
-            logger.debug("rows(%d)[]='%s'", len(rows), type(rows))
-            if len(rows) == 0:
-                logger.debug("Returned zero bytes, domain='%s' - BREAK!", domain)
-                break
-            elif len(rows) != step:
-                logger.debug("Fetched %d row(s) but expected: %d'", len(rows), step)
-                offset = offset + (step - len(rows))
-            else:
-                logger.debug("Raising offset by step=%d", step)
-                offset = offset + step
-
-            count = 0
-            logger.debug("Checking %d row(s) of instances ...", len(rows))
-            for instance in rows:
-                # Is it there?
-                logger.debug("instance[]='%s'", type(instance))
-                if not isinstance(instance, dict):
-                    logger.warning("instance[]='%s' has not expected type 'dict' - SKIPPED!", type(instance))
-                    continue
-                elif "host" not in instance:
-                    logger.warning("instance()=%d has no element 'host' - SKIPPED!", len(instance))
-                    continue
-                elif not isinstance(instance["host"], str):
-                    logger.warning("instance[host][]='%s' has not expected type 'str' - SKIPPED!", type(instance["host"]))
-                    continue
-
-                logger.debug("instance[host]='%s' - BEFORE!", instance["host"])
-                blocked = tidyup.domain(instance["host"]) if instance["host"] != "" else None
-                logger.debug("blocked='%s' - AFTER!", blocked)
-
-                if blocked in [None, ""]:
-                    logger.warning("instance[host]='%s' is None or empty after tidyup.domain() - SKIPPED!", instance["host"])
-                    continue
-                elif not domain_helper.is_wanted(blocked):
-                    logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
-                    continue
-                elif "isBlocked" in instance and instance["isBlocked"] and not dict_helper.has_key(blocklist, "blocked", blocked):
-                    count = count + 1
-                    logger.debug("Appending blocker='%s',blocked='%s',block_level='reject'", domain, blocked)
-                    blocklist.append({
-                        "blocker"    : domain,
-                        "blocked"    : blocked,
-                        "reason"     : None,
-                        "block_level": "rejected",
-                    })
-
-            logger.debug("count=%d", count)
-            if count == 0:
-                logger.debug("API is no more returning new instances, aborting loop!")
-                break
-
-        except network.exceptions as exception:
-            logger.warning("Caught error, exiting loop: domain='%s',exception[%s]='%s'", domain, type(exception), str(exception))
-            instances.set_last_error(domain, exception)
-            offset = 0
-            break
-
     logger.debug("blocklist()=%d - EXIT!", len(blocklist))
     return blocklist

diff --git a/recheck.sh b/recheck.sh
index 7dc83803184ec1a9616188525f91f428a6ea903e..5fc32437f7f41e7a56edfde2e9bd57ce4e7f59db 100755 (executable)
--- a/recheck.sh
+++ b/recheck.sh
@@ -21,7 +21,7 @@ fi
 for DOMAIN in ${DOMAIN_LIST};
 do
        echo "$0: DOMAIN='${DOMAIN}'"
-       ./fba.py recheck_obfuscation --domain="${DOMAIN}" --force
+       ./fba.py recheck_obfuscation --domain="${DOMAIN}" --force-all
        STATUS="$?"
        if [ "${STATUS}" != "0" ]
        then