From 79feebd71b899c30571ff867cc77cea55339fc2f Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Tue, 26 Aug 2025 16:01:56 +0200 Subject: [PATCH] Continued: - let network.post_json_api() return the failed "JSON" reply and not raise the exception again - this way, many of try/except blocks can be saved --- fba/commands.py | 147 +++++++++++++------------- fba/http/network.py | 1 - fba/networks/lemmy.py | 56 +++++----- fba/networks/misskey.py | 222 ++++++++++++++++++++-------------------- 4 files changed, 209 insertions(+), 217 deletions(-) diff --git a/fba/commands.py b/fba/commands.py index a1f1cce..d42a909 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -195,59 +195,58 @@ def fetch_bkali(args: argparse.Namespace) -> int: sources.update(source_domain) domains = {} - try: - logger.info("Fetching domainlist from source_domain='%s' ...", source_domain) - fetched = network.post_json_api( - source_domain, - "/v1/graphql", - json.dumps({ - "query": "query domainlist {nodeinfo(order_by: {domain: asc}) {domain}}" - }) - ) - logger.debug("fetched[]='%s'", type(fetched)) - if "error_message" in fetched: - logger.warning("post_json_api() for 'gql.sources.bka.li' returned error message='%s' - EXIT!", fetched["error_message"]) - return 100 - elif "json" not in fetched: - logger.warning("post_json_api() returned fetched[]='%s' with missing 'json' element - EXIT!", type(fetched)) - return 101 - elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]: - logger.warning("post_json_api() returned error: '%s' - EXIT!", fetched["json"]["error"]["message"]) - return 102 - - rows = fetched["json"] + logger.info("Fetching domainlist from source_domain='%s' ...", source_domain) + fetched = network.post_json_api( + source_domain, + "/v1/graphql", + json.dumps({ + "query": "query domainlist {nodeinfo(order_by: {domain: asc}) {domain}}" + }) + ) - logger.debug("rows(%d)[]='%s'", len(rows), type(rows)) - if len(rows) == 0: - raise RuntimeError("WARNING: Returned no records") - elif "data" not in rows: - raise KeyError(f"WARNING: rows()={len(rows)} does not contain key 'data'") - elif "nodeinfo" not in rows["data"]: - raise KeyError(f"WARNING: rows()={len(rows['data'])} does not contain key 'nodeinfo'") - - logger.info("Checking %d rows ...", len(rows["data"]["nodeinfo"])) - for entry in rows["data"]["nodeinfo"]: - logger.debug("entry[%s]='%s'", type(entry), entry) - if "domain" not in entry: - logger.warning("entry()=%d does not contain 'domain' - SKIPPED!", len(entry)) - continue - elif entry["domain"] in [None, ""]: - logger.debug("entry[domain]='%s' is empty - SKIPPED!", entry["domain"]) - continue - elif not domain_helper.is_wanted(entry["domain"]): - logger.debug("entry[domain]='%s' is not wanted - SKIPPED!", entry["domain"]) - continue - elif instances.is_registered(entry["domain"]): - logger.debug("entry[domain]='%s' is already registered - SKIPPED!", entry["domain"]) - continue + logger.debug("fetched[]='%s'", type(fetched)) + if "error_message" in fetched: + logger.warning("post_json_api() for '%s' returned error message='%s' - EXIT!", source_domain, fetched["error_message"]) + return 100 + elif "json" not in fetched: + logger.warning("post_json_api() returned fetched[]='%s' with missing 'json' element - EXIT!", type(fetched)) + return 101 + elif "exception" in fetched: + logger.warning("post_json_api() for '%s' has caused an exception='%s' - EXIT!", source_domain, fetched["exception"]) + return 102 + elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]: + logger.warning("post_json_api() returned error: '%s' - EXIT!", fetched["json"]["error"]["message"]) + return 103 - logger.debug("Adding domain='%s' ...", entry["domain"]) - domains.append(entry["domain"]) + rows = fetched["json"] + + logger.debug("rows(%d)[]='%s'", len(rows), type(rows)) + if len(rows) == 0: + raise RuntimeError("WARNING: Returned no records") + elif "data" not in rows: + raise KeyError(f"WARNING: rows()={len(rows)} does not contain key 'data'") + elif "nodeinfo" not in rows["data"]: + raise KeyError(f"WARNING: rows()={len(rows['data'])} does not contain key 'nodeinfo'") + + logger.info("Checking %d rows ...", len(rows["data"]["nodeinfo"])) + for entry in rows["data"]["nodeinfo"]: + logger.debug("entry[%s]='%s'", type(entry), entry) + if "domain" not in entry: + logger.warning("entry()=%d does not contain 'domain' - SKIPPED!", len(entry)) + continue + elif entry["domain"] in [None, ""]: + logger.debug("entry[domain]='%s' is empty - SKIPPED!", entry["domain"]) + continue + elif not domain_helper.is_wanted(entry["domain"]): + logger.debug("entry[domain]='%s' is not wanted - SKIPPED!", entry["domain"]) + continue + elif instances.is_registered(entry["domain"]): + logger.debug("entry[domain]='%s' is already registered - SKIPPED!", entry["domain"]) + continue - except network.exceptions as exception: - logger.warning("Cannot fetch graphql,exception[%s]:'%s' - EXIT!", type(exception), str(exception)) - return 102 + logger.debug("Adding domain='%s' ...", entry["domain"]) + domains.append(entry["domain"]) logger.debug("domains()=%d", len(domains)) if len(domains) > 0: @@ -556,36 +555,32 @@ def fetch_observer(args: argparse.Namespace) -> int: continue nodes = [] - try: - logger.debug("Fetching table data for software='%s' ...", software) - raw = network.post_json_api( - f"api.{source_domain}", - "/", - json.dumps({ - "query": "{nodes(softwarename:\"" + software + "\"){domain}}" - }) - ) - - logger.debug("raw[%s]()=%d", type(raw), len(raw)) - if "exception" in raw: - logger.warning("source_domain='%s' has caused an exception: '%s' - raising again ...", raw["domain"], type(raw["exception"])) - raise raw["exception"] - elif "error_message" in raw: - logger.warning("source_domain='%s' has caused error message: '%s' - SKIPPED!", raw["domain"], raw["error_message"]) - continue - elif "data" not in raw["json"]: - logger.warning("Cannot find key 'nodes' in raw[json]()=%d", len(raw["json"])) - continue - elif "nodes" not in raw["json"]["data"]: - logger.warning("Cannot find key 'nodes' in raw[json][data]()=%d", len(raw["json"]["data"])) - continue - nodes = raw["json"]["data"]["nodes"] - logger.debug("nodes()=%d", len(nodes)) + logger.debug("Fetching table data for software='%s' ...", software) + raw = network.post_json_api( + f"api.{source_domain}", + "/", + json.dumps({ + "query": "{nodes(softwarename:\"" + software + "\"){domain}}" + }) + ) - except network.exceptions as exception: - logger.warning("Cannot fetch software='%s' from source_domain='%s': '%s'", software, source_domain, type(exception)) + logger.debug("raw[%s]()=%d", type(raw), len(raw)) + if "exception" in raw: + logger.warning("source_domain='%s' has caused an exception: '%s' - raising again ...", raw["domain"], type(raw["exception"])) + raise raw["exception"] + elif "error_message" in raw: + logger.warning("source_domain='%s' has caused error message: '%s' - SKIPPED!", raw["domain"], raw["error_message"]) + continue + elif "data" not in raw["json"]: + logger.warning("Cannot find key 'nodes' in raw[json]()=%d", len(raw["json"])) continue + elif "nodes" not in raw["json"]["data"]: + logger.warning("Cannot find key 'nodes' in raw[json][data]()=%d", len(raw["json"]["data"])) + continue + + nodes = raw["json"]["data"]["nodes"] + logger.debug("nodes()=%d", len(nodes)) logger.info("Checking %d nodes,software='%s' ...", len(nodes), software) for node in nodes: diff --git a/fba/http/network.py b/fba/http/network.py index 7da2efd..778bfc9 100644 --- a/fba/http/network.py +++ b/fba/http/network.py @@ -130,7 +130,6 @@ def post_json_api(domain: str, path: str, data: str = "", headers: dict = {}) -> logger.debug("Invoking instances.set_last_error(%s,%s) ...", domain, exception) instances.set_last_error(domain, exception) - raise exception logger.debug("Returning json_reply(%d)[]='%s' - EXIT!", len(json_reply), type(json_reply)) return json_reply diff --git a/fba/networks/lemmy.py b/fba/networks/lemmy.py index c6e993b..a0887ae 100644 --- a/fba/networks/lemmy.py +++ b/fba/networks/lemmy.py @@ -94,36 +94,34 @@ def fetch_peers(domain: str, origin: str) -> list: logger.debug("Returning empty list ... - EXIT!") return [] - try: - logger.debug("Fetching '/api/v3/site' from domain='%s' ...", domain) - data = network.get_json_api( - domain, - "/api/v3/site", - headers=headers, - timeout=config.timeout - ) - - logger.debug("data[]='%s'", type(data)) - if "error_message" in data: - logger.warning("Could not reach any JSON API: domain='%s',error_message='%s'", domain, data["error_message"]) - instances.set_last_error(domain, data) - elif "federated_instances" in data["json"] and isinstance(data["json"]["federated_instances"], dict): - logger.debug("Found federated_instances for domain='%s'", domain) - peers = peers + federation.add_peers(data["json"]["federated_instances"]) - logger.debug("peers()=%d after adding", len(peers)) - - logger.debug("Marking domain='%s' as successfully handled ...", domain) - instances.set_success(domain) - - logger.debug("peers()=%d", len(peers)) - if len(peers) == 0: - logger.debug("Fetching instances for domain='%s' from /instances ...", domain) - peers = fetch_instances(domain, origin) - logger.debug("peers()=%d after fetch_instances(%s, %s)", len(peers), domain, origin) + logger.debug("Fetching '/api/v3/site' from domain='%s' ...", domain) + data = network.get_json_api( + domain, + "/api/v3/site", + headers=headers, + timeout=config.timeout + ) + + logger.debug("data[]='%s'", type(data)) + if "exception" in data: + logger.warning("Could not reach any JSON API: domain='%s',exception='%s'", domain, data["exception"]) + instances.set_last_error(domain, data) + elif "error_message" in data: + logger.warning("Could not reach any JSON API: domain='%s',error_message='%s'", domain, data["error_message"]) + instances.set_last_error(domain, data) + elif "federated_instances" in data["json"] and isinstance(data["json"]["federated_instances"], dict): + logger.debug("Found federated_instances for domain='%s'", domain) + peers = peers + federation.add_peers(data["json"]["federated_instances"]) + logger.debug("peers()=%d after adding", len(peers)) + + logger.debug("Marking domain='%s' as successfully handled ...", domain) + instances.set_success(domain) - except network.exceptions as exception: - logger.warning("Exception during fetching JSON: domain='%s',exception[%s]:'%s'", domain, type(exception), str(exception)) - instances.set_last_error(domain, exception) + logger.debug("peers()=%d", len(peers)) + if len(peers) == 0: + logger.debug("Fetching instances for domain='%s' from /instances ...", domain) + peers = fetch_instances(domain, origin) + logger.debug("peers()=%d after fetch_instances(%s, %s)", len(peers), domain, origin) logger.debug("peers()=%d - EXIT!", len(peers)) return peers diff --git a/fba/networks/misskey.py b/fba/networks/misskey.py index 7ed7065..9ac3864 100644 --- a/fba/networks/misskey.py +++ b/fba/networks/misskey.py @@ -88,7 +88,11 @@ def fetch_peers(domain: str) -> list: # Check records logger.debug("fetched[]='%s'", type(fetched)) - if "error_message" in fetched: + if "exception" in fetched: + logger.warning("post_json_api() for domain='%s' raised an exception: '%s'", domain, fetched['exception']) + instances.set_last_error(domain, fetched) + break + elif "error_message" in fetched: logger.warning("post_json_api() for domain='%s' returned error message: '%s'", domain, fetched['error_message']) instances.set_last_error(domain, fetched) break @@ -172,117 +176,113 @@ def fetch_blocks(domain: str) -> list: # instances page-by-page since it doesn't support sending them all at once logger.debug("Fetching misskey blocks from domain='%s' ...", domain) while True: - logger.debug("offset=%d", offset) - try: - logger.debug("Fetching offset=%d from domain='%s' ...", offset, domain) - if offset == 0: - logger.debug("Sending JSON API request to domain='%s',step=%d ...", domain, step) - fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({ - "allowPartial": True, - "sort" : "+pubSub", - "host" : None, - "limit" : step - }), headers) - else: - logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d ...", domain, step, offset) - fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({ - "allowPartial": True, - "sort" : "+pubSub", - "host" : None, - "limit" : step, - "offset" : offset - 1 - }), headers) - - logger.debug("fetched[]='%s'", type(fetched)) - if "error_message" in fetched: - logger.warning("post_json_api() for domain='%s' returned error message: '%s'", domain, fetched['error_message']) - instances.set_last_error(domain, fetched) - break - elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]: - logger.warning("post_json_api() returned error: '%s'", fetched["json"]["error"]["message"]) - instances.set_last_error(domain, fetched["json"]["error"]["message"]) - break - - rows = fetched["json"] - - logger.debug("rows(%d)[]='%s'", len(rows), type(rows)) - if len(rows) == 0: - logger.debug("Returned zero bytes, domain='%s' - BREAK!", domain) - break - elif len(rows) != step: - logger.debug("Fetched %d row(s) but expected: %d", len(rows), step) - offset = offset + (step - len(rows)) + logger.debug("Fetching offset=%d from domain='%s' ...", offset, domain) + if offset == 0: + logger.debug("Sending JSON API request to domain='%s',step=%d ...", domain, step) + fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({ + "allowPartial": True, + "sort" : "+pubSub", + "host" : None, + "limit" : step + }), headers) + else: + logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d ...", domain, step, offset) + fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({ + "allowPartial": True, + "sort" : "+pubSub", + "host" : None, + "limit" : step, + "offset" : offset - 1 + }), headers) + + logger.debug("fetched[]='%s'", type(fetched)) + if "exception" in fetched: + logger.warning("post_json_api() for domain='%s' caused an exception: '%s'", domain, fetched['exception']) + instances.set_last_error(domain, fetched) + break + elif "error_message" in fetched: + logger.warning("post_json_api() for domain='%s' returned error message: '%s'", domain, fetched['error_message']) + instances.set_last_error(domain, fetched) + break + elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]: + logger.warning("post_json_api() returned error: '%s'", fetched["json"]["error"]["message"]) + instances.set_last_error(domain, fetched["json"]["error"]["message"]) + break + + rows = fetched["json"] + + logger.debug("rows(%d)[]='%s'", len(rows), type(rows)) + if len(rows) == 0: + logger.debug("Returned zero bytes, domain='%s' - BREAK!", domain) + break + elif len(rows) != step: + logger.debug("Fetched %d row(s) but expected: %d", len(rows), step) + offset = offset + (step - len(rows)) + else: + logger.debug("Raising offset by step=%d", step) + offset = offset + step + + count = 0 + logger.debug("Checking %d row(s) of instances ...", len(rows)) + for instance in rows: + # Is it there? + logger.debug("instance[]='%s'", type(instance)) + if "host" not in instance: + logger.warning("instance(%d)='%s' has no key 'host' - SKIPPED!", len(instance), instance) + continue + elif not isinstance(instance["host"], str): + logger.warning("instance[host][]='%s' is not of expected type 'str' - SKIPPED!", type(instance["host"])) + continue + elif instance["host"] == "": + logger.warning("instance[host] is an empty string,domain='%s' - SKIPPED!", domain) + continue + + logger.debug("instance[host]='%s' - BEFORE!", instance["host"]) + blocked = tidyup.domain(instance["host"]) + logger.debug("blocked[%s]='%s' - AFTER!", type(blocked), blocked) + + if blocked in [None, ""]: + logger.warning("instance[host]='%s' is None or empty after tidyup.domain() - SKIPPED!", instance["host"]) + continue + elif blocked.find("*") == -1 and blocked.find("?") == -1 and not domain_helper.is_wanted(blocked): + logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked) + continue + elif "isSuspended" in instance and instance["isSuspended"] and not dict_helper.has_key(blocklist, "blocked", blocked): + count = count + 1 + logger.debug("Appending blocker='%s',blocked='%s',block_level='suspended' ... #1", domain, blocked) + blocklist.append({ + "blocker" : domain, + "blocked" : blocked, + "reason" : None, + "block_level": "suspended", + }) + elif "isBlocked" in instance and instance["isBlocked"] and not dict_helper.has_key(blocklist, "blocked", blocked): + count = count + 1 + logger.debug("Appending blocker='%s',blocked='%s',block_level='suspended' ... #2", domain, blocked) + blocklist.append({ + "blocker" : domain, + "blocked" : blocked, + "reason" : None, + "block_level": "suspended", + }) + elif "isSilenced" in instance and instance["isSilenced"] and not dict_helper.has_key(blocklist, "blocked", blocked): + count = count + 1 + logger.debug("Appending blocker='%s',blocked='%s',block_level='silenced' ...", domain, blocked) + blocklist.append({ + "blocker" : domain, + "blocked" : blocked, + "reason" : None, + "block_level": "silenced", + }) else: - logger.debug("Raising offset by step=%d", step) - offset = offset + step - - count = 0 - logger.debug("Checking %d row(s) of instances ...", len(rows)) - for instance in rows: - # Is it there? - logger.debug("instance[]='%s'", type(instance)) - if "host" not in instance: - logger.warning("instance(%d)='%s' has no key 'host' - SKIPPED!", len(instance), instance) - continue - elif not isinstance(instance["host"], str): - logger.warning("instance[host][]='%s' is not of expected type 'str' - SKIPPED!", type(instance["host"])) - continue - elif instance["host"] == "": - logger.warning("instance[host] is an empty string,domain='%s' - SKIPPED!", domain) - continue - - logger.debug("instance[host]='%s' - BEFORE!", instance["host"]) - blocked = tidyup.domain(instance["host"]) - logger.debug("blocked[%s]='%s' - AFTER!", type(blocked), blocked) - - if blocked in [None, ""]: - logger.warning("instance[host]='%s' is None or empty after tidyup.domain() - SKIPPED!", instance["host"]) - continue - elif blocked.find("*") == -1 and blocked.find("?") == -1 and not domain_helper.is_wanted(blocked): - logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked) - continue - elif "isSuspended" in instance and instance["isSuspended"] and not dict_helper.has_key(blocklist, "blocked", blocked): - count = count + 1 - logger.debug("Appending blocker='%s',blocked='%s',block_level='suspended' ... #1", domain, blocked) - blocklist.append({ - "blocker" : domain, - "blocked" : blocked, - "reason" : None, - "block_level": "suspended", - }) - elif "isBlocked" in instance and instance["isBlocked"] and not dict_helper.has_key(blocklist, "blocked", blocked): - count = count + 1 - logger.debug("Appending blocker='%s',blocked='%s',block_level='suspended' ... #2", domain, blocked) - blocklist.append({ - "blocker" : domain, - "blocked" : blocked, - "reason" : None, - "block_level": "suspended", - }) - elif "isSilenced" in instance and instance["isSilenced"] and not dict_helper.has_key(blocklist, "blocked", blocked): - count = count + 1 - logger.debug("Appending blocker='%s',blocked='%s',block_level='silenced' ...", domain, blocked) - blocklist.append({ - "blocker" : domain, - "blocked" : blocked, - "reason" : None, - "block_level": "silenced", - }) - else: - count = count + 1 - logger.debug("domain='%s',blocked='%s' is not marked suspended, blocked or silenced - SKIPPED!", domain, blocked) - continue - - logger.debug("count=%d", count) - if count == 0: - logger.debug("API is no more returning new instances, aborting loop! domain='%s'", domain) - instances.set_last_offset(domain, 0) - break - - except network.exceptions as exception: - logger.warning("Caught error, exiting loop: domain='%s',exception[%s]='%s'", domain, type(exception), str(exception)) - instances.set_last_offset(domain, offset) - instances.set_last_error(domain, exception) + count = count + 1 + logger.debug("domain='%s',blocked='%s' is not marked suspended, blocked or silenced - SKIPPED!", domain, blocked) + continue + + logger.debug("count=%d", count) + if count == 0: + logger.debug("API is no more returning new instances, aborting loop! domain='%s'", domain) + instances.set_last_offset(domain, 0) break sleep = default_rng.integers(low=config.get("low_sleep"), high=config.get("high_sleep")) -- 2.39.5