From 5af800628de088e864497096d78c8bc035d67aa3 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Thu, 8 Jun 2023 20:35:52 +0200 Subject: [PATCH] Continued: - moved fba.fetch_misskey_blocks() to misskey.fetch_blocks() - avoided calling tidyup_reason() on NoneType --- fba/commands.py | 4 +- fba/fba.py | 148 +------------------------------------- fba/federation/misskey.py | 144 +++++++++++++++++++++++++++++++++++++ fba/federation/pleroma.py | 2 +- 4 files changed, 149 insertions(+), 149 deletions(-) diff --git a/fba/commands.py b/fba/commands.py index 53ad518..799d6db 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -159,7 +159,7 @@ def fetch_blocks(args: argparse.Namespace): if software == "friendica": json = fba.fetch_friendica_blocks(blocker) elif software == "misskey": - json = fba.fetch_misskey_blocks(blocker) + json = misskey.fetch_blocks(blocker) print(f"INFO: Checking {len(json.items())} entries from blocker='{blocker}',software='{software}' ...") for block_level, blocklist in json.items(): @@ -175,7 +175,7 @@ def fetch_blocks(args: argparse.Namespace): blocked, reason = block.values() # DEBUG: print(f"DEBUG: blocked='{blocked}',reason='{reason}' - BEFORE!") blocked = fba.tidyup_domain(blocked) - reason = fba.tidyup_reason(reason) + reason = fba.tidyup_reason(reason) if reason != None else None # DEBUG: print(f"DEBUG: blocked='{blocked}',reason='{reason}' - AFTER!") if blocked == "": diff --git a/fba/fba.py b/fba/fba.py index e52ddc3..1eddf5d 100644 --- a/fba/fba.py +++ b/fba/fba.py @@ -793,154 +793,10 @@ def fetch_friendica_blocks(domain: str) -> dict: "reject": blocked } -def fetch_misskey_blocks(domain: str) -> dict: - # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") - if type(domain) != str: - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError(f"Parameter 'domain' is empty") - - # DEBUG: print("DEBUG: Fetching misskey blocks from domain:", domain) - blocklist = { - "suspended": [], - "blocked" : [] - } - - offset = 0 - step = config.get("misskey_limit") - while True: - # iterating through all "suspended" (follow-only in its terminology) - # instances page-by-page, since that troonware doesn't support - # sending them all at once - try: - # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...") - if offset == 0: - # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset) - fetched = post_json_api(domain, "/api/federation/instances", json.dumps({ - "sort" : "+pubAt", - "host" : None, - "suspended": True, - "limit" : step - }), { - "Origin": domain - }) - else: - # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset) - fetched = post_json_api(domain, "/api/federation/instances", json.dumps({ - "sort" : "+pubAt", - "host" : None, - "suspended": True, - "limit" : step, - "offset" : offset - 1 - }), { - "Origin": domain - }) - - # DEBUG: print("DEBUG: fetched():", len(fetched)) - if len(fetched) == 0: - # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain) - break - elif len(fetched) != config.get("misskey_limit"): - # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'") - offset = offset + (config.get("misskey_limit") - len(fetched)) - else: - # DEBUG: print("DEBUG: Raising offset by step:", step) - offset = offset + step - - count = 0 - for instance in fetched: - # Is it there? - if instance["isSuspended"] and not has_key(blocklist["suspended"], "domain", instance): - count = count + 1 - blocklist["suspended"].append( - { - "domain": tidyup_domain(instance["host"]), - # no reason field, nothing - "reason": None - } - ) - - # DEBUG: print(f"DEBUG: count={count}") - if count == 0: - # DEBUG: print(f"DEBUG: API is no more returning new instances, aborting loop!") - break - - except BaseException as e: - print("WARNING: Caught error, exiting loop:", domain, e) - instances.update_last_error(domain, e) - offset = 0 - break - - while True: - # same shit, different asshole ("blocked" aka full suspend) - try: - if offset == 0: - # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset) - fetched = post_json_api(domain,"/api/federation/instances", json.dumps({ - "sort" : "+pubAt", - "host" : None, - "blocked": True, - "limit" : step - }), { - "Origin": domain - }) - else: - # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset) - fetched = post_json_api(domain,"/api/federation/instances", json.dumps({ - "sort" : "+pubAt", - "host" : None, - "blocked": True, - "limit" : step, - "offset" : offset - 1 - }), { - "Origin": domain - }) - - # DEBUG: print("DEBUG: fetched():", len(fetched)) - if len(fetched) == 0: - # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain) - break - elif len(fetched) != config.get("misskey_limit"): - # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'") - offset = offset + (config.get("misskey_limit") - len(fetched)) - else: - # DEBUG: print("DEBUG: Raising offset by step:", step) - offset = offset + step - - count = 0 - for instance in fetched: - # Is it there? - if instance["isBlocked"] and not has_key(blocklist["blocked"], "domain", instance): - count = count + 1 - blocklist["blocked"].append({ - "domain": tidyup_domain(instance["host"]), - "reason": None - }) - - # DEBUG: print(f"DEBUG: count={count}") - if count == 0: - # DEBUG: print(f"DEBUG: API is no more returning new instances, aborting loop!") - break - - except BaseException as e: - print("ERROR: Exception during POST:", domain, e) - instances.update_last_error(domain, e) - offset = 0 - break - - # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") - instances.update_last_instance_fetch(domain) - - # DEBUG: print("DEBUG: Returning for domain,blocked(),suspended():", domain, len(blocklist["blocked"]), len(blocklist["suspended"])) - return { - "reject" : blocklist["blocked"], - "followers_only": blocklist["suspended"] - } - def tidyup_reason(reason: str) -> str: # DEBUG: print(f"DEBUG: reason='{reason}' - CALLED!") if type(reason) != str: - raise ValueError(f"Parameter reason[]={type(reason)} is not expected") + raise ValueError(f"Parameter reason[]={type(reason)} is not 'str'") # Strip string reason = reason.strip() @@ -954,7 +810,7 @@ def tidyup_reason(reason: str) -> str: def tidyup_domain(domain: str) -> str: # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") if type(domain) != str: - raise ValueError(f"Parameter domain[]={type(domain)} is not expected") + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") # All lower-case and strip spaces out + last dot domain = domain.lower().strip().rstrip(".") diff --git a/fba/federation/misskey.py b/fba/federation/misskey.py index 8ea0b2b..598e80e 100644 --- a/fba/federation/misskey.py +++ b/fba/federation/misskey.py @@ -106,3 +106,147 @@ def fetch_peers(domain: str) -> list: # DEBUG: print(f"DEBUG: Returning peers[]='{type(peers)}'") return peers + +def fetch_blocks(domain: str) -> dict: + print(f"DEBUG: domain='{domain}' - CALLED!") + if type(domain) != str: + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError(f"Parameter 'domain' is empty") + + print("DEBUG: Fetching misskey blocks from domain:", domain) + blocklist = { + "suspended": [], + "blocked" : [] + } + + offset = 0 + step = config.get("misskey_limit") + while True: + # iterating through all "suspended" (follow-only in its terminology) + # instances page-by-page, since that troonware doesn't support + # sending them all at once + try: + print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...") + if offset == 0: + print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset) + fetched = fba.post_json_api(domain, "/api/federation/instances", json.dumps({ + "sort" : "+pubAt", + "host" : None, + "suspended": True, + "limit" : step + }), { + "Origin": domain + }) + else: + print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset) + fetched = fba.post_json_api(domain, "/api/federation/instances", json.dumps({ + "sort" : "+pubAt", + "host" : None, + "suspended": True, + "limit" : step, + "offset" : offset - 1 + }), { + "Origin": domain + }) + + print("DEBUG: fetched():", len(fetched)) + if len(fetched) == 0: + print("DEBUG: Returned zero bytes, exiting loop:", domain) + break + elif len(fetched) != config.get("misskey_limit"): + print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'") + offset = offset + (config.get("misskey_limit") - len(fetched)) + else: + print("DEBUG: Raising offset by step:", step) + offset = offset + step + + count = 0 + for instance in fetched: + # Is it there? + if instance["isSuspended"] and not fba.has_key(blocklist["suspended"], "domain", instance): + count = count + 1 + blocklist["suspended"].append( + { + "domain": fba.tidyup_domain(instance["host"]), + # no reason field, nothing + "reason": None + } + ) + + print(f"DEBUG: count={count}") + if count == 0: + print(f"DEBUG: API is no more returning new instances, aborting loop!") + break + + except BaseException as e: + print("WARNING: Caught error, exiting loop:", domain, e) + instances.update_last_error(domain, e) + offset = 0 + break + + while True: + # same shit, different asshole ("blocked" aka full suspend) + try: + if offset == 0: + print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset) + fetched = fba.post_json_api(domain, "/api/federation/instances", json.dumps({ + "sort" : "+pubAt", + "host" : None, + "blocked": True, + "limit" : step + }), { + "Origin": domain + }) + else: + print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset) + fetched = fba.post_json_api(domain, "/api/federation/instances", json.dumps({ + "sort" : "+pubAt", + "host" : None, + "blocked": True, + "limit" : step, + "offset" : offset - 1 + }), { + "Origin": domain + }) + + print("DEBUG: fetched():", len(fetched)) + if len(fetched) == 0: + print("DEBUG: Returned zero bytes, exiting loop:", domain) + break + elif len(fetched) != config.get("misskey_limit"): + print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'") + offset = offset + (config.get("misskey_limit") - len(fetched)) + else: + print("DEBUG: Raising offset by step:", step) + offset = offset + step + + count = 0 + for instance in fetched: + # Is it there? + if instance["isBlocked"] and not fba.has_key(blocklist["blocked"], "domain", instance): + count = count + 1 + blocklist["blocked"].append({ + "domain": fba.tidyup_domain(instance["host"]), + "reason": None + }) + + print(f"DEBUG: count={count}") + if count == 0: + print(f"DEBUG: API is no more returning new instances, aborting loop!") + break + + except BaseException as e: + print("ERROR: Exception during POST:", domain, e) + instances.update_last_error(domain, e) + offset = 0 + break + + print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") + instances.update_last_instance_fetch(domain) + + print(f"DEBUG: Returning for domain='{domain}',blocked()={len(blocklist['blocked'])},suspended()={len(blocklist['suspended'])}") + return { + "reject" : blocklist["blocked"], + "followers_only": blocklist["suspended"] + } diff --git a/fba/federation/pleroma.py b/fba/federation/pleroma.py index 7094ae4..a934572 100644 --- a/fba/federation/pleroma.py +++ b/fba/federation/pleroma.py @@ -152,7 +152,7 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): for blocked, reason in info.items(): # DEBUG: print(f"DEBUG: blocked='{blocked}',reason='{reason}' - BEFORE!") blocked = fba.tidyup_domain(blocked) - reason = fba.tidyup_reason(reason) + reason = fba.tidyup_reason(reason) if reason != None else None # DEBUG: print(f"DEBUG: blocked='{blocked}',reason='{reason}' - AFTER!") if blocked == "": -- 2.39.5