From bcea1e2462903e5af1610f5b727545b8dae838fc Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Fri, 19 May 2023 19:58:01 +0200 Subject: [PATCH] Continued: - added more debug lines - if zero sized list is being returned, quit the loop, again raised exceptions are NOT there for your flow-control! --- fba.py | 49 ++++++++++++++++++++++++++++++++++++++----------- fetch_blocks.py | 12 ++++-------- 2 files changed, 42 insertions(+), 19 deletions(-) diff --git a/fba.py b/fba.py index 4f2e05d..f19c108 100644 --- a/fba.py +++ b/fba.py @@ -42,14 +42,15 @@ def get_peers(domain: str) -> str: # NOISY-DEBUG: print("DEBUG: Returning peers[]:", type(peers)) return peers -def post_json_api(domain: str, path: str, data: str) -> str: - # NOISY-DEBUG: print("DEBUG: Sending POST to domain,path,data():", domain, path, len(data)) - doc = reqto.post(f"https://{domain}{path}", data=data, headers=headers, timeout=5).json() +def post_json_api(domain: str, path: str, data: str) -> list: + # NOISY-DEBUG: print("DEBUG: Sending POST to domain,path,data:", domain, path, data) + res = reqto.post(f"https://{domain}{path}", data=data, headers=headers, timeout=5) - if doc == []: - print("WARNING: Cannot query JSON API:", domain, path) + if not res.ok: + print("WARNING: Cannot query JSON API:", domain, path, data, res.status_code) raise + doc = res.json() # NOISY-DEBUG: print("DEBUG: Returning doc():", len(doc)) return doc @@ -58,21 +59,29 @@ def determine_software(domain: str) -> str: software = None try: res = reqto.get(f"https://{domain}/nodeinfo/2.1.json", headers=headers, timeout=5) + + # NOISY-DEBUG: print("DEBUG: domain,res.ok,res.status_code:", domain, res.ok, res.status_code) if res.status_code == 404: res = reqto.get(f"https://{domain}/nodeinfo/2.0", headers=headers, timeout=5) + if res.status_code == 404: res = reqto.get(f"https://{domain}/nodeinfo/2.0.json", headers=headers, timeout=5) + if res.ok and "text/html" in res.headers["content-type"]: res = reqto.get(f"https://{domain}/nodeinfo/2.1", headers=headers, timeout=5) + if res.ok: - if res.json()["software"]["name"] in ["akkoma", "rebased"]: + json = res.json() + # NOISY-DEBUG: print("DEBUG: json():", len(json)) + + if json["software"]["name"] in ["akkoma", "rebased"]: software = "pleroma" - elif res.json()["software"]["name"] in ["hometown", "ecko"]: + elif json["software"]["name"] in ["hometown", "ecko"]: software = "mastodon" - elif res.json()["software"]["name"] in ["calckey", "groundpolis", "foundkey", "cherrypick"]: + elif json["software"]["name"] in ["calckey", "groundpolis", "foundkey", "cherrypick"]: software = "misskey" else: - software = res.json()["software"]["name"] + software = json["software"]["name"] elif res.status_code == 404: res = reqto.get(f"https://{domain}/api/v1/instance", headers=headers, timeout=5) if res.ok: @@ -286,6 +295,7 @@ def get_misskey_blocks(domain: str) -> dict: # sending them all at once try: if counter == 0: + # NOISY-DEBUG: print("DEBUG: Sending JSON API request to domain,step,counter:", domain, step, counter) doc = post_json_api(domain, "/api/federation/instances/", json.dumps({ "sort": "+caughtAt", "host": None, @@ -293,6 +303,7 @@ def get_misskey_blocks(domain: str) -> dict: "limit": step })) else: + # NOISY-DEBUG: print("DEBUG: Sending JSON API request to domain,step,counter:", domain, step, counter) doc = post_json_api(domain, "/api/federation/instances/", json.dumps({ "sort": "+caughtAt", "host": None, @@ -301,6 +312,11 @@ def get_misskey_blocks(domain: str) -> dict: "offset": counter-1 })) + # NOISY-DEBUG: print("DEBUG: doc():", len(doc)) + if len(doc) == 0: + # NOISY-DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain) + break + for instance in doc: # just in case if instance["isSuspended"]: @@ -311,8 +327,12 @@ def get_misskey_blocks(domain: str) -> dict: "reason": "" } ) + + # NOISY-DEBUG: print("DEBUG: Raising counter by step:", step) counter = counter + step + except: + print("WARNING: Caught error, exiting loop:", domain) counter = 0 break @@ -320,6 +340,7 @@ def get_misskey_blocks(domain: str) -> dict: # same shit, different asshole ("blocked" aka full suspend) try: if counter == 0: + # NOISY-DEBUG: print("DEBUG: Sending JSON API request to domain,step,counter:", domain, step, counter) doc = post_json_api(domain,"/api/federation/instances", json.dumps({ "sort": "+caughtAt", "host": None, @@ -327,6 +348,7 @@ def get_misskey_blocks(domain: str) -> dict: "limit": step })) else: + # NOISY-DEBUG: print("DEBUG: Sending JSON API request to domain,step,counter:", domain, step, counter) doc = post_json_api(domain,"/api/federation/instances", json.dumps({ "sort": "+caughtAt", "host": None, @@ -335,11 +357,16 @@ def get_misskey_blocks(domain: str) -> dict: "offset": counter-1 })) + # NOISY-DEBUG: print("DEBUG: doc():", len(doc)) + if len(doc) == 0: + # NOISY-DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain) + break + for instance in doc: if instance["isBlocked"]: blocks["blocked"].append({ - "domain": instance["host"], - "reason": "" + "domain": instance["host"], + "reason": "" }) counter = counter + step diff --git a/fetch_blocks.py b/fetch_blocks.py index cf9d97c..170e52a 100644 --- a/fetch_blocks.py +++ b/fetch_blocks.py @@ -23,8 +23,7 @@ for blocker, software in fba.c.fetchall(): print("INFO: blocker:", blocker) try: # Blocks - federation = reqto.get( - f"https://{blocker}/nodeinfo/2.1.json", headers=fba.headers, timeout=5 + federation = reqto.get(f"https://{blocker}/nodeinfo/2.1.json", headers=fba.headers, timeout=5 ).json()["metadata"]["federation"] if "enabled" in federation: @@ -173,9 +172,7 @@ for blocker, software in fba.c.fetchall(): reqheaders = fba.headers # NOISY-DEBUG: print("DEBUG: Quering API domain_blocks:", blocker) - blocks = reqto.get( - f"https://{blocker}/api/v1/instance/domain_blocks", headers=reqheaders, timeout=5 - ).json() + blocks = reqto.get(f"https://{blocker}/api/v1/instance/domain_blocks", headers=reqheaders, timeout=5).json() # NOISY-DEBUG: print("DEBUG: blocks():", len(blocks)) for block in blocks: @@ -347,15 +344,14 @@ for blocker, software in fba.c.fetchall(): print("INFO: blocker:", blocker) try: # Blocks - federation = reqto.get( - f"https://{blocker}/api/v1/instance/peers?filter=suspended", headers=fba.headers, timeout=5 - ).json() + federation = reqto.get(f"https://{blocker}/api/v1/instance/peers?filter=suspended", headers=fba.headers, timeout=5).json() if (federation == None): print("WARNING: No valid response:", blocker); elif "error" in federation: print("WARNING: API returned error:", federation["error"]) else: + # NOISY-DEBUG: print("DEBUG: Checking fenderation():", len(federation)) for peer in federation: blocked = peer["domain"].lower() # NOISY-DEBUG: print("DEBUG: BEFORE-blocked:", blocked) -- 2.39.5