From 3564aa4eaacad3679b79c4a03f81c33033c0d7f1 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Sat, 10 Jun 2023 19:56:16 +0200 Subject: [PATCH] Continued: - ops, don't copy whole HTML to last_error_message - also need to initialize local variable 'peers' first --- fba/federation.py | 5 +++- fba/network.py | 6 ++-- fba/networks/misskey.py | 64 +++++++++++++++++++++++++---------------- 3 files changed, 47 insertions(+), 28 deletions(-) diff --git a/fba/federation.py b/fba/federation.py index 8888682..a912234 100644 --- a/fba/federation.py +++ b/fba/federation.py @@ -120,6 +120,9 @@ def fetch_peers(domain: str, software: str) -> list: # DEBUG: print(f"DEBUG: Invoking peertube.fetch_peers({domain}) ...") return peertube.fetch_peers(domain) + # Init peers variable + peers = list() + # DEBUG: print(f"DEBUG: Checking CSRF for domain='{domain}'") headers = csrf.determine(domain, dict()) @@ -130,8 +133,8 @@ def fetch_peers(domain: str, software: str) -> list: headers, (config.get("connection_timeout"), config.get("read_timeout")) ) - # DEBUG: print(f"DEBUG: data[]='{type(data)}'") + # DEBUG: print(f"DEBUG: data[]='{type(data)}'") if "error_message" in data: # DEBUG: print("DEBUG: Was not able to fetch peers, trying alternative ...") data = network.get_json_api( diff --git a/fba/network.py b/fba/network.py index ec7e4bc..7c0bb8a 100644 --- a/fba/network.py +++ b/fba/network.py @@ -67,7 +67,7 @@ def post_json_api(domain: str, path: str, data: str, headers: dict = {}) -> dict if not response.ok or response.status_code >= 400: print(f"WARNING: Cannot query JSON API: domain='{domain}',path='{path}',data()={len(data)},response.status_code='{response.status_code}',json_reply[]='{type(json_reply)}'") json_reply["status_code"] = response.status_code - json_reply["error_message"] = response.text + json_reply["error_message"] = response.reason instances.update_last_error(domain, response) except requests.exceptions.ConnectionError as exception: @@ -101,7 +101,7 @@ def fetch_api_url(url: str, timeout: tuple) -> dict: if not response.ok or response.status_code >= 400: print(f"WARNING: Cannot query JSON API: url='{url}',response.status_code='{response.status_code}',json_reply[]='{type(json_reply)}'") json_reply["status_code"] = response.status_code - json_reply["error_message"] = response.text + json_reply["error_message"] = response.reason except requests.exceptions.ConnectionError as exception: # DEBUG: print(f"DEBUG: Fetching '{url}' failed. exception[{type(exception)}]='{str(exception)}'") @@ -152,7 +152,7 @@ def get_json_api(domain: str, path: str, headers: dict, timeout: tuple) -> dict: if not response.ok or response.status_code >= 400: print(f"WARNING: Cannot query JSON API: domain='{domain}',path='{path}',response.status_code='{response.status_code}',json_reply[]='{type(json_reply)}'") json_reply["status_code"] = response.status_code - json_reply["error_message"] = response.text + json_reply["error_message"] = response.reason instances.update_last_error(domain, response) # DEBUG: print(f"DEBUG: Returning json_reply({len(json_reply)})=[]:{type(json_reply)}") diff --git a/fba/networks/misskey.py b/fba/networks/misskey.py index 52c32e3..68d3b26 100644 --- a/fba/networks/misskey.py +++ b/fba/networks/misskey.py @@ -59,30 +59,33 @@ def fetch_peers(domain: str) -> list: "offset": offset - 1 }), headers) - # DEBUG: print(f"DEBUG: fetched()={len(fetched)}") - if len(fetched) == 0: + # Check records + # DEBUG: print(f"DEBUG: fetched[]={type(fetched)}") + if "error_message" in fetched: + print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}") + instances.update_last_error(domain, fetched) + break + elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]: + print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}") + instances.update_last_error(domain, fetched["json"]["error"]["message"]) + break + + rows = fetched["json"] + + # DEBUG: print(f"DEBUG: rows()={len(rows)}") + if len(rows) == 0: # DEBUG: print(f"DEBUG: Returned zero bytes, exiting loop, domain='{domain}'") break - elif len(fetched) != config.get("misskey_limit"): - # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'") - offset = offset + (config.get("misskey_limit") - len(fetched)) + elif len(rows) != config.get("misskey_limit"): + # DEBUG: print(f"DEBUG: Fetched '{len(rows)}' row(s) but expected: '{config.get('misskey_limit')}'") + offset = offset + (config.get("misskey_limit") - len(rows)) else: # DEBUG: print(f"DEBUG: Raising offset by step={step}") offset = offset + step - # Check records - # DEBUG: print(f"DEBUG: fetched({len(fetched)})[]={type(fetched)}") - if isinstance(fetched, dict) and "error" in fetched and "message" in fetched["error"]: - print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}") - instances.update_last_error(domain, fetched["error"]["message"]) - break - elif "error_message" in fetched: - print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}") - instances.update_last_error(domain, fetched) - break - already = 0 - for row in fetched["json"]: + # DEBUG: print(f"DEBUG: rows({len(rows)})[]={type(rows)}") + for row in rows: # DEBUG: print(f"DEBUG: row()={len(row)}") if not "host" in row: print(f"WARNING: row()={len(row)} does not contain key 'host': {row},domain='{domain}'") @@ -101,7 +104,7 @@ def fetch_peers(domain: str) -> list: # DEBUG: print(f"DEBUG: Adding peer: '{row['host']}'") peers.append(row["host"]) - if already == len(fetched): + if already == len(rows): # DEBUG: print(f"DEBUG: Host returned same set of '{already}' instances, aborting loop!") break @@ -155,19 +158,32 @@ def fetch_blocks(domain: str) -> dict: "offset" : offset - 1 }), headers) - # DEBUG: print("DEBUG: fetched():", len(fetched)) - if len(fetched) == 0: + # Check records + # DEBUG: print(f"DEBUG: fetched[]={type(fetched)}") + if "error_message" in fetched: + print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}") + instances.update_last_error(domain, fetched) + break + elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]: + print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}") + instances.update_last_error(domain, fetched["json"]["error"]["message"]) + break + + rows = fetched["json"] + + # DEBUG: print("DEBUG: rows():", len(rows)) + if len(rows) == 0: # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain) break - elif len(fetched) != config.get("misskey_limit"): - # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'") - offset = offset + (config.get("misskey_limit") - len(fetched)) + elif len(rows) != config.get("misskey_limit"): + # DEBUG: print(f"DEBUG: Fetched '{len(rows)}' row(s) but expected: '{config.get('misskey_limit')}'") + offset = offset + (config.get("misskey_limit") - len(rows)) else: # DEBUG: print("DEBUG: Raising offset by step:", step) offset = offset + step count = 0 - for instance in fetched: + for instance in rows: # Is it there? if instance["isSuspended"] and not dicts.has_key(blocklist["suspended"], "domain", instance): count = count + 1 -- 2.39.5