From 79346e1147b9b356060871ff19d4200a79ba807c Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Sun, 4 Jun 2023 23:32:58 +0200 Subject: [PATCH] Continued: - renamed 'res' to 'response' - catch exception, log error and raise it again --- api.py | 22 +++---- fba/fba.py | 147 ++++++++++++++++++++++++----------------------- fetch_blocks.py | 10 ++-- fetch_fba_rss.py | 8 +-- 4 files changed, 96 insertions(+), 91 deletions(-) diff --git a/api.py b/api.py index 177eb5f..94fdc38 100644 --- a/api.py +++ b/api.py @@ -133,9 +133,9 @@ def mutual(domains: list[str] = Query()): "bw": "*." + domains[1], }, ) - res = fba.cursor.fetchone() + response = fba.cursor.fetchone() - if res is not None: + if response is not None: # Blocks found return JSONResponse(status_code=418, content={}) @@ -147,22 +147,22 @@ def index(request: Request, blockers: int = None, blocked: int = None, reference scores = None if blockers != None and blockers > 0: - res = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/top.json?blockers={blockers}") + response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/top.json?blockers={blockers}") elif blocked != None and blocked > 0: - res = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/top.json?blocked={blocked}") + response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/top.json?blocked={blocked}") elif reference != None and reference > 0: - res = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/top.json?reference={reference}") + response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/top.json?reference={reference}") elif software != None and software > 0: - res = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/top.json?software={software}") + response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/top.json?software={software}") elif originator != None and originator > 0: - res = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/top.json?originator={originator}") + response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/top.json?originator={originator}") else: raise HTTPException(status_code=400, detail="No filter specified") - if res == None: + if response == None: raise HTTPException(status_code=500, detail="Could not determine scores") - elif not res.ok: - raise HTTPException(status_code=res.status_code, detail=res.text) + elif not response.ok: + raise HTTPException(status_code=response.status_code, detail=response.text) return templates.TemplateResponse("scoreboard.html", { "base_url" : config.get("base_url"), @@ -174,7 +174,7 @@ def index(request: Request, blockers: int = None, blocked: int = None, reference "reference" : reference, "software" : software, "originator": originator, - "scores" : fba.json_from_response(res) + "scores" : fba.json_from_response(response) }) @router.get(config.get("base_url") + "/") diff --git a/fba/fba.py b/fba/fba.py index e81e478..d8abc9e 100644 --- a/fba/fba.py +++ b/fba/fba.py @@ -371,30 +371,30 @@ def update_last_blocked(domain: str): # DEBUG: print("DEBUG: EXIT!") -def log_error(domain: str, res: any): - # DEBUG: print("DEBUG: domain,res[]:", domain, type(res)) +def log_error(domain: str, response: requests.models.Response): + # DEBUG: print("DEBUG: domain,response[]:", domain, type(response)) if type(domain) != str: raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": raise ValueError(f"Parameter 'domain' cannot be empty") try: - # DEBUG: print("DEBUG: BEFORE res[]:", type(res)) - if isinstance(res, BaseException) or isinstance(res, json.decoder.JSONDecodeError): - res = str(res) + # DEBUG: print("DEBUG: BEFORE response[]:", type(response)) + if isinstance(response, BaseException) or isinstance(response, json.decoder.JSONDecodeError): + response = str(response) - # DEBUG: print("DEBUG: AFTER res[]:", type(res)) - if type(res) is str: + # DEBUG: print("DEBUG: AFTER response[]:", type(response)) + if type(response) is str: cursor.execute("INSERT INTO error_log (domain, error_code, error_message, created) VALUES (?, 999, ?, ?)",[ domain, - res, + response, time.time() ]) else: cursor.execute("INSERT INTO error_log (domain, error_code, error_message, created) VALUES (?, ?, ?, ?)",[ domain, - res.status_code, - res.reason, + response.status_code, + response.reason, time.time() ]) @@ -407,32 +407,32 @@ def log_error(domain: str, res: any): # DEBUG: print("DEBUG: EXIT!") -def update_last_error(domain: str, res: any): - # DEBUG: print("DEBUG: domain,res[]:", domain, type(res)) +def update_last_error(domain: str, response: requests.models.Response): + # DEBUG: print("DEBUG: domain,response[]:", domain, type(response)) if type(domain) != str: raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": raise ValueError(f"Parameter 'domain' cannot be empty") - # DEBUG: print("DEBUG: BEFORE res[]:", type(res)) - if isinstance(res, BaseException) or isinstance(res, json.decoder.JSONDecodeError): - res = f"{type}:str(res)" + # DEBUG: print("DEBUG: BEFORE response[]:", type(response)) + if isinstance(response, BaseException) or isinstance(response, json.decoder.JSONDecodeError): + response = f"{type}:str(response)" - # DEBUG: print("DEBUG: AFTER res[]:", type(res)) - if type(res) is str: - # DEBUG: print(f"DEBUG: Setting last_error_details='{res}'"); + # DEBUG: print("DEBUG: AFTER response[]:", type(response)) + if type(response) is str: + # DEBUG: print(f"DEBUG: Setting last_error_details='{response}'"); instances.set("last_status_code" , domain, 999) - instances.set("last_error_details", domain, res) + instances.set("last_error_details", domain, response) else: - # DEBUG: print(f"DEBUG: Setting last_error_details='{res.reason}'"); - instances.set("last_status_code" , domain, res.status_code) - instances.set("last_error_details", domain, res.reason) + # DEBUG: print(f"DEBUG: Setting last_error_details='{response.reason}'"); + instances.set("last_status_code" , domain, response.status_code) + instances.set("last_error_details", domain, response.reason) # Running pending updated # DEBUG: print(f"DEBUG: Invoking instances.update_instance_data({domain}) ...") instances.update_instance_data(domain) - log_error(domain, res) + log_error(domain, response) # DEBUG: print("DEBUG: EXIT!") @@ -561,15 +561,15 @@ def get_peers(domain: str, software: str) -> list: elif software == "lemmy": # DEBUG: print(f"DEBUG: domain='{domain}' is Lemmy, fetching JSON ...") try: - res = get_response(domain, "/api/v3/site", api_headers, (config.get("connection_timeout"), config.get("read_timeout"))) + response = get_response(domain, "/api/v3/site", api_headers, (config.get("connection_timeout"), config.get("read_timeout"))) - data = json_from_response(res) + data = json_from_response(response) - # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code='{res.status_code}',data[]='{type(data)}'") - if not res.ok or res.status_code >= 400: + # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code='{response.status_code}',data[]='{type(data)}'") + if not response.ok or response.status_code >= 400: print("WARNING: Could not reach any JSON API:", domain) - update_last_error(domain, res) - elif res.ok and isinstance(data, list): + update_last_error(domain, response) + elif response.ok and isinstance(data, list): # DEBUG: print(f"DEBUG: domain='{domain}' returned a list: '{data}'") sys.exit(255) elif "federated_instances" in data: @@ -578,7 +578,7 @@ def get_peers(domain: str, software: str) -> list: # DEBUG: print("DEBUG: Added instance(s) to peers") else: print("WARNING: JSON response does not contain 'federated_instances':", domain) - update_last_error(domain, res) + update_last_error(domain, response) except BaseException as e: print(f"WARNING: Exception during fetching JSON: domain='{domain}',exception[{type(e)}]:'{str(e)}'") @@ -599,11 +599,11 @@ def get_peers(domain: str, software: str) -> list: # DEBUG: print(f"DEBUG: domain='{domain}',mode='{mode}'") while True: try: - res = get_response(domain, "/api/v1/server/{mode}?start={start}&count=100", headers, (config.get("connection_timeout"), config.get("read_timeout"))) + response = get_response(domain, "/api/v1/server/{mode}?start={start}&count=100", headers, (config.get("connection_timeout"), config.get("read_timeout"))) - data = json_from_response(res) - # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code='{res.status_code}',data[]='{type(data)}'") - if res.ok and isinstance(data, dict): + data = json_from_response(response) + # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code='{response.status_code}',data[]='{type(data)}'") + if response.ok and isinstance(data, dict): # DEBUG: print("DEBUG: Success, data:", len(data)) if "data" in data: # DEBUG: print(f"DEBUG: Found {len(data['data'])} record(s).") @@ -636,21 +636,21 @@ def get_peers(domain: str, software: str) -> list: # DEBUG: print(f"DEBUG: Fetching get_peers_url='{get_peers_url}' from '{domain}' ...") try: - res = get_response(domain, get_peers_url, api_headers, (config.get("connection_timeout"), config.get("read_timeout"))) + response = get_response(domain, get_peers_url, api_headers, (config.get("connection_timeout"), config.get("read_timeout"))) - data = json_from_response(res) + data = json_from_response(response) - # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'") - if not res.ok or res.status_code >= 400: + # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'") + if not response.ok or response.status_code >= 400: # DEBUG: print(f"DEBUG: Was not able to fetch '{get_peers_url}', trying alternative ...") - res = get_response(domain, "/api/v3/site", api_headers, (config.get("connection_timeout"), config.get("read_timeout"))) + response = get_response(domain, "/api/v3/site", api_headers, (config.get("connection_timeout"), config.get("read_timeout"))) - data = json_from_response(res) - # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'") - if not res.ok or res.status_code >= 400: + data = json_from_response(response) + # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'") + if not response.ok or response.status_code >= 400: print("WARNING: Could not reach any JSON API:", domain) - update_last_error(domain, res) - elif res.ok and isinstance(data, list): + update_last_error(domain, response) + elif response.ok and isinstance(data, list): # DEBUG: print(f"DEBUG: domain='{domain}' returned a list: '{data}'") sys.exit(255) elif "federated_instances" in data: @@ -659,7 +659,7 @@ def get_peers(domain: str, software: str) -> list: # DEBUG: print("DEBUG: Added instance(s) to peers") else: print("WARNING: JSON response does not contain 'federated_instances':", domain) - update_last_error(domain, res) + update_last_error(domain, response) else: # DEBUG: print("DEBUG: Querying API was successful:", domain, len(data)) peers = data @@ -692,13 +692,13 @@ def post_json_api(domain: str, path: str, parameter: str, extra_headers: dict = # DEBUG: print("DEBUG: Sending POST to domain,path,parameter:", domain, path, parameter, extra_headers) data = {} try: - res = reqto.post(f"https://{domain}{path}", data=parameter, headers={**api_headers, **extra_headers}, timeout=(config.get("connection_timeout"), config.get("read_timeout"))) + response = reqto.post(f"https://{domain}{path}", data=parameter, headers={**api_headers, **extra_headers}, timeout=(config.get("connection_timeout"), config.get("read_timeout"))) - data = json_from_response(res) - # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'") - if not res.ok or res.status_code >= 400: - print(f"WARNING: Cannot query JSON API: domain='{domain}',path='{path}',parameter()={len(parameter)},res.status_code='{res.status_code}',data[]='{type(data)}'") - update_last_error(domain, res) + data = json_from_response(response) + # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'") + if not response.ok or response.status_code >= 400: + print(f"WARNING: Cannot query JSON API: domain='{domain}',path='{path}',parameter()={len(parameter)},response.status_code='{response.status_code}',data[]='{type(data)}'") + update_last_error(domain, response) except BaseException as e: print(f"WARNING: Some error during post(): domain='{domain}',path='{path}',parameter()={len(parameter)},exception[{type(e)}]:'{str(e)}'") @@ -740,21 +740,21 @@ def fetch_nodeinfo(domain: str, path: str = None) -> list: try: # DEBUG: print("DEBUG: Fetching request:", request) - res = reqto.get(request, headers=api_headers, timeout=(config.get("nodeinfo_connection_timeout"), config.get("nodeinfo_read_timeout"))) + response = reqto.get(request, headers=api_headers, timeout=(config.get("nodeinfo_connection_timeout"), config.get("nodeinfo_read_timeout"))) - data = json_from_response(res) - # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'") - if res.ok and isinstance(data, dict): + data = json_from_response(response) + # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'") + if response.ok and isinstance(data, dict): # DEBUG: print("DEBUG: Success:", request) instances.set("detection_mode", domain, "STATIC_CHECK") instances.set("nodeinfo_url" , domain, request) break - elif res.ok and isinstance(data, list): + elif response.ok and isinstance(data, list): # DEBUG: print(f"DEBUG: domain='{domain}' returned a list: '{data}'") sys.exit(255) - elif not res.ok or res.status_code >= 400: + elif not response.ok or response.status_code >= 400: print("WARNING: Failed fetching nodeinfo from domain:", domain) - update_last_error(domain, res) + update_last_error(domain, response) continue except BaseException as e: @@ -776,11 +776,11 @@ def fetch_wellknown_nodeinfo(domain: str) -> list: data = {} try: - res = get_response(domain, "/.well-known/nodeinfo", api_headers, (config.get("nodeinfo_connection_timeout"), config.get("nodeinfo_read_timeout"))) + response = get_response(domain, "/.well-known/nodeinfo", api_headers, (config.get("nodeinfo_connection_timeout"), config.get("nodeinfo_read_timeout"))) - data = json_from_response(res) - # DEBUG: print("DEBUG: domain,res.ok,data[]:", domain, res.ok, type(data)) - if res.ok and isinstance(data, dict): + data = json_from_response(response) + # DEBUG: print("DEBUG: domain,response.ok,data[]:", domain, response.ok, type(data)) + if response.ok and isinstance(data, dict): nodeinfo = data # DEBUG: print("DEBUG: Found entries:", len(nodeinfo), domain) if "links" in nodeinfo: @@ -789,11 +789,11 @@ def fetch_wellknown_nodeinfo(domain: str) -> list: # DEBUG: print("DEBUG: rel,href:", link["rel"], link["href"]) if link["rel"] in nodeinfo_identifier: # DEBUG: print("DEBUG: Fetching nodeinfo from:", link["href"]) - res = reqto.get(link["href"]) + response = reqto.get(link["href"]) - data = json_from_response(res) - # DEBUG: print("DEBUG: href,res.ok,res.status_code:", link["href"], res.ok, res.status_code) - if res.ok and isinstance(data, dict): + data = json_from_response(response) + # DEBUG: print("DEBUG: href,response.ok,response.status_code:", link["href"], response.ok, response.status_code) + if response.ok and isinstance(data, dict): # DEBUG: print("DEBUG: Found JSON nodeinfo():", len(data)) instances.set("detection_mode", domain, "AUTO_DISCOVERY") instances.set("nodeinfo_url" , domain, link["href"]) @@ -829,10 +829,10 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str: # DEBUG: print(f"DEBUG: Fetching path='{path}' from '{domain}' ...") response = get_response(domain, path, headers, (config.get("connection_timeout"), config.get("read_timeout"))) - # DEBUG: print("DEBUG: domain,res.ok,res.status_code,res.text[]:", domain, res.ok, res.status_code, type(res.text)) - if res.ok and res.status_code < 300 and len(res.text) > 0: + # DEBUG: print("DEBUG: domain,response.ok,response.status_code,response.text[]:", domain, response.ok, response.status_code, type(response.text)) + if response.ok and response.status_code < 300 and len(response.text) > 0: # DEBUG: print("DEBUG: Search for :", domain) - doc = bs4.BeautifulSoup(res.text, "html.parser") + doc = bs4.BeautifulSoup(response.text, "html.parser") # DEBUG: print("DEBUG: doc[]:", type(doc)) generator = doc.find("meta", {"name": "generator"}) @@ -1481,8 +1481,13 @@ def get_response(domain: str, path: str, headers: dict, timeout: list) -> reques elif type(path) != str: raise ValueError(f"Parameter path[]='{type(path)}' is not 'str'") - # DEBUG: print(f"DEBUG: Sending request to '{domain}{path}' ...") - response = reqto.get(f"https://{domain}{path}", headers=headers, timeout=timeout); + try: + # DEBUG: print(f"DEBUG: Sending request to '{domain}{path}' ...") + response = reqto.get(f"https://{domain}{path}", headers=headers, timeout=timeout); + except requests.exceptions.ConnectionError as e: + # DEBUG: print(f"DEBUG: Fetching '{path}' from '{domain}' failed. exception[{type(e)}]='{str(e)}'") + update_last_error(domain, e) + raise e # DEBUG: print(f"DEBUG: response[]='{type(response)}' - EXXIT!") return response diff --git a/fetch_blocks.py b/fetch_blocks.py index 424f61a..bb03061 100755 --- a/fetch_blocks.py +++ b/fetch_blocks.py @@ -115,7 +115,7 @@ for blocker, software, origin, nodeinfo_url in rows: nodeinfo_url = searchres[1] # DEBUG: print("DEBUG: Looked up domain:", blocked) elif not validators.domain(blocked): - print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domai name - skipped!") + print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!") continue # DEBUG: print("DEBUG: Looking up instance by domain:", blocked) @@ -188,7 +188,7 @@ for blocker, software, origin, nodeinfo_url in rows: origin = searchres[1] nodeinfo_url = searchres[2] elif not validators.domain(blocked): - print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domai name - skipped!") + print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!") continue # DEBUG: print("DEBUG: Looking up instance by domain:", blocked) @@ -312,7 +312,7 @@ for blocker, software, origin, nodeinfo_url in rows: # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., blocker='{blocker}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'") fba.add_instance(blocked, blocker, sys.argv[0], nodeinfo_url) elif not validators.domain(blocked): - print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domai name - skipped!") + print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!") continue # DEBUG: print("DEBUG: Looking up instance by domain:", blocked) @@ -414,7 +414,7 @@ for blocker, software, origin, nodeinfo_url in rows: origin = searchres[1] nodeinfo_url = searchres[2] elif not validators.domain(blocked): - print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domai name - skipped!") + print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!") continue # DEBUG: print("DEBUG: Looking up instance by domain:", blocked) @@ -482,7 +482,7 @@ for blocker, software, origin, nodeinfo_url in rows: origin = searchres[1] nodeinfo_url = searchres[2] elif not validators.domain(blocked): - print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domai name - skipped!") + print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!") continue # DEBUG: print("DEBUG: Looking up instance by domain:", blocked) diff --git a/fetch_fba_rss.py b/fetch_fba_rss.py index 6553bde..c7c28e6 100755 --- a/fetch_fba_rss.py +++ b/fetch_fba_rss.py @@ -29,12 +29,12 @@ feed = sys.argv[1] domains = list() try: print(f"INFO: Fetch FBA-specific RSS feed='{feed}' ...") - res = reqto.get(feed, headers=fba.headers, timeout=(config.get("connection_timeout"), config.get("read_timeout"))) + response = reqto.get(feed, headers=fba.headers, timeout=(config.get("connection_timeout"), config.get("read_timeout"))) - # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code='{res.status_code}',res.text()={len(res.text)}") - if res.ok and res.status_code < 300 and len(res.text) > 0: + # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code='{response.status_code}',response.text()={len(response.text)}") + if response.ok and response.status_code < 300 and len(response.text) > 0: # DEBUG: print(f"DEBUG: Parsing RSS feed ...") - rss = rss_parser.Parser.parse(res.text) + rss = rss_parser.Parser.parse(response.text) for item in rss.channel.items: # DEBUG: print(f"DEBUG: item.link={item.link}") domain = item.link.split("=")[1] -- 2.39.5