From: Roland Häder Date: Sat, 10 Jun 2023 14:14:54 +0000 (+0200) Subject: WIP: X-Git-Url: https://git.mxchange.org/?a=commitdiff_plain;h=a993efe50352f30ef4fb2d5e3a2aab640a129019;p=fba.git WIP: - more refacturing away from "generic except blocks" --- diff --git a/api.py b/api.py index 1275d8f..a9c0e4d 100644 --- a/api.py +++ b/api.py @@ -31,6 +31,7 @@ import validators from fba import config from fba import fba +from fba import network router = fastapi.FastAPI(docs_url=config.get("base_url") + "/docs", redoc_url=config.get("base_url") + "/redoc") templates = Jinja2Templates(directory="templates") @@ -157,6 +158,8 @@ def api_mutual(domains: list[str] = Query()): @router.get(config.get("base_url") + "/scoreboard") def scoreboard(request: Request, blockers: int = None, blocked: int = None, reference: int = None, software: int = None, command: int = None, error_code: int = None): + response = None + if blockers is not None and blockers > 0: response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/top.json?blockers={blockers}") elif blocked is not None and blocked > 0: @@ -188,7 +191,7 @@ def scoreboard(request: Request, blockers: int = None, blocked: int = None, refe "software" : software, "command" : command, "error_code": error_code, - "scores" : fba.json_from_response(response) + "scores" : network.json_from_response(response) }) @router.get(config.get("base_url") + "/") diff --git a/fba/commands.py b/fba/commands.py index d85c1eb..3f37fae 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -25,6 +25,7 @@ import atoma import bs4 import markdown import reqto +import requests import validators from fba import blacklist @@ -105,8 +106,12 @@ def fetch_bkali(args: argparse.Namespace): print(f"INFO: Adding {len(domains)} new instances ...") for domain in domains: - print(f"INFO: Fetching instances from domain='{domain}' ...") - federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) + try: + print(f"INFO: Fetching instances from domain='{domain}' ...") + federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) + except requests.exceptions.Timeout as ex: + print(f"WARNING: Timeout during fetching instances from domain='{domain}'") + instances.update_last_error(domain, ex) # DEBUG: print("DEBUG: EXIT!") @@ -312,8 +317,12 @@ def fetch_cs(args: argparse.Namespace): blocks.add_instance('chaos.social', row["domain"], row["reason"], block_level) if not instances.is_registered(row["domain"]): - print(f"INFO: Fetching instances from domain='{row['domain']}' ...") - federation.fetch_instances(row["domain"], 'chaos.social', None, inspect.currentframe().f_code.co_name) + try: + print(f"INFO: Fetching instances from domain='{row['domain']}' ...") + federation.fetch_instances(row["domain"], 'chaos.social', None, inspect.currentframe().f_code.co_name) + except requests.exceptions.Timeout as ex: + print(f"WARNING: Timeout during fetching instances from domain='{row['domain']}'") + instances.update_last_error(row["domain"], ex) # DEBUG: print("DEBUG: Committing changes ...") fba.connection.commit() @@ -356,8 +365,12 @@ def fetch_fba_rss(args: argparse.Namespace): print(f"INFO: Adding {len(domains)} new instances ...") for domain in domains: - print(f"INFO: Fetching instances from domain='{domain}' ...") - federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) + try: + print(f"INFO: Fetching instances from domain='{domain}' ...") + federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) + except requests.exceptions.Timeout as ex: + print(f"WARNING: Timeout during fetching instances from domain='{domain}'") + instances.update_last_error(domain, ex) # DEBUG: print("DEBUG: EXIT!") @@ -405,8 +418,12 @@ def fetch_fbabot_atom(args: argparse.Namespace): print(f"INFO: Adding {len(domains)} new instances ...") for domain in domains: - print(f"INFO: Fetching instances from domain='{domain}' ...") - federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) + try: + print(f"INFO: Fetching instances from domain='{domain}' ...") + federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) + except requests.exceptions.Timeout as ex: + print(f"WARNING: Timeout during fetching instances from domain='{domain}'") + instances.update_last_error(domain, ex) # DEBUG: print("DEBUG: EXIT!") @@ -415,7 +432,13 @@ def fetch_instances(args: argparse.Namespace): locking.acquire() # Initial fetch - federation.fetch_instances(args.domain, None, None, inspect.currentframe().f_code.co_name) + try: + print(f"INFO: Fetching instances from args.domain='{args.domain}' ...") + federation.fetch_instances(args.domain, None, None, inspect.currentframe().f_code.co_name) + except requests.exceptions.Timeout as ex: + print(f"WARNING: Timeout during fetching instances from args.domain='{args.domain}'") + instances.update_last_error(args.domain, ex) + return if args.single: # DEBUG: print("DEBUG: Not fetching more instances - EXIT!") @@ -434,8 +457,12 @@ def fetch_instances(args: argparse.Namespace): print("WARNING: domain is blacklisted:", row[0]) continue - print(f"INFO: Fetching instances for instance '{row[0]}' ('{row[2]}') of origin='{row[1]}',nodeinfo_url='{row[3]}'") - federation.fetch_instances(row[0], row[1], row[2], inspect.currentframe().f_code.co_name, row[3]) + try: + print(f"INFO: Fetching instances for instance '{row[0]}' ('{row[2]}') of origin='{row[1]}',nodeinfo_url='{row[3]}'") + federation.fetch_instances(row[0], row[1], row[2], inspect.currentframe().f_code.co_name, row[3]) + except requests.exceptions.Timeout as ex: + print(f"WARNING: Timeout during fetching instances from domain='{row[0]}'") + instances.update_last_error(row[0], ex) # DEBUG: print("DEBUG: EXIT!") @@ -463,7 +490,11 @@ def fetch_federater(args: argparse.Namespace): # DEBUG: print(f"DEBUG: domain='{row['#domain']}' is already registered - skipped!") continue - print(f"INFO: Fetching instances for instane='{row['#domain']}' ...") - federation.fetch_instances(row["#domain"], None, None, inspect.currentframe().f_code.co_name) + try: + print(f"INFO: Fetching instances for instane='{row['#domain']}' ...") + federation.fetch_instances(row["#domain"], None, None, inspect.currentframe().f_code.co_name) + except requests.exceptions.Timeout as ex: + print(f"WARNING: Timeout during fetching instances from domain='{row['#domain']}'") + instances.update_last_error(row["#domain"], ex) # DEBUG: print("DEBUG: EXIT!") diff --git a/fba/csrf.py b/fba/csrf.py index 00ec0f3..4602c28 100644 --- a/fba/csrf.py +++ b/fba/csrf.py @@ -21,7 +21,7 @@ from fba import config from fba import network def determine(domain: str, headers: dict) -> dict: - print(f"DEBUG: domain='{domain}',headers()={len(headers)} - CALLED!") + # DEBUG: print(f"DEBUG: domain='{domain}',headers()={len(headers)} - CALLED!") if not isinstance(domain, str): raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": @@ -33,26 +33,26 @@ def determine(domain: str, headers: dict) -> dict: reqheaders = headers # Fetch / to check for meta tag indicating csrf - print(f"DEBUG: Fetching / from domain='{domain}' for CSRF check ...") + # DEBUG: print(f"DEBUG: Fetching / from domain='{domain}' for CSRF check ...") response = reqto.get( f"https://{domain}/", headers=network.web_headers, timeout=(config.get("connection_timeout"), config.get("read_timeout")) ) - print(f"DEBUG: response.ok='{response.ok}',response.status_code={response.status_code},response.text()={len(response.text)}") + # DEBUG: print(f"DEBUG: response.ok='{response.ok}',response.status_code={response.status_code},response.text()={len(response.text)}") if response.ok and len(response.text) > 0: meta = bs4.BeautifulSoup( response.text, "html.parser" ) - print(f"DEBUG: meta[]='{type(meta)}'") + # DEBUG: print(f"DEBUG: meta[]='{type(meta)}'") tag = meta.find("meta", attrs={"name": "csrf-token"}) - print(f"DEBUG: tag={tag}") + # DEBUG: print(f"DEBUG: tag={tag}") if tag is not None: - print(f"DEBUG: Adding CSRF token='{tag['content']}' for domain='{domain}'") + # DEBUG: print(f"DEBUG: Adding CSRF token='{tag['content']}' for domain='{domain}'") reqheaders["X-CSRF-Token"] = tag["content"] - print(f"DEBUG: reqheaders()={len(reqheaders)} - EXIT!") + # DEBUG: print(f"DEBUG: reqheaders()={len(reqheaders)} - EXIT!") return reqheaders diff --git a/fba/federation.py b/fba/federation.py index 085c37b..4db6e5c 100644 --- a/fba/federation.py +++ b/fba/federation.py @@ -122,8 +122,7 @@ def fetch_peers(domain: str, software: str) -> list: return peertube.fetch_peers(domain) # DEBUG: print(f"DEBUG: Fetching peers from '{domain}',software='{software}' ...") - data = list() - response = network.get_json_api( + data = network.get_json_api( domain, "/api/v1/instance/peers", (config.get("connection_timeout"), config.get("read_timeout")) @@ -141,16 +140,16 @@ def fetch_peers(domain: str, software: str) -> list: # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'") if "error_message" in data: print(f"WARNING: Could not reach any JSON API at domain='{domain}',status_code='{data['status_code']}',error_message='{data['error_message']}'") - elif "federated_instances" in data: + elif "federated_instances" in data["json"]: # DEBUG: print(f"DEBUG: Found federated_instances for domain='{domain}'") - peers = peers + add_peers(data["federated_instances"]) + peers = peers + add_peers(data["json"]["federated_instances"]) # DEBUG: print("DEBUG: Added instance(s) to peers") else: print("WARNING: JSON response does not contain 'federated_instances':", domain) instances.update_last_error(domain, response) else: # DEBUG: print("DEBUG: Querying API was successful:", domain, len(data)) - peers = data + peers = data["json"] # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'") instances.set_data("total_peers", domain, len(peers)) @@ -226,7 +225,7 @@ def fetch_wellknown_nodeinfo(domain: str) -> list: ) if "error_message" not in data: - nodeinfo = data + nodeinfo = data["json"] # DEBUG: print("DEBUG: Found entries:", len(nodeinfo), domain) if "links" in nodeinfo: # DEBUG: print("DEBUG: Found links in nodeinfo():", len(nodeinfo["links"])) @@ -242,7 +241,7 @@ def fetch_wellknown_nodeinfo(domain: str) -> list: data = network.json_from_response(response) # DEBUG: print("DEBUG: href,response.ok,response.status_code:", link["href"], response.ok, response.status_code) - if response.ok and isinstance(data, dict): + if response.ok and len(data) > 0: # DEBUG: print("DEBUG: Found JSON nodeinfo():", len(data)) instances.set_data("detection_mode", domain, "AUTO_DISCOVERY") instances.set_data("nodeinfo_url" , domain, link["href"]) diff --git a/fba/instances.py b/fba/instances.py index 27a0821..c872c2b 100644 --- a/fba/instances.py +++ b/fba/instances.py @@ -201,8 +201,9 @@ def add(domain: str, origin: str, command: str, path: str = None): # DEBUG: print("DEBUG: domain,origin,command,path:", domain, origin, command, path) software = federation.determine_software(domain, path) + # DEBUG: print("DEBUG: Determined software:", software) - if domain.find("/c/") > 0 and software == "lemmy": + if software == "lemmy" and domain.find("/c/") > 0: domain = domain.split("/c/")[0] if is_registered(domain): print(f"WARNING: domain='{domain}' already registered after cutting off user part. - EXIT!") @@ -257,33 +258,33 @@ def update_last_nodeinfo(domain: str): # DEBUG: print("DEBUG: EXIT!") def update_last_error(domain: str, response: requests.models.Response): - # DEBUG: print("DEBUG: domain,response[]:", domain, type(response)) + print("DEBUG: domain,response[]:", domain, type(response)) if not isinstance(domain, str): raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") - # DEBUG: print("DEBUG: BEFORE response[]:", type(response)) + print("DEBUG: BEFORE response[]:", type(response)) if isinstance(response, BaseException) or isinstance(response, json.decoder.JSONDecodeError): response = f"response[{type(response)}]='{str(response)}'" - # DEBUG: print("DEBUG: AFTER response[]:", type(response)) + print("DEBUG: AFTER response[]:", type(response)) if isinstance(response, str): - # DEBUG: print(f"DEBUG: Setting last_error_details='{response}'") + print(f"DEBUG: Setting last_error_details='{response}'") set_data("last_status_code" , domain, 999) set_data("last_error_details", domain, response) else: - # DEBUG: print(f"DEBUG: Setting last_error_details='{response.reason}'") + print(f"DEBUG: Setting last_error_details='{response.reason}'") set_data("last_status_code" , domain, response.status_code) set_data("last_error_details", domain, response.reason) # Running pending updated - # DEBUG: print(f"DEBUG: Invoking update_data({domain}) ...") + print(f"DEBUG: Invoking update_data({domain}) ...") update_data(domain) fba.log_error(domain, response) - # DEBUG: print("DEBUG: EXIT!") + print("DEBUG: EXIT!") def is_registered(domain: str) -> bool: # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") diff --git a/fba/network.py b/fba/network.py index e77a658..9a30d58 100644 --- a/fba/network.py +++ b/fba/network.py @@ -51,7 +51,9 @@ def post_json_api(domain: str, path: str, data: str, headers: dict = {}) -> dict # DEBUG: print(f"DEBUG: Determining if CSRF header needs to be sent for domain='{domain}' ...") headers = csrf.determine(domain, {**api_headers, **headers}) - json_reply = {} + json_reply = { + "status_code": 200, + } try: # DEBUG: print(f"DEBUG: Sending POST to domain='{domain}',path='{path}',data='{data}',headers({len(headers)})={headers}") @@ -62,6 +64,15 @@ def post_json_api(domain: str, path: str, data: str, headers: dict = {}) -> dict timeout=(config.get("connection_timeout"), config.get("read_timeout")) ) + json_reply["json"] = json_from_response(response) + + # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},json_reply[]='{type(json_reply)}'") + if not response.ok or response.status_code >= 400: + print(f"WARNING: Cannot query JSON API: domain='{domain}',path='{path}',data()={len(data)},response.status_code='{response.status_code}',json_reply[]='{type(json_reply)}'") + json_reply["status_code"] = response.status_code + json_reply["error_message"] = response.text + instances.update_last_error(domain, response) + except requests.exceptions.ConnectionError as exception: # DEBUG: print(f"DEBUG: Fetching '{path}' from '{domain}' failed. exception[{type(exception)}]='{str(exception)}'") json_reply["status_code"] = 999 @@ -69,15 +80,6 @@ def post_json_api(domain: str, path: str, data: str, headers: dict = {}) -> dict instances.update_last_error(domain, exception) raise exception - json_reply = json_from_response(response) - - # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},json_reply[]='{type(json_reply)}'") - if not response.ok or response.status_code >= 400: - print(f"WARNING: Cannot query JSON API: domain='{domain}',path='{path}',data()={len(data)},response.status_code='{response.status_code}',json_reply[]='{type(json_reply)}'") - json_reply["status_code"] = response.status_code - json_reply["error_message"] = response.text - instances.update_last_error(domain, response) - # DEBUG: print(f"DEBUG: Returning json_reply({len(json_reply)})=[]:{type(json_reply)}") return json_reply @@ -116,7 +118,7 @@ def get_json_api(domain: str, path: str, timeout: tuple) -> dict: instances.update_last_error(domain, exception) raise exception - json_reply = json_from_response(response) + json_reply["json"] = json_from_response(response) # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},json_reply[]='{type(json_reply)}'") if not response.ok or response.status_code >= 400: @@ -209,7 +211,7 @@ def json_from_response(response: requests.models.Response) -> list: if not isinstance(response, requests.models.Response): raise ValueError(f"Parameter response[]='{type(response)}' is not type of 'Response'") - data = list() + data = dict() if response.text.strip() != "": # DEBUG: print(f"DEBUG: response.text()={len(response.text)} is not empty, invoking response.json() ...") try: diff --git a/fba/networks/lemmy.py b/fba/networks/lemmy.py index f819fb0..f2cc4e4 100644 --- a/fba/networks/lemmy.py +++ b/fba/networks/lemmy.py @@ -39,9 +39,9 @@ def fetch_peers(domain: str) -> list: if "error_message" in data: print("WARNING: Could not reach any JSON API:", domain) instances.update_last_error(domain, response) - elif "federated_instances" in data: + elif "federated_instances" in data["json"]: # DEBUG: print(f"DEBUG: Found federated_instances for domain='{domain}'") - peers = peers + federation.add_peers(data["federated_instances"]) + peers = peers + federation.add_peers(data["json"]["federated_instances"]) # DEBUG: print("DEBUG: Added instance(s) to peers") else: print("WARNING: JSON response does not contain 'federated_instances':", domain) diff --git a/fba/networks/mastodon.py b/fba/networks/mastodon.py index 23f2536..25d11f4 100644 --- a/fba/networks/mastodon.py +++ b/fba/networks/mastodon.py @@ -140,16 +140,17 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): } # DEBUG: print("DEBUG: Querying API domain_blocks:", domain) - blocklist = network.get_json_api( + data = network.get_json_api( domain, "/api/v1/instance/domain_blocks", (config.get("connection_timeout"), config.get("read_timeout")) ) - if "error_message" in blocklist: + if "error_message" in data: print(f"WARNING: Was not able to fetch domain_blocks from domain='{domain}': status_code='{data['status_code']}',error_message='{data['error_message']}'") - instances.update_last_error(domain, blocklist) + instances.update_last_error(domain, data) + blocklist = data["json"] print(f"INFO: Checking {len(blocklist)} entries from domain='{domain}',software='mastodon' ...") for block in blocklist: entry = { diff --git a/fba/networks/peertube.py b/fba/networks/peertube.py index 3a02078..83640da 100644 --- a/fba/networks/peertube.py +++ b/fba/networks/peertube.py @@ -39,10 +39,10 @@ def fetch_peers(domain: str) -> list: print(f"DEBUG: data['{type(data)}']='{data}'") if "error_message" not in data: - print("DEBUG: Success, data:", len(data)) - if "data" in data: + print("DEBUG: Success, data[json]:", len(data["json"])) + if "data" in data["json"]: print(f"DEBUG: Found {len(data['data'])} record(s).") - for record in data["data"]: + for record in data["json"]["data"]: print(f"DEBUG: record()={len(record)}") if mode in record and "host" in record[mode]: print(f"DEBUG: Found host={record[mode]['host']}, adding ...") @@ -50,7 +50,7 @@ def fetch_peers(domain: str) -> list: else: print(f"WARNING: record from '{domain}' has no '{mode}' or 'host' record: {record}") - if len(data["data"]) < 100: + if len(data["json"]["data"]) < 100: print("DEBUG: Reached end of JSON response:", domain) break