From: Roland Häder Date: Mon, 12 Jun 2023 04:25:42 +0000 (+0200) Subject: Continued: X-Git-Url: https://git.mxchange.org/?a=commitdiff_plain;h=22c205985577ac7e7e78da9e1974a12f7474df84;p=fba.git Continued: - also update last_instance_fetch when software type isn't determined yet - include command name in log messages --- diff --git a/fba/commands.py b/fba/commands.py index 1c0f74e..74656c4 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -118,7 +118,7 @@ def fetch_bkali(args: argparse.Namespace) -> int: print(f"INFO: Fetching instances from domain='{domain}' ...") federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) except network.exceptions as exception: - print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{domain}'") + print(f"WARNING: Exception '{type(exception)}' during fetching instances (fetch_bkali) from domain='{domain}'") instances.update_last_error(domain, exception) # DEBUG: print("DEBUG: EXIT!") @@ -342,7 +342,7 @@ def fetch_cs(args: argparse.Namespace): print(f"INFO: Fetching instances from domain='{row['domain']}' ...") federation.fetch_instances(row["domain"], 'chaos.social', None, inspect.currentframe().f_code.co_name) except network.exceptions as exception: - print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{row['domain']}'") + print(f"WARNING: Exception '{type(exception)}' during fetching instances (fetch_cs) from domain='{row['domain']}'") instances.update_last_error(row["domain"], exception) # DEBUG: print("DEBUG: Committing changes ...") @@ -390,7 +390,7 @@ def fetch_fba_rss(args: argparse.Namespace): print(f"INFO: Fetching instances from domain='{domain}' ...") federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) except network.exceptions as exception: - print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{domain}'") + print(f"WARNING: Exception '{type(exception)}' during fetching instances (fetch_fba_rss) from domain='{domain}'") instances.update_last_error(domain, exception) # DEBUG: print("DEBUG: EXIT!") @@ -443,7 +443,7 @@ def fetch_fbabot_atom(args: argparse.Namespace): print(f"INFO: Fetching instances from domain='{domain}' ...") federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) except network.exceptions as exception: - print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{domain}'") + print(f"WARNING: Exception '{type(exception)}' during fetching instances (fetch_fbabot_atom) from domain='{domain}'") instances.update_last_error(domain, exception) # DEBUG: print("DEBUG: EXIT!") @@ -457,8 +457,9 @@ def fetch_instances(args: argparse.Namespace) -> int: print(f"INFO: Fetching instances from args.domain='{args.domain}' ...") federation.fetch_instances(args.domain, None, None, inspect.currentframe().f_code.co_name) except network.exceptions as exception: - print(f"WARNING: Exception '{type(exception)}' during fetching instances from args.domain='{args.domain}'") + print(f"WARNING: Exception '{type(exception)}' during fetching instances (fetch_instances) from args.domain='{args.domain}'") instances.update_last_error(args.domain, exception) + return 100 if args.single: @@ -482,7 +483,7 @@ def fetch_instances(args: argparse.Namespace) -> int: print(f"INFO: Fetching instances for instance '{row[0]}' ('{row[2]}') of origin='{row[1]}',nodeinfo_url='{row[3]}'") federation.fetch_instances(row[0], row[1], row[2], inspect.currentframe().f_code.co_name, row[3]) except network.exceptions as exception: - print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{row[0]}'") + print(f"WARNING: Exception '{type(exception)}' during fetching instances (fetch_instances) from domain='{row[0]}'") instances.update_last_error(row[0], exception) # DEBUG: print("DEBUG: EXIT!") @@ -549,9 +550,8 @@ def fetch_oliphant(args: argparse.Namespace): # DEBUG: print(f"DEBUG: response[]='{type(response)}'") if response.ok and response.content != "": # DEBUG: print(f"DEBUG: Fetched {len(response.content)} Bytes, parsing CSV ...") - #print(f"DEBUG: response.content={response.content}") reader = csv.DictReader(response.content.decode('utf-8').splitlines(), dialect="unix") - #, fieldnames='domain,severity,reject_media,reject_reports,public_comment,obfuscate' + # DEBUG: print(f"DEBUG: reader[]='{type(reader)}'") for row in reader: if not validators.domain(row["#domain"]): @@ -568,7 +568,7 @@ def fetch_oliphant(args: argparse.Namespace): print(f"INFO: Fetching instances for instane='{row['#domain']}' ...") federation.fetch_instances(row["#domain"], block["blocker"], None, inspect.currentframe().f_code.co_name) except network.exceptions as exception: - print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{row['#domain']}'") + print(f"WARNING: Exception '{type(exception)}' during fetching instances (fetch_oliphant) from domain='{row['#domain']}'") instances.update_last_error(row["#domain"], exception) # DEBUG: print("DEBUG: EXIT!") diff --git a/fba/federation.py b/fba/federation.py index 5e63317..80d7a71 100644 --- a/fba/federation.py +++ b/fba/federation.py @@ -41,7 +41,7 @@ nodeinfo_identifier = [ "http://nodeinfo.diaspora.software/ns/schema/1.0", ] -def fetch_instances(domain: str, origin: str, software: str, script: str, path: str = None): +def fetch_instances(domain: str, origin: str, software: str, command: str, path: str = None): # DEBUG: print(f"DEBUG: domain='{domain}',origin='{origin}',software='{software}',path='{path}' - CALLED!") if not isinstance(domain, str): raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") @@ -50,22 +50,25 @@ def fetch_instances(domain: str, origin: str, software: str, script: str, path: elif not isinstance(origin, str) and origin is not None: raise ValueError(f"Parameter origin[]='{type(origin)}' is not 'str'") elif software is None: + # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") + instances.update_last_instance_fetch(domain) + # DEBUG: print(f"DEBUG: software for domain='{domain}' is not set, determining ...") software = determine_software(domain, path) # DEBUG: print(f"DEBUG: Determined software='{software}' for domain='{domain}'") elif not isinstance(software, str): raise ValueError(f"Parameter software[]='{type(software)}' is not 'str'") - elif not isinstance(script, str): - raise ValueError(f"Parameter script[]='{type(script)}' is not 'str'") - elif domain == "": - raise ValueError("Parameter 'domain' is empty") + elif not isinstance(command, str): + raise ValueError(f"Parameter command[]='{type(command)}' is not 'str'") + elif command == "": + raise ValueError("Parameter 'command' is empty") if domain.split(".")[-1] == "arpa": print(f"WARNING: domain='{domain}' is a reversed .arpa domain and should not be used generally.") return elif not instances.is_registered(domain): # DEBUG: print("DEBUG: Adding new domain:", domain, origin) - instances.add(domain, origin, script, path) + instances.add(domain, origin, command, path) # DEBUG: print("DEBUG: Fetching instances for domain:", domain, software) peerlist = fetch_peers(domain, software) @@ -107,7 +110,7 @@ def fetch_instances(domain: str, origin: str, software: str, script: str, path: continue elif not instances.is_registered(instance): # DEBUG: print("DEBUG: Adding new instance:", instance, domain) - instances.add(instance, domain, script) + instances.add(instance, domain, command) # DEBUG: print("DEBUG: EXIT!") @@ -197,7 +200,7 @@ def fetch_nodeinfo(domain: str, path: str = None) -> dict: # DEBUG: print(f"DEBUG: Fetching nodeinfo from domain='{domain}' ...") nodeinfo = fetch_wellknown_nodeinfo(domain) - # DEBUG: print(f"DEBUG: nodeinfo[{type(nodeinfo)}]()='{len(nodeinfo)}'") + # DEBUG: print(f"DEBUG: nodeinfo[{type(nodeinfo)}]({len(nodeinfo)}='{nodeinfo}'") if "error_message" not in nodeinfo and "json" in nodeinfo and len(nodeinfo["json"]) > 0: # DEBUG: print(f"DEBUG: Found nodeinfo[json]()={len(nodeinfo['json'])} - EXIT!") return nodeinfo["json"] @@ -266,7 +269,7 @@ def fetch_wellknown_nodeinfo(domain: str) -> dict: # DEBUG: print(f"DEBUG: Checking CSRF for domain='{domain}'") headers = csrf.determine(domain, dict()) except network.exceptions as exception: - print(f"WARNING: Exception '{type(exception)}' during checking CSRF (fetch_wellknown,{__name__}) - EXIT!") + print(f"WARNING: Exception '{type(exception)}' during checking CSRF (fetch_wellknown_nodeinfo,{__name__}) - EXIT!") return { "status_code" : 500, "error_message": type(exception), diff --git a/fba/instances.py b/fba/instances.py index 96a4cd6..5b5046f 100644 --- a/fba/instances.py +++ b/fba/instances.py @@ -161,6 +161,7 @@ def update_last_instance_fetch(domain: str): # DEBUG: print("DEBUG: EXIT!") def update_last_blocked(domain: str): + # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") if not isinstance(domain, str): raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": diff --git a/fba/networks/friendica.py b/fba/networks/friendica.py index 6fedb47..dc63120 100644 --- a/fba/networks/friendica.py +++ b/fba/networks/friendica.py @@ -47,8 +47,12 @@ def fetch_blocks(domain: str) -> dict: blocklist = doc.find(id="about_blocklist") except network.exceptions as exception: - print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{domain}'") + print(f"WARNING: Exception '{type(exception)}' during fetching instances (friendica) from domain='{domain}'") instances.update_last_error(domain, exception) + + if instances.has_pending(domain): + instances.update_data(domain) + return dict() # Prevents exceptions: diff --git a/fba/networks/peertube.py b/fba/networks/peertube.py index f8813d1..c25cf8c 100644 --- a/fba/networks/peertube.py +++ b/fba/networks/peertube.py @@ -45,7 +45,7 @@ def fetch_peers(domain: str) -> list: while True: data = network.get_json_api( domain, - "/api/v1/server/{mode}?start={start}&count=100", + f"/api/v1/server/{mode}?start={start}&count=100", headers, (config.get("connection_timeout"), config.get("read_timeout")) ) @@ -54,8 +54,10 @@ def fetch_peers(domain: str) -> list: if "error_message" not in data: print("DEBUG: Success, data[json]:", len(data["json"])) if "data" in data["json"]: - print(f"DEBUG: Found {len(data['data'])} record(s).") - for record in data["json"]["data"]: + rows = data["json"]["data"] + + print(f"DEBUG: Found {len(rows)} record(s).") + for record in rows: print(f"DEBUG: record()={len(record)}") if mode in record and "host" in record[mode]: print(f"DEBUG: Found host={record[mode]['host']}, adding ...") @@ -63,7 +65,7 @@ def fetch_peers(domain: str) -> list: else: print(f"WARNING: record from '{domain}' has no '{mode}' or 'host' record: {record}") - if len(data["json"]["data"]) < 100: + if len(rows) < 100: print(f"DEBUG: Reached end of JSON response, domain='{domain}'") break