X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;f=fba%2Ffederation.py;h=da7277cbce40668754ef15c8d802595a7ad23ff9;hb=c8b6b6a4aea21b3fefc8fd4900391d59751ff814;hp=9f1e0098cc4f09f21a844259488bfcd0e04a800a;hpb=f955b100e984685baeb0c27b0705148d765bb9ce;p=fba.git diff --git a/fba/federation.py b/fba/federation.py index 9f1e009..da7277c 100644 --- a/fba/federation.py +++ b/fba/federation.py @@ -44,23 +44,26 @@ nodeinfo_identifier = [ def fetch_instances(domain: str, origin: str, software: str, script: str, path: str = None): # DEBUG: print(f"DEBUG: domain='{domain}',origin='{origin}',software='{software}',path='{path}' - CALLED!") if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") elif not isinstance(origin, str) and origin is not None: - raise ValueError(f"Parameter origin[]={type(origin)} is not 'str'") + raise ValueError(f"Parameter origin[]='{type(origin)}' is not 'str'") elif software is None: # DEBUG: print(f"DEBUG: software for domain='{domain}' is not set, determining ...") software = determine_software(domain, path) # DEBUG: print(f"DEBUG: Determined software='{software}' for domain='{domain}'") elif not isinstance(software, str): - raise ValueError(f"Parameter software[]={type(software)} is not 'str'") + raise ValueError(f"Parameter software[]='{type(software)}' is not 'str'") elif not isinstance(script, str): - raise ValueError(f"Parameter script[]={type(script)} is not 'str'") + raise ValueError(f"Parameter script[]='{type(script)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") - if not instances.is_registered(domain): + if domain.split(".")[-1] == "arpa": + print(f"WARNING: domain='{domain}' is a reversed .arpa domain and should not be used generally.") + return + elif not instances.is_registered(domain): # DEBUG: print("DEBUG: Adding new domain:", domain, origin) instances.add(domain, origin, script, path) @@ -74,8 +77,9 @@ def fetch_instances(domain: str, origin: str, software: str, script: str, path: # DEBUG: print(f"DEBUG: domain='{domain}' has pending nodeinfo data, flushing ...") instances.update_data(domain) - print(f"INFO: Checking {len(peerlist)} instances from {domain} ...") + print(f"INFO: Checking {len(peerlist)} instances from domain='{domain}' ...") for instance in peerlist: + # DEBUG: print(f"DEBUG: instance='{instance}'") if instance is None: # Skip "None" types as tidup.domain() cannot parse them continue @@ -85,17 +89,23 @@ def fetch_instances(domain: str, origin: str, software: str, script: str, path: # DEBUG: print(f"DEBUG: instance='{instance}' - AFTER") if instance == "": - print("WARNING: Empty instance after tidyup.domain(), domain:", domain) + print(f"WARNING: Empty instance after tidyup.domain(), domain='{domain}'") continue elif not validators.domain(instance.split("/")[0]): print(f"WARNING: Bad instance='{instance}' from domain='{domain}',origin='{origin}',software='{software}'") continue + elif instance.split(".")[-1] == "arpa": + print(f"WARNING: instance='{instance}' is a reversed .arpa domain and should not be used generally.") + continue elif blacklist.is_blacklisted(instance): # DEBUG: print("DEBUG: instance is blacklisted:", instance) continue # DEBUG: print("DEBUG: Handling instance:", instance) - if not instances.is_registered(instance): + if instance.split(".")[-1] == "arpa": + print(f"WARNING: instance='{instance}' is a reversed .arpa domain and should not be used generally.") + continue + elif not instances.is_registered(instance): # DEBUG: print("DEBUG: Adding new instance:", instance, domain) instances.add(instance, domain, script) @@ -104,11 +114,11 @@ def fetch_instances(domain: str, origin: str, software: str, script: str, path: def fetch_peers(domain: str, software: str) -> list: # DEBUG: print(f"DEBUG: domain({len(domain)})={domain},software={software} - CALLED!") if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") elif not isinstance(software, str) and software is not None: - raise ValueError(f"software[]={type(software)} is not 'str'") + raise ValueError(f"software[]='{type(software)}' is not 'str'") if software == "misskey": # DEBUG: print(f"DEBUG: Invoking misskey.fetch_peers({domain}) ...") @@ -122,14 +132,15 @@ def fetch_peers(domain: str, software: str) -> list: # Init peers variable peers = list() + # No CSRF by default, you don't have to add network.api_headers by yourself here headers = tuple() - # DEBUG: print(f"DEBUG: Checking CSRF for domain='{domain}'") try: - headers = csrf.determine(domain, dict()) + # DEBUG: print(f"DEBUG: Checking CSRF for domain='{domain}'") + headers = csrf.determine(domain, dict()) except network.exceptions as exception: - print(f"WARNING: Exception '{type(exception)}' during checking CSRF - EXIT!") - return + print(f"WARNING: Exception '{type(exception)}' during checking CSRF (fetch_peers,{__name__}) - EXIT!") + return peers # DEBUG: print(f"DEBUG: Fetching peers from '{domain}',software='{software}' ...") data = network.get_json_api( @@ -173,31 +184,37 @@ def fetch_peers(domain: str, software: str) -> list: # DEBUG: print("DEBUG: Returning peers[]:", type(peers)) return peers -def fetch_nodeinfo(domain: str, path: str = None) -> list: +def fetch_nodeinfo(domain: str, path: str = None) -> dict: # DEBUG: print(f"DEBUG: domain='{domain}',path={path} - CALLED!") if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") elif not isinstance(path, str) and path is not None: - raise ValueError(f"Parameter path[]={type(path)} is not 'str'") + raise ValueError(f"Parameter path[]='{type(path)}' is not 'str'") # DEBUG: print(f"DEBUG: Fetching nodeinfo from domain='{domain}' ...") nodeinfo = fetch_wellknown_nodeinfo(domain) - # DEBUG: print(f"DEBUG: nodeinfo({len(nodeinfo)})={nodeinfo}") - if len(nodeinfo) > 0: - # DEBUG: print("DEBUG: nodeinfo()={len(nodeinfo))} - EXIT!") - return nodeinfo + # DEBUG: print(f"DEBUG: nodeinfo[{type(nodeinfo)}]()='{len(nodeinfo)}'") + if "error_message" not in nodeinfo and "json" in nodeinfo and len(nodeinfo["json"]) > 0: + # DEBUG: print(f"DEBUG: Found nodeinfo[json]()={len(nodeinfo['json'])} - EXIT!") + return nodeinfo["json"] + # No CSRF by default, you don't have to add network.api_headers by yourself here headers = tuple() + data = dict() - # DEBUG: print(f"DEBUG: Checking CSRF for domain='{domain}'") try: - headers = csrf.determine(domain, dict()) + # DEBUG: print(f"DEBUG: Checking CSRF for domain='{domain}'") + headers = csrf.determine(domain, dict()) except network.exceptions as exception: - print(f"WARNING: Exception '{type(exception)}' during checking CSRF - EXIT!") - return + print(f"WARNING: Exception '{type(exception)}' during checking CSRF (nodeinfo,{__name__}) - EXIT!") + return { + "status_code" : 500, + "error_message": f"exception[{type(exception)}]='{str(exception)}'", + "exception" : exception, + } request_paths = [ "/nodeinfo/2.1.json", @@ -209,6 +226,7 @@ def fetch_nodeinfo(domain: str, path: str = None) -> list: ] for request in request_paths: + # DEBUG: print(f"DEBUG: path[{type(path)}]='{path}',request='{request}'") if path is not None and path != "" and path != request: # DEBUG: print(f"DEBUG: path='{path}' does not match request='{request}' - SKIPPED!") continue @@ -233,21 +251,26 @@ def fetch_nodeinfo(domain: str, path: str = None) -> list: # DEBUG: print(f"DEBUG: data()={len(data)} - EXIT!") return data -def fetch_wellknown_nodeinfo(domain: str) -> list: +def fetch_wellknown_nodeinfo(domain: str) -> dict: # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") + # No CSRF by default, you don't have to add network.api_headers by yourself here headers = tuple() - # DEBUG: print(f"DEBUG: Checking CSRF for domain='{domain}'") try: - headers = csrf.determine(domain, dict()) + # DEBUG: print(f"DEBUG: Checking CSRF for domain='{domain}'") + headers = csrf.determine(domain, dict()) except network.exceptions as exception: - print(f"WARNING: Exception '{type(exception)}' during checking CSRF - EXIT!") - return + print(f"WARNING: Exception '{type(exception)}' during checking CSRF (fetch_wellknown,{__name__}) - EXIT!") + return { + "status_code" : 500, + "error_message": type(exception), + "exception" : exception, + } # DEBUG: print("DEBUG: Fetching .well-known info for domain:", domain) data = network.get_json_api( @@ -290,11 +313,11 @@ def fetch_wellknown_nodeinfo(domain: str) -> list: def fetch_generator_from_path(domain: str, path: str = "/") -> str: # DEBUG: print(f"DEBUG: domain({len(domain)})={domain},path={path} - CALLED!") if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") elif not isinstance(path, str): - raise ValueError(f"path[]={type(path)} is not 'str'") + raise ValueError(f"path[]='{type(path)}' is not 'str'") elif path == "": raise ValueError("Parameter 'path' is empty") @@ -325,7 +348,7 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str: print(f"INFO: domain='{domain}' has og:site_name='{software}'") instances.set_data("detection_mode", domain, "SITE_NAME") - # DEBUG: print(f"DEBUG: software[]={type(software)}") + # DEBUG: print(f"DEBUG: software[]='{type(software)}'") if isinstance(software, str) and software == "": # DEBUG: print(f"DEBUG: Corrected empty string to None for software of domain='{domain}'") software = None @@ -333,7 +356,7 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str: # DEBUG: print(f"DEBUG: software='{software}' may contain a version number, domain='{domain}', removing it ...") software = fba.remove_version(software) - # DEBUG: print(f"DEBUG: software[]={type(software)}") + # DEBUG: print(f"DEBUG: software[]='{type(software)}'") if isinstance(software, str) and " powered by " in software: # DEBUG: print(f"DEBUG: software='{software}' has 'powered by' in it") software = fba.remove_version(fba.strip_powered_by(software)) @@ -353,11 +376,11 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str: def determine_software(domain: str, path: str = None) -> str: # DEBUG: print(f"DEBUG: domain({len(domain)})={domain},path={path} - CALLED!") if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") elif not isinstance(path, str) and path is not None: - raise ValueError(f"Parameter path[]={type(path)} is not 'str'") + raise ValueError(f"Parameter path[]='{type(path)}' is not 'str'") # DEBUG: print("DEBUG: Determining software for domain,path:", domain, path) software = None @@ -365,28 +388,29 @@ def determine_software(domain: str, path: str = None) -> str: # DEBUG: print(f"DEBUG: Fetching nodeinfo from '{domain}' ...") data = fetch_nodeinfo(domain, path) - # DEBUG: print("DEBUG: data[]:", type(data)) - if "error_message" in data: - # DEBUG: print("DEBUG: Could not determine software type:", domain) + # DEBUG: print(f"DEBUG: data[{type(data)}]='{data}'") + if "exception" in data: + # Continue raising it + raise data["exception"] + elif "error_message" in data: + # DEBUG: print(f"DEBUG: Returned error_message during fetching nodeinfo: '{data['error_message']}',status_code='{data['status_code']}'") return fetch_generator_from_path(domain) - - # DEBUG: print("DEBUG: data():", len(data), data) - if "status" in data["json"] and data["json"]["status"] == "error" and "message" in data["json"]: - print("WARNING: JSON response is an error:", data["json"]["message"]) - instances.update_last_error(domain, data["json"]["message"]) + elif "status" in data and data["status"] == "error" and "message" in data: + print("WARNING: JSON response is an error:", data["message"]) + instances.update_last_error(domain, data["message"]) return fetch_generator_from_path(domain) - elif "message" in data["json"]: + elif "message" in data: print("WARNING: JSON response contains only a message:", data["message"]) - instances.update_last_error(domain, data["json"]["message"]) + instances.update_last_error(domain, data["message"]) return fetch_generator_from_path(domain) - elif "software" not in data["json"] or "name" not in data["json"]["software"]: + elif "software" not in data or "name" not in data["software"]: # DEBUG: print(f"DEBUG: JSON response from domain='{domain}' does not include [software][name], fetching / ...") software = fetch_generator_from_path(domain) # DEBUG: print(f"DEBUG: Generator for domain='{domain}' is: {software}, EXIT!") return software - software = tidyup.domain(data["json"]["software"]["name"]) + software = tidyup.domain(data["software"]["name"]) # DEBUG: print("DEBUG: sofware after tidyup.domain():", software) if software in ["akkoma", "rebased"]: @@ -414,12 +438,12 @@ def determine_software(domain: str, path: str = None) -> str: # DEBUG: print(f"DEBUG: software='{software}' has ' see ' in it") software = fba.strip_until(software, " see ") - # DEBUG: print(f"DEBUG: software[]={type(software)}") + # DEBUG: print(f"DEBUG: software[]='{type(software)}'") if software == "": print("WARNING: tidyup.domain() left no software name behind:", domain) software = None - # DEBUG: print(f"DEBUG: software[]={type(software)}") + # DEBUG: print(f"DEBUG: software[]='{type(software)}'") if str(software) == "": # DEBUG: print(f"DEBUG: software for '{domain}' was not detected, trying generator ...") software = fetch_generator_from_path(domain) @@ -427,7 +451,7 @@ def determine_software(domain: str, path: str = None) -> str: # DEBUG: print(f"DEBUG: software='{software}' may contain a version number, domain='{domain}', removing it ...") software = fba.remove_version(software) - # DEBUG: print(f"DEBUG: software[]={type(software)}") + # DEBUG: print(f"DEBUG: software[]='{type(software)}'") if isinstance(software, str) and "powered by" in software: # DEBUG: print(f"DEBUG: software='{software}' has 'powered by' in it") software = fba.remove_version(fba.strip_powered_by(software)) @@ -436,15 +460,15 @@ def determine_software(domain: str, path: str = None) -> str: return software def find_domains(tag: bs4.element.Tag) -> list: - # DEBUG: print(f"DEBUG: tag[]={type(tag)} - CALLED!") + # DEBUG: print(f"DEBUG: tag[]='{type(tag)}' - CALLED!") if not isinstance(tag, bs4.element.Tag): - raise ValueError(f"Parameter tag[]={type(tag)} is not type of bs4.element.Tag") + raise ValueError(f"Parameter tag[]='{type(tag)}' is not type of bs4.element.Tag") elif len(tag.select("tr")) == 0: raise KeyError("No table rows found in table!") domains = list() for element in tag.select("tr"): - # DEBUG: print(f"DEBUG: element[]={type(element)}") + # DEBUG: print(f"DEBUG: element[]='{type(element)}'") if not element.find("td"): # DEBUG: print("DEBUG: Skipping element, no found") continue