- raise ValueError(f"Parameter 'domain' cannot be empty")
- elif type(path) != str:
- raise ValueError(f"path[]={type(path)} is not 'str'")
- elif path == "":
- raise ValueError(f"path cannot be empty")
- elif type(parameter) != str:
- raise ValueError(f"parameter[]={type(parameter)} is not 'str'")
-
- # DEBUG: print("DEBUG: Sending POST to domain,path,parameter:", domain, path, parameter, extra_headers)
- data = {}
- try:
- res = reqto.post(f"https://{domain}{path}", data=parameter, headers={**api_headers, **extra_headers}, timeout=(config.get("connection_timeout"), config.get("read_timeout")))
-
- data = res.json()
- # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'")
- if not res.ok or res.status_code >= 400:
- print(f"WARNING: Cannot query JSON API: domain='{domain}',path='{path}',parameter()={len(parameter)},res.status_code='{res.status_code}',data[]='{type(data)}'")
- update_last_error(domain, res)
-
- except BaseException as e:
- print(f"WARNING: Some error during post(): domain='{domain}',path='{path}',parameter()={len(parameter)},exception[{type(e)}]:'{str(e)}'")
-
- # DEBUG: print(f"DEBUG: Returning data({len(data)})=[]:{type(data)}")
- return data
-
-def fetch_nodeinfo(domain: str, path: str = None) -> list:
- # DEBUG: print(f"DEBUG: domain='{domain}',path={path} - CALLED!")
- if type(domain) != str:
- raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
- elif domain == "":
- raise ValueError(f"Parameter 'domain' cannot be empty")
- elif type(path) != str and path != None:
- raise ValueError(f"Parameter path[]={type(path)} is not 'str'")
-
- # DEBUG: print("DEBUG: Fetching nodeinfo from domain,path:", domain, path)
- nodeinfo = fetch_wellknown_nodeinfo(domain)
-
- # DEBUG: print(f"DEBUG: nodeinfo({len(nodeinfo)})={nodeinfo}")
- if len(nodeinfo) > 0:
- # DEBUG: print("DEBUG: nodeinfo()={len(nodeinfo))} - EXIT!")
- return nodeinfo
-
- requests = [
- f"https://{domain}/nodeinfo/2.1.json",
- f"https://{domain}/nodeinfo/2.1",
- f"https://{domain}/nodeinfo/2.0.json",
- f"https://{domain}/nodeinfo/2.0",
- f"https://{domain}/nodeinfo/1.0",
- f"https://{domain}/api/v1/instance"
- ]
-
- data = {}
- for request in requests:
- if path != None and path != "" and request != path:
- # DEBUG: print(f"DEBUG: path='{path}' does not match request='{request}' - SKIPPED!")
- continue
-
- try:
- # DEBUG: print("DEBUG: Fetching request:", request)
- res = reqto.get(request, headers=api_headers, timeout=(config.get("connection_timeout"), config.get("read_timeout")))
-
- data = res.json()
- # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'")
- if res.ok and isinstance(data, dict):
- # DEBUG: print("DEBUG: Success:", request)
- instances.set_instance_data("detection_mode", domain, "STATIC_CHECK")
- instances.set_instance_data("nodeinfo_url" , domain, request)
- break
- elif res.ok and isinstance(data, list):
- # DEBUG: print(f"DEBUG: domain='{domain}' returned a list: '{data}'")
- sys.exit(255)
- elif not res.ok or res.status_code >= 400:
- print("WARNING: Failed fetching nodeinfo from domain:", domain)
- update_last_error(domain, res)
- continue
-
- except BaseException as e:
- # DEBUG: print("DEBUG: Cannot fetch API request:", request)
- update_last_error(domain, e)
- pass
-
- # DEBUG: print(f"DEBUG: data()={len(data)} - EXIT!")
- return data
-
-def fetch_wellknown_nodeinfo(domain: str) -> list:
- # DEBUG: print(f"DEBUG: domain={domain} - CALLED!")
- if type(domain) != str:
- raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
- elif domain == "":
- raise ValueError(f"Parameter 'domain' cannot be empty")
-
- # DEBUG: print("DEBUG: Fetching .well-known info for domain:", domain)
- data = {}
-
- try:
- res = reqto.get(f"https://{domain}/.well-known/nodeinfo", headers=api_headers, timeout=(config.get("connection_timeout"), config.get("read_timeout")))
-
- data = res.json()
- # DEBUG: print("DEBUG: domain,res.ok,data[]:", domain, res.ok, type(data))
- if res.ok and isinstance(data, dict):
- nodeinfo = data
- # DEBUG: print("DEBUG: Found entries:", len(nodeinfo), domain)
- if "links" in nodeinfo:
- # DEBUG: print("DEBUG: Found links in nodeinfo():", len(nodeinfo["links"]))
- for link in nodeinfo["links"]:
- # DEBUG: print("DEBUG: rel,href:", link["rel"], link["href"])
- if link["rel"] in nodeinfo_identifier:
- # DEBUG: print("DEBUG: Fetching nodeinfo from:", link["href"])
- res = reqto.get(link["href"])
-
- data = res.json()
- # DEBUG: print("DEBUG: href,res.ok,res.status_code:", link["href"], res.ok, res.status_code)
- if res.ok and isinstance(data, dict):
- # DEBUG: print("DEBUG: Found JSON nodeinfo():", len(data))
- instances.set_instance_data("detection_mode", domain, "AUTO_DISCOVERY")
- instances.set_instance_data("nodeinfo_url" , domain, link["href"])
- break
- else:
- print("WARNING: Unknown 'rel' value:", domain, link["rel"])
- else:
- print("WARNING: nodeinfo does not contain 'links':", domain)
-
- except BaseException as e:
- print("WARNING: Failed fetching .well-known info:", domain)
- update_last_error(domain, e)
- pass
-
- # DEBUG: print("DEBUG: Returning data[]:", type(data))
- return data
-
-def fetch_generator_from_path(domain: str, path: str = "/") -> str:
- # DEBUG: print(f"DEBUG: domain({len(domain)})={domain},path={path} - CALLED!")
- if type(domain) != str:
- raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
- elif domain == "":
- raise ValueError(f"Parameter 'domain' cannot be empty")
- elif type(path) != str:
- raise ValueError(f"path[]={type(path)} is not 'str'")
- elif path == "":
- raise ValueError(f"Parameter 'domain' cannot be empty")
-
- # DEBUG: print(f"DEBUG: domain='{domain}',path='{path}' - CALLED!")
- software = None
-
- try:
- # DEBUG: print(f"DEBUG: Fetching path='{path}' from '{domain}' ...")
- res = reqto.get(f"https://{domain}{path}", headers=headers, timeout=(config.get("connection_timeout"), config.get("read_timeout")))
-
- # DEBUG: print("DEBUG: domain,res.ok,res.status_code,res.text[]:", domain, res.ok, res.status_code, type(res.text))
- if res.ok and res.status_code < 300 and len(res.text) > 0:
- # DEBUG: print("DEBUG: Search for <meta name='generator'>:", domain)
- doc = bs4.BeautifulSoup(res.text, "html.parser")
-
- # DEBUG: print("DEBUG: doc[]:", type(doc))
- generator = doc.find("meta", {"name": "generator"})
- site_name = doc.find("meta", {"property": "og:site_name"})
-
- # DEBUG: print(f"DEBUG: generator='{generator}',site_name='{site_name}'")
- if isinstance(generator, bs4.element.Tag):
- # DEBUG: print("DEBUG: Found generator meta tag:", domain)
- software = tidyup_domain(generator.get("content"))
- print(f"INFO: domain='{domain}' is generated by '{software}'")
- instances.set_instance_data("detection_mode", domain, "GENERATOR")
- remove_pending_error(domain)
- elif isinstance(site_name, bs4.element.Tag):
- # DEBUG: print("DEBUG: Found property=og:site_name:", domain)
- sofware = tidyup_domain(site_name.get("content"))
- print(f"INFO: domain='{domain}' has og:site_name='{software}'")
- instances.set_instance_data("detection_mode", domain, "SITE_NAME")
- remove_pending_error(domain)
-
- except BaseException as e:
- # DEBUG: print(f"DEBUG: Cannot fetch / from '{domain}':", e)
- update_last_error(domain, e)
- pass
-
- # DEBUG: print(f"DEBUG: software[]={type(software)}")
- if type(software) is str and software == "":
- # DEBUG: print(f"DEBUG: Corrected empty string to None for software of domain='{domain}'")
- software = None
- elif type(software) is str and ("." in software or " " in software):
- # DEBUG: print(f"DEBUG: software='{software}' may contain a version number, domain='{domain}', removing it ...")
- software = remove_version(software)
-
- # DEBUG: print(f"DEBUG: software[]={type(software)}")
- if type(software) is str and " powered by " in software:
- # DEBUG: print(f"DEBUG: software='{software}' has 'powered by' in it")
- software = remove_version(strip_powered_by(software))
- elif type(software) is str and " hosted on " in software:
- # DEBUG: print(f"DEBUG: software='{software}' has 'hosted on' in it")
- software = remove_version(strip_hosted_on(software))
- elif type(software) is str and " by " in software:
- # DEBUG: print(f"DEBUG: software='{software}' has ' by ' in it")
- software = strip_until(software, " by ")
- elif type(software) is str and " see " in software:
- # DEBUG: print(f"DEBUG: software='{software}' has ' see ' in it")
- software = strip_until(software, " see ")
-
- # DEBUG: print(f"DEBUG: software='{software}' - EXIT!")
- return software
-
-def determine_software(domain: str, path: str = None) -> str:
- # DEBUG: print(f"DEBUG: domain({len(domain)})={domain},path={path} - CALLED!")
- if type(domain) != str:
- raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
- elif domain == "":
- raise ValueError(f"Parameter 'domain' cannot be empty")
- elif type(path) != str and path != None:
- raise ValueError(f"Parameter path[]={type(path)} is not 'str'")
-
- # DEBUG: print("DEBUG: Determining software for domain,path:", domain, path)
- software = None
-
- # DEBUG: print(f"DEBUG: Fetching nodeinfo from '{domain}' ...")
- data = fetch_nodeinfo(domain, path)
-
- # DEBUG: print("DEBUG: data[]:", type(data))
- if not isinstance(data, dict) or len(data) == 0:
- # DEBUG: print("DEBUG: Could not determine software type:", domain)
- return fetch_generator_from_path(domain)
-
- # DEBUG: print("DEBUG: data():", len(data), data)
- if "status" in data and data["status"] == "error" and "message" in data:
- print("WARNING: JSON response is an error:", data["message"])
- update_last_error(domain, data["message"])
- return fetch_generator_from_path(domain)
- elif "message" in data:
- print("WARNING: JSON response contains only a message:", data["message"])
- update_last_error(domain, data["message"])
- return fetch_generator_from_path(domain)
- elif "software" not in data or "name" not in data["software"]:
- # DEBUG: print(f"DEBUG: JSON response from domain='{domain}' does not include [software][name], fetching / ...")
- software = fetch_generator_from_path(domain)
-
- # DEBUG: print(f"DEBUG: Generator for domain='{domain}' is: {software}, EXIT!")
- return software
-
- software = tidyup_domain(data["software"]["name"])
-
- # DEBUG: print("DEBUG: sofware after tidyup_domain():", software)
- if software in ["akkoma", "rebased"]:
- # DEBUG: print("DEBUG: Setting pleroma:", domain, software)
- software = "pleroma"
- elif software in ["hometown", "ecko"]:
- # DEBUG: print("DEBUG: Setting mastodon:", domain, software)
- software = "mastodon"
- elif software in ["calckey", "groundpolis", "foundkey", "cherrypick", "meisskey"]:
- # DEBUG: print("DEBUG: Setting misskey:", domain, software)
- software = "misskey"
- elif software.find("/") > 0:
- print("WARNING: Spliting of slash:", software)
- software = software.split("/")[-1];
- elif software.find("|") > 0:
- print("WARNING: Spliting of pipe:", software)
- software = tidyup_domain(software.split("|")[0]);
- elif "powered by" in software:
- # DEBUG: print(f"DEBUG: software='{software}' has 'powered by' in it")
- software = strip_powered_by(software)
- elif type(software) is str and " by " in software:
- # DEBUG: print(f"DEBUG: software='{software}' has ' by ' in it")
- software = strip_until(software, " by ")
- elif type(software) is str and " see " in software:
- # DEBUG: print(f"DEBUG: software='{software}' has ' see ' in it")
- software = strip_until(software, " see ")
-
- # DEBUG: print(f"DEBUG: software[]={type(software)}")
- if software == "":
- print("WARNING: tidyup_domain() left no software name behind:", domain)
- software = None
-
- # DEBUG: print(f"DEBUG: software[]={type(software)}")
- if str(software) == "":
- # DEBUG: print(f"DEBUG: software for '{domain}' was not detected, trying generator ...")
- software = fetch_generator_from_path(domain)
- elif len(str(software)) > 0 and ("." in software or " " in software):
- # DEBUG: print(f"DEBUG: software='{software}' may contain a version number, domain='{domain}', removing it ...")
- software = remove_version(software)
-
- # DEBUG: print(f"DEBUG: software[]={type(software)}")
- if type(software) is str and "powered by" in software:
- # DEBUG: print(f"DEBUG: software='{software}' has 'powered by' in it")
- software = remove_version(strip_powered_by(software))
-
- # DEBUG: print("DEBUG: Returning domain,software:", domain, software)
- return software
-
-def update_block_reason(reason: str, blocker: str, blocked: str, block_level: str):
- # DEBUG: print(f"DEBUG: reason='{reason}',blocker={blocker},blocked={blocked},block_level={block_level} - CALLED!")
- if type(reason) != str and reason != None:
- raise ValueError(f"Parameter reason[]='{type(reason)}' is not 'str'")
- elif type(blocker) != str: