# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
-import sys
-
import bs4
import validators
from fba import blacklist
from fba import config
+from fba import csrf
from fba import fba
from fba import instances
from fba import network
def fetch_instances(domain: str, origin: str, software: str, script: str, path: str = None):
# DEBUG: print(f"DEBUG: domain='{domain}',origin='{origin}',software='{software}',path='{path}' - CALLED!")
if not isinstance(domain, str):
- raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
+ raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
elif domain == "":
raise ValueError("Parameter 'domain' is empty")
elif not isinstance(origin, str) and origin is not None:
- raise ValueError(f"Parameter origin[]={type(origin)} is not 'str'")
+ raise ValueError(f"Parameter origin[]='{type(origin)}' is not 'str'")
elif software is None:
# DEBUG: print(f"DEBUG: software for domain='{domain}' is not set, determining ...")
software = determine_software(domain, path)
# DEBUG: print(f"DEBUG: Determined software='{software}' for domain='{domain}'")
elif not isinstance(software, str):
- raise ValueError(f"Parameter software[]={type(software)} is not 'str'")
+ raise ValueError(f"Parameter software[]='{type(software)}' is not 'str'")
elif not isinstance(script, str):
- raise ValueError(f"Parameter script[]={type(script)} is not 'str'")
+ raise ValueError(f"Parameter script[]='{type(script)}' is not 'str'")
elif domain == "":
raise ValueError("Parameter 'domain' is empty")
def fetch_peers(domain: str, software: str) -> list:
# DEBUG: print(f"DEBUG: domain({len(domain)})={domain},software={software} - CALLED!")
if not isinstance(domain, str):
- raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
+ raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
elif domain == "":
raise ValueError("Parameter 'domain' is empty")
elif not isinstance(software, str) and software is not None:
- raise ValueError(f"software[]={type(software)} is not 'str'")
+ raise ValueError(f"software[]='{type(software)}' is not 'str'")
if software == "misskey":
# DEBUG: print(f"DEBUG: Invoking misskey.fetch_peers({domain}) ...")
# DEBUG: print(f"DEBUG: Invoking peertube.fetch_peers({domain}) ...")
return peertube.fetch_peers(domain)
+ # Init peers variable
+ peers = list()
+ headers = tuple()
+
+ # DEBUG: print(f"DEBUG: Checking CSRF for domain='{domain}'")
+ try:
+ headers = csrf.determine(domain, dict())
+ except network.exceptions as exception:
+ print(f"WARNING: Exception '{type(exception)}' during checking CSRF - EXIT!")
+ return
+
# DEBUG: print(f"DEBUG: Fetching peers from '{domain}',software='{software}' ...")
data = network.get_json_api(
domain,
"/api/v1/instance/peers",
+ headers,
(config.get("connection_timeout"), config.get("read_timeout"))
)
- # DEBUG: print(f"DEBUG: data[]='{type(data)}'")
+ # DEBUG: print(f"DEBUG: data[]='{type(data)}'")
if "error_message" in data:
# DEBUG: print("DEBUG: Was not able to fetch peers, trying alternative ...")
data = network.get_json_api(
domain,
"/api/v3/site",
+ headers,
(config.get("connection_timeout"), config.get("read_timeout"))
)
peers = peers + add_peers(data["json"]["federated_instances"])
# DEBUG: print("DEBUG: Added instance(s) to peers")
else:
- print("WARNING: JSON response does not contain 'federated_instances':", domain)
- instances.update_last_error(domain, response)
+ message = "JSON response does not contain 'federated_instances' or 'error_message'"
+ print(f"WARNING: {message},domain='{domain}'")
+ instances.update_last_error(domain, message)
else:
# DEBUG: print("DEBUG: Querying API was successful:", domain, len(data))
peers = data["json"]
def fetch_nodeinfo(domain: str, path: str = None) -> list:
# DEBUG: print(f"DEBUG: domain='{domain}',path={path} - CALLED!")
if not isinstance(domain, str):
- raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
+ raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
elif domain == "":
raise ValueError("Parameter 'domain' is empty")
elif not isinstance(path, str) and path is not None:
- raise ValueError(f"Parameter path[]={type(path)} is not 'str'")
+ raise ValueError(f"Parameter path[]='{type(path)}' is not 'str'")
# DEBUG: print(f"DEBUG: Fetching nodeinfo from domain='{domain}' ...")
nodeinfo = fetch_wellknown_nodeinfo(domain)
# DEBUG: print("DEBUG: nodeinfo()={len(nodeinfo))} - EXIT!")
return nodeinfo
+ headers = tuple()
+
+ # DEBUG: print(f"DEBUG: Checking CSRF for domain='{domain}'")
+ try:
+ headers = csrf.determine(domain, dict())
+ except network.exceptions as exception:
+ print(f"WARNING: Exception '{type(exception)}' during checking CSRF - EXIT!")
+ return
+
request_paths = [
"/nodeinfo/2.1.json",
"/nodeinfo/2.1",
data = network.get_json_api(
domain,
request,
+ headers,
(config.get("nodeinfo_connection_timeout"), config.get("nodeinfo_read_timeout"))
)
def fetch_wellknown_nodeinfo(domain: str) -> list:
# DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
if not isinstance(domain, str):
- raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
+ raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
elif domain == "":
raise ValueError("Parameter 'domain' is empty")
+ headers = tuple()
+
+ # DEBUG: print(f"DEBUG: Checking CSRF for domain='{domain}'")
+ try:
+ headers = csrf.determine(domain, dict())
+ except network.exceptions as exception:
+ print(f"WARNING: Exception '{type(exception)}' during checking CSRF - EXIT!")
+ return
+
# DEBUG: print("DEBUG: Fetching .well-known info for domain:", domain)
data = network.get_json_api(
domain,
"/.well-known/nodeinfo",
+ headers,
(config.get("nodeinfo_connection_timeout"), config.get("nodeinfo_read_timeout"))
)
def fetch_generator_from_path(domain: str, path: str = "/") -> str:
# DEBUG: print(f"DEBUG: domain({len(domain)})={domain},path={path} - CALLED!")
if not isinstance(domain, str):
- raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
+ raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
elif domain == "":
raise ValueError("Parameter 'domain' is empty")
elif not isinstance(path, str):
- raise ValueError(f"path[]={type(path)} is not 'str'")
+ raise ValueError(f"path[]='{type(path)}' is not 'str'")
elif path == "":
raise ValueError("Parameter 'path' is empty")
print(f"INFO: domain='{domain}' has og:site_name='{software}'")
instances.set_data("detection_mode", domain, "SITE_NAME")
- # DEBUG: print(f"DEBUG: software[]={type(software)}")
+ # DEBUG: print(f"DEBUG: software[]='{type(software)}'")
if isinstance(software, str) and software == "":
# DEBUG: print(f"DEBUG: Corrected empty string to None for software of domain='{domain}'")
software = None
# DEBUG: print(f"DEBUG: software='{software}' may contain a version number, domain='{domain}', removing it ...")
software = fba.remove_version(software)
- # DEBUG: print(f"DEBUG: software[]={type(software)}")
+ # DEBUG: print(f"DEBUG: software[]='{type(software)}'")
if isinstance(software, str) and " powered by " in software:
# DEBUG: print(f"DEBUG: software='{software}' has 'powered by' in it")
software = fba.remove_version(fba.strip_powered_by(software))
def determine_software(domain: str, path: str = None) -> str:
# DEBUG: print(f"DEBUG: domain({len(domain)})={domain},path={path} - CALLED!")
if not isinstance(domain, str):
- raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
+ raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
elif domain == "":
raise ValueError("Parameter 'domain' is empty")
elif not isinstance(path, str) and path is not None:
- raise ValueError(f"Parameter path[]={type(path)} is not 'str'")
+ raise ValueError(f"Parameter path[]='{type(path)}' is not 'str'")
# DEBUG: print("DEBUG: Determining software for domain,path:", domain, path)
software = None
data = fetch_nodeinfo(domain, path)
# DEBUG: print("DEBUG: data[]:", type(data))
- if not isinstance(data, dict) or len(data) == 0:
+ if "error_message" in data:
# DEBUG: print("DEBUG: Could not determine software type:", domain)
return fetch_generator_from_path(domain)
# DEBUG: print("DEBUG: data():", len(data), data)
- if "status" in data and data["status"] == "error" and "message" in data:
- print("WARNING: JSON response is an error:", data["message"])
- instances.update_last_error(domain, data["message"])
+ if "status" in data["json"] and data["json"]["status"] == "error" and "message" in data["json"]:
+ print("WARNING: JSON response is an error:", data["json"]["message"])
+ instances.update_last_error(domain, data["json"]["message"])
return fetch_generator_from_path(domain)
- elif "message" in data:
+ elif "message" in data["json"]:
print("WARNING: JSON response contains only a message:", data["message"])
- instances.update_last_error(domain, data["message"])
+ instances.update_last_error(domain, data["json"]["message"])
return fetch_generator_from_path(domain)
- elif "software" not in data or "name" not in data["software"]:
+ elif "software" not in data["json"] or "name" not in data["json"]["software"]:
# DEBUG: print(f"DEBUG: JSON response from domain='{domain}' does not include [software][name], fetching / ...")
software = fetch_generator_from_path(domain)
# DEBUG: print(f"DEBUG: Generator for domain='{domain}' is: {software}, EXIT!")
return software
- software = tidyup.domain(data["software"]["name"])
+ software = tidyup.domain(data["json"]["software"]["name"])
# DEBUG: print("DEBUG: sofware after tidyup.domain():", software)
if software in ["akkoma", "rebased"]:
# DEBUG: print(f"DEBUG: software='{software}' has ' see ' in it")
software = fba.strip_until(software, " see ")
- # DEBUG: print(f"DEBUG: software[]={type(software)}")
+ # DEBUG: print(f"DEBUG: software[]='{type(software)}'")
if software == "":
print("WARNING: tidyup.domain() left no software name behind:", domain)
software = None
- # DEBUG: print(f"DEBUG: software[]={type(software)}")
+ # DEBUG: print(f"DEBUG: software[]='{type(software)}'")
if str(software) == "":
# DEBUG: print(f"DEBUG: software for '{domain}' was not detected, trying generator ...")
software = fetch_generator_from_path(domain)
# DEBUG: print(f"DEBUG: software='{software}' may contain a version number, domain='{domain}', removing it ...")
software = fba.remove_version(software)
- # DEBUG: print(f"DEBUG: software[]={type(software)}")
+ # DEBUG: print(f"DEBUG: software[]='{type(software)}'")
if isinstance(software, str) and "powered by" in software:
# DEBUG: print(f"DEBUG: software='{software}' has 'powered by' in it")
software = fba.remove_version(fba.strip_powered_by(software))
return software
def find_domains(tag: bs4.element.Tag) -> list:
- # DEBUG: print(f"DEBUG: tag[]={type(tag)} - CALLED!")
+ # DEBUG: print(f"DEBUG: tag[]='{type(tag)}' - CALLED!")
if not isinstance(tag, bs4.element.Tag):
- raise ValueError(f"Parameter tag[]={type(tag)} is not type of bs4.element.Tag")
+ raise ValueError(f"Parameter tag[]='{type(tag)}' is not type of bs4.element.Tag")
elif len(tag.select("tr")) == 0:
raise KeyError("No table rows found in table!")
domains = list()
for element in tag.select("tr"):
- # DEBUG: print(f"DEBUG: element[]={type(element)}")
+ # DEBUG: print(f"DEBUG: element[]='{type(element)}'")
if not element.find("td"):
# DEBUG: print("DEBUG: Skipping element, no <td> found")
continue