Continued:

[fba.git] / fba / http / federation.py

diff --git a/fba/http/federation.py b/fba/http/federation.py
index 07cbe0ab547bbd961d5351330d0f8aee652ec42b..136a4ea3158b848c2af41e20b64fa4ca09b597cd 100644 (file)
--- a/fba/http/federation.py
+++ b/fba/http/federation.py
@@ -18,11 +18,10 @@ import logging
 from urllib.parse import urlparse
 
 import bs4
 from urllib.parse import urlparse
 
 import bs4

+import requests

 import validators
 
 import validators
 

-from fba import csrf
-from fba import utils
-
+from fba.helpers import blacklist

 from fba.helpers import config
 from fba.helpers import cookies
 from fba.helpers import domain as domain_helper
 from fba.helpers import config
 from fba.helpers import cookies
 from fba.helpers import domain as domain_helper


@@ -30,88 +29,138 @@ from fba.helpers import software as software_helper
 from fba.helpers import tidyup
 from fba.helpers import version
 
 from fba.helpers import tidyup
 from fba.helpers import version
 

+from fba.http import csrf

 from fba.http import network
 from fba.http import network

+from fba.http import nodeinfo

 
 

+from fba.models import blocks

 from fba.models import instances
 
 from fba.networks import lemmy
 from fba.networks import misskey
 from fba.networks import peertube
 
 from fba.models import instances
 
 from fba.networks import lemmy
 from fba.networks import misskey
 from fba.networks import peertube
 

+# Depth counter, being raised and lowered
+_DEPTH = 0
+
+# API paths
+_api_paths = [
+    "/api/v1/instance/peers",
+    "/api/v3/site",
+]
+

 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
 def fetch_instances(domain: str, origin: str, software: str, command: str, path: str = None):
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
 def fetch_instances(domain: str, origin: str, software: str, command: str, path: str = None):

-    logger.debug("domain='%s',origin='%s',software='%s',command='%s',path='%s' - CALLED!", domain, origin, software, command, path)
+    global _DEPTH
+    logger.debug("domain='%s',origin='%s',software='%s',command='%s',path='%s',_DEPTH=%d - CALLED!", domain, origin, software, command, path, _DEPTH)

     domain_helper.raise_on(domain)
 
     domain_helper.raise_on(domain)
 

-    if not isinstance(origin, str) and origin is not None:
-        raise ValueError(f"Parameter origin[]='{type(origin)}' is not 'str'")
+    if blacklist.is_blacklisted(domain):
+        raise Exception(f"domain='{domain}' is blacklisted but function was invoked")
+    elif not isinstance(origin, str) and origin is not None:
+        raise ValueError(f"Parameter origin[]='{type(origin)}' is not of type 'str'")

     elif not isinstance(command, str):
     elif not isinstance(command, str):

-        raise ValueError(f"Parameter command[]='{type(command)}' is not 'str'")
+        raise ValueError(f"Parameter command[]='{type(command)}' is not of type 'str'")

     elif command == "":
         raise ValueError("Parameter 'command' is empty")
     elif command == "":
         raise ValueError("Parameter 'command' is empty")

-    elif software is None:
+    elif command in ["fetch_blocks", "fetch_cs", "fetch_bkali", "fetch_relays", "fetch_fedipact", "fetch_joinmobilizon", "fetch_joinmisskey", "fetch_joinfediverse", "fetch_relaylist"] and origin is None:
+        raise ValueError(f"Parameter command='{command}' but origin is None, please fix invoking this function.")
+    elif not isinstance(path, str) and path is not None:
+        raise ValueError(f"Parameter path[]='{type(path)}' is not of type 'str'")
+    elif path is not None and not path.startswith("/"):
+        raise ValueError(f"path='{path}' does not start with a slash")
+    elif _DEPTH > 0 and instances.is_recent(domain, "last_instance_fetch"):
+        raise ValueError(f"domain='{domain}' has recently been fetched but function was invoked")
+    elif software is None and not instances.is_recent(domain, "last_nodeinfo"):

         try:
         try:

-            logger.debug("Software for domain='%s' is not set, determining ...", domain)
+            logger.debug("Software for domain='%s',path='%s' is not set, determining ...", domain, path)

             software = determine_software(domain, path)
         except network.exceptions as exception:
             logger.warning("Exception '%s' during determining software type", type(exception))
             instances.set_last_error(domain, exception)
 
         logger.debug("Determined software='%s' for domain='%s'", software, domain)
             software = determine_software(domain, path)
         except network.exceptions as exception:
             logger.warning("Exception '%s' during determining software type", type(exception))
             instances.set_last_error(domain, exception)
 
         logger.debug("Determined software='%s' for domain='%s'", software, domain)

+    elif software is None:
+        logger.debug("domain='%s' has unknown software or nodeinfo has recently being fetched", domain)

     elif not isinstance(software, str):
     elif not isinstance(software, str):

-        raise ValueError(f"Parameter software[]='{type(software)}' is not 'str'")
+        raise ValueError(f"Parameter software[]='{type(software)}' is not of type 'str'")
+
+    # Increase depth
+    _DEPTH = _DEPTH + 1

 
     logger.debug("Checking if domain='%s' is registered ...", domain)
     if not instances.is_registered(domain):
         logger.debug("Adding new domain='%s',origin='%s',command='%s',path='%s',software='%s'", domain, origin, command, path, software)
         instances.add(domain, origin, command, path, software)
 
 
     logger.debug("Checking if domain='%s' is registered ...", domain)
     if not instances.is_registered(domain):
         logger.debug("Adding new domain='%s',origin='%s',command='%s',path='%s',software='%s'", domain, origin, command, path, software)
         instances.add(domain, origin, command, path, software)
 

+        logger.debug("software='%s'", software)
+        if software_helper.is_relay(software):
+            logger.debug("software='%s' is a relay software - EXIT!", software)
+            _DEPTH = _DEPTH - 1
+            return
+

     logger.debug("Updating last_instance_fetch for domain='%s' ...", domain)
     instances.set_last_instance_fetch(domain)
 
     peerlist = list()
     logger.debug("Updating last_instance_fetch for domain='%s' ...", domain)
     instances.set_last_instance_fetch(domain)
 
     peerlist = list()

-    try:
+    logger.debug("software='%s'", software)
+    if software is not None:

         logger.debug("Fetching instances for domain='%s',software='%s',origin='%s'", domain, software, origin)
         peerlist = fetch_peers(domain, software, origin)
         logger.debug("Fetching instances for domain='%s',software='%s',origin='%s'", domain, software, origin)
         peerlist = fetch_peers(domain, software, origin)

-    except network.exceptions as exception:
-        logger.warning("Cannot fetch peers from domain='%s': '%s'", domain, type(exception))

 
     logger.debug("peerlist[]='%s'", type(peerlist))
     if isinstance(peerlist, list):
         logger.debug("Invoking instances.set_total_peerlist(%s,%d) ...", domain, len(peerlist))
         instances.set_total_peers(domain, peerlist)
 
 
     logger.debug("peerlist[]='%s'", type(peerlist))
     if isinstance(peerlist, list):
         logger.debug("Invoking instances.set_total_peerlist(%s,%d) ...", domain, len(peerlist))
         instances.set_total_peers(domain, peerlist)
 

-    logger.debug("peerlist[]='%s'", type(peerlist))
-    if peerlist is None or len(peerlist) == 0:
-        logger.warning("Cannot fetch peers: domain='%s'", domain)
+    logger.debug("Invoking cookies.clear(%s) ...", domain)
+    cookies.clear(domain)

 
 

+    logger.debug("peerlist[]='%s'", type(peerlist))
+    if peerlist is None:
+        logger.warning("Cannot fetch peers: domain='%s',software='%s'", domain, software)

         if instances.has_pending(domain):
             logger.debug("Flushing updates for domain='%s' ...", domain)
         if instances.has_pending(domain):
             logger.debug("Flushing updates for domain='%s' ...", domain)

-            instances.update_data(domain)
-
-        logger.debug("Invoking cookies.clear(%s) ...", domain)
-        cookies.clear(domain)
+            instances.update(domain)

 
 

+        _DEPTH = _DEPTH - 1

         logger.debug("EXIT!")
         return
         logger.debug("EXIT!")
         return

+    elif len(peerlist) == 0:
+        logger.info("domain='%s' returned an empty peer list.", domain)
+        if instances.has_pending(domain):
+            logger.debug("Flushing updates for domain='%s' ...", domain)
+            instances.update(domain)
+
+        _DEPTH = _DEPTH - 1
+        logger.debug("domain='%s',software='%s' has an empty peer list returned - EXIT!", domain, software)
+        return

 
 

-    logger.info("Checking %d instance(s) from domain='%s',software='%s' ...", len(peerlist), domain, software)
+    logger.info("Checking %d instance(s) from domain='%s',software='%s',depth=%d ...", len(peerlist), domain, software, _DEPTH)

     for instance in peerlist:
     for instance in peerlist:

-        logger.debug("instance='%s'", instance)
-        if instance is None:
-            # Skip "None" types as tidup.domain() cannot parse them
+        logger.debug("instance[%s]='%s'", type(instance), instance)
+        if instance is None or instance == "":
+            logger.debug("instance[%s]='%s' is either None or empty - SKIPPED!", type(instance), instance)

             continue
 
         logger.debug("instance='%s' - BEFORE!", instance)
             continue
 
         logger.debug("instance='%s' - BEFORE!", instance)

-        instance = tidyup.domain(instance)
+        instance = tidyup.domain(instance) if isinstance(instance, str) and instance != "" else None

         logger.debug("instance='%s' - AFTER!", instance)
 
         logger.debug("instance='%s' - AFTER!", instance)
 

-        if instance == "":
-            logger.warning("Empty instance after tidyup.domain(), domain='%s'", domain)
+        if instance is None or instance == "":
+            logger.warning("instance='%s' is empty after tidyup.domain(), domain='%s'", instance, domain)

             continue
             continue

-        elif not utils.is_domain_wanted(instance):
+        elif ".." in instance:
+            logger.warning("instance='%s' contains double-dot, removing ...", instance)
+            instance = instance.replace("..", ".")
+
+        logger.debug("instance='%s' - BEFORE!", instance)
+        instance = instance.encode("idna").decode("utf-8")
+        logger.debug("instance='%s' - AFTER!", instance)
+
+        if not domain_helper.is_wanted(instance):

             logger.debug("instance='%s' is not wanted - SKIPPED!", instance)
             continue
         elif instance.find("/profile/") > 0 or instance.find("/users/") > 0 or (instances.is_registered(instance.split("/")[0]) and instance.find("/c/") > 0):
             logger.debug("instance='%s' is not wanted - SKIPPED!", instance)
             continue
         elif instance.find("/profile/") > 0 or instance.find("/users/") > 0 or (instances.is_registered(instance.split("/")[0]) and instance.find("/c/") > 0):


@@ -121,25 +170,43 @@ def fetch_instances(domain: str, origin: str, software: str, command: str, path:
             logger.debug("instance='%s' is a link to a tag - SKIPPED!", instance)
             continue
         elif not instances.is_registered(instance):
             logger.debug("instance='%s' is a link to a tag - SKIPPED!", instance)
             continue
         elif not instances.is_registered(instance):

-            logger.debug("Adding new instance='%s',domain='%s',command='%s'", instance, domain, command)
-            instances.add(instance, domain, command)
-
-    logger.debug("Invoking cookies.clear(%s) ...", domain)
-    cookies.clear(domain)
+            logger.debug("Checking if domain='%s' has pending updates ...", domain)
+            if instances.has_pending(domain):
+                logger.debug("Flushing updates for domain='%s' ...", domain)
+                instances.update(domain)
+
+            logger.debug("instance='%s',origin='%s',_DEPTH=%d reached!", instance, origin, _DEPTH)
+            if _DEPTH <= config.get("max_crawl_depth") and len(peerlist) >= config.get("min_peers_length"):
+                logger.debug("Fetching instance='%s',origin='%s',command='%s',path='%s',_DEPTH=%d ...", instance, domain, command, path, _DEPTH)
+                fetch_instances(instance, domain, None, command, path)
+            else:
+                logger.debug("Adding instance='%s',domain='%s',command='%s',_DEPTH=%d ...", instance, domain, command, _DEPTH)
+                instances.add(instance, domain, command)

 
     logger.debug("Checking if domain='%s' has pending updates ...", domain)
     if instances.has_pending(domain):
         logger.debug("Flushing updates for domain='%s' ...", domain)
 
     logger.debug("Checking if domain='%s' has pending updates ...", domain)
     if instances.has_pending(domain):
         logger.debug("Flushing updates for domain='%s' ...", domain)

-        instances.update_data(domain)
+        instances.update(domain)

 
 

+    _DEPTH = _DEPTH - 1

     logger.debug("EXIT!")
 
 def fetch_peers(domain: str, software: str, origin: str) -> list:
     logger.debug("domain='%s',software='%s',origin='%s' - CALLED!", domain, software, origin)
     domain_helper.raise_on(domain)
 
     logger.debug("EXIT!")
 
 def fetch_peers(domain: str, software: str, origin: str) -> list:
     logger.debug("domain='%s',software='%s',origin='%s' - CALLED!", domain, software, origin)
     domain_helper.raise_on(domain)
 

-    if not isinstance(software, str) and software is not None:
-        raise ValueError(f"software[]='{type(software)}' is not 'str'")
+    if blacklist.is_blacklisted(domain):
+        raise Exception(f"domain='{domain}' is blacklisted but function was invoked")
+    elif not isinstance(software, str) and software is not None:
+        raise ValueError(f"Parameter software[]='{type(software)}' is not of type 'str'")
+    elif isinstance(software, str) and software == "":
+        raise ValueError("Parameter 'software' is empty")
+    elif software_helper.is_relay(software):
+        raise ValueError(f"domain='{domain}' is of software='{software}' and isn't supported here.")
+    elif not isinstance(origin, str) and origin is not None:
+        raise ValueError(f"Parameter origin[]='{type(origin)}' is not of type 'str'")
+    elif isinstance(origin, str) and origin == "":
+        raise ValueError("Parameter 'origin' is empty")

 
     if software == "misskey":
         logger.debug("Invoking misskey.fetch_peers(%s) ...", domain)
 
     if software == "misskey":
         logger.debug("Invoking misskey.fetch_peers(%s) ...", domain)


@@ -158,29 +225,26 @@ def fetch_peers(domain: str, software: str, origin: str) -> list:
         logger.debug("Checking CSRF for domain='%s'", domain)
         headers = csrf.determine(domain, dict())
     except network.exceptions as exception:
         logger.debug("Checking CSRF for domain='%s'", domain)
         headers = csrf.determine(domain, dict())
     except network.exceptions as exception:

-        logger.warning("Exception '%s' during checking CSRF (fetch_peers,%s) - EXIT!", type(exception), __name__)
+        logger.warning("Exception '%s' during checking CSRF (fetch_peers,%s)", type(exception), __name__)

         instances.set_last_error(domain, exception)
         instances.set_last_error(domain, exception)

-        return list()

 
 

-    paths = [
-        "/api/v1/instance/peers",
-        "/api/v3/site",
-    ]
+        logger.debug("Returning empty list ... - EXIT!")
+        return list()

 
     # Init peers variable
     peers = list()
 
 
     # Init peers variable
     peers = list()
 

-    logger.debug("Checking %d paths ...", len(paths))
-    for path in paths:
+    logger.debug("Checking %d API paths ...", len(_api_paths))
+    for path in _api_paths:

         logger.debug("Fetching path='%s' from domain='%s',software='%s' ...", path, domain, software)
         data = network.get_json_api(
             domain,
             path,
         logger.debug("Fetching path='%s' from domain='%s',software='%s' ...", path, domain, software)
         data = network.get_json_api(
             domain,
             path,

-            headers,
-            (config.get("connection_timeout"), config.get("read_timeout"))
+            headers=headers,
+            timeout=(config.get("connection_timeout"), config.get("read_timeout"))

         )
 
         )
 

-        logger.debug("data[]='%s'", type(data))
+        logger.debug("data(%d)[]='%s'", len(data), type(data))

         if "error_message" in data:
             logger.debug("Was not able to fetch peers from path='%s',domain='%s' ...", path, domain)
             instances.set_last_error(domain, data)
         if "error_message" in data:
             logger.debug("Was not able to fetch peers from path='%s',domain='%s' ...", path, domain)
             instances.set_last_error(domain, data)


@@ -193,7 +257,7 @@ def fetch_peers(domain: str, software: str, origin: str) -> list:
             break
 
     if not isinstance(peers, list):
             break
 
     if not isinstance(peers, list):

-        logger.warning("peers[]='%s' is not 'list', maybe bad API response?", type(peers))
+        logger.warning("peers[]='%s' is not of type 'list', maybe bad API response?", type(peers))

         peers = list()
 
     logger.debug("Invoking instances.set_total_peers(%s,%d) ...", domain, len(peers))
         peers = list()
 
     logger.debug("Invoking instances.set_total_peers(%s,%d) ...", domain, len(peers))


@@ -202,220 +266,51 @@ def fetch_peers(domain: str, software: str, origin: str) -> list:
     logger.debug("peers()=%d - EXIT!", len(peers))
     return peers
 
     logger.debug("peers()=%d - EXIT!", len(peers))
     return peers
 

-def fetch_nodeinfo(domain: str, path: str = None) -> dict:
-    logger.debug("domain='%s',path='%s' - CALLED!", domain, path)
-    domain_helper.raise_on(domain)
-
-    if not isinstance(path, str) and path is not None:
-        raise ValueError(f"Parameter path[]='{type(path)}' is not 'str'")
-
-    logger.debug("Fetching nodeinfo from domain='%s' ...", domain)
-    nodeinfo = fetch_wellknown_nodeinfo(domain)
-
-    logger.debug("nodeinfo[%s](%d='%s'", type(nodeinfo), len(nodeinfo), nodeinfo)
-    if "error_message" not in nodeinfo and "json" in nodeinfo and len(nodeinfo["json"]) > 0:
-        logger.debug("Found nodeinfo[json]()=%d - EXIT!", len(nodeinfo['json']))
-        return nodeinfo
-
-    # No CSRF by default, you don't have to add network.api_headers by yourself here
-    headers = tuple()
-    data = dict()
-
-    try:
-        logger.debug("Checking CSRF for domain='%s'", domain)
-        headers = csrf.determine(domain, dict())
-    except network.exceptions as exception:
-        logger.warning("Exception '%s' during checking CSRF (nodeinfo,%s) - EXIT!", type(exception), __name__)
-        instances.set_last_error(domain, exception)
-        instances.set_software(domain, None)
-        instances.set_detection_mode(domain, None)
-        instances.set_nodeinfo_url(domain, None)
-        return {
-            "status_code"  : 500,
-            "error_message": f"exception[{type(exception)}]='{str(exception)}'",
-            "exception"    : exception,
-        }
-
-    request_paths = [
-       "/nodeinfo/2.1.json",
-       "/nodeinfo/2.1",
-       "/nodeinfo/2.0.json",
-       "/nodeinfo/2.0",
-       "/nodeinfo/1.0.json",
-       "/nodeinfo/1.0",
-       "/api/v1/instance",
-    ]
-
-    for request in request_paths:
-        logger.debug("request='%s'", request)
-        http_url  = f"http://{domain}{path}"
-        https_url = f"https://{domain}{path}"
-
-        logger.debug("path[%s]='%s',request='%s',http_url='%s',https_url='%s'", type(path), path, request, http_url, https_url)
-        if path is None or path in [request, http_url, https_url]:
-            logger.debug("Fetching request='%s' from domain='%s' ...", request, domain)
-            if path in [http_url, https_url]:
-                logger.debug("domain='%s',path='%s' has protocol in path, splitting ...", domain, path)
-                components = urlparse(path)
-                path = components.path
-
-            data = network.get_json_api(
-                domain,
-                request,
-                headers,
-                (config.get("nodeinfo_connection_timeout"), config.get("nodeinfo_read_timeout"))
-            )
-
-            logger.debug("data[]='%s'", type(data))
-            if "error_message" not in data and "json" in data:
-                logger.debug("Success: request='%s' - Setting detection_mode=STATIC_CHECK ...", request)
-                instances.set_detection_mode(domain, "STATIC_CHECK")
-                instances.set_nodeinfo_url(domain, request)
-                break
-
-            logger.warning("Failed fetching nodeinfo from domain='%s',status_code='%s',error_message='%s'", domain, data['status_code'], data['error_message'])
-
-    logger.debug("data()=%d - EXIT!", len(data))
-    return data
-
-def fetch_wellknown_nodeinfo(domain: str) -> dict:
-    logger.debug("domain='%s' - CALLED!", domain)
-    domain_helper.raise_on(domain)
-
-    # "rel" identifiers (no real URLs)
-    nodeinfo_identifier = [
-        "https://nodeinfo.diaspora.software/ns/schema/2.1",
-        "http://nodeinfo.diaspora.software/ns/schema/2.1",
-        "https://nodeinfo.diaspora.software/ns/schema/2.0",
-        "http://nodeinfo.diaspora.software/ns/schema/2.0",
-        "https://nodeinfo.diaspora.software/ns/schema/1.1",
-        "http://nodeinfo.diaspora.software/ns/schema/1.1",
-        "https://nodeinfo.diaspora.software/ns/schema/1.0",
-        "http://nodeinfo.diaspora.software/ns/schema/1.0",
-    ]
-
-    # No CSRF by default, you don't have to add network.api_headers by yourself here
-    headers = tuple()
-
-    try:
-        logger.debug("Checking CSRF for domain='%s'", domain)
-        headers = csrf.determine(domain, dict())
-    except network.exceptions as exception:
-        logger.warning("Exception '%s' during checking CSRF (fetch_wellknown_nodeinfo,%s) - EXIT!", type(exception), __name__)
-        instances.set_last_error(domain, exception)
-        return {
-            "status_code"  : 500,
-            "error_message": type(exception),
-            "exception"    : exception,
-        }
-
-    logger.debug("Fetching .well-known info for domain='%s'", domain)
-    data = network.get_json_api(
-        domain,
-        "/.well-known/nodeinfo",
-        headers,
-        (config.get("nodeinfo_connection_timeout"), config.get("nodeinfo_read_timeout"))
-    )
-
-    logger.debug("data[]='%s'", type(data))
-    if "error_message" not in data:
-        nodeinfo = data["json"]
-
-        logger.debug("Marking domain='%s' as successfully handled ...", domain)
-        instances.set_success(domain)
-
-        logger.debug("Found entries: nodeinfo()=%d,domain='%s'", len(nodeinfo), domain)
-        if "links" in nodeinfo:
-            logger.debug("Found nodeinfo[links]()=%d record(s),", len(nodeinfo["links"]))
-            for niid in nodeinfo_identifier:
-                data = dict()
-
-                logger.debug("Checking niid='%s' ...", niid)
-                for link in nodeinfo["links"]:
-                    logger.debug("link[%s]='%s'", type(link), link)
-                    if not isinstance(link, dict) or not "rel" in link:
-                        logger.warning("link[]='%s' is not 'dict' or no element 'rel' found", type(link))
-                    elif link["rel"] == niid:
-                        # Default is that 'href' has a complete URL, but some hosts don't send that
-                        logger.debug("link[href]='%s' matches niid='%s'", link["href"], niid)
-                        url = link["href"]
-                        components = urlparse(link["href"])
-
-                        logger.debug("components[%s]='%s'", type(components), components)
-                        if components.scheme == "" and components.netloc == "":
-                            logger.warning("link[href]='%s' has no scheme and host name in it, prepending from domain='%s'", link['href'], domain)
-                            url = f"https://{domain}{url}"
-                            components = urlparse(url)
-                        elif components.netloc == "":
-                            logger.warning("link[href]='%s' has no netloc set, setting domain='%s'", link["href"], domain)
-                            url = f"{components.scheme}://{domain}{components.path}"
-                            components = urlparse(url)
-
-                        if not utils.is_domain_wanted(components.netloc):
-                            logger.debug("components.netloc='%s' is not wanted - SKIPPED!", components.netloc)
-                            continue
-
-                        logger.debug("Fetching nodeinfo from url='%s' ...", url)
-                        data = network.fetch_api_url(
-                            url,
-                            (config.get("connection_timeout"), config.get("read_timeout"))
-                         )
-
-                        logger.debug("link[href]='%s',data[]='%s'", link["href"], type(data))
-                        if "error_message" not in data and "json" in data:
-                            logger.debug("Found JSON data()=%d,link[href]='%s' - Setting detection_mode=AUTO_DISCOVERY ...", len(data), link["href"])
-                            instances.set_detection_mode(domain, "AUTO_DISCOVERY")
-                            instances.set_nodeinfo_url(domain, link["href"])
-
-                            logger.debug("Marking domain='%s' as successfully handled ...", domain)
-                            instances.set_success(domain)
-                            break
-                        else:
-                            logger.debug("Setting last error for domain='%s',data[]='%s'", domain, type(data))
-                            instances.set_last_error(domain, data)
-
-                logger.debug("data()=%d", len(data))
-                if "error_message" not in data and "json" in data:
-                    logger.debug("Auto-discovery successful: domain='%s'", domain)
-                    break
-        else:
-            logger.warning("nodeinfo does not contain 'links': domain='%s'", domain)
-
-    logger.debug("Returning data[]='%s' - EXIT!", type(data))
-    return data
-

 def fetch_generator_from_path(domain: str, path: str = "/") -> str:
 def fetch_generator_from_path(domain: str, path: str = "/") -> str:

-    logger.debug("domain(%d)='%s',path='%s' - CALLED!", len(domain), domain, path)
+    logger.debug("domain='%s',path='%s' - CALLED!", domain, path)

     domain_helper.raise_on(domain)
 
     domain_helper.raise_on(domain)
 

-    if not isinstance(path, str):
-        raise ValueError(f"path[]='{type(path)}' is not 'str'")
+    if blacklist.is_blacklisted(domain):
+        raise Exception(f"domain='{domain}' is blacklisted but function was invoked")
+    elif not isinstance(path, str):
+        raise ValueError(f"path[]='{type(path)}' is not of type 'str'")

     elif path == "":
         raise ValueError("Parameter 'path' is empty")
     elif path == "":
         raise ValueError("Parameter 'path' is empty")

+    elif not path.startswith("/"):
+        raise ValueError(f"path='{path}' does not start with / but should")

 
 

-    logger.debug("domain='%s',path='%s' - CALLED!", domain, path)

     software = None
 
     logger.debug("Fetching path='%s' from domain='%s' ...", path, domain)
     response = network.fetch_response(
     software = None
 
     logger.debug("Fetching path='%s' from domain='%s' ...", path, domain)
     response = network.fetch_response(

-        domain, path,
-        network.web_headers,
-        (config.get("connection_timeout"), config.get("read_timeout")),
+        domain,
+        path,
+        headers=network.web_headers,
+        timeout=(config.get("connection_timeout"), config.get("read_timeout")),

         allow_redirects=True
     )
         allow_redirects=True
     )

-    components = urlparse(response.url)

 
     logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text))
 
     logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text))

-    if response.ok and response.status_code < 300 and response.text.find("<html") > 0 and components.netloc == domain:
+    if ((response.ok and response.status_code == 200) or response.status_code == 410) and response.text.find("<html") > 0 and domain_helper.is_in_url(domain, response.url):

         logger.debug("Parsing response.text()=%d Bytes ...", len(response.text))
         doc = bs4.BeautifulSoup(response.text, "html.parser")
 
         logger.debug("doc[]='%s'", type(doc))
         logger.debug("Parsing response.text()=%d Bytes ...", len(response.text))
         doc = bs4.BeautifulSoup(response.text, "html.parser")
 
         logger.debug("doc[]='%s'", type(doc))

+        platform  = doc.find("meta", {"property": "og:platform"})

         generator = doc.find("meta", {"name"    : "generator"})
         site_name = doc.find("meta", {"property": "og:site_name"})
         generator = doc.find("meta", {"name"    : "generator"})
         site_name = doc.find("meta", {"property": "og:site_name"})

+        app_name  = doc.find("meta", {"name"    : "application-name"})
+
+        logger.debug("generator[]='%s',site_name[]='%s',platform[]='%s',app_name[]='%s'", type(generator), type(site_name), type(platform), type(app_name))
+        if isinstance(platform, bs4.element.Tag) and isinstance(platform.get("content"), str) and platform.get("content") != "":
+            logger.debug("Found property=og:platform, domain='%s'", domain)
+            software = tidyup.domain(platform.get("content"))
+            logger.debug("software[%s]='%s' after tidyup.domain() ...", type(software), software)

 
 

-        logger.debug("generator[]='%s',site_name[]='%s'", type(generator), type(site_name))
-        if isinstance(generator, bs4.element.Tag) and isinstance(generator.get("content"), str):
+            if software is not None and software != "":
+                logger.debug("domain='%s' has og:platform='%s' - Setting detection_mode=PLATFORM ...", domain, software)
+                instances.set_detection_mode(domain, "PLATFORM")
+        elif isinstance(generator, bs4.element.Tag) and isinstance(generator.get("content"), str) and generator.get("content") != "":

             logger.debug("Found generator meta tag: domain='%s'", domain)
             software = tidyup.domain(generator.get("content"))
 
             logger.debug("Found generator meta tag: domain='%s'", domain)
             software = tidyup.domain(generator.get("content"))
 


@@ -423,7 +318,15 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str:
             if software is not None and software != "":
                 logger.info("domain='%s' is generated by software='%s' - Setting detection_mode=GENERATOR ...", domain, software)
                 instances.set_detection_mode(domain, "GENERATOR")
             if software is not None and software != "":
                 logger.info("domain='%s' is generated by software='%s' - Setting detection_mode=GENERATOR ...", domain, software)
                 instances.set_detection_mode(domain, "GENERATOR")

-        elif isinstance(site_name, bs4.element.Tag) and isinstance(site_name.get("content"), str):
+        elif isinstance(app_name, bs4.element.Tag) and isinstance(app_name.get("content"), str) and app_name.get("content") != "":
+            logger.debug("Found property=og:app_name, domain='%s'", domain)
+            software = tidyup.domain(app_name.get("content"))
+
+            logger.debug("software[%s]='%s'", type(software), software)
+            if software is not None and software != "":
+                logger.debug("domain='%s' has application-name='%s' - Setting detection_mode=app_name ...", domain, software)
+                instances.set_detection_mode(domain, "APP_NAME")
+        elif isinstance(site_name, bs4.element.Tag) and isinstance(site_name.get("content"), str) and site_name.get("content") != "":

             logger.debug("Found property=og:site_name, domain='%s'", domain)
             software = tidyup.domain(site_name.get("content"))
 
             logger.debug("Found property=og:site_name, domain='%s'", domain)
             software = tidyup.domain(site_name.get("content"))
 


@@ -431,13 +334,26 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str:
             if software is not None and software != "":
                 logger.debug("domain='%s' has og:site_name='%s' - Setting detection_mode=SITE_NAME ...", domain, software)
                 instances.set_detection_mode(domain, "SITE_NAME")
             if software is not None and software != "":
                 logger.debug("domain='%s' has og:site_name='%s' - Setting detection_mode=SITE_NAME ...", domain, software)
                 instances.set_detection_mode(domain, "SITE_NAME")

-    elif domain != components.netloc:
-        logger.warning("domain='%s' doesn't match components.netloc='%s', maybe redirect to other domain?", domain, components.netloc)
-        message = f"Redirect from domain='{domain}' to components.netloc='{components.netloc}'"
+    elif not domain_helper.is_in_url(domain, response.url):
+        logger.warning("domain='%s' doesn't match response.url='%s', maybe redirect to other domain?", domain, response.url)
+
+        components = urlparse(response.url)
+        domain2 = components.netloc.lower().split(":")[0]
+
+        logger.debug("domain2='%s'", domain2)
+        if not domain_helper.is_wanted(domain2):
+            logger.debug("domain2='%s' is not wanted - EXIT!", domain2)
+            return None
+        elif not instances.is_registered(domain2):
+            logger.info("components.netloc='%s' is not registered, adding ...", components.netloc)
+            instances.add(domain2, domain, "redirect_target")
+
+        message = f"Redirect from domain='{domain}' to response.url='{response.url}'"

         instances.set_last_error(domain, message)
         instances.set_software(domain, None)
         instances.set_detection_mode(domain, None)
         instances.set_nodeinfo_url(domain, None)
         instances.set_last_error(domain, message)
         instances.set_software(domain, None)
         instances.set_detection_mode(domain, None)
         instances.set_nodeinfo_url(domain, None)

+

         raise requests.exceptions.TooManyRedirects(message)
 
     logger.debug("software[]='%s'", type(software))
         raise requests.exceptions.TooManyRedirects(message)
 
     logger.debug("software[]='%s'", type(software))


@@ -448,36 +364,38 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str:
         logger.debug("software='%s' may contain a version number, domain='%s', removing it ...", software, domain)
         software = version.remove(software)
 
         logger.debug("software='%s' may contain a version number, domain='%s', removing it ...", software, domain)
         software = version.remove(software)
 

-    logger.debug("software[]='%s'", type(software))
+    logger.debug("software[%s]='%s'", type(software), software)

     if isinstance(software, str) and "powered by " in software:
         logger.debug("software='%s' has 'powered by' in it", software)
     if isinstance(software, str) and "powered by " in software:
         logger.debug("software='%s' has 'powered by' in it", software)

-        software = version.remove(version.strip_powered_by(software))
+        software = version.remove(software_helper.strip_powered_by(software))

     elif isinstance(software, str) and " hosted on " in software:
         logger.debug("software='%s' has 'hosted on' in it", software)
     elif isinstance(software, str) and " hosted on " in software:
         logger.debug("software='%s' has 'hosted on' in it", software)

-        software = version.remove(version.strip_hosted_on(software))
+        software = version.remove(software_helper.strip_hosted_on(software))

     elif isinstance(software, str) and " by " in software:
         logger.debug("software='%s' has ' by ' in it", software)
     elif isinstance(software, str) and " by " in software:
         logger.debug("software='%s' has ' by ' in it", software)

-        software = version.strip_until(software, " by ")
+        software = software_helper.strip_until(software, " by ")

     elif isinstance(software, str) and " see " in software:
         logger.debug("software='%s' has ' see ' in it", software)
     elif isinstance(software, str) and " see " in software:
         logger.debug("software='%s' has ' see ' in it", software)

-        software = version.strip_until(software, " see ")
+        software = software_helper.strip_until(software, " see ")

 
 

-    logger.debug("software='%s' - EXIT!", software)
+    logger.debug("software[%s]='%s' - EXIT!", type(software), software)

     return software
 
 def determine_software(domain: str, path: str = None) -> str:
     return software
 
 def determine_software(domain: str, path: str = None) -> str:

-    logger.debug("domain(%d)='%s',path='%s' - CALLED!", len(domain), domain, path)
+    logger.debug("domain='%s',path='%s' - CALLED!", domain, path)

     domain_helper.raise_on(domain)
 
     domain_helper.raise_on(domain)
 

-    if not isinstance(path, str) and path is not None:
-        raise ValueError(f"Parameter path[]='{type(path)}' is not 'str'")
+    if blacklist.is_blacklisted(domain):
+        raise Exception(f"domain='{domain}' is blacklisted but function was invoked")
+    elif not isinstance(path, str) and path is not None:
+        raise ValueError(f"Parameter path[]='{type(path)}' is not of type 'str'")
+    elif path is not None and not path.startswith("/"):
+        raise ValueError(f"path='{path}' does not start with a slash")

 
 

-    logger.debug("Determining software for domain='%s',path='%s'", domain, path)
+    logger.debug("Fetching nodeinfo from domain='%s',path='%s' ...", domain, path)
+    data = nodeinfo.fetch(domain, path)

     software = None
 
     software = None
 

-    logger.debug("Fetching nodeinfo from domain='%s' ...", domain)
-    data = fetch_nodeinfo(domain, path)
-

     logger.debug("data[%s]='%s'", type(data), data)
     if "exception" in data:
         # Continue raising it
     logger.debug("data[%s]='%s'", type(data), data)
     if "exception" in data:
         # Continue raising it


@@ -491,7 +409,7 @@ def determine_software(domain: str, path: str = None) -> str:
         logger.debug("domain='%s',path='%s',data[json] found ...", domain, path)
         data = data["json"]
     else:
         logger.debug("domain='%s',path='%s',data[json] found ...", domain, path)
         data = data["json"]
     else:

-        logger.debug("JSON response from domain='%s' does not include [software][name], fetching / ...", domain)
+        logger.debug("Auto-detection for domain='%s' was failing, fetching / ...", domain)

         software = fetch_generator_from_path(domain)
         logger.debug("Generator for domain='%s' is: '%s'", domain, software)
 
         software = fetch_generator_from_path(domain)
         logger.debug("Generator for domain='%s' is: '%s'", domain, software)
 


@@ -515,6 +433,10 @@ def determine_software(domain: str, path: str = None) -> str:
         logger.debug("Invoking fetch_generator_from_path(%s) ...", domain)
         software = fetch_generator_from_path(domain)
         logger.debug("Generator for domain='%s' is: '%s'", domain, software)
         logger.debug("Invoking fetch_generator_from_path(%s) ...", domain)
         software = fetch_generator_from_path(domain)
         logger.debug("Generator for domain='%s' is: '%s'", domain, software)

+    elif "server" in data and "software" in data["server"]:
+        logger.debug("Found data[server][software]='%s' for domain='%s'", data["server"]["software"].lower(), domain)
+        software = data["server"]["software"].lower()
+        logger.debug("Detected software for domain='%s' is: '%s'", domain, software)

     elif "software" not in data or "name" not in data["software"]:
         logger.debug("JSON response from domain='%s' does not include [software][name] - Resetting detection_mode,nodeinfo_url ...", domain)
         instances.set_detection_mode(domain, None)
     elif "software" not in data or "name" not in data["software"]:
         logger.debug("JSON response from domain='%s' does not include [software][name] - Resetting detection_mode,nodeinfo_url ...", domain)
         instances.set_detection_mode(domain, None)


@@ -525,7 +447,7 @@ def determine_software(domain: str, path: str = None) -> str:
         logger.debug("Generator for domain='%s' is: '%s'", domain, software)
 
     logger.debug("software[%s]='%s'", type(software), software)
         logger.debug("Generator for domain='%s' is: '%s'", domain, software)
 
     logger.debug("software[%s]='%s'", type(software), software)

-    if software is None:
+    if software is None or software == "":

         logger.debug("Returning None - EXIT!")
         return None
 
         logger.debug("Returning None - EXIT!")
         return None
 


@@ -543,9 +465,11 @@ def determine_software(domain: str, path: str = None) -> str:
     logger.debug("software[]='%s'", type(software))
     if isinstance(software, str) and "powered by" in software:
         logger.debug("software='%s' has 'powered by' in it", software)
     logger.debug("software[]='%s'", type(software))
     if isinstance(software, str) and "powered by" in software:
         logger.debug("software='%s' has 'powered by' in it", software)

-        software = version.remove(version.strip_powered_by(software))
+        software = version.remove(software_helper.strip_powered_by(software))
+
+    software = software.strip()

 
 

-    logger.debug("software='%s' - EXIT!", software)
+    logger.debug("software[%s]='%s' - EXIT!", type(software), software)

     return software
 
 def find_domains(tag: bs4.element.Tag) -> list:
     return software
 
 def find_domains(tag: bs4.element.Tag) -> list:


@@ -567,11 +491,11 @@ def find_domains(tag: bs4.element.Tag) -> list:
 
         logger.debug("domain='%s',reason='%s'", domain, reason)
 
 
         logger.debug("domain='%s',reason='%s'", domain, reason)
 

-        if not utils.is_domain_wanted(domain):
+        if not domain_helper.is_wanted(domain):

             logger.debug("domain='%s' is blacklisted - SKIPPED!", domain)
             continue
         elif domain == "gab.com/.ai, develop.gab.com":
             logger.debug("domain='%s' is blacklisted - SKIPPED!", domain)
             continue
         elif domain == "gab.com/.ai, develop.gab.com":

-            logger.debug("Multiple domains detected in one row")
+            logger.debug("Multiple gab.com domains detected in one row")

             domains.append({
                 "domain": "gab.com",
                 "reason": reason,
             domains.append({
                 "domain": "gab.com",
                 "reason": reason,


@@ -601,11 +525,11 @@ def find_domains(tag: bs4.element.Tag) -> list:
 def add_peers(rows: dict) -> list:
     logger.debug("rows[]='%s' - CALLED!", type(rows))
     if not isinstance(rows, dict):
 def add_peers(rows: dict) -> list:
     logger.debug("rows[]='%s' - CALLED!", type(rows))
     if not isinstance(rows, dict):

-        raise ValueError(f"Parameter rows[]='{type(rows)}' is not 'dict'")
+        raise ValueError(f"Parameter rows[]='{type(rows)}' is not of type 'dict'")

 
     peers = list()
     for key in ["linked", "allowed", "blocked"]:
 
     peers = list()
     for key in ["linked", "allowed", "blocked"]:

-        logger.debug("Checking key='%s'", key)
+        logger.debug("key='%s'", key)

         if key not in rows or rows[key] is None:
             logger.debug("Cannot find key='%s' or it is NoneType - SKIPPED!", key)
             continue
         if key not in rows or rows[key] is None:
             logger.debug("Cannot find key='%s' or it is NoneType - SKIPPED!", key)
             continue


@@ -626,7 +550,7 @@ def add_peers(rows: dict) -> list:
                 raise ValueError(f"peer[]='{type(peer)}' is not supported,key='{key}'")
 
             logger.debug("peer[%s]='%s' - AFTER!", type(peer), peer)
                 raise ValueError(f"peer[]='{type(peer)}' is not supported,key='{key}'")
 
             logger.debug("peer[%s]='%s' - AFTER!", type(peer), peer)

-            if not utils.is_domain_wanted(peer):
+            if not domain_helper.is_wanted(peer):

                 logger.debug("peer='%s' is not wanted - SKIPPED!", peer)
                 continue
 
                 logger.debug("peer='%s' is not wanted - SKIPPED!", peer)
                 continue
 


@@ -635,3 +559,101 @@ def add_peers(rows: dict) -> list:
 
     logger.debug("peers()=%d - EXIT!", len(peers))
     return peers
 
     logger.debug("peers()=%d - EXIT!", len(peers))
     return peers

+
+def fetch_blocks(domain: str) -> list:
+    logger.debug("domain='%s' - CALLED!", domain)
+    domain_helper.raise_on(domain)
+
+    if not instances.is_registered(domain):
+        raise Exception(f"domain='{domain}' is not registered but function is invoked.")
+    elif blacklist.is_blacklisted(domain):
+        raise Exception(f"domain='{domain}' is blacklisted but function was invoked")
+
+    # Init block list
+    blocklist = list()
+
+    # No CSRF by default, you don't have to add network.api_headers by yourself here
+    headers = tuple()
+
+    try:
+        logger.debug("Checking CSRF for domain='%s'", domain)
+        headers = csrf.determine(domain, dict())
+    except network.exceptions as exception:
+        logger.warning("Exception '%s' during checking CSRF (fetch_blocks,%s)", type(exception), __name__)
+        instances.set_last_error(domain, exception)
+
+        logger.debug("Returning empty list ... - EXIT!")
+        return list()
+
+    try:
+        # json endpoint for newer mastodongs
+        logger.info("Fetching domain_blocks from domain='%s' ...", domain)
+        data = network.get_json_api(
+            domain,
+            "/api/v1/instance/domain_blocks",
+            headers=headers,
+            timeout=(config.get("connection_timeout"), config.get("read_timeout"))
+        )
+        rows = list()
+
+        logger.debug("data(%d)[]='%s'", len(data), type(data))
+        if "error_message" in data:
+            logger.debug("Was not able to fetch domain_blocks from domain='%s': status_code=%d,error_message='%s'", domain, data['status_code'], data['error_message'])
+            instances.set_last_error(domain, data)
+
+            logger.debug("blocklist()=%d - EXIT!", len(blocklist))
+            return blocklist
+        elif "json" in data and "error" in data["json"]:
+            logger.warning("JSON API returned error message: '%s'", data["json"]["error"])
+            instances.set_last_error(domain, data)
+
+            logger.debug("blocklist()=%d - EXIT!", len(blocklist))
+            return blocklist
+        else:
+            # Getting blocklist
+            rows = data["json"]
+
+            logger.debug("Marking domain='%s' as successfully handled ...", domain)
+            instances.set_success(domain)
+
+        logger.debug("rows(%d)[]='%s'", len(rows), type(rows))
+        if len(rows) > 0:
+            logger.debug("Checking %d entries from domain='%s' ...", len(rows), domain)
+            for block in rows:
+                # Check type
+                logger.debug("block[]='%s'", type(block))
+                if not isinstance(block, dict):
+                    logger.debug("block[]='%s' is of type 'dict' - SKIPPED!", type(block))
+                    continue
+                elif "domain" not in block:
+                    logger.warning("block()=%d does not contain element 'domain' - SKIPPED!", len(block))
+                    continue
+                elif "severity" not in block:
+                    logger.warning("block()=%d does not contain element 'severity' - SKIPPED!", len(block))
+                    continue
+                elif block["severity"] in ["accept", "accepted"]:
+                    logger.debug("block[domain]='%s' has unwanted severity level '%s' - SKIPPED!", block["domain"], block["severity"])
+                    continue
+                elif "digest" in block and not validators.hashes.sha256(block["digest"]):
+                    logger.warning("block[domain]='%s' has invalid block[digest]='%s' - SKIPPED!", block["domain"], block["digest"])
+                    continue
+
+                reason = tidyup.reason(block["comment"]) if "comment" in block and block["comment"] is not None and block["comment"] != "" else None
+
+                logger.debug("Appending blocker='%s',blocked='%s',reason='%s',block_level='%s' ...", domain, block["domain"], reason, block["severity"])
+                blocklist.append({
+                    "blocker"    : domain,
+                    "blocked"    : block["domain"],
+                    "digest"     : block["digest"] if "digest" in block else None,
+                    "reason"     : reason,
+                    "block_level": blocks.alias_block_level(block["severity"]),
+                })
+        else:
+            logger.debug("domain='%s' has no block list", domain)
+
+    except network.exceptions as exception:
+        logger.warning("domain='%s',exception[%s]='%s'", domain, type(exception), str(exception))
+        instances.set_last_error(domain, exception)
+
+    logger.debug("blocklist()=%d - EXIT!", len(blocklist))
+    return blocklist