Continued:

[fba.git] / fba / http / nodeinfo.py

diff --git a/fba/http/nodeinfo.py b/fba/http/nodeinfo.py
index 7d7620ad9d658240216011bb2108dbe5f76fc0da..5626812a29c776532d7408a0a456ec39c89b207b 100644 (file)
--- a/fba/http/nodeinfo.py
+++ b/fba/http/nodeinfo.py
@@ -17,11 +17,10 @@ import logging
 
 from urllib.parse import urlparse
 
-from fba import csrf
-
 from fba.helpers import config
 from fba.helpers import domain as domain_helper
 
+from fba.http import csrf
 from fba.http import network
 
 from fba.models import instances
@@ -31,23 +30,29 @@ _DEPTH = 0
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
-def fetch_nodeinfo(domain: str, path: str = None) -> dict:
-    logger.debug("domain='%s',path='%s' - CALLED!", domain, path)
+def fetch(domain: str, path: str = None, update_mode: bool = True) -> dict:
+    logger.debug("domain='%s',path='%s',update_mode='%s' - CALLED!", domain, path, update_mode)
     domain_helper.raise_on(domain)
 
     if not isinstance(path, str) and path is not None:
         raise ValueError(f"Parameter path[]='{type(path)}' is not of type 'str'")
+    elif not isinstance(update_mode, bool) and update_mode is not None:
+        raise ValueError(f"Parameter update_mode[]='{type(update_mode)}' is not of type 'bool'")
 
-    logger.debug("Fetching nodeinfo from domain='%s' ...", domain)
-    data = fetch_wellknown_nodeinfo(domain)
+    if path is None and update_mode:
+        logger.debug("Fetching well-known nodeinfo from domain='%s' ...", domain)
+        data = fetch_wellknown_nodeinfo(domain)
 
-    logger.debug("data[%s](%d)='%s'", type(data), len(data), data)
-    if "error_message" not in data and "json" in data and len(data["json"]) > 0:
-        logger.debug("Invoking instances.set_last_nodeinfo(%s) ...", domain)
-        instances.set_last_nodeinfo(domain)
+        logger.debug("data[%s](%d)='%s'", type(data), len(data), data)
+        if "exception" in data:
+            logger.warning("Exception returned: '%s', raising again ...", type(data["exception"]))
+            raise data["exception"]
+        elif "error_message" not in data and "json" in data and len(data["json"]) > 0:
+            logger.debug("Invoking instances.set_last_nodeinfo(%s) ...", domain)
+            instances.set_last_nodeinfo(domain)
 
-        logger.debug("Found data[json]()=%d - EXIT!", len(data['json']))
-        return data
+            logger.debug("Found data[json]()=%d - EXIT!", len(data['json']))
+            return data
 
     # No CSRF by default, you don't have to add network.api_headers by yourself here
     headers = tuple()
@@ -77,17 +82,11 @@ def fetch_nodeinfo(domain: str, path: str = None) -> dict:
 
     for request in request_paths:
         logger.debug("request='%s'", request)
-        http_url  = f"http://{domain}{str(path)}"
-        https_url = f"https://{domain}{str(path)}"
+        http_url  = f"http://{domain}{str(path) if path is not None else '/'}"
+        https_url = f"https://{domain}{str(path) if path is not None else '/'}"
 
         logger.debug("path[%s]='%s',request='%s',http_url='%s',https_url='%s'", type(path), path, request, http_url, https_url)
         if path is None or path in [request, http_url, https_url]:
-            logger.debug("path='%s',http_url='%s',https_url='%s'", path, http_url, https_url)
-            if path in [http_url, https_url]:
-                logger.debug("domain='%s',path='%s' has protocol in path, splitting ...", domain, path)
-                components = urlparse(path)
-                path = components.path
-
             logger.debug("Fetching request='%s' from domain='%s' ...", request, domain)
             data = network.get_json_api(
                 domain,
@@ -100,8 +99,9 @@ def fetch_nodeinfo(domain: str, path: str = None) -> dict:
             if "error_message" not in data and "json" in data:
                 logger.debug("Success: request='%s' - Setting detection_mode=STATIC_CHECK ...", request)
                 instances.set_last_nodeinfo(domain)
-                instances.set_detection_mode(domain, "STATIC_CHECK")
-                instances.set_nodeinfo_url(domain, request)
+                if update_mode:
+                    instances.set_detection_mode(domain, "STATIC_CHECK")
+                    instances.set_nodeinfo_url(domain, "https://{domain}{request}")
                 break
 
             logger.warning("Failed fetching nodeinfo from domain='%s',status_code='%s',error_message='%s'", domain, data['status_code'], data['error_message'])
@@ -197,7 +197,7 @@ def fetch_wellknown_nodeinfo(domain: str) -> dict:
 
                 # Default is that 'href' has a complete URL, but some hosts don't send that
                 logger.debug("link[rel]='%s' matches niid='%s'", link["rel"], niid)
-                url = link["href"]
+                url = link["href"].lower()
                 components = urlparse(url)
 
                 logger.debug("components[%s]='%s'", type(components), components)
@@ -210,9 +210,10 @@ def fetch_wellknown_nodeinfo(domain: str) -> dict:
                     url = f"{components.scheme}://{domain}{components.path}"
                     components = urlparse(url)
 
-                logger.debug("components.netloc[]='%s'", type(components.netloc))
-                if not domain_helper.is_wanted(components.netloc):
-                    logger.debug("components.netloc='%s' is not wanted - SKIPPED!", components.netloc)
+                domain2 = components.netloc.lower().split(":")[0]
+                logger.debug("domain2='%s'", domain2)
+                if not domain_helper.is_wanted(domain2):
+                    logger.debug("domain2='%s' is not wanted - SKIPPED!", domain2)
                     continue
 
                 logger.debug("Fetching nodeinfo from url='%s' ...", url)