Continued:

[fba.git] / fba / http / federation.py

diff --git a/fba/http/federation.py b/fba/http/federation.py
index 75f006b29df666659f425c491a7f81ef10136947..8440f40636b88dd285275414db5081e6667b55f9 100644 (file)
--- a/fba/http/federation.py
+++ b/fba/http/federation.py
@@ -48,7 +48,6 @@ logger = logging.getLogger(__name__)
 def fetch_instances(domain: str, origin: str, software: str, command: str, path: str = None):
     global _DEPTH
     logger.debug("domain='%s',origin='%s',software='%s',command='%s',path='%s',_DEPTH=%d - CALLED!", domain, origin, software, command, path, _DEPTH)
-    _DEPTH = _DEPTH + 1
     domain_helper.raise_on(domain)
 
     if not isinstance(origin, str) and origin is not None:
@@ -59,7 +58,11 @@ def fetch_instances(domain: str, origin: str, software: str, command: str, path:
         raise ValueError("Parameter 'command' is empty")
     elif command in ["fetch_blocks", "fetch_cs", "fetch_bkali", "fetch_relays", "fetch_fedipact", "fetch_joinmobilizon", "fetch_joinmisskey", "fetch_joinfediverse"] and origin is None:
         raise ValueError(f"Parameter command='{command}' but origin is None, please fix invoking this function.")
-    elif software is None:
+    elif not isinstance(path, str) and path is not None:
+        raise ValueError(f"Parameter path[]='{type(path)}' is not of type 'str'")
+    elif _DEPTH > 0 and instances.is_recent(domain, "last_instance_fetch"):
+        raise ValueError(f"domain='{domain}' has recently been fetched but function was invoked")
+    elif software is None and not instances.is_recent(domain, "last_nodeinfo"):
         try:
             logger.debug("Software for domain='%s' is not set, determining ...", domain)
             software = determine_software(domain, path)
@@ -68,10 +71,13 @@ def fetch_instances(domain: str, origin: str, software: str, command: str, path:
             instances.set_last_error(domain, exception)
 
         logger.debug("Determined software='%s' for domain='%s'", software, domain)
+    elif software is None:
+        logger.debug("domain='%s' has unknown software or nodeinfo has recently being fetched", domain)
     elif not isinstance(software, str):
         raise ValueError(f"Parameter software[]='{type(software)}' is not of type 'str'")
-    elif not isinstance(path, str) and path is not None:
-        raise ValueError(f"Parameter path[]='{type(path)}' is not of type 'str'")
+
+    # Increase depth
+    _DEPTH = _DEPTH + 1
 
     logger.debug("Checking if domain='%s' is registered ...", domain)
     if not instances.is_registered(domain):
@@ -101,7 +107,7 @@ def fetch_instances(domain: str, origin: str, software: str, command: str, path:
 
         if instances.has_pending(domain):
             logger.debug("Flushing updates for domain='%s' ...", domain)
-            instances.update_data(domain)
+            instances.update(domain)
 
         logger.debug("Invoking cookies.clear(%s) ...", domain)
         cookies.clear(domain)
@@ -124,6 +130,9 @@ def fetch_instances(domain: str, origin: str, software: str, command: str, path:
         if instance == "":
             logger.warning("Empty instance after tidyup.domain(), domain='%s'", domain)
             continue
+        elif ".." in instance:
+            logger.warning("instance='%s' contains double-dot, removing ...", instance)
+            instance = instance.replace("..", ".")
 
         logger.debug("instance='%s' - BEFORE!", instance)
         instance = instance.encode("idna").decode("utf-8")
@@ -142,7 +151,7 @@ def fetch_instances(domain: str, origin: str, software: str, command: str, path:
             logger.debug("Checking if domain='%s' has pending updates ...", domain)
             if instances.has_pending(domain):
                 logger.debug("Flushing updates for domain='%s' ...", domain)
-                instances.update_data(domain)
+                instances.update(domain)
 
             logger.debug("instance='%s',origin='%s',_DEPTH=%d reached!", instance, origin, _DEPTH)
             if _DEPTH <= config.get("max_crawl_depth") and len(peerlist) >= config.get("min_peers_length"):
@@ -158,7 +167,7 @@ def fetch_instances(domain: str, origin: str, software: str, command: str, path:
     logger.debug("Checking if domain='%s' has pending updates ...", domain)
     if instances.has_pending(domain):
         logger.debug("Flushing updates for domain='%s' ...", domain)
-        instances.update_data(domain)
+        instances.update(domain)
 
     _DEPTH = _DEPTH - 1
     logger.debug("EXIT!")
@@ -345,19 +354,19 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str:
     logger.debug("software='%s' - EXIT!", software)
     return software
 
-def determine_software(domain: str, path: str = None) -> str:
-    logger.debug("domain='%s',path='%s' - CALLED!", domain, path)
+def determine_software(domain: str, path: str = None, nodeinfo_url: str = None) -> str:
+    logger.debug("domain='%s',path='%s',nodeinfo_url='%s' - CALLED!", domain, path, nodeinfo_url)
     domain_helper.raise_on(domain)
 
     if not isinstance(path, str) and path is not None:
         raise ValueError(f"Parameter path[]='{type(path)}' is not of type 'str'")
+    elif not isinstance(nodeinfo_url, str) and nodeinfo_url is not None:
+        raise ValueError(f"Parameter nodeinfo_url[]='{type(nodeinfo_url)}' is not of type 'str'")
 
-    logger.debug("Determining software for domain='%s',path='%s'", domain, path)
+    logger.debug("Fetching nodeinfo from domain='%s',path='%s',nodeinfo_url='%s' ...", domain, path, nodeinfo_url)
+    data = nodeinfo.fetch(domain, path, nodeinfo_url)
     software = None
 
-    logger.debug("Fetching nodeinfo from domain='%s' ...", domain)
-    data = nodeinfo.fetch_nodeinfo(domain, path)
-
     logger.debug("data[%s]='%s'", type(data), data)
     if "exception" in data:
         # Continue raising it
@@ -429,6 +438,8 @@ def determine_software(domain: str, path: str = None) -> str:
         logger.debug("software='%s' has 'powered by' in it", software)
         software = version.remove(software_helper.strip_powered_by(software))
 
+    software = software.strip()
+
     logger.debug("software='%s' - EXIT!", software)
     return software