Continued:

[fba.git] / fba / http / federation.py

diff --git a/fba/http/federation.py b/fba/http/federation.py
index 5fb9608491c1c73daf053fe7e530f35735ba4762..50f66c37eb0ccea179e207a05010d11d213ed54c 100644 (file)
--- a/fba/http/federation.py
+++ b/fba/http/federation.py
@@ -84,6 +84,11 @@ def fetch_instances(domain: str, origin: str, software: str, command: str, path:
         logger.debug("Adding new domain='%s',origin='%s',command='%s',path='%s',software='%s'", domain, origin, command, path, software)
         instances.add(domain, origin, command, path, software)
 
+        logger.debug("software='%s'", software)
+        if software_helper.is_relay(software):
+            logger.debug("software='%s' is a relay software - EXIT!", software)
+            return
+
     logger.debug("Updating last_instance_fetch for domain='%s' ...", domain)
     instances.set_last_instance_fetch(domain)
 
@@ -257,7 +262,6 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str:
     elif path == "":
         raise ValueError("Parameter 'path' is empty")
 
-    logger.debug("domain='%s',path='%s' - CALLED!", domain, path)
     software = None
 
     logger.debug("Fetching path='%s' from domain='%s' ...", path, domain)
@@ -281,7 +285,7 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str:
         app_name  = doc.find("meta", {"name"    : "application-name"})
 
         logger.debug("generator[]='%s',site_name[]='%s',platform[]='%s',app_name[]='%s'", type(generator), type(site_name), type(platform), type(app_name))
-        if isinstance(platform, bs4.element.Tag) and isinstance(platform.get("content"), str):
+        if isinstance(platform, bs4.element.Tag) and isinstance(platform.get("content"), str) and platform.get("content") != "":
             logger.debug("Found property=og:platform, domain='%s'", domain)
             software = tidyup.domain(platform.get("content"))
 
@@ -289,7 +293,7 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str:
             if software is not None and software != "":
                 logger.debug("domain='%s' has og:platform='%s' - Setting detection_mode=PLATFORM ...", domain, software)
                 instances.set_detection_mode(domain, "PLATFORM")
-        elif isinstance(generator, bs4.element.Tag) and isinstance(generator.get("content"), str):
+        elif isinstance(generator, bs4.element.Tag) and isinstance(generator.get("content"), str) and generator.get("content") != "":
             logger.debug("Found generator meta tag: domain='%s'", domain)
             software = tidyup.domain(generator.get("content"))
 
@@ -297,7 +301,7 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str:
             if software is not None and software != "":
                 logger.info("domain='%s' is generated by software='%s' - Setting detection_mode=GENERATOR ...", domain, software)
                 instances.set_detection_mode(domain, "GENERATOR")
-        elif isinstance(app_name, bs4.element.Tag) and isinstance(app_name.get("content"), str):
+        elif isinstance(app_name, bs4.element.Tag) and isinstance(app_name.get("content"), str) and app_name.get("content") != "":
             logger.debug("Found property=og:app_name, domain='%s'", domain)
             software = tidyup.domain(app_name.get("content"))
 
@@ -305,7 +309,7 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str:
             if software is not None and software != "":
                 logger.debug("domain='%s' has application-name='%s' - Setting detection_mode=app_name ...", domain, software)
                 instances.set_detection_mode(domain, "APP_NAME")
-        elif isinstance(site_name, bs4.element.Tag) and isinstance(site_name.get("content"), str):
+        elif isinstance(site_name, bs4.element.Tag) and isinstance(site_name.get("content"), str) and site_name.get("content") != "":
             logger.debug("Found property=og:site_name, domain='%s'", domain)
             software = tidyup.domain(site_name.get("content"))
 
@@ -320,9 +324,12 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str:
         domain2 = components.netloc.lower().split(":")[0]
 
         logger.debug("domain2='%s'", domain2)
-        if not instances.is_registered(domain2):
+        if not domain_helper.is_wanted(domain2):
+            logger.debug("domain2='%s' is not wanted - EXIT!", domain2)
+            return None
+        elif not instances.is_registered(domain2):
             logger.info("components.netloc='%s' is not registered, adding ...", components.netloc)
-            fetch_instances(domain2, domain, None, "redirect_target")
+            instances.add(domain2, domain, "redirect_target")
 
         message = f"Redirect from domain='{domain}' to response.url='{response.url}'"
         instances.set_last_error(domain, message)
@@ -419,7 +426,7 @@ def determine_software(domain: str, path: str = None) -> str:
         logger.debug("Generator for domain='%s' is: '%s'", domain, software)
 
     logger.debug("software[%s]='%s'", type(software), software)
-    if software is None:
+    if software is None or software == "":
         logger.debug("Returning None - EXIT!")
         return None
 
@@ -610,7 +617,7 @@ def fetch_blocks(domain: str) -> list:
                 blocklist.append({
                     "blocker"    : domain,
                     "blocked"    : block["domain"],
-                    "hash"       : block["digest"] if "digest" in block else None,
+                    "digest"     : block["digest"] if "digest" in block else None,
                     "reason"     : reason,
                     "block_level": blocks.alias_block_level(block["severity"]),
                 })