Continued:

- federation.fetch_instances() must encode domain to IDNA, too
- renamed variables

diff --git a/fba/helpers/processing.py b/fba/helpers/processing.py
index f21ff535cb17f67f57408bffabffbf0c3eaf2331..09f223c9d3b2a73ae53d136c4b8a564bc0b0e986 100644 (file)
--- a/fba/helpers/processing.py
+++ b/fba/helpers/processing.py
@@ -339,8 +339,8 @@ def csv_instance(instance: str, url: str, command: str) -> None:
         logger.debug("Invoking commit() ...")
         database.connection.commit()
 
-    logger.debug("Invoking instances.set_total_instances(%s, domains()=%d) ...", instance, len(domains))
-    instances.set_total_instances(instance, domains)
+    logger.debug("Invoking instances.set_total_peers(%s, domains()=%d) ...", instance, len(domains))
+    instances.set_total_peers(instance, domains)
 
     logger.debug("Checking if instance='%s' has pending updates ...", instance)
     if instances.has_pending(instance):

diff --git a/fba/http/federation.py b/fba/http/federation.py
index 7c996e944246e0d06c373bc9fdaf583dac33fd84..805adced237ee8620267175de070bb6cd0333700 100644 (file)
--- a/fba/http/federation.py
+++ b/fba/http/federation.py
@@ -92,13 +92,17 @@ def fetch_instances(domain: str, origin: str, software: str, command: str, path:
     elif not isinstance(software, str):
         raise TypeError(f"Parameter software[]='{type(software)}' has not expected type 'str'")
 
+    logger.debug("domain='%s' - BEFORE!", domain)
+    instance = domain_helper.encode_idna(domain.split("?")[0])
+    logger.debug("instance='%s' - AFTER!", instance)
+
     # Increase depth
     _DEPTH = _DEPTH + 1
 
-    logger.debug("Checking if domain='%s' is registered ...", domain)
-    if not instances.is_registered(domain):
-        logger.debug("Adding new domain='%s',origin='%s',command='%s',path='%s',software='%s'", domain, origin, command, path, software)
-        instances.add(domain, origin, command, path, software)
+    logger.debug("Checking if instance='%s' is registered ...", instance)
+    if not instances.is_registered(instance):
+        logger.debug("Adding new instance='%s',origin='%s',command='%s',path='%s',software='%s'", instance, origin, command, path, software)
+        instances.add(instance, origin, command, path, software)
 
         logger.debug("software='%s'", software)
         if software is not None and software_helper.is_relay(software):
@@ -106,78 +110,78 @@ def fetch_instances(domain: str, origin: str, software: str, command: str, path:
             _DEPTH = _DEPTH - 1
             return
 
-    logger.debug("Updating last_instance_fetch for domain='%s' ...", domain)
-    instances.set_last_instance_fetch(domain)
+    logger.debug("Updating last_instance_fetch for instance='%s' ...", instance)
+    instances.set_last_instance_fetch(instance)
 
     peerlist = []
     logger.debug("software='%s'", software)
     if software is not None and not software_helper.is_relay(software):
         try:
-            logger.debug("Fetching instances for domain='%s',software='%s',origin='%s'", domain, software, origin)
-            peerlist = fetch_peers(domain, software, origin)
+            logger.debug("Fetching instances for instance='%s',software='%s',origin='%s'", instance, software, origin)
+            peerlist = fetch_peers(instance, software, origin)
         except network.exceptions as exception:
             _DEPTH = _DEPTH - 1
             raise exception
 
     logger.debug("peerlist[]='%s'", type(peerlist))
     if isinstance(peerlist, list):
-        logger.debug("Invoking instances.set_total_peerlist(%s,%d) ...", domain, len(peerlist))
-        instances.set_total_peers(domain, peerlist)
+        logger.debug("Invoking instances.set_total_peerlist(%s,%d) ...", instance, len(peerlist))
+        instances.set_total_peers(instance, peerlist)
 
-    logger.debug("Invoking cookies.clear(%s) ...", domain)
-    cookies.clear(domain)
+    logger.debug("Invoking cookies.clear(%s) ...", instance)
+    cookies.clear(instance)
 
     logger.debug("peerlist[]='%s'", type(peerlist))
     if peerlist is None:
-        logger.warning("Cannot fetch peers: domain='%s',software='%s'", domain, software)
-        if instances.has_pending(domain):
-            logger.debug("Flushing updates for domain='%s' ...", domain)
-            instances.update(domain)
+        logger.warning("Cannot fetch peers: instance='%s',software='%s'", instance, software)
+        if instances.has_pending(instance):
+            logger.debug("Flushing updates for instance='%s' ...", instance)
+            instances.update(instance)
 
         _DEPTH = _DEPTH - 1
         logger.debug("EXIT!")
         return
     elif len(peerlist) == 0:
-        logger.info("domain='%s' returned an empty peer list.", domain)
-        if instances.has_pending(domain):
-            logger.debug("Flushing updates for domain='%s' ...", domain)
-            instances.update(domain)
+        logger.info("instance='%s' returned an empty peer list.", instance)
+        if instances.has_pending(instance):
+            logger.debug("Flushing updates for instance='%s' ...", instance)
+            instances.update(instance)
 
         _DEPTH = _DEPTH - 1
-        logger.debug("domain='%s',software='%s' has an empty peer list returned - EXIT!", domain, software)
+        logger.debug("instance='%s',software='%s' has an empty peer list returned - EXIT!", instance, software)
         return
 
-    logger.info("Checking %d instance(s) from domain='%s',software='%s',depth=%d ...", len(peerlist), domain, software, _DEPTH)
-    for instance in peerlist:
-        logger.debug("instance[%s]='%s'", type(instance), instance)
-        if instance in [None, ""]:
-            logger.debug("instance[%s]='%s' is either None or empty - SKIPPED!", type(instance), instance)
+    logger.info("Checking %d instance(s) from instance='%s',software='%s',depth=%d ...", len(peerlist), instance, software, _DEPTH)
+    for peer in peerlist:
+        logger.debug("peer[%s]='%s'", type(peer), peer)
+        if peer in [None, ""]:
+            logger.debug("peer[%s]='%s' is either None or empty - SKIPPED!", type(peer), peer)
             continue
-        elif isinstance(instance, dict) and "url" in instance:
-            logger.debug("Found instance[url]='%s', extracting domain/host name ...", instance["url"])
-            if not validators.url(instance["url"]):
-                logger.warning("instance[url]='%s' is not a valid URL - SKIPPED!", instance["url"])
+        elif isinstance(peer, dict) and "url" in peer:
+            logger.debug("Found peer[url]='%s', extracting domain/host name ...", peer["url"])
+            if not validators.url(peer["url"]):
+                logger.warning("peer[url]='%s' is not a valid URL - SKIPPED!", peer["url"])
                 continue
 
-            components = urllib.parse.urlparse(instance["url"])
+            components = urllib.parse.urlparse(peer["url"])
             logger.debug("components[%s]()=%d", type(components), len(components))
 
-            instance = components.netloc.lower().split(":")[0]
-            logger.debug("instance='%s'", instance)
+            peer = components.netloc.lower().split(":")[0]
+            logger.debug("peer='%s'", peer)
 
-        logger.debug("instance='%s' - BEFORE!", instance)
-        instance = tidyup.domain(instance) if isinstance(instance, str) and instance != "" else None
-        logger.debug("instance='%s' - AFTER!", instance)
+        logger.debug("peer='%s' - BEFORE!", peer)
+        peer = tidyup.domain(peer) if isinstance(peer, str) and peer != "" else None
+        logger.debug("peer='%s' - AFTER!", peer)
 
-        if instance in [None, ""]:
-            logger.debug("instance[%s]='%s' is empty after tidyup.domain(), domain='%s'", type(instance), instance, domain)
+        if peer in [None, ""]:
+            logger.debug("peer[%s]='%s' is empty after tidyup.domain(), domain='%s'", type(peer), peer, domain)
             continue
-        elif ".." in instance:
-            logger.warning("instance='%s' contains double-dot, removing ...", instance)
-            instance = instance.replace("..", ".")
+        elif ".." in peer:
+            logger.warning("peer='%s' contains double-dot, removing ...", peer)
+            peer = peer.replace("..", ".")
 
-        probe = instance.split("/")[0]
-        logger.debug("instance='%s',probe='%s'", instance, probe)
+        probe = peer.split("/")[0]
+        logger.debug("peer='%s',probe='%s'", peer, probe)
         if not validators.domain(probe, rfc_2782=True):
             logger.warning("probe='%s' is not a valid domain - SKIPPED!", probe)
             continue
@@ -185,32 +189,32 @@ def fetch_instances(domain: str, origin: str, software: str, command: str, path:
             logger.debug("probe='%s' has an unwanted TLD - SKIPPED!", probe)
             continue
 
-        logger.debug("instance='%s' - BEFORE!", instance)
-        instance = domain_helper.encode_idna(instance.split("?")[0])
-        logger.debug("instance='%s' - AFTER!", instance)
+        logger.debug("peer='%s' - BEFORE!", peer)
+        peer = domain_helper.encode_idna(peer.split("?")[0])
+        logger.debug("peer='%s' - AFTER!", peer)
 
-        if not domain_helper.is_wanted(instance):
-            logger.debug("instance='%s' is not wanted - SKIPPED!", instance)
+        if not domain_helper.is_wanted(peer):
+            logger.debug("peer='%s' is not wanted - SKIPPED!", peer)
             continue
-        elif instance.find("/profile/") > 0 or instance.find("/users/") > 0 or (instances.is_registered(instance.split("/")[0]) and instance.find("/c/") > 0):
-            logger.debug("instance='%s' is a link to a single user profile - SKIPPED!", instance)
+        elif peer.find("/profile/") > 0 or peer.find("/users/") > 0 or (instances.is_registered(peer.split("/")[0]) and peer.find("/c/") > 0):
+            logger.debug("peer='%s' is a link to a single user profile - SKIPPED!", peer)
             continue
-        elif instance.find("/tag/") > 0:
-            logger.debug("instance='%s' is a link to a tag - SKIPPED!", instance)
+        elif peer.find("/tag/") > 0:
+            logger.debug("peer='%s' is a link to a tag - SKIPPED!", peer)
             continue
-        elif not instances.is_registered(instance):
+        elif not instances.is_registered(peer):
             logger.debug("Checking if domain='%s' has pending updates ...", domain)
             if instances.has_pending(domain):
                 logger.debug("Flushing updates for domain='%s' ...", domain)
                 instances.update(domain)
 
-            logger.debug("instance='%s',origin='%s',_DEPTH=%d reached!", instance, origin, _DEPTH)
+            logger.debug("peer='%s',origin='%s',_DEPTH=%d reached!", peer, origin, _DEPTH)
             if _DEPTH <= _max_crawl_depth and len(peerlist) >= _min_peers_length:
-                logger.debug("Fetching instance='%s',origin='%s',command='%s',path='%s',_DEPTH=%d ...", instance, domain, command, path, _DEPTH)
-                fetch_instances(instance, domain, None, command, path)
+                logger.debug("Fetching peer='%s',origin='%s',command='%s',path='%s',_DEPTH=%d ...", peer, domain, command, path, _DEPTH)
+                fetch_instances(peer, domain, None, command, path)
             else:
-                logger.debug("Adding instance='%s',domain='%s',command='%s',_DEPTH=%d ...", instance, domain, command, _DEPTH)
-                instances.add(instance, domain, command)
+                logger.debug("Adding peer='%s',domain='%s',command='%s',_DEPTH=%d ...", peer, domain, command, _DEPTH)
+                instances.add(peer, domain, command)
 
     logger.debug("Checking if domain='%s' has pending updates ...", domain)
     if instances.has_pending(domain):