Continued:

- max "crawl" depth and min peerlist size to go deeper is now configurable
- for example for low-memory systems, keep max_crawl_depth small and
  min_peers_length big
- the default values may cause python3 to consume ~550 MB RAM
- so you can practially say each depth adds another MB RAM usage

diff --git a/config.defaults.json b/config.defaults.json
index b0c3dbb509aff78f891ce1303ea6387f953b699a..09c3293906e9a468a6cc22901941fceeb3922715 100644 (file)
--- a/config.defaults.json
+++ b/config.defaults.json
@@ -25,5 +25,7 @@
     "rss_limit"               : 50,
     "api_limit"               : 500,
     "theme"                   : "light",
-    "instances_social_api_key": ""
+    "instances_social_api_key": "",
+    "max_crawl_depth"         : 500,
+    "min_peers_length"        : 100
 }

diff --git a/fba/http/federation.py b/fba/http/federation.py
index 0d8cdf8b2bf5a51d5bce16ee1616e8017aa9f113..ae01d1ff03d803ab331edbc77e93c14c020b6842 100644 (file)
--- a/fba/http/federation.py
+++ b/fba/http/federation.py
@@ -108,8 +108,8 @@ def fetch_instances(domain: str, origin: str, software: str, command: str, path:
     logger.info("Checking %d instance(s) from domain='%s',software='%s' ...", len(peerlist), domain, software)
     for instance in peerlist:
         logger.debug("instance='%s'", instance)
-        if instance is None:
-            # Skip "None" types as tidup.domain() cannot parse them
+        if instance is None or instance == "":
+            logger.debug("instance[%s]='%s' is either None or empty - SKIPPED!", type(instance), instance)
             continue
 
         logger.debug("instance='%s' - BEFORE!", instance)
@@ -140,7 +140,7 @@ def fetch_instances(domain: str, origin: str, software: str, command: str, path:
                 instances.update_data(domain)
 
             logger.debug("instance='%s',origin='%s',_DEPTH=%d reached!", instance, origin, _DEPTH)
-            if _DEPTH <= 500 and len(peerlist) >= 100:
+            if _DEPTH <= config.get("max_crawl_depth") and len(peerlist) >= config.get("min_peers_length"):
                 logger.debug("Fetching instance='%s',origin='%s',command='%s',path='%s',_DEPTH=%d ...", instance, domain, command, path, _DEPTH)
                 fetch_instances(instance, domain, None, command, path)
             else: