From: Roland Häder <roland@mxchange.org>
Date: Wed, 12 Jul 2023 09:05:03 +0000 (+0200)
Subject: Continued:
X-Git-Url: https://git.mxchange.org/?a=commitdiff_plain;h=d316ff7bb21d6b8032919d6b403f2c9319937188;p=fba.git

Continued:
- max "crawl" depth and min peerlist size to go deeper is now configurable
- for example for low-memory systems, keep max_crawl_depth small and
  min_peers_length big
- the default values may cause python3 to consume ~550 MB RAM
- so you can practially say each depth adds another MB RAM usage
---

diff --git a/config.defaults.json b/config.defaults.json
index b0c3dbb..09c3293 100644
--- a/config.defaults.json
+++ b/config.defaults.json
@@ -25,5 +25,7 @@
     "rss_limit"               : 50,
     "api_limit"               : 500,
     "theme"                   : "light",
-    "instances_social_api_key": ""
+    "instances_social_api_key": "",
+    "max_crawl_depth"         : 500,
+    "min_peers_length"        : 100
 }
diff --git a/fba/http/federation.py b/fba/http/federation.py
index 0d8cdf8..ae01d1f 100644
--- a/fba/http/federation.py
+++ b/fba/http/federation.py
@@ -108,8 +108,8 @@ def fetch_instances(domain: str, origin: str, software: str, command: str, path:
     logger.info("Checking %d instance(s) from domain='%s',software='%s' ...", len(peerlist), domain, software)
     for instance in peerlist:
         logger.debug("instance='%s'", instance)
-        if instance is None:
-            # Skip "None" types as tidup.domain() cannot parse them
+        if instance is None or instance == "":
+            logger.debug("instance[%s]='%s' is either None or empty - SKIPPED!", type(instance), instance)
             continue
 
         logger.debug("instance='%s' - BEFORE!", instance)
@@ -140,7 +140,7 @@ def fetch_instances(domain: str, origin: str, software: str, command: str, path:
                 instances.update_data(domain)
 
             logger.debug("instance='%s',origin='%s',_DEPTH=%d reached!", instance, origin, _DEPTH)
-            if _DEPTH <= 500 and len(peerlist) >= 100:
+            if _DEPTH <= config.get("max_crawl_depth") and len(peerlist) >= config.get("min_peers_length"):
                 logger.debug("Fetching instance='%s',origin='%s',command='%s',path='%s',_DEPTH=%d ...", instance, domain, command, path, _DEPTH)
                 fetch_instances(instance, domain, None, command, path)
             else: