From d316ff7bb21d6b8032919d6b403f2c9319937188 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Wed, 12 Jul 2023 11:05:03 +0200 Subject: [PATCH] Continued: - max "crawl" depth and min peerlist size to go deeper is now configurable - for example for low-memory systems, keep max_crawl_depth small and min_peers_length big - the default values may cause python3 to consume ~550 MB RAM - so you can practially say each depth adds another MB RAM usage --- config.defaults.json | 4 +++- fba/http/federation.py | 6 +++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/config.defaults.json b/config.defaults.json index b0c3dbb..09c3293 100644 --- a/config.defaults.json +++ b/config.defaults.json @@ -25,5 +25,7 @@ "rss_limit" : 50, "api_limit" : 500, "theme" : "light", - "instances_social_api_key": "" + "instances_social_api_key": "", + "max_crawl_depth" : 500, + "min_peers_length" : 100 } diff --git a/fba/http/federation.py b/fba/http/federation.py index 0d8cdf8..ae01d1f 100644 --- a/fba/http/federation.py +++ b/fba/http/federation.py @@ -108,8 +108,8 @@ def fetch_instances(domain: str, origin: str, software: str, command: str, path: logger.info("Checking %d instance(s) from domain='%s',software='%s' ...", len(peerlist), domain, software) for instance in peerlist: logger.debug("instance='%s'", instance) - if instance is None: - # Skip "None" types as tidup.domain() cannot parse them + if instance is None or instance == "": + logger.debug("instance[%s]='%s' is either None or empty - SKIPPED!", type(instance), instance) continue logger.debug("instance='%s' - BEFORE!", instance) @@ -140,7 +140,7 @@ def fetch_instances(domain: str, origin: str, software: str, command: str, path: instances.update_data(domain) logger.debug("instance='%s',origin='%s',_DEPTH=%d reached!", instance, origin, _DEPTH) - if _DEPTH <= 500 and len(peerlist) >= 100: + if _DEPTH <= config.get("max_crawl_depth") and len(peerlist) >= config.get("min_peers_length"): logger.debug("Fetching instance='%s',origin='%s',command='%s',path='%s',_DEPTH=%d ...", instance, domain, command, path, _DEPTH) fetch_instances(instance, domain, None, command, path) else: -- 2.39.5