Continued:

- some variables fixed (e.g. no row[domain] but source_domain does exist
- added some random sleep for fetch_observer() as they seem to block to quick
  responses on automatic requests

diff --git a/fba/commands.py b/fba/commands.py
index c3e7bc782dc5f0399e5b7464d230e45089170e87..5dfdcedb104a31a5756ee16fd7b9f0b869f2e20a 100644 (file)
--- a/fba/commands.py
+++ b/fba/commands.py
@@ -18,6 +18,7 @@ import csv
 import inspect
 import json
 import logging
+import numpy
 import time
 
 from urllib.parse import urlparse
@@ -62,6 +63,8 @@ logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 #logger.setLevel(logging.DEBUG)
 
+default_rng = numpy.random.default_rng()
+
 def check_instance(args: argparse.Namespace) -> int:
     logger.debug("args.domain='%s' - CALLED!", args.domain)
 
@@ -549,10 +552,10 @@ def fetch_observer(args: argparse.Namespace) -> int:
 
             logger.debug("raw[%s]()=%d", type(raw), len(raw))
             if "exception" in raw:
-                logger.warning("row[domain]='%s' has caused an exception: '%s' - raising again ...", row["domain"], type(raw["exception"]))
+                logger.warning("source_domain='%s' has caused an exception: '%s' - raising again ...", raw["domain"], type(raw["exception"]))
                 raise raw["exception"]
             elif "error_message" in raw:
-                logger.warning("row[domain]='%s' has caused error message: '%s' - SKIPPED!", row["domain"], raw["error_message"])
+                logger.warning("source_domain='%s' has caused error message: '%s' - SKIPPED!", raw["domain"], raw["error_message"])
                 continue
             elif not "data" in raw["json"]:
                 logger.warning("Cannot find key 'nodes' in raw[json]()=%d", len(raw["json"]))
@@ -600,6 +603,9 @@ def fetch_observer(args: argparse.Namespace) -> int:
             logger.info("Fetching instances for domain='%s',software='%s' ...", domain, software)
             federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
 
+        logger.debug("Random sleep to mitigate source's automatic blocking (sorry) ...")
+        time.sleep(default_rng.integers(low=1, high=3))
+
     logger.debug("Success! - EXIT!")
     return 0
 

diff --git a/fba/http/network.py b/fba/http/network.py
index 227e76fbe6ae2f7f579ff2532fd9c4c509b4bbd3..a2e3b430fd85343cb1c07983c305dd7cf572bf13 100644 (file)
--- a/fba/http/network.py
+++ b/fba/http/network.py
@@ -200,7 +200,8 @@ def get_json_api(domain: str, path: str, headers: dict, timeout: tuple) -> dict:
 
         logger.debug("Invoking instances.set_last_error(%s,%s) ...", domain, exception)
         instances.set_last_error(domain, exception)
-        raise exception
+        logger.debug("Returning json_reply(%d)[]='%s' during an exception: '%s' - EXIT!", len(json_reply), type(json_reply), exception)
+        return json_reply
 
     logger.debug("response.ok='%s',response.status_code=%d,response.reason='%s'", response.ok, response.status_code, response.reason)
     if not response.ok or response.status_code > 200:
@@ -367,6 +368,7 @@ def fetch_url(url: str, headers: dict, timeout: tuple) -> requests.models.Respon
 
 def fetch_json_rows(hostname: str, path: str, headers: dict = {}, rows_key: str = None):
     logger.debug("hostname='%s',path='%s',headers()=%d,rows_key='%s' - CALLED!", hostname, path, len(headers), rows_key)
+
     if not isinstance(hostname, str):
         raise ValueError(f"hostname[]='{type(hostname)}' is not of type 'str'")
     elif hostname == "":