Continued:

- renamed functions in 'cache' module as they are already in proper module
- tried to prevent error if an empty (0 Byte) result is returned

diff --git a/fba/cache.py b/fba/cache.py
index 843e35395695328e009715c48a17df80cc2b1c97..25adde245d0346d28a2e561b9860189dc5b2a694 100644 (file)
--- a/fba/cache.py
+++ b/fba/cache.py
@@ -14,20 +14,19 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-
 # Cache for redundant SQL queries
 _cache = {}
 
 ##### Cache #####
 
-def is_cache_initialized(key: str) -> bool:
+def key_exists(key: str) -> bool:
     return key in _cache
 
-def set_all_cache_key(key: str, rows: list, value: any):
+def set_all(key: str, rows: list, value: any):
     # NOISY-DEBUG: print(f"DEBUG: key='{key}',rows()={len(rows)},value[]={type(value)} - CALLED!")
     if type(key) != str:
         raise ValueError("Parameter key[]='{type(key)}' is not 'str'")
-    elif not is_cache_initialized(key):
+    elif not key_exists(key):
         # NOISY-DEBUG: print(f"DEBUG: Cache for key='{key}' not initialized.")
         _cache[key] = {}
 
@@ -41,23 +40,23 @@ def set_all_cache_key(key: str, rows: list, value: any):
 
     # NOISY-DEBUG: print("DEBUG: EXIT!")
 
-def set_cache_key(key: str, sub: str, value: any):
+def set_sub_key(key: str, sub: str, value: any):
     if type(key) != str:
         raise ValueError("Parameter key[]='{type(key)}' is not 'str'")
     elif type(sub) != str:
         raise ValueError("Parameter sub[]='{type(sub)}' is not 'str'")
-    elif not is_cache_initialized(key):
+    elif not key_exists(key):
         print(f"WARNING: Bad method call, key='{key}' is not initialized yet.")
         raise Exception(f"Cache for key='{key}' is not initialized, but function called")
 
     _cache[key][sub] = value
 
-def is_cache_key_set(key: str, sub: str) -> bool:
+def sub_key_exists(key: str, sub: str) -> bool:
     if type(key) != str:
         raise ValueError("Parameter key[]='{type(key)}' is not 'str'")
     elif type(sub) != str:
         raise ValueError("Parameter sub[]='{type(sub)}' is not 'str'")
-    elif not is_cache_initialized(key):
+    elif not key_exists(key):
         print(f"WARNING: Bad method call, key='{key}' is not initialized yet.")
         raise Exception(f"Cache for key='{key}' is not initialized, but function called")
 

diff --git a/fba/fba.py b/fba/fba.py
index 7d24b5540ed90774f27e106c2cbb5f5ebf170965..2025397906b15b39428a2e849740185edeb1485f 100644 (file)
--- a/fba/fba.py
+++ b/fba/fba.py
@@ -362,7 +362,7 @@ def update_last_blocked(domain: str):
         raise ValueError(f"Parameter 'domain' cannot be empty")
 
     # DEBUG: print("DEBUG: Updating last_blocked for domain", domain)
-    instances.set_instance_data("last_blocked", domain, time.time())
+    instances.set("last_blocked", domain, time.time())
 
     # Running pending updated
     # DEBUG: print(f"DEBUG: Invoking instances.update_instance_data({domain}) ...")
@@ -415,17 +415,17 @@ def update_last_error(domain: str, res: any):
 
     # DEBUG: print("DEBUG: BEFORE res[]:", type(res))
     if isinstance(res, BaseException) or isinstance(res, json.JSONDecodeError):
-        res = str(res)
+        res = f"{type}:str(res)"
 
     # DEBUG: print("DEBUG: AFTER res[]:", type(res))
     if type(res) is str:
         # DEBUG: print(f"DEBUG: Setting last_error_details='{res}'");
-        instances.set_instance_data("last_status_code"  , domain, 999)
-        instances.set_instance_data("last_error_details", domain, res)
+        instances.set("last_status_code"  , domain, 999)
+        instances.set("last_error_details", domain, res)
     else:
         # DEBUG: print(f"DEBUG: Setting last_error_details='{res.reason}'");
-        instances.set_instance_data("last_status_code"  , domain, res.status_code)
-        instances.set_instance_data("last_error_details", domain, res.reason)
+        instances.set("last_status_code"  , domain, res.status_code)
+        instances.set("last_error_details", domain, res.reason)
 
     # Running pending updated
     # DEBUG: print(f"DEBUG: Invoking instances.update_instance_data({domain}) ...")
@@ -443,7 +443,7 @@ def update_last_instance_fetch(domain: str):
         raise ValueError(f"Parameter 'domain' cannot be empty")
 
     # DEBUG: print("DEBUG: Updating last_instance_fetch for domain:", domain)
-    instances.set_instance_data("last_instance_fetch", domain, time.time())
+    instances.set("last_instance_fetch", domain, time.time())
 
     # Running pending updated
     # DEBUG: print(f"DEBUG: Invoking instances.update_instance_data({domain}) ...")
@@ -459,8 +459,8 @@ def update_last_nodeinfo(domain: str):
         raise ValueError(f"Parameter 'domain' cannot be empty")
 
     # DEBUG: print("DEBUG: Updating last_nodeinfo for domain:", domain)
-    instances.set_instance_data("last_nodeinfo", domain, time.time())
-    instances.set_instance_data("last_updated" , domain, time.time())
+    instances.set("last_nodeinfo", domain, time.time())
+    instances.set("last_updated" , domain, time.time())
 
     # Running pending updated
     # DEBUG: print(f"DEBUG: Invoking instances.update_instance_data({domain}) ...")
@@ -546,7 +546,7 @@ def get_peers(domain: str, software: str) -> list:
                 break
 
         # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'")
-        instances.set_instance_data("total_peers", domain, len(peers))
+        instances.set("total_peers", domain, len(peers))
 
         # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...")
         update_last_instance_fetch(domain)
@@ -578,7 +578,7 @@ def get_peers(domain: str, software: str) -> list:
             print(f"WARNING: Exception during fetching JSON: domain='{domain}',exception[{type(e)}]:'{str(e)}'")
 
         # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'")
-        instances.set_instance_data("total_peers", domain, len(peers))
+        instances.set("total_peers", domain, len(peers))
 
         # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...")
         update_last_instance_fetch(domain)
@@ -620,7 +620,7 @@ def get_peers(domain: str, software: str) -> list:
                     print(f"WARNING: Exception during fetching JSON: domain='{domain}',exception[{type(e)}]:'{str(e)}'")
 
         # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'")
-        instances.set_instance_data("total_peers", domain, len(peers))
+        instances.set("total_peers", domain, len(peers))
 
         # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...")
         update_last_instance_fetch(domain)
@@ -638,7 +638,10 @@ def get_peers(domain: str, software: str) -> list:
             # DEBUG: print(f"DEBUG: Was not able to fetch '{get_peers_url}', trying alternative ...")
             res = reqto.get(f"https://{domain}/api/v3/site", headers=api_headers, timeout=(config.get("connection_timeout"), config.get("read_timeout")))
 
-            data = res.json()
+            data = json.dumps({})
+            if res.text.strip() != "":
+                data = res.json()
+
             # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'")
             if not res.ok or res.status_code >= 400:
                 print("WARNING: Could not reach any JSON API:", domain)
@@ -662,7 +665,7 @@ def get_peers(domain: str, software: str) -> list:
         update_last_error(domain, e)
 
     # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'")
-    instances.set_instance_data("total_peers", domain, len(peers))
+    instances.set("total_peers", domain, len(peers))
 
     # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...")
     update_last_instance_fetch(domain)
@@ -678,7 +681,7 @@ def post_json_api(domain: str, path: str, parameter: str, extra_headers: dict =
     elif type(path) != str:
         raise ValueError(f"path[]={type(path)} is not 'str'")
     elif path == "":
-        raise ValueError(f"path cannot be empty")
+        raise ValueError("Parameter 'path' cannot be empty")
     elif type(parameter) != str:
         raise ValueError(f"parameter[]={type(parameter)} is not 'str'")
 
@@ -739,8 +742,8 @@ def fetch_nodeinfo(domain: str, path: str = None) -> list:
             # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'")
             if res.ok and isinstance(data, dict):
                 # DEBUG: print("DEBUG: Success:", request)
-                instances.set_instance_data("detection_mode", domain, "STATIC_CHECK")
-                instances.set_instance_data("nodeinfo_url"  , domain, request)
+                instances.set("detection_mode", domain, "STATIC_CHECK")
+                instances.set("nodeinfo_url"  , domain, request)
                 break
             elif res.ok and isinstance(data, list):
                 # DEBUG: print(f"DEBUG: domain='{domain}' returned a list: '{data}'")
@@ -788,8 +791,8 @@ def fetch_wellknown_nodeinfo(domain: str) -> list:
                         # DEBUG: print("DEBUG: href,res.ok,res.status_code:", link["href"], res.ok, res.status_code)
                         if res.ok and isinstance(data, dict):
                             # DEBUG: print("DEBUG: Found JSON nodeinfo():", len(data))
-                            instances.set_instance_data("detection_mode", domain, "AUTO_DISCOVERY")
-                            instances.set_instance_data("nodeinfo_url"  , domain, link["href"])
+                            instances.set("detection_mode", domain, "AUTO_DISCOVERY")
+                            instances.set("nodeinfo_url"  , domain, link["href"])
                             break
                     else:
                         print("WARNING: Unknown 'rel' value:", domain, link["rel"])
@@ -836,13 +839,13 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str:
                 # DEBUG: print("DEBUG: Found generator meta tag:", domain)
                 software = tidyup_domain(generator.get("content"))
                 print(f"INFO: domain='{domain}' is generated by '{software}'")
-                instances.set_instance_data("detection_mode", domain, "GENERATOR")
+                instances.set("detection_mode", domain, "GENERATOR")
                 remove_pending_error(domain)
             elif isinstance(site_name, bs4.element.Tag):
                 # DEBUG: print("DEBUG: Found property=og:site_name:", domain)
                 sofware = tidyup_domain(site_name.get("content"))
                 print(f"INFO: domain='{domain}' has og:site_name='{software}'")
-                instances.set_instance_data("detection_mode", domain, "SITE_NAME")
+                instances.set("detection_mode", domain, "SITE_NAME")
                 remove_pending_error(domain)
 
     except BaseException as e:
@@ -1094,19 +1097,19 @@ def is_instance_registered(domain: str) -> bool:
         raise ValueError(f"Parameter 'domain' cannot be empty")
 
     # NOISY-DEBUG: # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
-    if not cache.is_cache_initialized("is_registered"):
+    if not cache.key_exists("is_registered"):
         # NOISY-DEBUG: # DEBUG: print(f"DEBUG: Cache for 'is_registered' not initialized, fetching all rows ...")
         try:
             cursor.execute("SELECT domain FROM instances")
 
             # Check Set all
-            cache.set_all_cache_key("is_registered", cursor.fetchall(), True)
+            cache.set_all("is_registered", cursor.fetchall(), True)
         except BaseException as e:
             print(f"ERROR: failed SQL query: domain='{domain}',exception[{type(e)}]:'{str(e)}'")
             sys.exit(255)
 
     # Is cache found?
-    registered = cache.is_cache_key_set("is_registered", domain)
+    registered = cache.sub_key_exists("is_registered", domain)
 
     # NOISY-DEBUG: # DEBUG: print(f"DEBUG: registered='{registered}' - EXIT!")
     return registered
@@ -1148,12 +1151,12 @@ def add_instance(domain: str, origin: str, originator: str, path: str = None):
             ),
         )
 
-        cache.set_cache_key("is_registered", domain, True)
+        cache.set_sub_key("is_registered", domain, True)
 
         if instances.has_pending_instance_data(domain):
             # DEBUG: print(f"DEBUG: domain='{domain}' has pending nodeinfo being updated ...")
-            instances.set_instance_data("last_status_code"  , domain, None)
-            instances.set_instance_data("last_error_details", domain, None)
+            instances.set("last_status_code"  , domain, None)
+            instances.set("last_error_details", domain, None)
             instances.update_instance_data(domain)
             remove_pending_error(domain)
 

diff --git a/fba/instances.py b/fba/instances.py
index 16b0e332972af6d69da65074ff8bab886d1adfeb..3312fa824a3bc359f59e6b6eb87dc801636a6136 100644 (file)
--- a/fba/instances.py
+++ b/fba/instances.py
@@ -44,7 +44,7 @@ _pending = {
     "last_error_details" : {},
 }
 
-def set_instance_data(key: str, domain: str, value: any):
+def set(key: str, domain: str, value: any):
     # NOISY-DEBUG: print(f"DEBUG: key='{key}',domain='{domain}',value[]='{type(value)}' - CALLED!")
     if type(key) != str:
         raise ValueError("Parameter key[]='{type(key)}' is not 'str'")