Continued:

- some APIs return the same result at the end all over again
- so we need to check if the domain is already part of the created list
- for that purpose has_element() is introduced

diff --git a/fba/cache.py b/fba/cache.py
index 790d88f8772ad3b146702d8b213380de5a79377e..22c2b00ba2ae725d4416d08b5f40f624fb5c4cfc 100644 (file)
--- a/fba/cache.py
+++ b/fba/cache.py
@@ -23,22 +23,22 @@ def key_exists(key: str) -> bool:
     return key in _cache
 
 def set_all(key: str, rows: list, value: any):
-    # NOISY-DEBUG: print(f"DEBUG: key='{key}',rows()={len(rows)},value[]={type(value)} - CALLED!")
+    # DEBUG: print(f"DEBUG: key='{key}',rows()={len(rows)},value[]={type(value)} - CALLED!")
     if type(key) != str:
         raise ValueError("Parameter key[]='{type(key)}' is not 'str'")
     elif not key_exists(key):
-        # NOISY-DEBUG: print(f"DEBUG: Cache for key='{key}' not initialized.")
+        # DEBUG: print(f"DEBUG: Cache for key='{key}' not initialized.")
         _cache[key] = {}
 
     for sub in rows:
-        # NOISY-DEBUG: print(f"DEBUG: Setting key='{key}',sub[{type(sub)}]='{sub}'")
+        # DEBUG: print(f"DEBUG: Setting key='{key}',sub[{type(sub)}]='{sub}'")
 
         if isinstance(sub, tuple):
             _cache[key][sub[0]] = value
         else:
             print(f"WARNING: Unsupported type row[]='{type(row)}'")
 
-    # NOISY-DEBUG: print("DEBUG: EXIT!")
+    # DEBUG: print("DEBUG: EXIT!")
 
 def set_sub_key(key: str, sub: str, value: any):
     if type(key) != str:

diff --git a/fba/fba.py b/fba/fba.py
index d8abc9ee5c911078756b811b42607555226163b9..5350a0e6df19be181e7da26d6e477d62372dc857 100644 (file)
--- a/fba/fba.py
+++ b/fba/fba.py
@@ -125,6 +125,7 @@ def is_primitive(var: any) -> bool:
     return type(var) in {int, str, float, bool} or var == None
 
 def fetch_instances(domain: str, origin: str, software: str, script: str, path: str = None):
+    # DEBUG: print(f"DEBUG: domain={domain},origin={origin},software={software},path={path} - CALLED!")
     if type(domain) != str:
         raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
     elif domain == "":
@@ -136,7 +137,6 @@ def fetch_instances(domain: str, origin: str, software: str, script: str, path:
     elif domain == "":
         raise ValueError(f"Parameter 'domain' cannot be empty")
 
-    # DEBUG: print("DEBUG: domain,origin,software,path:", domain, origin, software, path)
     if not is_instance_registered(domain):
         # DEBUG: print("DEBUG: Adding new domain:", domain, origin)
         add_instance(domain, origin, script, path)
@@ -157,9 +157,9 @@ def fetch_instances(domain: str, origin: str, software: str, script: str, path:
             # Skip "None" types as tidup() cannot parse them
             continue
 
-        # DEBUG: print(f"DEBUG: instance[{type(instance}]={instance} - BEFORE")
+        # DEBUG: print(f"DEBUG: instance='{instance}' - BEFORE")
         instance = tidyup_domain(instance)
-        # DEBUG: print(f"DEBUG: instance[{type(instance}]={instance} - AFTER")
+        # DEBUG: print(f"DEBUG: instance='{instance}' - AFTER")
 
         if instance == "":
             print("WARNING: Empty instance after tidyup_domain(), domain:", domain)
@@ -508,7 +508,7 @@ def get_peers(domain: str, software: str) -> list:
                     "Origin": domain
                 })
 
-            # DEBUG: print(f"DEBUG: fetched()={len(fetched))}")
+            # DEBUG: print(f"DEBUG: fetched()={len(fetched)}")
             if len(fetched) == 0:
                 # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
                 break
@@ -1100,9 +1100,9 @@ def is_instance_registered(domain: str) -> bool:
     elif domain == "":
         raise ValueError(f"Parameter 'domain' cannot be empty")
 
-    # NOISY-DEBUG: # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
+    # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
     if not cache.key_exists("is_registered"):
-        # NOISY-DEBUG: # DEBUG: print(f"DEBUG: Cache for 'is_registered' not initialized, fetching all rows ...")
+        # DEBUG: print(f"DEBUG: Cache for 'is_registered' not initialized, fetching all rows ...")
         try:
             cursor.execute("SELECT domain FROM instances")
 
@@ -1115,7 +1115,7 @@ def is_instance_registered(domain: str) -> bool:
     # Is cache found?
     registered = cache.sub_key_exists("is_registered", domain)
 
-    # NOISY-DEBUG: # DEBUG: print(f"DEBUG: registered='{registered}' - EXIT!")
+    # DEBUG: print(f"DEBUG: registered='{registered}' - EXIT!")
     return registered
 
 def add_instance(domain: str, origin: str, originator: str, path: str = None):
@@ -1359,7 +1359,7 @@ def get_misskey_blocks(domain: str) -> dict:
 
             for instance in fetched:
                 # just in case
-                if instance["isSuspended"]:
+                if instance["isSuspended"] and not has_element(blocks["suspended"], "domain", instance):
                     blocks["suspended"].append(
                         {
                             "domain": tidyup_domain(instance["host"]),
@@ -1411,7 +1411,7 @@ def get_misskey_blocks(domain: str) -> dict:
                 offset = offset + step
 
             for instance in fetched:
-                if instance["isBlocked"]:
+                if instance["isBlocked"] and not has_element(blocks["blocked"], "domain", instance):
                     blocks["blocked"].append({
                         "domain": tidyup_domain(instance["host"]),
                         "reason": None
@@ -1478,8 +1478,12 @@ def get_response(domain: str, path: str, headers: dict, timeout: list) -> reques
     # DEBUG: print(f"DEBUG: domain='{domain}',path='{path}',headers()={len(headers)},timeout={timeout} - CALLED!")
     if type(domain) != str:
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
+    elif domain == "":
+        raise ValueError("Parameter 'domain' cannot be empty")
     elif type(path) != str:
         raise ValueError(f"Parameter path[]='{type(path)}' is not 'str'")
+    elif path == "":
+        raise ValueError("Parameter 'path' cannot be empty")
 
     try:
         # DEBUG: print(f"DEBUG: Sending request to '{domain}{path}' ...")
@@ -1491,3 +1495,25 @@ def get_response(domain: str, path: str, headers: dict, timeout: list) -> reques
 
     # DEBUG: print(f"DEBUG: response[]='{type(response)}' - EXXIT!")
     return response
+
+def has_element(elements: list, key: str, value: any) -> bool:
+    # DEBUG: print(f"DEBUG: element()={len(element)},key='{key}',value[]='{type(value)}' - CALLED!")
+    if type(key) != str:
+        raise ValueError(f"Parameter key[]='{type(key)}' is not 'str'")
+    elif key == "":
+        raise ValueError("Parameter 'key' cannot be empty")
+
+    has = False
+    # DEBUG: print(f"DEBUG: Checking elements()={len(elements)} ...")
+    for element in elements:
+        # DEBUG: print(f"DEBUG: element[]='{type(element)}'")
+        if type(element) != dict:
+            raise ValueError(f"element[]='{type(element)}' is not 'dict'")
+        elif not key in element:
+            raise KeyError(f"Cannot find key='{key}'")
+        elif element[key] == value:
+            has = True
+            break
+
+    # DEBUG: print(f"DEBUG: has={has} - EXIT!")
+    return has

diff --git a/fba/instances.py b/fba/instances.py
index b21e85b2f8a2e8362ecbc49c885f6b5928c8fb80..f900bb1ea3e3fe6ec65839884d4beeb044155ead 100644 (file)
--- a/fba/instances.py
+++ b/fba/instances.py
@@ -45,7 +45,7 @@ _pending = {
 }
 
 def set(key: str, domain: str, value: any):
-    # NOISY-DEBUG: print(f"DEBUG: key='{key}',domain='{domain}',value[]='{type(value)}' - CALLED!")
+    # DEBUG: print(f"DEBUG: key='{key}',domain='{domain}',value[]='{type(value)}' - CALLED!")
     if type(key) != str:
         raise ValueError("Parameter key[]='{type(key)}' is not 'str'")
     elif key == "":
@@ -62,7 +62,7 @@ def set(key: str, domain: str, value: any):
     # Set it
     _pending[key][domain] = value
 
-    # DEBUG: print("DEBUG: EXIT!")
+    # DEBUG: # DEBUG: print("DEBUG: EXIT!")
 
 def has_pending_instance_data(domain: str) -> bool:
     # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
@@ -91,7 +91,7 @@ def update_instance_data(domain: str):
         raise Exception(f"Domain '{domain}' has no pending instance data, but function invoked")
 
     # DEBUG: print(f"DEBUG: Updating instance data for domain='{domain}' ...")
-    sql_string = ''
+    sql_string = ""
     fields = list()
     for key in _pending:
         # DEBUG: print("DEBUG: key:", key)
@@ -103,7 +103,7 @@ def update_instance_data(domain: str):
     fields.append(time.time())
     fields.append(domain)
 
-    if sql_string == '':
+    if sql_string == "":
         raise ValueError(f"No fields have been set, but method invoked, domain='{domain}'")
 
     # DEBUG: print(f"DEBUG: sql_string='{sql_string}',fields()={len(fields)}")

diff --git a/fetch_blocks.py b/fetch_blocks.py
index bb03061bdde2e2b6deb054f2aca4e188838aad3c..c12c26f742f691d5d33edcb93d7a570ff892666f 100755 (executable)
--- a/fetch_blocks.py
+++ b/fetch_blocks.py
@@ -138,7 +138,7 @@ for blocker, software, origin, nodeinfo_url in rows:
                                         "reason" : None
                                     })
                         else:
-                            print(f"DEBUG: Updating block last seen for blocker='{blocker}',blocked='{blocked}' ...")
+                            # DEBUG: print(f"DEBUG: Updating block last seen for blocker='{blocker}',blocked='{blocked}' ...")
                             fba.update_last_seen(blocker, blocked, block_level)
 
             # DEBUG: print("DEBUG: Committing changes ...")
@@ -331,11 +331,10 @@ for blocker, software, origin, nodeinfo_url in rows:
                         fba.block_instance(blocker, blocking, reason, block_level)
 
                         if block_level == "reject":
-                            blockdict.append(
-                                {
-                                    "blocked": blocked,
-                                    "reason" : reason
-                                })
+                            blockdict.append({
+                                "blocked": blocked,
+                                "reason" : reason
+                            })
                     else:
                         # DEBUG: print(f"DEBUG: Updating block last seen and reason for blocker='{blocker}',blocking='{blocking}' ...")
                         fba.update_last_seen(blocker, blocking, block_level)
@@ -429,13 +428,12 @@ for blocker, software, origin, nodeinfo_url in rows:
                         fba.block_instance(blocker, blocked, reason, block_level)
 
                         if block_level == "reject":
-                            blockdict.append(
-                                {
-                                    "blocked": blocked,
-                                    "reason" : reason
-                                })
+                            blockdict.append({
+                                "blocked": blocked,
+                                "reason" : reason
+                            })
                     else:
-                        print(f"DEBUG: Updating block last seen and reason for blocker='{blocker}',blocked='{blocked}' ...")
+                        # DEBUG: print(f"DEBUG: Updating block last seen and reason for blocker='{blocker}',blocked='{blocked}' ...")
                         fba.update_last_seen(blocker, blocked, block_level)
                         fba.update_block_reason(reason, blocker, blocked, block_level)
 
@@ -497,13 +495,12 @@ for blocker, software, origin, nodeinfo_url in rows:
                         # DEBUG: print(f"DEBUG: blocker='{blocker}' is blocking '{blocked}' for unknown reason at this point")
                         fba.block_instance(blocker, blocked, "unknown", "reject")
 
-                        blockdict.append(
-                            {
-                                "blocked": blocked,
-                                "reason" : None
-                            })
+                        blockdict.append({
+                            "blocked": blocked,
+                            "reason" : None
+                        })
                     else:
-                        print(f"DEBUG: Updating block last seen for blocker='{blocker}',blocked='{blocked}' ...")
+                        # DEBUG: print(f"DEBUG: Updating block last seen for blocker='{blocker}',blocked='{blocked}' ...")
                         fba.update_last_seen(blocker, blocked, "reject")
 
                     if "public_comment" in peer: