X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;ds=sidebyside;f=fba%2Ffba.py;h=bac2d5d80ed5e8ab5d8e92c820d7848d752fa238;hb=5b7cee454bb523f7227da1f9fef06ed6beb16861;hp=228f98cfd32f2dcb292cb63d2f69257701e0087f;hpb=d4e620c6fb05c9c1741a89c412fe1e14cfb37940;p=fba.git

diff --git a/fba/fba.py b/fba/fba.py
index 228f98c..bac2d5d 100644
--- a/fba/fba.py
+++ b/fba/fba.py
@@ -15,18 +15,19 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 import bs4
-from fba import cache
 import hashlib
 import re
 import reqto
+import requests
 import json
 import sqlite3
 import sys
 import time
 import validators
 
-with open("config.json") as f:
-    config = json.loads(f.read())
+from fba import cache
+from fba import config
+from fba import instances
 
 # Don't check these, known trolls/flooders/testing/developing
 blacklist = [
@@ -64,40 +65,15 @@ nodeinfo_identifier = [
 
 # HTTP headers for non-API requests
 headers = {
-    "User-Agent": config["useragent"],
+    "User-Agent": config.get("useragent"),
 }
 
 # HTTP headers for API requests
 api_headers = {
-    "User-Agent": config["useragent"],
+    "User-Agent": config.get("useragent"),
     "Content-Type": "application/json",
 }
 
-# Found info from node, such as nodeinfo URL, detection mode that needs to be
-# written to database. Both arrays must be filled at the same time or else
-# update_instance_data() will fail
-instance_data = {
-    # Detection mode: 'AUTO_DISCOVERY', 'STATIC_CHECKS' or 'GENERATOR'
-    # NULL means all detection methods have failed (maybe still reachable instance)
-    "detection_mode"     : {},
-    # Found nodeinfo URL
-    "nodeinfo_url"       : {},
-    # Found total peers
-    "total_peers"        : {},
-    # Last fetched instances
-    "last_instance_fetch": {},
-    # Last updated
-    "last_updated"       : {},
-    # Last blocked
-    "last_blocked"       : {},
-    # Last nodeinfo (fetched)
-    "last_nodeinfo"      : {},
-    # Last status code
-    "last_status_code"   : {},
-    # Last error details
-    "last_error_details" : {},
-}
-
 language_mapping = {
     # English -> English
     "Silenced instances"            : "Silenced servers",
@@ -149,6 +125,7 @@ def is_primitive(var: any) -> bool:
     return type(var) in {int, str, float, bool} or var == None
 
 def fetch_instances(domain: str, origin: str, software: str, script: str, path: str = None):
+    # DEBUG: print(f"DEBUG: domain={domain},origin={origin},software={software},path={path} - CALLED!")
     if type(domain) != str:
         raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
     elif domain == "":
@@ -160,7 +137,6 @@ def fetch_instances(domain: str, origin: str, software: str, script: str, path:
     elif domain == "":
         raise ValueError(f"Parameter 'domain' cannot be empty")
 
-    # DEBUG: print("DEBUG: domain,origin,software,path:", domain, origin, software, path)
     if not is_instance_registered(domain):
         # DEBUG: print("DEBUG: Adding new domain:", domain, origin)
         add_instance(domain, origin, script, path)
@@ -171,9 +147,9 @@ def fetch_instances(domain: str, origin: str, software: str, script: str, path:
     if (peerlist is None):
         print("ERROR: Cannot fetch peers:", domain)
         return
-    elif has_pending_instance_data(domain):
+    elif instances.has_pending_instance_data(domain):
         # DEBUG: print(f"DEBUG: domain='{domain}' has pending nodeinfo data, flushing ...")
-        update_instance_data(domain)
+        instances.update_instance_data(domain)
 
     print(f"INFO: Checking {len(peerlist)} instances from {domain} ...")
     for instance in peerlist:
@@ -181,9 +157,9 @@ def fetch_instances(domain: str, origin: str, software: str, script: str, path:
             # Skip "None" types as tidup() cannot parse them
             continue
 
-        # DEBUG: print(f"DEBUG: instance[{type(instance}]={instance} - BEFORE")
+        # DEBUG: print(f"DEBUG: instance='{instance}' - BEFORE")
         instance = tidyup_domain(instance)
-        # DEBUG: print(f"DEBUG: instance[{type(instance}]={instance} - AFTER")
+        # DEBUG: print(f"DEBUG: instance='{instance}' - AFTER")
 
         if instance == "":
             print("WARNING: Empty instance after tidyup_domain(), domain:", domain)
@@ -206,26 +182,6 @@ def fetch_instances(domain: str, origin: str, software: str, script: str, path:
 
     # DEBUG: print("DEBUG: EXIT!")
 
-def set_instance_data(key: str, domain: str, value: any):
-    # NOISY-DEBUG: print(f"DEBUG: key='{key}',domain='{domain}',value[]='{type(value)}' - CALLED!")
-    if type(key) != str:
-        raise ValueError("Parameter key[]='{type(key)}' is not 'str'")
-    elif key == "":
-        raise ValueError(f"Parameter 'key' cannot be empty")
-    elif type(domain) != str:
-        raise ValueError("Parameter domain[]='{type(domain)}' is not 'str'")
-    elif domain == "":
-        raise ValueError(f"Parameter 'domain' cannot be empty")
-    elif not key in instance_data:
-        raise ValueError(f"key='{key}' not found in instance_data")
-    elif not is_primitive(value):
-        raise ValueError(f"value[]='{type(value)}' is not a primitive type")
-
-    # Set it
-    instance_data[key][domain] = value
-
-    # DEBUG: print("DEBUG: EXIT!")
-
 def add_peers(rows: dict) -> list:
     # DEBUG: print(f"DEBUG: rows()={len(rows)} - CALLED!")
     peers = list()
@@ -407,146 +363,76 @@ def update_last_blocked(domain: str):
         raise ValueError(f"Parameter 'domain' cannot be empty")
 
     # DEBUG: print("DEBUG: Updating last_blocked for domain", domain)
-    set_instance_data("last_blocked", domain, time.time())
+    instances.set("last_blocked", domain, time.time())
 
     # Running pending updated
-    # DEBUG: print(f"DEBUG: Invoking update_instance_data({domain}) ...")
-    update_instance_data(domain)
+    # DEBUG: print(f"DEBUG: Invoking instances.update_instance_data({domain}) ...")
+    instances.update_instance_data(domain)
 
     # DEBUG: print("DEBUG: EXIT!")
 
-def has_pending_instance_data(domain: str) -> bool:
-    # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
+def log_error(domain: str, response: requests.models.Response):
+    # DEBUG: print("DEBUG: domain,response[]:", domain, type(response))
     if type(domain) != str:
         raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
     elif domain == "":
         raise ValueError(f"Parameter 'domain' cannot be empty")
 
-    has_pending = False
-    for key in instance_data:
-        # DEBUG: print(f"DEBUG: key='{key}',domain='{domain}',instance_data[key]()='{len(instance_data[key])}'")
-        if domain in instance_data[key]:
-            has_pending = True
-            break
-
-    # DEBUG: print(f"DEBUG: has_pending='{has_pending}' - EXIT!")
-    return has_pending
-
-def update_instance_data(domain: str):
-    # DEBUG: print(f"DEBUG: domain={domain} - CALLED!")
-    if type(domain) != str:
-        raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
-    elif domain == "":
-        raise ValueError(f"Parameter 'domain' cannot be empty")
-    elif not has_pending_instance_data(domain):
-        raise Exception(f"Domain '{domain}' has no pending instance data, but function invoked")
-
-    # DEBUG: print(f"DEBUG: Updating nodeinfo for domain='{domain}' ...")
-    sql_string = ''
-    fields = list()
-    for key in instance_data:
-        # DEBUG: print("DEBUG: key:", key)
-        if domain in instance_data[key]:
-           # DEBUG: print(f"DEBUG: Adding '{instance_data[key][domain]}' for key='{key}' ...")
-           fields.append(instance_data[key][domain])
-           sql_string += f" {key} = ?,"
-
-    fields.append(domain)
-
-    if sql_string == '':
-        raise ValueError(f"No fields have been set, but method invoked, domain='{domain}'")
-
-    # DEBUG: print(f"DEBUG: sql_string='{sql_string}',fields()={len(fields)}")
-    sql_string = "UPDATE instances SET" + sql_string + " last_updated = TIME() WHERE domain = ? LIMIT 1"
-    # DEBUG: print("DEBUG: sql_string:", sql_string)
-
     try:
-        # DEBUG: print("DEBUG: Executing SQL:", sql_string)
-        cursor.execute(sql_string, fields)
-
-        # DEBUG: print(f"DEBUG: Success! (rowcount={cursor.rowcount })")
-        if cursor.rowcount == 0:
-            print(f"WARNING: Did not update any rows: domain='{domain}',fields()={len(fields)} - EXIT!")
-            return
-
-        connection.commit()
-
-        # DEBUG: print("DEBUG: Deleting instance_data for domain:", domain)
-        for key in instance_data:
-            try:
-                # DEBUG: print("DEBUG: Deleting key:", key)
-                del instance_data[key][domain]
-            except:
-                pass
-
-    except BaseException as e:
-        print(f"ERROR: failed SQL query: domain='{domain}',sql_string='{sql_string}',exception[{type(e)}]:'{str(e)}'")
-        sys.exit(255)
+        # DEBUG: print("DEBUG: BEFORE response[]:", type(response))
+        if isinstance(response, BaseException) or isinstance(response, json.decoder.JSONDecodeError):
+            response = str(response)
 
-    # DEBUG: print("DEBUG: EXIT!")
-
-def log_error(domain: str, res: any):
-    # DEBUG: print("DEBUG: domain,res[]:", domain, type(res))
-    if type(domain) != str:
-        raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
-    elif domain == "":
-        raise ValueError(f"Parameter 'domain' cannot be empty")
-
-    try:
-        # DEBUG: print("DEBUG: BEFORE res[]:", type(res))
-        if isinstance(res, BaseException) or isinstance(res, json.JSONDecodeError):
-            res = str(res)
-
-        # DEBUG: print("DEBUG: AFTER res[]:", type(res))
-        if type(res) is str:
+        # DEBUG: print("DEBUG: AFTER response[]:", type(response))
+        if type(response) is str:
             cursor.execute("INSERT INTO error_log (domain, error_code, error_message, created) VALUES (?, 999, ?, ?)",[
                 domain,
-                res,
+                response,
                 time.time()
             ])
         else:
             cursor.execute("INSERT INTO error_log (domain, error_code, error_message, created) VALUES (?, ?, ?, ?)",[
                 domain,
-                res.status_code,
-                res.reason,
+                response.status_code,
+                response.reason,
                 time.time()
             ])
 
         # Cleanup old entries
-        # DEBUG: print(f"DEBUG: Purging old records (distance: {config['error_log_cleanup']})")
-        cursor.execute("DELETE FROM error_log WHERE created < ?", [time.time() - config["error_log_cleanup"]])
+        # DEBUG: print(f"DEBUG: Purging old records (distance: {config.get('error_log_cleanup')})")
+        cursor.execute("DELETE FROM error_log WHERE created < ?", [time.time() - config.get("error_log_cleanup")])
     except BaseException as e:
         print(f"ERROR: failed SQL query: domain='{domain}',exception[{type(e)}]:'{str(e)}'")
         sys.exit(255)
 
     # DEBUG: print("DEBUG: EXIT!")
 
-def update_last_error(domain: str, res: any):
-    # DEBUG: print("DEBUG: domain,res[]:", domain, type(res))
+def update_last_error(domain: str, response: requests.models.Response):
+    # DEBUG: print("DEBUG: domain,response[]:", domain, type(response))
     if type(domain) != str:
         raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
     elif domain == "":
         raise ValueError(f"Parameter 'domain' cannot be empty")
 
-    # DEBUG: print("DEBUG: BEFORE res[]:", type(res))
-    if isinstance(res, BaseException) or isinstance(res, json.JSONDecodeError):
-        res = str(res)
+    # DEBUG: print("DEBUG: BEFORE response[]:", type(response))
+    if isinstance(response, BaseException) or isinstance(response, json.decoder.JSONDecodeError):
+        response = f"{type}:str(response)"
 
-    # DEBUG: print("DEBUG: AFTER res[]:", type(res))
-    if type(res) is str:
-        # DEBUG: print(f"DEBUG: Setting last_error_details='{res}'");
-        set_instance_data("last_status_code"  , domain, 999)
-        set_instance_data("last_error_details", domain, res)
+    # DEBUG: print("DEBUG: AFTER response[]:", type(response))
+    if type(response) is str:
+        # DEBUG: print(f"DEBUG: Setting last_error_details='{response}'");
+        instances.set("last_status_code"  , domain, 999)
+        instances.set("last_error_details", domain, response)
     else:
-        # DEBUG: print(f"DEBUG: Setting last_error_details='{res.reason}'");
-        set_instance_data("last_status_code"  , domain, res.status_code)
-        set_instance_data("last_error_details", domain, res.reason)
+        # DEBUG: print(f"DEBUG: Setting last_error_details='{response.reason}'");
+        instances.set("last_status_code"  , domain, response.status_code)
+        instances.set("last_error_details", domain, response.reason)
 
     # Running pending updated
-    # DEBUG: print(f"DEBUG: Invoking update_instance_data({domain}) ...")
-    update_instance_data(domain)
+    # DEBUG: print(f"DEBUG: Invoking instances.update_instance_data({domain}) ...")
+    instances.update_instance_data(domain)
 
-    log_error(domain, res)
+    log_error(domain, response)
 
     # DEBUG: print("DEBUG: EXIT!")
 
@@ -558,11 +444,11 @@ def update_last_instance_fetch(domain: str):
         raise ValueError(f"Parameter 'domain' cannot be empty")
 
     # DEBUG: print("DEBUG: Updating last_instance_fetch for domain:", domain)
-    set_instance_data("last_instance_fetch", domain, time.time())
+    instances.set("last_instance_fetch", domain, time.time())
 
     # Running pending updated
-    # DEBUG: print(f"DEBUG: Invoking update_instance_data({domain}) ...")
-    update_instance_data(domain)
+    # DEBUG: print(f"DEBUG: Invoking instances.update_instance_data({domain}) ...")
+    instances.update_instance_data(domain)
 
     # DEBUG: print("DEBUG: EXIT!")
 
@@ -574,12 +460,12 @@ def update_last_nodeinfo(domain: str):
         raise ValueError(f"Parameter 'domain' cannot be empty")
 
     # DEBUG: print("DEBUG: Updating last_nodeinfo for domain:", domain)
-    set_instance_data("last_nodeinfo", domain, time.time())
-    set_instance_data("last_updated" , domain, time.time())
+    instances.set("last_nodeinfo", domain, time.time())
+    instances.set("last_updated" , domain, time.time())
 
     # Running pending updated
-    # DEBUG: print(f"DEBUG: Invoking update_instance_data({domain}) ...")
-    update_instance_data(domain)
+    # DEBUG: print(f"DEBUG: Invoking instances.update_instance_data({domain}) ...")
+    instances.update_instance_data(domain)
 
     # DEBUG: print("DEBUG: EXIT!")
 
@@ -592,13 +478,12 @@ def get_peers(domain: str, software: str) -> list:
     elif type(software) != str and software != None:
         raise ValueError(f"software[]={type(software)} is not 'str'")
 
-    # DEBUG: print(f"DEBUG: domain='{domain}',software='{software}' - CALLED!")
     peers = list()
 
     if software == "misskey":
         # DEBUG: print(f"DEBUG: domain='{domain}' is misskey, sending API POST request ...")
         offset = 0
-        step = config["misskey_offset"]
+        step = config.get("misskey_limit")
 
         # iterating through all "suspended" (follow-only in its terminology)
         # instances page-by-page, since that troonware doesn't support
@@ -610,22 +495,26 @@ def get_peers(domain: str, software: str) -> list:
                     "sort" : "+pubAt",
                     "host" : None,
                     "limit": step
-                }), {"Origin": domain})
+                }), {
+                    "Origin": domain
+                })
             else:
                 fetched = post_json_api(domain, "/api/federation/instances", json.dumps({
                     "sort"  : "+pubAt",
                     "host"  : None,
                     "limit" : step,
                     "offset": offset - 1
-                }), {"Origin": domain})
+                }), {
+                    "Origin": domain
+                })
 
-            # DEBUG: print("DEBUG: fetched():", len(fetched))
+            # DEBUG: print(f"DEBUG: fetched()={len(fetched)}")
             if len(fetched) == 0:
                 # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
                 break
-            elif len(fetched) != config["misskey_offset"]:
-                # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config['misskey_offset']}'")
-                offset = offset + (config["misskey_offset"] - len(fetched))
+            elif len(fetched) != config.get("misskey_limit"):
+                # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'")
+                offset = offset + (config.get("misskey_limit") - len(fetched))
             else:
                 # DEBUG: print("DEBUG: Raising offset by step:", step)
                 offset = offset + step
@@ -637,6 +526,7 @@ def get_peers(domain: str, software: str) -> list:
                 update_last_error(domain, fetched["error"]["message"])
                 break
 
+            already = 0
             for row in fetched:
                 # DEBUG: print(f"DEBUG: row():{len(row)}")
                 if not "host" in row:
@@ -648,12 +538,20 @@ def get_peers(domain: str, software: str) -> list:
                 elif is_blacklisted(row["host"]):
                     # DEBUG: print(f"DEBUG: row[host]='{row['host']}' is blacklisted. domain='{domain}'")
                     continue
+                elif row["host"] in peers:
+                    # DEBUG: print(f"DEBUG: Not adding row[host]='{row['host']}', already found.")
+                    already = already + 1
+                    continue
 
                 # DEBUG: print(f"DEBUG: Adding peer: '{row['host']}'")
                 peers.append(row["host"])
 
+            if already == len(fetched):
+                print(f"WARNING: Host returned same set of '{already}' instances, aborting loop!")
+                break
+
         # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'")
-        set_instance_data("total_peers", domain, len(peers))
+        instances.set("total_peers", domain, len(peers))
 
         # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...")
         update_last_instance_fetch(domain)
@@ -663,14 +561,15 @@ def get_peers(domain: str, software: str) -> list:
     elif software == "lemmy":
         # DEBUG: print(f"DEBUG: domain='{domain}' is Lemmy, fetching JSON ...")
         try:
-            res = reqto.get(f"https://{domain}/api/v3/site", headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"]))
+            response = get_response(domain, "/api/v3/site", api_headers, (config.get("connection_timeout"), config.get("read_timeout")))
 
-            data = res.json()
-            # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code='{res.status_code}',data[]='{type(data)}'")
-            if not res.ok or res.status_code >= 400:
+            data = json_from_response(response)
+
+            # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code='{response.status_code}',data[]='{type(data)}'")
+            if not response.ok or response.status_code >= 400:
                 print("WARNING: Could not reach any JSON API:", domain)
-                update_last_error(domain, res)
-            elif res.ok and isinstance(data, list):
+                update_last_error(domain, response)
+            elif response.ok and isinstance(data, list):
                 # DEBUG: print(f"DEBUG: domain='{domain}' returned a list: '{data}'")
                 sys.exit(255)
             elif "federated_instances" in data:
@@ -679,13 +578,13 @@ def get_peers(domain: str, software: str) -> list:
                 # DEBUG: print("DEBUG: Added instance(s) to peers")
             else:
                 print("WARNING: JSON response does not contain 'federated_instances':", domain)
-                update_last_error(domain, res)
+                update_last_error(domain, response)
 
         except BaseException as e:
             print(f"WARNING: Exception during fetching JSON: domain='{domain}',exception[{type(e)}]:'{str(e)}'")
 
         # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'")
-        set_instance_data("total_peers", domain, len(peers))
+        instances.set("total_peers", domain, len(peers))
 
         # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...")
         update_last_instance_fetch(domain)
@@ -700,11 +599,11 @@ def get_peers(domain: str, software: str) -> list:
             # DEBUG: print(f"DEBUG: domain='{domain}',mode='{mode}'")
             while True:
                 try:
-                    res = reqto.get(f"https://{domain}/api/v1/server/{mode}?start={start}&count=100", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"]))
+                    response = get_response(domain, "/api/v1/server/{mode}?start={start}&count=100", headers, (config.get("connection_timeout"), config.get("read_timeout")))
 
-                    data = res.json()
-                    # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code='{res.status_code}',data[]='{type(data)}'")
-                    if res.ok and isinstance(data, dict):
+                    data = json_from_response(response)
+                    # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code='{response.status_code}',data[]='{type(data)}'")
+                    if response.ok and isinstance(data, dict):
                         # DEBUG: print("DEBUG: Success, data:", len(data))
                         if "data" in data:
                             # DEBUG: print(f"DEBUG: Found {len(data['data'])} record(s).")
@@ -727,7 +626,7 @@ def get_peers(domain: str, software: str) -> list:
                     print(f"WARNING: Exception during fetching JSON: domain='{domain}',exception[{type(e)}]:'{str(e)}'")
 
         # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'")
-        set_instance_data("total_peers", domain, len(peers))
+        instances.set("total_peers", domain, len(peers))
 
         # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...")
         update_last_instance_fetch(domain)
@@ -737,20 +636,21 @@ def get_peers(domain: str, software: str) -> list:
 
     # DEBUG: print(f"DEBUG: Fetching get_peers_url='{get_peers_url}' from '{domain}' ...")
     try:
-        res = reqto.get(f"https://{domain}{get_peers_url}", headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"]))
+        response = get_response(domain, get_peers_url, api_headers, (config.get("connection_timeout"), config.get("read_timeout")))
+
+        data = json_from_response(response)
 
-        data = res.json()
-        # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'")
-        if not res.ok or res.status_code >= 400:
+        # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'")
+        if not response.ok or response.status_code >= 400:
             # DEBUG: print(f"DEBUG: Was not able to fetch '{get_peers_url}', trying alternative ...")
-            res = reqto.get(f"https://{domain}/api/v3/site", headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"]))
+            response = get_response(domain, "/api/v3/site", api_headers, (config.get("connection_timeout"), config.get("read_timeout")))
 
-            data = res.json()
-            # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'")
-            if not res.ok or res.status_code >= 400:
+            data = json_from_response(response)
+            # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'")
+            if not response.ok or response.status_code >= 400:
                 print("WARNING: Could not reach any JSON API:", domain)
-                update_last_error(domain, res)
-            elif res.ok and isinstance(data, list):
+                update_last_error(domain, response)
+            elif response.ok and isinstance(data, list):
                 # DEBUG: print(f"DEBUG: domain='{domain}' returned a list: '{data}'")
                 sys.exit(255)
             elif "federated_instances" in data:
@@ -759,7 +659,7 @@ def get_peers(domain: str, software: str) -> list:
                 # DEBUG: print("DEBUG: Added instance(s) to peers")
             else:
                 print("WARNING: JSON response does not contain 'federated_instances':", domain)
-                update_last_error(domain, res)
+                update_last_error(domain, response)
         else:
             # DEBUG: print("DEBUG: Querying API was successful:", domain, len(data))
             peers = data
@@ -769,7 +669,7 @@ def get_peers(domain: str, software: str) -> list:
         update_last_error(domain, e)
 
     # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'")
-    set_instance_data("total_peers", domain, len(peers))
+    instances.set("total_peers", domain, len(peers))
 
     # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...")
     update_last_instance_fetch(domain)
@@ -785,20 +685,20 @@ def post_json_api(domain: str, path: str, parameter: str, extra_headers: dict =
     elif type(path) != str:
         raise ValueError(f"path[]={type(path)} is not 'str'")
     elif path == "":
-        raise ValueError(f"path cannot be empty")
+        raise ValueError("Parameter 'path' cannot be empty")
     elif type(parameter) != str:
         raise ValueError(f"parameter[]={type(parameter)} is not 'str'")
 
     # DEBUG: print("DEBUG: Sending POST to domain,path,parameter:", domain, path, parameter, extra_headers)
     data = {}
     try:
-        res = reqto.post(f"https://{domain}{path}", data=parameter, headers={**api_headers, **extra_headers}, timeout=(config["connection_timeout"], config["read_timeout"]))
+        response = reqto.post(f"https://{domain}{path}", data=parameter, headers={**api_headers, **extra_headers}, timeout=(config.get("connection_timeout"), config.get("read_timeout")))
 
-        data = res.json()
-        # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'")
-        if not res.ok or res.status_code >= 400:
-            print(f"WARNING: Cannot query JSON API: domain='{domain}',path='{path}',parameter()={len(parameter)},res.status_code='{res.status_code}',data[]='{type(data)}'")
-            update_last_error(domain, res)
+        data = json_from_response(response)
+        # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'")
+        if not response.ok or response.status_code >= 400:
+            print(f"WARNING: Cannot query JSON API: domain='{domain}',path='{path}',parameter()={len(parameter)},response.status_code='{response.status_code}',data[]='{type(data)}'")
+            update_last_error(domain, response)
 
     except BaseException as e:
         print(f"WARNING: Some error during post(): domain='{domain}',path='{path}',parameter()={len(parameter)},exception[{type(e)}]:'{str(e)}'")
@@ -840,21 +740,21 @@ def fetch_nodeinfo(domain: str, path: str = None) -> list:
 
         try:
             # DEBUG: print("DEBUG: Fetching request:", request)
-            res = reqto.get(request, headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"]))
+            response = reqto.get(request, headers=api_headers, timeout=(config.get("nodeinfo_connection_timeout"), config.get("nodeinfo_read_timeout")))
 
-            data = res.json()
-            # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'")
-            if res.ok and isinstance(data, dict):
+            data = json_from_response(response)
+            # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'")
+            if response.ok and isinstance(data, dict):
                 # DEBUG: print("DEBUG: Success:", request)
-                set_instance_data("detection_mode", domain, "STATIC_CHECK")
-                set_instance_data("nodeinfo_url"  , domain, request)
+                instances.set("detection_mode", domain, "STATIC_CHECK")
+                instances.set("nodeinfo_url"  , domain, request)
                 break
-            elif res.ok and isinstance(data, list):
+            elif response.ok and isinstance(data, list):
                 # DEBUG: print(f"DEBUG: domain='{domain}' returned a list: '{data}'")
                 sys.exit(255)
-            elif not res.ok or res.status_code >= 400:
+            elif not response.ok or response.status_code >= 400:
                 print("WARNING: Failed fetching nodeinfo from domain:", domain)
-                update_last_error(domain, res)
+                update_last_error(domain, response)
                 continue
 
         except BaseException as e:
@@ -876,11 +776,11 @@ def fetch_wellknown_nodeinfo(domain: str) -> list:
     data = {}
 
     try:
-        res = reqto.get(f"https://{domain}/.well-known/nodeinfo", headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"]))
+        response = get_response(domain, "/.well-known/nodeinfo", api_headers, (config.get("nodeinfo_connection_timeout"), config.get("nodeinfo_read_timeout")))
 
-        data = res.json()
-        # DEBUG: print("DEBUG: domain,res.ok,data[]:", domain, res.ok, type(data))
-        if res.ok and isinstance(data, dict):
+        data = json_from_response(response)
+        # DEBUG: print("DEBUG: domain,response.ok,data[]:", domain, response.ok, type(data))
+        if response.ok and isinstance(data, dict):
             nodeinfo = data
             # DEBUG: print("DEBUG: Found entries:", len(nodeinfo), domain)
             if "links" in nodeinfo:
@@ -889,14 +789,14 @@ def fetch_wellknown_nodeinfo(domain: str) -> list:
                     # DEBUG: print("DEBUG: rel,href:", link["rel"], link["href"])
                     if link["rel"] in nodeinfo_identifier:
                         # DEBUG: print("DEBUG: Fetching nodeinfo from:", link["href"])
-                        res = reqto.get(link["href"])
+                        response = reqto.get(link["href"])
 
-                        data = res.json()
-                        # DEBUG: print("DEBUG: href,res.ok,res.status_code:", link["href"], res.ok, res.status_code)
-                        if res.ok and isinstance(data, dict):
+                        data = json_from_response(response)
+                        # DEBUG: print("DEBUG: href,response.ok,response.status_code:", link["href"], response.ok, response.status_code)
+                        if response.ok and isinstance(data, dict):
                             # DEBUG: print("DEBUG: Found JSON nodeinfo():", len(data))
-                            set_instance_data("detection_mode", domain, "AUTO_DISCOVERY")
-                            set_instance_data("nodeinfo_url"  , domain, link["href"])
+                            instances.set("detection_mode", domain, "AUTO_DISCOVERY")
+                            instances.set("nodeinfo_url"  , domain, link["href"])
                             break
                     else:
                         print("WARNING: Unknown 'rel' value:", domain, link["rel"])
@@ -927,12 +827,12 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str:
 
     try:
         # DEBUG: print(f"DEBUG: Fetching path='{path}' from '{domain}' ...")
-        res = reqto.get(f"https://{domain}{path}", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"]))
+        response = get_response(domain, path, headers, (config.get("connection_timeout"), config.get("read_timeout")))
 
-        # DEBUG: print("DEBUG: domain,res.ok,res.status_code,res.text[]:", domain, res.ok, res.status_code, type(res.text))
-        if res.ok and res.status_code < 300 and len(res.text) > 0:
+        # DEBUG: print("DEBUG: domain,response.ok,response.status_code,response.text[]:", domain, response.ok, response.status_code, type(response.text))
+        if response.ok and response.status_code < 300 and len(response.text) > 0:
             # DEBUG: print("DEBUG: Search for <meta name='generator'>:", domain)
-            doc = bs4.BeautifulSoup(res.text, "html.parser")
+            doc = bs4.BeautifulSoup(response.text, "html.parser")
 
             # DEBUG: print("DEBUG: doc[]:", type(doc))
             generator = doc.find("meta", {"name": "generator"})
@@ -943,13 +843,13 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str:
                 # DEBUG: print("DEBUG: Found generator meta tag:", domain)
                 software = tidyup_domain(generator.get("content"))
                 print(f"INFO: domain='{domain}' is generated by '{software}'")
-                set_instance_data("detection_mode", domain, "GENERATOR")
+                instances.set("detection_mode", domain, "GENERATOR")
                 remove_pending_error(domain)
             elif isinstance(site_name, bs4.element.Tag):
                 # DEBUG: print("DEBUG: Found property=og:site_name:", domain)
                 sofware = tidyup_domain(site_name.get("content"))
                 print(f"INFO: domain='{domain}' has og:site_name='{software}'")
-                set_instance_data("detection_mode", domain, "SITE_NAME")
+                instances.set("detection_mode", domain, "SITE_NAME")
                 remove_pending_error(domain)
 
     except BaseException as e:
@@ -1174,6 +1074,10 @@ def block_instance(blocker: str, blocked: str, reason: str, block_level: str):
     elif is_blacklisted(blocked):
         raise Exception(f"blocked='{blocked}' is blacklisted but function invoked")
 
+    if reason != None:
+        # Maybe needs cleaning
+        reason = tidyup_reason(reason)
+
     print(f"INFO: New block: blocker='{blocker}',blocked='{blocked}', reason='{reason}', block_level='{block_level}'")
     try:
         cursor.execute(
@@ -1187,8 +1091,6 @@ def block_instance(blocker: str, blocked: str, reason: str, block_level: str):
                  time.time()
              ),
         )
-
-        connection.commit()
     except BaseException as e:
         print(f"ERROR: failed SQL query: blocker='{blocker}',blocked='{blocked}',reason='{reason}',block_level='{block_level}',exception[{type(e)}]:'{str(e)}'")
         sys.exit(255)
@@ -1202,22 +1104,22 @@ def is_instance_registered(domain: str) -> bool:
     elif domain == "":
         raise ValueError(f"Parameter 'domain' cannot be empty")
 
-    # NOISY-DEBUG: # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
-    if not cache.is_cache_initialized("is_registered"):
-        # NOISY-DEBUG: # DEBUG: print(f"DEBUG: Cache for 'is_registered' not initialized, fetching all rows ...")
+    # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
+    if not cache.key_exists("is_registered"):
+        # DEBUG: print(f"DEBUG: Cache for 'is_registered' not initialized, fetching all rows ...")
         try:
             cursor.execute("SELECT domain FROM instances")
 
             # Check Set all
-            cache.set_all_cache_key("is_registered", cursor.fetchall(), True)
+            cache.set_all("is_registered", cursor.fetchall(), True)
         except BaseException as e:
             print(f"ERROR: failed SQL query: domain='{domain}',exception[{type(e)}]:'{str(e)}'")
             sys.exit(255)
 
     # Is cache found?
-    registered = cache.is_cache_key_set("is_registered", domain)
+    registered = cache.sub_key_exists("is_registered", domain)
 
-    # NOISY-DEBUG: # DEBUG: print(f"DEBUG: registered='{registered}' - EXIT!")
+    # DEBUG: print(f"DEBUG: registered='{registered}' - EXIT!")
     return registered
 
 def add_instance(domain: str, origin: str, originator: str, path: str = None):
@@ -1257,13 +1159,13 @@ def add_instance(domain: str, origin: str, originator: str, path: str = None):
             ),
         )
 
-        cache.set_cache_key("is_registered", domain, True)
+        cache.set_sub_key("is_registered", domain, True)
 
-        if has_pending_instance_data(domain):
+        if instances.has_pending_instance_data(domain):
             # DEBUG: print(f"DEBUG: domain='{domain}' has pending nodeinfo being updated ...")
-            set_instance_data("last_status_code"  , domain, None)
-            set_instance_data("last_error_details", domain, None)
-            update_instance_data(domain)
+            instances.set("last_status_code"  , domain, None)
+            instances.set("last_error_details", domain, None)
+            instances.update_instance_data(domain)
             remove_pending_error(domain)
 
         if domain in pending_errors:
@@ -1301,13 +1203,13 @@ def send_bot_post(instance: str, blocks: dict):
     if truncated:
         message = message + "(the list has been truncated to the first 20 entries)"
 
-    botheaders = {**api_headers, **{"Authorization": "Bearer " + config["bot_token"]}}
+    botheaders = {**api_headers, **{"Authorization": "Bearer " + config.get("bot_token")}}
 
     req = reqto.post(
-        f"{config['bot_instance']}/api/v1/statuses",
+        f"{config.get('bot_instance')}/api/v1/statuses",
         data={
             "status"      : message,
-            "visibility"  : config['bot_visibility'],
+            "visibility"  : config.get('bot_visibility'),
             "content_type": "text/plain"
         },
         headers=botheaders,
@@ -1333,7 +1235,7 @@ def get_mastodon_blocks(domain: str) -> dict:
 
     try:
         doc = bs4.BeautifulSoup(
-            reqto.get(f"https://{domain}/about", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])).text,
+            get_response(domain, "/about", headers, (config.get("connection_timeout"), config.get("read_timeout"))).text,
             "html.parser",
         )
     except BaseException as e:
@@ -1378,7 +1280,7 @@ def get_friendica_blocks(domain: str) -> dict:
 
     try:
         doc = bs4.BeautifulSoup(
-            reqto.get(f"https://{domain}/friendica", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])).text,
+            get_response(domain, "/friendica", headers, (config.get("connection_timeout"), config.get("read_timeout"))).text,
             "html.parser",
         )
     except BaseException as e:
@@ -1419,7 +1321,7 @@ def get_misskey_blocks(domain: str) -> dict:
     }
 
     offset = 0
-    step = config["misskey_offset"]
+    step = config.get("misskey_limit")
     while True:
         # iterating through all "suspended" (follow-only in its terminology)
         # instances page-by-page, since that troonware doesn't support
@@ -1433,7 +1335,9 @@ def get_misskey_blocks(domain: str) -> dict:
                     "host"     : None,
                     "suspended": True,
                     "limit"    : step
-                }), {"Origin": domain})
+                }), {
+                    "Origin": domain
+                })
             else:
                 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
                 fetched = post_json_api(domain, "/api/federation/instances", json.dumps({
@@ -1442,22 +1346,24 @@ def get_misskey_blocks(domain: str) -> dict:
                     "suspended": True,
                     "limit"    : step,
                     "offset"   : offset - 1
-                }), {"Origin": domain})
+                }), {
+                    "Origin": domain
+                })
 
             # DEBUG: print("DEBUG: fetched():", len(fetched))
             if len(fetched) == 0:
                 # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
                 break
-            elif len(fetched) != config["misskey_offset"]:
-                # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config['misskey_offset']}'")
-                offset = offset + (config["misskey_offset"] - len(fetched))
+            elif len(fetched) != config.get("misskey_limit"):
+                # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'")
+                offset = offset + (config.get("misskey_limit") - len(fetched))
             else:
                 # DEBUG: print("DEBUG: Raising offset by step:", step)
                 offset = offset + step
 
             for instance in fetched:
                 # just in case
-                if instance["isSuspended"]:
+                if instance["isSuspended"] and not has_element(blocks["suspended"], "domain", instance):
                     blocks["suspended"].append(
                         {
                             "domain": tidyup_domain(instance["host"]),
@@ -1482,7 +1388,9 @@ def get_misskey_blocks(domain: str) -> dict:
                     "host"   : None,
                     "blocked": True,
                     "limit"  : step
-                }), {"Origin": domain})
+                }), {
+                    "Origin": domain
+                })
             else:
                 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
                 fetched = post_json_api(domain,"/api/federation/instances", json.dumps({
@@ -1490,22 +1398,24 @@ def get_misskey_blocks(domain: str) -> dict:
                     "host"   : None,
                     "blocked": True,
                     "limit"  : step,
-                    "offset" : offset-1
-                }), {"Origin": domain})
+                    "offset" : offset - 1
+                }), {
+                    "Origin": domain
+                })
 
             # DEBUG: print("DEBUG: fetched():", len(fetched))
             if len(fetched) == 0:
                 # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
                 break
-            elif len(fetched) != config["misskey_offset"]:
-                # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config['misskey_offset']}'")
-                offset = offset + (config["misskey_offset"] - len(fetched))
+            elif len(fetched) != config.get("misskey_limit"):
+                # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'")
+                offset = offset + (config.get("misskey_limit") - len(fetched))
             else:
                 # DEBUG: print("DEBUG: Raising offset by step:", step)
                 offset = offset + step
 
             for instance in fetched:
-                if instance["isBlocked"]:
+                if instance["isBlocked"] and not has_element(blocks["blocked"], "domain", instance):
                     blocks["blocked"].append({
                         "domain": tidyup_domain(instance["host"]),
                         "reason": None
@@ -1526,18 +1436,32 @@ def get_misskey_blocks(domain: str) -> dict:
         "followers_only": blocks["suspended"]
     }
 
+def tidyup_reason(reason: str) -> str:
+    # DEBUG: print(f"DEBUG: reason='{reason}' - CALLED!")
+    if type(reason) != str:
+        raise ValueError(f"Parameter reason[]={type(reason)} is not expected")
+
+    # Strip string
+    reason = reason.strip()
+
+    # Replace â with "
+    reason = re.sub("â", "\"", reason)
+
+    #print(f"DEBUG: reason='{reason}' - EXIT!")
+    return reason
+
 def tidyup_domain(domain: str) -> str:
     # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
     if type(domain) != str:
         raise ValueError(f"Parameter domain[]={type(domain)} is not expected")
 
-    # All lower-case and strip spaces out
-    domain = domain.lower().strip()
+    # All lower-case and strip spaces out + last dot
+    domain = domain.lower().strip().rstrip(".")
 
     # No port number
     domain = re.sub("\:\d+$", "", domain)
 
-    # No protocol, sometimes with the slashes
+    # No protocol, sometimes without the slashes
     domain = re.sub("^https?\:(\/*)", "", domain)
 
     # No trailing slash
@@ -1551,3 +1475,115 @@ def tidyup_domain(domain: str) -> str:
 
     # DEBUG: print(f"DEBUG: domain='{domain}' - EXIT!")
     return domain
+
+def json_from_response(response: requests.models.Response) -> list:
+    # DEBUG: print(f"DEBUG: response[]={type(response)} - CALLED!")
+    if not isinstance(response, requests.models.Response):
+        raise ValueError(f"Parameter response[]='{type(response)}' is not type of 'Response'")
+
+    data = list()
+    if response.text.strip() != "":
+        # DEBUG: print(f"DEBUG: response.text()={len(response.text)} is not empty, invoking response.json() ...")
+        try:
+            data = response.json()
+        except json.decoder.JSONDecodeError:
+            pass
+
+    # DEBUG: print(f"DEBUG: data[]={type(data)} - EXIT!")
+    return data
+
+def get_response(domain: str, path: str, headers: dict, timeout: list) -> requests.models.Response:
+    # DEBUG: print(f"DEBUG: domain='{domain}',path='{path}',headers()={len(headers)},timeout={timeout} - CALLED!")
+    if type(domain) != str:
+        raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
+    elif domain == "":
+        raise ValueError("Parameter 'domain' cannot be empty")
+    elif type(path) != str:
+        raise ValueError(f"Parameter path[]='{type(path)}' is not 'str'")
+    elif path == "":
+        raise ValueError("Parameter 'path' cannot be empty")
+
+    try:
+        # DEBUG: print(f"DEBUG: Sending request to '{domain}{path}' ...")
+        response = reqto.get(f"https://{domain}{path}", headers=headers, timeout=timeout);
+    except requests.exceptions.ConnectionError as e:
+        # DEBUG: print(f"DEBUG: Fetching '{path}' from '{domain}' failed. exception[{type(e)}]='{str(e)}'")
+        update_last_error(domain, e)
+        raise e
+
+    # DEBUG: print(f"DEBUG: response[]='{type(response)}' - EXXIT!")
+    return response
+
+def has_element(elements: list, key: str, value: any) -> bool:
+    # DEBUG: print(f"DEBUG: element()={len(element)},key='{key}',value[]='{type(value)}' - CALLED!")
+    if type(key) != str:
+        raise ValueError(f"Parameter key[]='{type(key)}' is not 'str'")
+    elif key == "":
+        raise ValueError("Parameter 'key' cannot be empty")
+
+    has = False
+    # DEBUG: print(f"DEBUG: Checking elements()={len(elements)} ...")
+    for element in elements:
+        # DEBUG: print(f"DEBUG: element[]='{type(element)}'")
+        if type(element) != dict:
+            raise ValueError(f"element[]='{type(element)}' is not 'dict'")
+        elif not key in element:
+            raise KeyError(f"Cannot find key='{key}'")
+        elif element[key] == value:
+            has = True
+            break
+
+    # DEBUG: print(f"DEBUG: has={has} - EXIT!")
+    return has
+
+def find_domains(tag: bs4.element.Tag) -> list:
+    # DEBUG: print(f"DEBUG: tag[]={type(tag)} - CALLED!")
+    if not isinstance(tag, bs4.element.Tag):
+        raise ValueError(f"Parameter tag[]={type(tag)} is not type of bs4.element.Tag")
+    elif not isinstance(tag, bs4.element.Tag):
+        raise KeyError("Cannot find table with instances!")
+    elif len(tag.select("tr")) == 0:
+        raise KeyError("No table rows found in table!")
+
+    domains = list()
+    for element in tag.select("tr"):
+        # DEBUG: print(f"DEBUG: element[]={type(element)}")
+        if not element.find("td"):
+            # DEBUG: print("DEBUG: Skipping element, no <td> found")
+            continue
+
+        domain = tidyup_domain(element.find("td").text)
+        reason = tidyup_reason(element.findAll("td")[1].text)
+
+        # DEBUG: print(f"DEBUG: domain='{domain}',reason='{reason}'")
+
+        if is_blacklisted(domain):
+            print(f"WARNING: domain='{domain}' is blacklisted - skipped!")
+            continue
+        elif domain == "gab.com/.ai, develop.gab.com":
+            print(f"DEBUG: Multiple domains detected in one row")
+            domains.append({
+                "domain": "gab.com",
+                "reason": reason,
+            })
+            domains.append({
+                "domain": "gab.ai",
+                "reason": reason,
+            })
+            domains.append({
+                "domain": "develop.gab.com",
+                "reason": reason,
+            })
+            continue
+        elif not validators.domain(domain):
+            print(f"WARNING: domain='{domain}' is not a valid domain - skipped!")
+            continue
+
+        # DEBUG: print(f"DEBUG: Adding domain='{domain}' ...")
+        domains.append({
+            "domain": domain,
+            "reason": reason,
+        })
+
+    # DEBUG: print(f"DEBUG: domains()={len(domains)} - EXIT!")
+    return domains