]> git.mxchange.org Git - fba.git/blobdiff - fba.py
projects / fba.git / blobdiff
? search:
summary | shortlog | log | commit | commitdiff | tree
raw | inline | side by side
Continued:
[fba.git] / fba.py
diff --git a/fba.py b/fba.py
index 56514ad423e708045d86372c1f517d79a81674b2..d4f13b161671c716bd1f31601f1d7e0221b448a8 100644 (file)
--- a/fba.py
+++ b/fba.py
@@ -80,8 +80,6 @@ nodeinfos = {
     "detection_mode": {},
     # Found nodeinfo URL
     "nodeinfo_url": {},
-    # Where to fetch peers (other instances)
-    "get_peers_url": {},
 }
 
 language_mapping = {
@@ -316,11 +314,23 @@ def update_last_blocked(domain: str):
             print("WARNING: Did not update any rows:", domain)
 
     except BaseException as e:
-        print(f"ERROR: failed SQL query: domain='{domain}',exception:'{str(e)}'")
+        print(f"ERROR: failed SQL query: domain='{domain}',exception[{type(e)}]:'{str(e)}'")
         sys.exit(255)
 
     # DEBUG: print("DEBUG: EXIT!")
 
+def has_pending_nodeinfos(domain: str) -> bool:
+    # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
+    has_pending = False
+    for key in nodeinfos:
+        # DEBUG: print(f"DEBUG: key='{key}',domain='{domain}',nodeinfos[key]='{nodeinfos[key]}'")
+        if domain in nodeinfos[key]:
+            has_pending = True
+            break
+
+    # DEBUG: print(f"DEBUG: has_pending='{has_pending}' - EXIT!")
+    return has_pending
+
 def update_nodeinfos(domain: str):
     # DEBUG: print("DEBUG: Updating nodeinfo for domain:", domain)
     sql_string = ''
@@ -347,7 +357,7 @@ def update_nodeinfos(domain: str):
             print("WARNING: Did not update any rows:", domain)
 
     except BaseException as e:
-        print(f"ERROR: failed SQL query: domain='{domain}',sql='{sql}',exception:'{str(e)}'")
+        print(f"ERROR: failed SQL query: domain='{domain}',sql='{sql}',exception[{type(e)}]:'{str(e)}'")
         sys.exit(255)
 
     # DEBUG: print("DEBUG: Deleting nodeinfos for domain:", domain)
@@ -360,6 +370,37 @@ def update_nodeinfos(domain: str):
 
     # DEBUG: print("DEBUG: EXIT!")
 
+def log_error(domain: str, res: any):
+    # DEBUG: print("DEBUG: domain,res[]:", domain, type(res))
+    try:
+        # DEBUG: print("DEBUG: BEFORE res[]:", type(res))
+        if isinstance(res, BaseException) or isinstance(res, json.JSONDecodeError):
+            res = str(res)
+
+        # DEBUG: print("DEBUG: AFTER res[]:", type(res))
+        if type(res) is str:
+            cursor.execute("INSERT INTO error_log (domain, error_code, error_message, created) VALUES (?, 999, ?, ?)",[
+                domain,
+                res,
+                time.time()
+            ])
+        else:
+            cursor.execute("INSERT INTO error_log (domain, error_code, error_message, created) VALUES (?, ?, ?, ?)",[
+                domain,
+                res.status_code,
+                res.reason,
+                time.time()
+            ])
+
+        # Cleanup old entries
+        # DEBUG: print(f"DEBUG: Purging old records (distance: {config['error_log_cleanup'])")
+        cursor.execute("DELETE FROM error_log WHERE created < ?", [time.time() - config["error_log_cleanup"]])
+    except BaseException as e:
+        print(f"ERROR: failed SQL query: domain='{domain}',exception[{type(e)}]:'{str(e)}'")
+        sys.exit(255)
+
+    # DEBUG: print("DEBUG: EXIT!")
+
 def update_last_error(domain: str, res: any):
     # DEBUG: print("DEBUG: domain,res[]:", domain, type(res))
     try:
@@ -388,8 +429,10 @@ def update_last_error(domain: str, res: any):
             # DEBUG: print("DEBUG: Did not update any rows:", domain)
             pending_errors[domain] = res
 
+        log_error(domain, res)
+
     except BaseException as e:
-        print(f"ERROR: failed SQL query: domain='{domain}',exception:'{str(e)}'")
+        print(f"ERROR: failed SQL query: domain='{domain}',exception[{type(e)}]:'{str(e)}'")
         sys.exit(255)
 
     # DEBUG: print("DEBUG: EXIT!")
@@ -407,7 +450,7 @@ def update_last_instance_fetch(domain: str):
             print("WARNING: Did not update any rows:", domain)
 
     except BaseException as e:
-        print(f"ERROR: failed SQL query: domain='{domain}',exception:'{str(e)}'")
+        print(f"ERROR: failed SQL query: domain='{domain}',exception[{type(e)}]:'{str(e)}'")
         sys.exit(255)
 
     connection.commit()
@@ -426,21 +469,21 @@ def update_last_nodeinfo(domain: str):
             print("WARNING: Did not update any rows:", domain)
 
     except BaseException as e:
-        print(f"ERROR: failed SQL query: domain='{domain}',exception:'{str(e)}'")
+        print(f"ERROR: failed SQL query: domain='{domain}',exception[{type(e)}]:'{str(e)}'")
         sys.exit(255)
 
     connection.commit()
     # DEBUG: print("DEBUG: EXIT!")
 
 def get_peers(domain: str, software: str) -> list:
-    # DEBUG: print("DEBUG: Getting peers for domain:", domain, software)
+    # DEBUG: print(f"DEBUG: domain='{domain}',software='{software}' - CALLED!")
     peers = list()
 
     if software == "misskey":
         # DEBUG: print(f"DEBUG: domain='{domain}' is misskey, sending API POST request ...")
-
         offset = 0
         step = config["misskey_offset"]
+
         # iterating through all "suspended" (follow-only in its terminology)
         # instances page-by-page, since that troonware doesn't support
         # sending them all at once
@@ -477,6 +520,7 @@ def get_peers(domain: str, software: str) -> list:
                 print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}")
                 update_last_error(domain, fetched["error"]["message"])
                 break
+
             for row in fetched:
                 # DEBUG: print(f"DEBUG: row():{len(row)}")
                 if not "host" in row:
@@ -489,6 +533,7 @@ def get_peers(domain: str, software: str) -> list:
                 # DEBUG: print(f"DEBUG: Adding peer: '{row['host']}'")
                 peers.append(row["host"])
 
+        #print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...")
         update_last_instance_fetch(domain)
 
         # DEBUG: print("DEBUG: Returning peers[]:", type(peers))
@@ -515,8 +560,9 @@ def get_peers(domain: str, software: str) -> list:
                 update_last_error(domain, res)
 
         except BaseException as e:
-            print(f"WARNING: Exception during fetching JSON: domain='{domain}',exception:'{str(e)}'")
+            print(f"WARNING: Exception during fetching JSON: domain='{domain}',exception[{type(e)}]:'{str(e)}'")
 
+        #print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...")
         update_last_instance_fetch(domain)
 
         # DEBUG: print("DEBUG: Returning peers[]:", type(peers))
@@ -553,12 +599,13 @@ def get_peers(domain: str, software: str) -> list:
                         start = start + 100
 
                 except BaseException as e:
-                    print(f"WARNING: Exception during fetching JSON: domain='{domain}',exception:'{str(e)}'")
+                    print(f"WARNING: Exception during fetching JSON: domain='{domain}',exception[{type(e)}]:'{str(e)}'")
 
-            update_last_instance_fetch(domain)
+        #print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...")
+        update_last_instance_fetch(domain)
 
-            # DEBUG: print("DEBUG: Returning peers[]:", type(peers))
-            return peers
+        # DEBUG: print("DEBUG: Returning peers[]:", type(peers))
+        return peers
 
     # DEBUG: print(f"DEBUG: Fetching get_peers_url='{get_peers_url}' from '{domain}' ...")
     try:
@@ -588,12 +635,12 @@ def get_peers(domain: str, software: str) -> list:
         else:
             # DEBUG: print("DEBUG: Querying API was successful:", domain, len(data))
             peers = data
-            nodeinfos["get_peers_url"][domain] = get_peers_url
 
     except BaseException as e:
         print("WARNING: Some error during get():", domain, e)
         update_last_error(domain, e)
 
+    #print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...")
     update_last_instance_fetch(domain)
 
     # DEBUG: print("DEBUG: Returning peers[]:", type(peers))
@@ -610,8 +657,6 @@ def post_json_api(domain: str, path: str, parameter: str, extra_headers: dict =
         if not res.ok or res.status_code >= 400:
             print(f"WARNING: Cannot query JSON API: domain='{domain}',path='{path}',parameter()={len(parameter)},res.status_code='{res.status_code}',data[]='{type(data)}'")
             update_last_error(domain, res)
-        else:
-            update_last_nodeinfo(domain)
 
     except BaseException as e:
         print(f"WARNING: Some error during post(): domain='{domain}',path='{path}',parameter()={len(parameter)},exception[{type(e)}]:'{str(e)}'")
@@ -854,7 +899,7 @@ def update_block_reason(reason: str, blocker: str, blocked: str, block_level: st
             print("WARNING: Did not update any rows:", domain)
 
     except BaseException as e:
-        print(f"ERROR: failed SQL query: reason='{reason}',blocker='{blocker}',blocked='{blocked}',block_level='{block_level}',sql='{sql}',exception:'{str(e)}'")
+        print(f"ERROR: failed SQL query: reason='{reason}',blocker='{blocker}',blocked='{blocked}',block_level='{block_level}',sql='{sql}',exception[{type(e)}]:'{str(e)}'")
         sys.exit(255)
 
     # DEBUG: print("DEBUG: EXIT!")
@@ -876,7 +921,7 @@ def update_last_seen(blocker: str, blocked: str, block_level: str):
             print("WARNING: Did not update any rows:", domain)
 
     except BaseException as e:
-        print(f"ERROR: failed SQL query: last_seen='{last_seen}',blocker='{blocker}',blocked='{blocked}',block_level='{block_level}',exception:'{str(e)}'")
+        print(f"ERROR: failed SQL query: last_seen='{last_seen}',blocker='{blocker}',blocked='{blocked}',block_level='{block_level}',exception[{type(e)}]:'{str(e)}'")
         sys.exit(255)
 
     # DEBUG: print("DEBUG: EXIT!")
@@ -905,7 +950,7 @@ def block_instance(blocker: str, blocked: str, reason: str, block_level: str):
         )
 
     except BaseException as e:
-        print(f"ERROR: failed SQL query: blocker='{blocker}',blocked='{blocked}',reason='{reason}',block_level='{block_level}',first_seen='{first_seen}',last_seen='{last_seen}',exception:'{str(e)}'")
+        print(f"ERROR: failed SQL query: blocker='{blocker}',blocked='{blocked}',reason='{reason}',block_level='{block_level}',first_seen='{first_seen}',last_seen='{last_seen}',exception[{type(e)}]:'{str(e)}'")
         sys.exit(255)
 
     # DEBUG: print("DEBUG: EXIT!")
@@ -913,7 +958,7 @@ def block_instance(blocker: str, blocked: str, reason: str, block_level: str):
 def is_instance_registered(domain: str) -> bool:
     # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
     if not is_cache_initialized("is_registered"):
-        print(f"DEBUG: Cache for {__name__} not initialized, fetching all rows ...")
+        # DEBUG: print(f"DEBUG: Cache for 'is_registered' not initialized, fetching all rows ...")
         try:
             cursor.execute("SELECT domain FROM instances")
 
@@ -957,13 +1002,10 @@ def add_instance(domain: str, origin: str, originator: str, path: str = None):
 
         set_cache_key("is_registered", domain, True)
 
-        for key in nodeinfos:
-            # DEBUG: print(f"DEBUG: key='{key}',domain='{domain}',nodeinfos[key]={nodeinfos[key]}")
-            if domain in nodeinfos[key]:
-                # DEBUG: print(f"DEBUG: domain='{domain}' has pending nodeinfo being updated ...")
-                update_nodeinfos(domain)
-                remove_pending_error(domain)
-                break
+        if has_pending_nodeinfos(domain):
+            # DEBUG: print(f"DEBUG: domain='{domain}' has pending nodeinfo being updated ...")
+            update_nodeinfos(domain)
+            remove_pending_error(domain)
 
         if domain in pending_errors:
             # DEBUG: print("DEBUG: domain has pending error being updated:", domain)
@@ -971,7 +1013,7 @@ def add_instance(domain: str, origin: str, originator: str, path: str = None):
             remove_pending_error(domain)
 
     except BaseException as e:
-        print(f"ERROR: failed SQL query: domain='{domain}',exception:'{str(e)}'")
+        print(f"ERROR: failed SQL query: domain='{domain}',exception[{type(e)}]:'{str(e)}'")
         sys.exit(255)
     else:
         # DEBUG: print("DEBUG: Updating nodeinfo for domain:", domain)
@@ -1197,6 +1239,9 @@ def get_misskey_blocks(domain: str) -> dict:
             offset = 0
             break
 
+    #print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...")
+    update_last_instance_fetch(domain)
+
     # DEBUG: print("DEBUG: Returning for domain,blocked(),suspended():", domain, len(blocks["blocked"]), len(blocks["suspended"]))
     return {
         "reject"        : blocks["blocked"],
Personal clone of "fedi-block-api", if you like to use the command fetch_instances_social you need to get an API key from https://instances.social .