Continued:

- committing every single INSERT INTO will cause a lot of I/O load, let's
  combine them all into a single transaction

diff --git a/fba/fba.py b/fba/fba.py
index db002a90424fde0cf7c16ccdac2d7c6b9ed4c090..14d500b1a55429350c535bcea153382b0d14ebce 100644 (file)
--- a/fba/fba.py
+++ b/fba/fba.py
@@ -1074,8 +1074,6 @@ def block_instance(blocker: str, blocked: str, reason: str, block_level: str):
                  time.time()
              ),
         )
-
-        connection.commit()
     except BaseException as e:
         print(f"ERROR: failed SQL query: blocker='{blocker}',blocked='{blocked}',reason='{reason}',block_level='{block_level}',exception[{type(e)}]:'{str(e)}'")
         sys.exit(255)

diff --git a/fba/instances.py b/fba/instances.py
index 1271048913b147b44c7903f58bc7a1ba6f8953bb..9b0323a1faed8ea79206cc246b37e7f7b0072bde 100644 (file)
--- a/fba/instances.py
+++ b/fba/instances.py
@@ -116,6 +116,7 @@ def update_instance_data(domain: str):
             print(f"WARNING: Did not update any rows: domain='{domain}',fields()={len(fields)} - EXIT!")
             return
 
+        # DEBUG: print("DEBUG: Committing changes ...")
         fba.connection.commit()
 
         # DEBUG: print("DEBUG: Deleting instance_data for domain:", domain)

diff --git a/fetch_blocks.py b/fetch_blocks.py
index a9b8535305b2a2899a0f7da768569b13a31cc376..260945f35a2a58c60bda26e04a7784c1933901ee 100755 (executable)
--- a/fetch_blocks.py
+++ b/fetch_blocks.py
@@ -141,6 +141,7 @@ for blocker, software, origin, nodeinfo_url in rows:
                             print(f"DEBUG: Updating block last seen for blocker='{blocker}',blocked='{blocked}' ...")
                             fba.update_last_seen(blocker, blocked, block_level)
 
+            # DEBUG: print("DEBUG: Committing changes ...")
             fba.connection.commit()
 
             # Reasons
@@ -340,6 +341,7 @@ for blocker, software, origin, nodeinfo_url in rows:
                         fba.update_last_seen(blocker, blocking, block_level)
                         fba.update_block_reason(reason, blocker, blocking, block_level)
 
+            # DEBUG: print("DEBUG: Committing changes ...")
             fba.connection.commit()
         except Exception as e:
             print(f"ERROR: blocker='{blocker}',software='{software}',exception[{type(e)}]:'{str(e)}'")
@@ -437,6 +439,7 @@ for blocker, software, origin, nodeinfo_url in rows:
                         fba.update_last_seen(blocker, blocked, block_level)
                         fba.update_block_reason(reason, blocker, blocked, block_level)
 
+            # DEBUG: print("DEBUG: Committing changes ...")
             fba.connection.commit()
         except Exception as e:
             print(f"ERROR: blocker='{blocker}',software='{software}',exception[{type(e)}]:'{str(e)}'")
@@ -512,6 +515,7 @@ for blocker, software, origin, nodeinfo_url in rows:
                                 # DEBUG: print(f"DEBUG: Setting block reason for blocked='{blocked}':'{peer['public_comment']}'")
                                 entry["reason"] = peer["public_comment"]
 
+                # DEBUG: print("DEBUG: Committing changes ...")
                 fba.connection.commit()
         except Exception as e:
             print(f"ERROR: blocker='{blocker}',software='{software}',exception[{type(e)}]:'{str(e)}'")

diff --git a/fetch_cs.py b/fetch_cs.py
index 94e4d68f59ad8663b17233bfc0a8864e0c9eea10..e65738dbaa53e88bd2f238b5c2412181d1af36e1 100755 (executable)
--- a/fetch_cs.py
+++ b/fetch_cs.py
@@ -119,4 +119,7 @@ if len(domains) > 0:
                 # DEBUG: print(f"DEBUG: domain='{row['domain']}',block_level='{block_level}' blocked by chaos.social, adding ...")
                 fba.block_instance('chaos.social', row["domain"], row["reason"], block_level)
 
+    # DEBUG: print("DEBUG: Committing changes ...")
+    fba.connection.commit()
+
 boot.shutdown()