Continued:

[fba.git] / fetch_blocks.py

diff --git a/fetch_blocks.py b/fetch_blocks.py
index 2e0ff2d006f5adbe60b421978e67bee20efef6f5..bbccb931f121fc76052c7b01af8a09aef603f17d 100644 (file)
--- a/fetch_blocks.py
+++ b/fetch_blocks.py
@@ -22,13 +22,13 @@ import re
 import fba
 
 fba.cursor.execute(
-    "SELECT domain, software FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial', 'bookwyrm', 'takahe') AND (last_blocked IS NULL OR last_blocked < ?) ORDER BY rowid DESC", [time.time() - fba.config["recheck_block"]]
+    "SELECT domain, software, origin, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial', 'bookwyrm', 'takahe') AND (last_blocked IS NULL OR last_blocked < ?) ORDER BY rowid DESC", [time.time() - fba.config["recheck_block"]]
 )
 
 rows = fba.cursor.fetchall()
 print(f"INFO: Checking {len(rows)} entries ...")
-for blocker, software in rows:
-    # NOISY-DEBUG: print("DEBUG: BEFORE blocker,software:", blocker, software)
+for blocker, software, origin, nodeinfo_url in rows:
+    # NOISY-DEBUG: print("DEBUG: BEFORE blocker,software,origin,nodeinfo_url:", blocker, software, origin, nodeinfo_url)
     blockdict = []
     blocker = fba.tidyup(blocker)
     # NOISY-DEBUG: print("DEBUG: AFTER blocker,software:", blocker, software)
@@ -47,7 +47,7 @@ for blocker, software in rows:
         print("INFO: blocker:", blocker)
         try:
             # Blocks
-            json = fba.fetch_nodeinfo(blocker)
+            json = fba.fetch_nodeinfo(blocker, nodeinfo_url)
             if json is None:
                 print("WARNING: Could not fetch nodeinfo from blocker:", blocker)
                 continue
@@ -86,18 +86,19 @@ for blocker, software in rows:
                         if blocked.count("*") > 1:
                             # -ACK!-oma also started obscuring domains without hash
                             fba.cursor.execute(
-                                "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")]
+                                "SELECT domain, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")]
                             )
                             searchres = fba.cursor.fetchone()
                             # NOISY-DEBUG: print("DEBUG: searchres[]:", type(searchres))
                             if searchres != None:
                                 blocked = searchres[0]
+                                nodeinfo_url = searchres[1]
                                 # NOISY-DEBUG: print("DEBUG: Looked up domain:", blocked)
 
                         # NOISY-DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
                         if not fba.is_instance_registered(blocked):
                             # NOISY-DEBUG: print("DEBUG: Domain wasn't found, adding:", blocked, blocker)
-                            fba.add_instance(blocked, blocker, argv[0])
+                            fba.add_instance(blocked, blocker, origin, nodeinfo_url)
 
                         fba.cursor.execute(
                             "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ? LIMIT 1",
@@ -249,7 +250,7 @@ for blocker, software in rows:
 
                         if fba.cursor.fetchone() == None:
                             # NOISY-DEBUG: print("DEBUG: Hash wasn't found, adding:", blocked, blocker)
-                            fba.add_instance(blocked, blocker, argv[0])
+                            fba.add_instance(blocked, blocker, origin)
                     else:
                         # Doing the hash search for instance names as well to tidy up DB
                         fba.cursor.execute(
@@ -332,16 +333,18 @@ for blocker, software in rows:
                     if blocked.count("?") > 0:
                         # Some obscure them with question marks, not sure if that's dependent on version or not
                         fba.cursor.execute(
-                            "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("?", "_")]
+                            "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("?", "_")]
                         )
                         searchres = fba.cursor.fetchone()
                         if searchres != None:
                             blocked = searchres[0]
+                            origin = searchres[1]
+                            nodeinfo_url = searchres[2]
 
                     # NOISY-DEBUG: print("DEBUG: AFTER-blocked:", blocked)
                     if not fba.is_instance_registered(blocked):
                         # NOISY-DEBUG: print("DEBUG: Hash wasn't found, adding:", blocked, blocker)
-                        fba.add_instance(blocked, blocker)
+                        fba.add_instance(blocked, blocker, origin, nodeinfo_url)
 
                     fba.cursor.execute(
                         "SELECT * FROM blocks WHERE blocker = ? AND blocked = ?",
@@ -391,16 +394,18 @@ for blocker, software in rows:
                     elif blocked.count("*") > 0:
                         # GTS does not have hashes for obscured domains, so we have to guess it
                         fba.cursor.execute(
-                            "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")]
+                            "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")]
                         )
                         searchres = fba.cursor.fetchone()
 
                         if searchres != None:
                             blocked = searchres[0]
+                            origin = searchres[1]
+                            nodeinfo_url = searchres[2]
 
                     if not fba.is_instance_registered(blocked):
                         # NOISY-DEBUG: print("DEBUG: Domain wasn't found, adding:", blocked, blocker)
-                        fba.add_instance(blocked, blocker)
+                        fba.add_instance(blocked, blocker, origin, nodeinfo_url)
 
                     fba.cursor.execute(
                         "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ? LIMIT 1",