Continued:

- better warn about not deobsfucated domain names

diff --git a/fetch_blocks.py b/fetch_blocks.py
index c72d56eaf5a06d5014b535f0f0bf3bbfdb5af3ca..612b84f985fcf9ab521591fd8926b1bb61d2f129 100755 (executable)
--- a/fetch_blocks.py
+++ b/fetch_blocks.py
@@ -101,11 +101,14 @@ for blocker, software, origin, nodeinfo_url in rows:
                                 nodeinfo_url = searchres[1]
                                 # DEBUG: print("DEBUG: Looked up domain:", blocked)
                         elif not validators.domain(blocked):
-                            print(f"WARNING: blocked='{blocked}' is not a valid domai name - skipped!")
+                            print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domai name - skipped!")
                             continue
 
                         # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
-                        if not fba.is_instance_registered(blocked):
+                        if not validators.domain(blocked):
+                            print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
+                            continue
+                        elif not fba.is_instance_registered(blocked):
                             # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., blocker='{blocker}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'")
                             fba.add_instance(blocked, blocker, origin, nodeinfo_url)
 
@@ -175,11 +178,14 @@ for blocker, software, origin, nodeinfo_url in rows:
                                 origin = searchres[1]
                                 nodeinfo_url = searchres[2]
                         elif not validators.domain(blocked):
-                            print(f"WARNING: blocked='{blocked}' is not a valid domai name - skipped!")
+                            print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domai name - skipped!")
                             continue
 
                         # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
-                        if not fba.is_instance_registered(blocked):
+                        if not validators.domain(blocked):
+                            print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
+                            continue
+                        elif not fba.is_instance_registered(blocked):
                             # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., blocker='{blocker}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'")
                             fba.add_instance(blocked, blocker, origin, nodeinfo_url)
 
@@ -223,7 +229,7 @@ for blocker, software, origin, nodeinfo_url in rows:
                 # DEBUG: print("DEBUG: Querying API domain_blocks:", blocker)
                 blocks = reqto.get(f"https://{blocker}/api/v1/instance/domain_blocks", headers=reqheaders, timeout=(fba.config["connection_timeout"], fba.config["read_timeout"])).json()
 
-                print(f"INFO: Checking {len(blocks)} entries from blocker='{blocker}' ...")
+                print(f"INFO: Checking {len(blocks)} entries from blocker='{blocker}',software='{software}' ...")
                 for block in blocks:
                     entry = {
                         'domain': block['domain'],
@@ -246,7 +252,7 @@ for blocker, software, origin, nodeinfo_url in rows:
                 # DEBUG: print(f"DEBUG: Failed, trying mastodon-specific fetches: blocker='{blocker}',exception[{type(e)}]={str(e)}")
                 json = fba.get_mastodon_blocks(blocker)
 
-            print(f"INFO: Checking {len(json.items())} entries from blocker='{blocker}' ...")
+            print(f"INFO: Checking {len(json.items())} entries from blocker='{blocker}',software='{software}' ...")
             for block_level, blocks in json.items():
                 # DEBUG: print("DEBUG: blocker,block_level,blocks():", blocker, block_level, len(blocks))
                 block_level = fba.tidyup(block_level)
@@ -274,22 +280,31 @@ for blocker, software, origin, nodeinfo_url in rows:
                         )
                         searchres = fba.cursor.fetchone()
 
-                        if searchres != None:
-                            # DEBUG: print("DEBUG: Updating domain: ", searchres[0])
-                            blocked = searchres[0]
-                            origin = searchres[1]
-                            nodeinfo_url = searchres[2]
+                        if searchres == None:
+                            print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
+                            continue
+
+                        # DEBUG: print("DEBUG: Updating domain: ", searchres[0])
+                        blocked = searchres[0]
+                        origin = searchres[1]
+                        nodeinfo_url = searchres[2]
 
                         # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
-                        if not fba.is_instance_registered(blocked):
+                        if not validators.domain(blocked):
+                            print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
+                            continue
+                        elif not fba.is_instance_registered(blocked):
                             # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., blocker='{blocker}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'")
                             fba.add_instance(blocked, blocker, origin, nodeinfo_url)
                     elif not validators.domain(blocked):
-                        print(f"WARNING: blocked='{blocked}' is not a valid domai name - skipped!")
+                        print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domai name - skipped!")
                         continue
 
                     # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
-                    if not fba.is_instance_registered(blocked):
+                    if not validators.domain(blocked):
+                        print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
+                        continue
+                    elif not fba.is_instance_registered(blocked):
                         # DEBUG: print("DEBUG: Hash wasn't found, adding:", blocked, blocker)
                         fba.add_instance(blocked, blocker, origin, nodeinfo_url)
 
@@ -336,7 +351,7 @@ for blocker, software, origin, nodeinfo_url in rows:
                 print("WARNING: takahe is not fully supported for fetching blacklist!", blocker)
                 #json = fba.get_takahe_blocks(blocker)
 
-            print(f"INFO: Checking {len(json.items())} entries from blocker='{blocker}' ...")
+            print(f"INFO: Checking {len(json.items())} entries from blocker='{blocker}',software='{software}' ...")
             for block_level, blocks in json.items():
                 # DEBUG: print("DEBUG: blocker,block_level,blocks():", blocker, block_level, len(blocks))
                 block_level = fba.tidyup(block_level)
@@ -365,26 +380,37 @@ for blocker, software, origin, nodeinfo_url in rows:
 
                         searchres = fba.cursor.fetchone()
 
-                        if searchres != None:
-                            blocked = searchres[0]
-                            origin = searchres[1]
-                            nodeinfo_url = searchres[2]
+                        if searchres == None:
+                            print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
+                            continue
+
+                        blocked = searchres[0]
+                        origin = searchres[1]
+                        nodeinfo_url = searchres[2]
                     elif blocked.count("?") > 0:
                         # Some obscure them with question marks, not sure if that's dependent on version or not
                         fba.cursor.execute(
                             "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("?", "_")]
                         )
+
                         searchres = fba.cursor.fetchone()
-                        if searchres != None:
-                            blocked = searchres[0]
-                            origin = searchres[1]
-                            nodeinfo_url = searchres[2]
+
+                        if searchres == None:
+                            print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
+                            continue
+
+                        blocked = searchres[0]
+                        origin = searchres[1]
+                        nodeinfo_url = searchres[2]
                     elif not validators.domain(blocked):
-                        print(f"WARNING: blocked='{blocked}' is not a valid domai name - skipped!")
+                        print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domai name - skipped!")
                         continue
 
                     # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
-                    if not fba.is_instance_registered(blocked):
+                    if not validators.domain(blocked):
+                        print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
+                        continue
+                    elif not fba.is_instance_registered(blocked):
                         # DEBUG: print("DEBUG: Hash wasn't found, adding:", blocked, blocker)
                         fba.add_instance(blocked, blocker, origin, nodeinfo_url)
 
@@ -423,7 +449,7 @@ for blocker, software, origin, nodeinfo_url in rows:
             elif "error" in federation:
                 print("WARNING: API returned error:", federation["error"])
             else:
-                print(f"INFO: Checking {len(federation)} entries from blocker='{blocker}' ...")
+                print(f"INFO: Checking {len(federation)} entries from blocker='{blocker}',software='{software}' ...")
                 for peer in federation:
                     blocked = peer["domain"].lower()
                     # DEBUG: print("DEBUG: BEFORE blocked:", blocked)
@@ -443,16 +469,22 @@ for blocker, software, origin, nodeinfo_url in rows:
                         )
                         searchres = fba.cursor.fetchone()
 
-                        if searchres != None:
-                            blocked = searchres[0]
-                            origin = searchres[1]
-                            nodeinfo_url = searchres[2]
+                        if searchres == None:
+                            print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
+                            continue
+
+                        blocked = searchres[0]
+                        origin = searchres[1]
+                        nodeinfo_url = searchres[2]
                     elif not validators.domain(blocked):
-                        print(f"WARNING: blocked='{blocked}' is not a valid domai name - skipped!")
+                        print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domai name - skipped!")
                         continue
 
                     # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
-                    if not fba.is_instance_registered(blocked):
+                    if not validators.domain(blocked):
+                        print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
+                        continue
+                    elif not fba.is_instance_registered(blocked):
                         # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., blocker='{blocker}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'")
                         fba.add_instance(blocked, blocker, origin, nodeinfo_url)