Continued:

- do not include anything that is empty after being cleared (exception reason)

diff --git a/fetch_blocks.py b/fetch_blocks.py
index 7cc6fda20a9a6af474f9f009584af8d6820767d4..2ea4a8662a30380da72e189153d77556a6881207 100644 (file)
--- a/fetch_blocks.py
+++ b/fetch_blocks.py
@@ -10,9 +10,15 @@ fba.c.execute(
 )
 
 for blocker, software in fba.c.fetchall():
-    # NOISY-DEBUG: print("DEBUG: blocker,software:", blocker, software)
+    # NOISY-DEBUG: print("DEBUG: BEFORE-blocker,software:", blocker, software)
     blockdict = []
     blocker = fba.tidyup(blocker)
+    # NOISY-DEBUG: print("DEBUG: AFTER-blocker,software:", blocker, software)
+
+    if blocker == "":
+        print("WARNING: blocker is now empty!")
+        continue
+
     if software == "pleroma":
         print("INFO: blocker:", blocker)
         try:
@@ -25,6 +31,14 @@ for blocker, software in fba.c.fetchall():
                     {**federation["mrf_simple"],
                     **{"quarantined_instances": federation["quarantined_instances"]}}
                 ).items():
+                    # NOISY-DEBUG: print("DEBUG: block_level, blocks.items():", block_level, len(blocks.items()))
+                    block_level = fba.tidyup(block_level)
+                    # NOISY-DEBUG: print("DEBUG: BEFORE block_level:", block_level)
+
+                    if block_level == "":
+                        print("WARNING: block_level is now empty!")
+                        continue
+
                     for blocked in blocks:
                         # NOISY-DEBUG: print("DEBUG: BEFORE blocked:", blocked)
                         blocked = fba.tidyup(blocked)
@@ -43,6 +57,7 @@ for blocker, software in fba.c.fetchall():
                             # NOISY-DEBUG: print("DEBUG: searchres[]:", type(searchres))
                             if searchres != None:
                                 blocked = searchres[0]
+                                # NOISY-DEBUG: print("DEBUG: Looked up domain:", blocked)
 
                         # NOISY-DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
                         fba.c.execute(
@@ -64,15 +79,18 @@ for blocker, software in fba.c.fetchall():
                         )
 
                         if fba.c.fetchone() == None:
+                            # NOISY-DEBUG: print("DEBUG: Blocking:", blocker, blocked, block_level)
                             fba.block_instance(blocker, blocked, "unknown", block_level, timestamp, timestamp)
 
                             if block_level == "reject":
+                                # NOISY-DEBUG: print("DEBUG: Adding to blockdict:", blocked)
                                 blockdict.append(
                                     {
                                         "blocked": blocked,
                                         "reason": None
                                     })
                         else:
+                            # NOISY-DEBUG: print("DEBUG: Updating last_seen:", blocker, blocked, block_level)
                             fba.update_last_seen(timestamp, blocker, blocked, block_level)
 
             fba.conn.commit()
@@ -87,6 +105,13 @@ for blocker, software in fba.c.fetchall():
                     else {})}
                 ).items():
                     # NOISY-DEBUG: print("DEBUG: block_level, info.items():", block_level, len(info.items()))
+                    block_level = fba.tidyup(block_level)
+                    # NOISY-DEBUG: print("DEBUG: BEFORE block_level:", block_level)
+
+                    if block_level == "":
+                        print("WARNING: block_level is now empty!")
+                        continue
+
                     for blocked, reason in info.items():
                         # NOISY-DEBUG: print("DEBUG: BEFORE blocked:", blocked)
                         blocked = fba.tidyup(blocked)
@@ -95,8 +120,7 @@ for blocker, software in fba.c.fetchall():
                         if blocked == "":
                             print("WARNING: blocked is empty after fba.tidyup():", blocker, block_level)
                             continue
-
-                        if blocked.count("*") > 1:
+                        elif blocked.count("*") > 1:
                             # same domain guess as above, but for reasons field
                             fba.c.execute(
                                 "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", (blocked.replace("*", "_"),)
@@ -150,7 +174,11 @@ for blocker, software in fba.c.fetchall():
 
                 # NOISY-DEBUG: print("DEBUG: blocks():", len(blocks))
                 for block in blocks:
-                    entry = {'domain': block['domain'], 'hash': block['digest'], 'reason': block['comment']}
+                    entry = {
+                        'domain': block['domain'],
+                        'hash': block['digest'],
+                        'reason': block['comment']
+                    }
 
                     # NOISY-DEBUG: print("DEBUG: severity,domain,hash,comment:", block['severity'], block['domain'], block['digest'], block['comment'])
                     if block['severity'] == 'suspend':
@@ -170,14 +198,22 @@ for blocker, software in fba.c.fetchall():
             # NOISY-DEBUG: print("DEBUG: json.items():", blocker, len(json.items()))
             for block_level, blocks in json.items():
                 # NOISY-DEBUG: print("DEBUG: blocker,block_level,blocks():", blocker, block_level, len(blocks))
+                block_level = fba.tidyup(block_level)
+                # NOISY-DEBUG: print("DEBUG: AFTER-block_level:", block_level)
+                if block_level == "":
+                    print("WARNING: block_level is empty, blocker:", blocker)
+                    continue
+
                 for instance in blocks:
                     blocked, blocked_hash, reason = instance.values()
                     # NOISY-DEBUG: print("DEBUG: blocked,hash,reason:", blocked, blocked_hash, reason)
-
                     blocked = fba.tidyup(blocked)
-                    # NOISY-DEBUG: print("DEBUG: blocked:", blocked)
+                    # NOISY-DEBUG: print("DEBUG: AFTER-blocked:", blocked)
 
-                    if blocked.count("*") < 1:
+                    if blocked == "":
+                        print("WARNING: blocked is empty:", blocker)
+                        continue
+                    elif blocked.count("*") < 1:
                         # No obsfucation for this instance
                         fba.c.execute(
                             "SELECT hash FROM instances WHERE domain = ? LIMIT 1", (blocked,)
@@ -233,12 +269,24 @@ for blocker, software in fba.c.fetchall():
                 json = fba.get_friendica_blocks(blocker)
             elif software == "misskey":
                 json = fba.get_misskey_blocks(blocker)
+
             for block_level, blocks in json.items():
+                # NOISY-DEBUG: print("DEBUG: blocker,block_level,blocks():", blocker, block_level, len(blocks))
+                block_level = fba.tidyup(block_level)
+                # NOISY-DEBUG: print("DEBUG: AFTER-block_level:", block_level)
+                if block_level == "":
+                    print("WARNING: block_level is empty, blocker:", blocker)
+                    continue
+
                 for instance in blocks:
                     blocked, reason = instance.values()
+                    # NOISY-DEBUG: print("DEBUG: BEFORE-blocked:", blocked)
                     blocked = fba.tidyup(blocked)
+                    # NOISY-DEBUG: print("DEBUG: AFTER-blocked:", blocked)
 
-                    # NOISY-DEBUG: print("DEBUG: BEFORE-blocked:", blocked)
+                    if blocked == "":
+                        print("WARNING: blocked is empty:", blocker)
+                        continue
                     if blocked.count("*") > 0:
                         # Some friendica servers also obscure domains without hash
                         fba.c.execute(
@@ -308,8 +356,13 @@ for blocker, software in fba.c.fetchall():
                         break
 
                     blocked = peer["domain"].lower()
-                    # NOISY-DEBUG: print("DEBUG: blocked:", blocked)
+                    # NOISY-DEBUG: print("DEBUG: BEFORE-blocked:", blocked)
+                    blocked = fba.tidyup(blocked)
+                    # NOISY-DEBUG: print("DEBUG: AFTER-blocked:", blocked)
 
+                    if blocked == "":
+                        print("WARNING: blocked is empty:", blocker)
+                        continue
                     if blocked.count("*") > 0:
                         # GTS does not have hashes for obscured domains, so we have to guess it
                         fba.c.execute(