Continued:

[fba.git] / fba / fba.py

diff --git a/fba/fba.py b/fba/fba.py
index 5084f761df06e7d394d65fa08557f728cdf1d211..7d24b5540ed90774f27e106c2cbb5f5ebf170965 100644 (file)
--- a/fba/fba.py
+++ b/fba/fba.py
@@ -477,13 +477,12 @@ def get_peers(domain: str, software: str) -> list:
     elif type(software) != str and software != None:
         raise ValueError(f"software[]={type(software)} is not 'str'")
 
-    # DEBUG: print(f"DEBUG: domain='{domain}',software='{software}' - CALLED!")
     peers = list()
 
     if software == "misskey":
         # DEBUG: print(f"DEBUG: domain='{domain}' is misskey, sending API POST request ...")
         offset = 0
-        step = config.get("misskey_offset")
+        step = config.get("misskey_limit")
 
         # iterating through all "suspended" (follow-only in its terminology)
         # instances page-by-page, since that troonware doesn't support
@@ -504,13 +503,13 @@ def get_peers(domain: str, software: str) -> list:
                     "offset": offset - 1
                 }), {"Origin": domain})
 
-            # DEBUG: print("DEBUG: fetched():", len(fetched))
+            # DEBUG: print(f"DEBUG: fetched()={len(fetched))}")
             if len(fetched) == 0:
                 # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
                 break
-            elif len(fetched) != config.get("misskey_offset"):
-                # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_offset')}'")
-                offset = offset + (config.get("misskey_offset") - len(fetched))
+            elif len(fetched) != config.get("misskey_limit"):
+                # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'")
+                offset = offset + (config.get("misskey_limit") - len(fetched))
             else:
                 # DEBUG: print("DEBUG: Raising offset by step:", step)
                 offset = offset + step
@@ -522,6 +521,7 @@ def get_peers(domain: str, software: str) -> list:
                 update_last_error(domain, fetched["error"]["message"])
                 break
 
+            already = 0
             for row in fetched:
                 # DEBUG: print(f"DEBUG: row():{len(row)}")
                 if not "host" in row:
@@ -533,10 +533,18 @@ def get_peers(domain: str, software: str) -> list:
                 elif is_blacklisted(row["host"]):
                     # DEBUG: print(f"DEBUG: row[host]='{row['host']}' is blacklisted. domain='{domain}'")
                     continue
+                elif row["host"] in peers:
+                    # DEBUG: print(f"DEBUG: Not adding row[host]='{row['host']}', already found.")
+                    already = already + 1
+                    continue
 
                 # DEBUG: print(f"DEBUG: Adding peer: '{row['host']}'")
                 peers.append(row["host"])
 
+            if already == len(fetched):
+                print(f"WARNING: Host returned same set of '{already}' instances, aborting loop!")
+                break
+
         # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'")
         instances.set_instance_data("total_peers", domain, len(peers))
 
@@ -1302,7 +1310,7 @@ def get_misskey_blocks(domain: str) -> dict:
     }
 
     offset = 0
-    step = config.get("misskey_offset")
+    step = config.get("misskey_limit")
     while True:
         # iterating through all "suspended" (follow-only in its terminology)
         # instances page-by-page, since that troonware doesn't support
@@ -1331,9 +1339,9 @@ def get_misskey_blocks(domain: str) -> dict:
             if len(fetched) == 0:
                 # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
                 break
-            elif len(fetched) != config.get("misskey_offset"):
-                # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_offset')}'")
-                offset = offset + (config.get("misskey_offset") - len(fetched))
+            elif len(fetched) != config.get("misskey_limit"):
+                # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'")
+                offset = offset + (config.get("misskey_limit") - len(fetched))
             else:
                 # DEBUG: print("DEBUG: Raising offset by step:", step)
                 offset = offset + step
@@ -1380,9 +1388,9 @@ def get_misskey_blocks(domain: str) -> dict:
             if len(fetched) == 0:
                 # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
                 break
-            elif len(fetched) != config.get("misskey_offset"):
-                # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_offset')}'")
-                offset = offset + (config.get("misskey_offset") - len(fetched))
+            elif len(fetched) != config.get("misskey_limit"):
+                # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'")
+                offset = offset + (config.get("misskey_limit") - len(fetched))
             else:
                 # DEBUG: print("DEBUG: Raising offset by step:", step)
                 offset = offset + step
@@ -1420,7 +1428,7 @@ def tidyup_domain(domain: str) -> str:
     # No port number
     domain = re.sub("\:\d+$", "", domain)
 
-    # No protocol, sometimes with the slashes
+    # No protocol, sometimes without the slashes
     domain = re.sub("^https?\:(\/*)", "", domain)
 
     # No trailing slash