Continued:

- the value from "misskey_offset" went into "limit" parameter, so let's rename
  it to "misskey_limit"
- also misskey may return same set of results, so this need to be counted and
  then the while loop aborted
- also TIME() did NOT store timestamp, we need to use time.time() again ...

diff --git a/config.defaults.json b/config.defaults.json
index bbce48349139cf387129081dc8043f12938258d6..1b1cfb78eac2c9c9a1229a73e804f89a8f423d18 100644 (file)
--- a/config.defaults.json
+++ b/config.defaults.json
@@ -13,6 +13,6 @@
     "slogan"            : "### Your footer slogan ###",
     "recheck_instance"  : 172800,
     "recheck_block"     : 43200,
-    "misskey_offset"    : 100,
+    "misskey_limit"     : 100,
     "error_log_cleanup" : 604800
 }

diff --git a/fba/fba.py b/fba/fba.py
index 982cc628b3b3bb0eec1247e8aca425fb0616623b..7d24b5540ed90774f27e106c2cbb5f5ebf170965 100644 (file)
--- a/fba/fba.py
+++ b/fba/fba.py
@@ -477,13 +477,12 @@ def get_peers(domain: str, software: str) -> list:
     elif type(software) != str and software != None:
         raise ValueError(f"software[]={type(software)} is not 'str'")
 
-    # DEBUG: print(f"DEBUG: domain='{domain}',software='{software}' - CALLED!")
     peers = list()
 
     if software == "misskey":
         # DEBUG: print(f"DEBUG: domain='{domain}' is misskey, sending API POST request ...")
         offset = 0
-        step = config.get("misskey_offset")
+        step = config.get("misskey_limit")
 
         # iterating through all "suspended" (follow-only in its terminology)
         # instances page-by-page, since that troonware doesn't support
@@ -504,13 +503,13 @@ def get_peers(domain: str, software: str) -> list:
                     "offset": offset - 1
                 }), {"Origin": domain})
 
-            # DEBUG: print("DEBUG: fetched():", len(fetched))
+            # DEBUG: print(f"DEBUG: fetched()={len(fetched))}")
             if len(fetched) == 0:
                 # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
                 break
-            elif len(fetched) != config.get("misskey_offset"):
-                # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_offset')}'")
-                offset = offset + (config.get("misskey_offset") - len(fetched))
+            elif len(fetched) != config.get("misskey_limit"):
+                # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'")
+                offset = offset + (config.get("misskey_limit") - len(fetched))
             else:
                 # DEBUG: print("DEBUG: Raising offset by step:", step)
                 offset = offset + step
@@ -522,6 +521,7 @@ def get_peers(domain: str, software: str) -> list:
                 update_last_error(domain, fetched["error"]["message"])
                 break
 
+            already = 0
             for row in fetched:
                 # DEBUG: print(f"DEBUG: row():{len(row)}")
                 if not "host" in row:
@@ -533,10 +533,18 @@ def get_peers(domain: str, software: str) -> list:
                 elif is_blacklisted(row["host"]):
                     # DEBUG: print(f"DEBUG: row[host]='{row['host']}' is blacklisted. domain='{domain}'")
                     continue
+                elif row["host"] in peers:
+                    # DEBUG: print(f"DEBUG: Not adding row[host]='{row['host']}', already found.")
+                    already = already + 1
+                    continue
 
                 # DEBUG: print(f"DEBUG: Adding peer: '{row['host']}'")
                 peers.append(row["host"])
 
+            if already == len(fetched):
+                print(f"WARNING: Host returned same set of '{already}' instances, aborting loop!")
+                break
+
         # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'")
         instances.set_instance_data("total_peers", domain, len(peers))
 
@@ -1302,7 +1310,7 @@ def get_misskey_blocks(domain: str) -> dict:
     }
 
     offset = 0
-    step = config.get("misskey_offset")
+    step = config.get("misskey_limit")
     while True:
         # iterating through all "suspended" (follow-only in its terminology)
         # instances page-by-page, since that troonware doesn't support
@@ -1331,9 +1339,9 @@ def get_misskey_blocks(domain: str) -> dict:
             if len(fetched) == 0:
                 # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
                 break
-            elif len(fetched) != config.get("misskey_offset"):
-                # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_offset')}'")
-                offset = offset + (config.get("misskey_offset") - len(fetched))
+            elif len(fetched) != config.get("misskey_limit"):
+                # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'")
+                offset = offset + (config.get("misskey_limit") - len(fetched))
             else:
                 # DEBUG: print("DEBUG: Raising offset by step:", step)
                 offset = offset + step
@@ -1380,9 +1388,9 @@ def get_misskey_blocks(domain: str) -> dict:
             if len(fetched) == 0:
                 # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
                 break
-            elif len(fetched) != config.get("misskey_offset"):
-                # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_offset')}'")
-                offset = offset + (config.get("misskey_offset") - len(fetched))
+            elif len(fetched) != config.get("misskey_limit"):
+                # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'")
+                offset = offset + (config.get("misskey_limit") - len(fetched))
             else:
                 # DEBUG: print("DEBUG: Raising offset by step:", step)
                 offset = offset + step

diff --git a/fba/instances.py b/fba/instances.py
index 2b507fe109a55eaf88dc089c4f71da49ecfdbb87..16b0e332972af6d69da65074ff8bab886d1adfeb 100644 (file)
--- a/fba/instances.py
+++ b/fba/instances.py
@@ -14,8 +14,10 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-from fba import fba
 import sys
+import time
+
+from fba import fba
 
 # Found info from node, such as nodeinfo URL, detection mode that needs to be
 # written to database. Both arrays must be filled at the same time or else
@@ -98,13 +100,14 @@ def update_instance_data(domain: str):
            fields.append(_pending[key][domain])
            sql_string += f" {key} = ?,"
 
+    fields.append(time.time())
     fields.append(domain)
 
     if sql_string == '':
         raise ValueError(f"No fields have been set, but method invoked, domain='{domain}'")
 
     # DEBUG: print(f"DEBUG: sql_string='{sql_string}',fields()={len(fields)}")
-    sql_string = "UPDATE instances SET" + sql_string + " last_updated = TIME() WHERE domain = ? LIMIT 1"
+    sql_string = "UPDATE instances SET" + sql_string + " last_updated = ? WHERE domain = ? LIMIT 1"
     # DEBUG: print("DEBUG: sql_string:", sql_string)
 
     try: