From a022e75f4294f75c51943bc89df8c42d7b1a93d2 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Sun, 4 Jun 2023 11:47:07 +0200 Subject: [PATCH] Continued: - the value from "misskey_offset" went into "limit" parameter, so let's rename it to "misskey_limit" - also misskey may return same set of results, so this need to be counted and then the while loop aborted - also TIME() did NOT store timestamp, we need to use time.time() again ... --- config.defaults.json | 2 +- fba/fba.py | 34 +++++++++++++++++++++------------- fba/instances.py | 7 +++++-- 3 files changed, 27 insertions(+), 16 deletions(-) diff --git a/config.defaults.json b/config.defaults.json index bbce483..1b1cfb7 100644 --- a/config.defaults.json +++ b/config.defaults.json @@ -13,6 +13,6 @@ "slogan" : "### Your footer slogan ###", "recheck_instance" : 172800, "recheck_block" : 43200, - "misskey_offset" : 100, + "misskey_limit" : 100, "error_log_cleanup" : 604800 } diff --git a/fba/fba.py b/fba/fba.py index 982cc62..7d24b55 100644 --- a/fba/fba.py +++ b/fba/fba.py @@ -477,13 +477,12 @@ def get_peers(domain: str, software: str) -> list: elif type(software) != str and software != None: raise ValueError(f"software[]={type(software)} is not 'str'") - # DEBUG: print(f"DEBUG: domain='{domain}',software='{software}' - CALLED!") peers = list() if software == "misskey": # DEBUG: print(f"DEBUG: domain='{domain}' is misskey, sending API POST request ...") offset = 0 - step = config.get("misskey_offset") + step = config.get("misskey_limit") # iterating through all "suspended" (follow-only in its terminology) # instances page-by-page, since that troonware doesn't support @@ -504,13 +503,13 @@ def get_peers(domain: str, software: str) -> list: "offset": offset - 1 }), {"Origin": domain}) - # DEBUG: print("DEBUG: fetched():", len(fetched)) + # DEBUG: print(f"DEBUG: fetched()={len(fetched))}") if len(fetched) == 0: # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain) break - elif len(fetched) != config.get("misskey_offset"): - # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_offset')}'") - offset = offset + (config.get("misskey_offset") - len(fetched)) + elif len(fetched) != config.get("misskey_limit"): + # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'") + offset = offset + (config.get("misskey_limit") - len(fetched)) else: # DEBUG: print("DEBUG: Raising offset by step:", step) offset = offset + step @@ -522,6 +521,7 @@ def get_peers(domain: str, software: str) -> list: update_last_error(domain, fetched["error"]["message"]) break + already = 0 for row in fetched: # DEBUG: print(f"DEBUG: row():{len(row)}") if not "host" in row: @@ -533,10 +533,18 @@ def get_peers(domain: str, software: str) -> list: elif is_blacklisted(row["host"]): # DEBUG: print(f"DEBUG: row[host]='{row['host']}' is blacklisted. domain='{domain}'") continue + elif row["host"] in peers: + # DEBUG: print(f"DEBUG: Not adding row[host]='{row['host']}', already found.") + already = already + 1 + continue # DEBUG: print(f"DEBUG: Adding peer: '{row['host']}'") peers.append(row["host"]) + if already == len(fetched): + print(f"WARNING: Host returned same set of '{already}' instances, aborting loop!") + break + # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'") instances.set_instance_data("total_peers", domain, len(peers)) @@ -1302,7 +1310,7 @@ def get_misskey_blocks(domain: str) -> dict: } offset = 0 - step = config.get("misskey_offset") + step = config.get("misskey_limit") while True: # iterating through all "suspended" (follow-only in its terminology) # instances page-by-page, since that troonware doesn't support @@ -1331,9 +1339,9 @@ def get_misskey_blocks(domain: str) -> dict: if len(fetched) == 0: # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain) break - elif len(fetched) != config.get("misskey_offset"): - # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_offset')}'") - offset = offset + (config.get("misskey_offset") - len(fetched)) + elif len(fetched) != config.get("misskey_limit"): + # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'") + offset = offset + (config.get("misskey_limit") - len(fetched)) else: # DEBUG: print("DEBUG: Raising offset by step:", step) offset = offset + step @@ -1380,9 +1388,9 @@ def get_misskey_blocks(domain: str) -> dict: if len(fetched) == 0: # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain) break - elif len(fetched) != config.get("misskey_offset"): - # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_offset')}'") - offset = offset + (config.get("misskey_offset") - len(fetched)) + elif len(fetched) != config.get("misskey_limit"): + # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'") + offset = offset + (config.get("misskey_limit") - len(fetched)) else: # DEBUG: print("DEBUG: Raising offset by step:", step) offset = offset + step diff --git a/fba/instances.py b/fba/instances.py index 2b507fe..16b0e33 100644 --- a/fba/instances.py +++ b/fba/instances.py @@ -14,8 +14,10 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from fba import fba import sys +import time + +from fba import fba # Found info from node, such as nodeinfo URL, detection mode that needs to be # written to database. Both arrays must be filled at the same time or else @@ -98,13 +100,14 @@ def update_instance_data(domain: str): fields.append(_pending[key][domain]) sql_string += f" {key} = ?," + fields.append(time.time()) fields.append(domain) if sql_string == '': raise ValueError(f"No fields have been set, but method invoked, domain='{domain}'") # DEBUG: print(f"DEBUG: sql_string='{sql_string}',fields()={len(fields)}") - sql_string = "UPDATE instances SET" + sql_string + " last_updated = TIME() WHERE domain = ? LIMIT 1" + sql_string = "UPDATE instances SET" + sql_string + " last_updated = ? WHERE domain = ? LIMIT 1" # DEBUG: print("DEBUG: sql_string:", sql_string) try: -- 2.39.5