From 73c2f55e1428189cf43ca59e8c14b5cd712fee07 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Mon, 22 May 2023 11:02:07 +0200 Subject: [PATCH] Continued: - try to update last error details and with a fake status_code --- fba.py | 91 +++++++++++++++++++++++++++++++--------------- fetch_instances.py | 27 ++++++++------ 2 files changed, 77 insertions(+), 41 deletions(-) diff --git a/fba.py b/fba.py index 5b27c36..dcdcea4 100644 --- a/fba.py +++ b/fba.py @@ -20,6 +20,9 @@ blacklist = [ "ngrok.io", ] +pending_errors = { +} + nodeinfos = [ "http://nodeinfo.diaspora.software/ns/schema/2.1", "http://nodeinfo.diaspora.software/ns/schema/2.0", @@ -54,22 +57,36 @@ def update_last_blocked(domain: str): domain ]) - except: - print("ERROR: failed SQL query:", domain) + if cursor.rowcount == 0: + print("WARNING: Did not update any rows:", domain) + + except BaseException as e: + print("ERROR: failed SQL query:", domain, e) sys.exit(255) def update_last_error(domain: str, res: any): # NOISY-DEBUG: print("DEBUG: domain,res.status_code:", domain, res.status_code, res.reason) try: - cursor.execute("UPDATE instances SET last_status_code = ?, last_error_details = ?, last_updated = ? WHERE domain = ?", [ - res.status_code, - res.reason, - time.time(), - domain - ]) + if type(res) is str: + cursor.execute("UPDATE instances SET last_status_code = 999, last_error_details = ?, last_updated = ? WHERE domain = ?", [ + res, + time.time(), + domain + ]) + else: + cursor.execute("UPDATE instances SET last_status_code = ?, last_error_details = ?, last_updated = ? WHERE domain = ?", [ + res.status_code, + res.reason, + time.time(), + domain + ]) - except: - print("ERROR: failed SQL query:", domain) + if cursor.rowcount == 0: + # NOISY-DEBUG: print("DEBUG: Did not update any rows:", domain) + pending_errors[domain] = res + + except BaseException as e: + print("ERROR: failed SQL query:", domain, e) sys.exit(255) def update_last_nodeinfo(domain: str): @@ -81,8 +98,11 @@ def update_last_nodeinfo(domain: str): domain ]) - except: - print("ERROR: failed SQL query:", domain) + if cursor.rowcount == 0: + print("WARNING: Did not update any rows:", domain) + + except BaseException as e: + print("ERROR: failed SQL query:", domain, e) sys.exit(255) connection.commit() @@ -159,7 +179,7 @@ def fetch_nodeinfo(domain: str) -> list: json = res.json() break elif not res.ok or res.status_code >= 400: - # NOISY-DEBUG: print("DEBUG: Failed fetching nodeinfo from domain:", domain) + print("WARNING: Failed fetching nodeinfo from domain:", domain) update_last_error(domain, res) continue @@ -170,6 +190,7 @@ def fetch_nodeinfo(domain: str) -> list: # NOISY-DEBUG: print("DEBUG: json[]:", type(json)) if json is None or len(json) == 0: print("WARNING: Failed fetching nodeinfo from domain:", domain) + update_last_error(domain, "Cannot fetch nodeinfo") # NOISY-DEBUG: print("DEBUG: Returning json[]:", type(json)) return json @@ -203,6 +224,7 @@ def fetch_wellknown_nodeinfo(domain: str) -> list: except: print("WARNING: Failed fetching .well-known info:", domain) + update_last_error(domain, "Cannot fetch .well-known") pass # NOISY-DEBUG: print("DEBUG: Returning json[]:", type(json)) @@ -275,6 +297,9 @@ def update_block_reason(reason: str, blocker: str, blocked: str, block_level: st ), ) + if cursor.rowcount == 0: + print("WARNING: Did not update any rows:", domain) + except: print("ERROR: failed SQL query:", reason, blocker, blocked, block_level) sys.exit(255) @@ -292,6 +317,9 @@ def update_last_seen(blocker: str, blocked: str, block_level: str): ) ) + if cursor.rowcount == 0: + print("WARNING: Did not update any rows:", domain) + except: print("ERROR: failed SQL query:", last_seen, blocker, blocked, block_level) sys.exit(255) @@ -349,8 +377,13 @@ def add_instance(domain: str, origin: str, originator: str): ), ) - except: - print("ERROR: failed SQL query:", domain) + if domain in pending_errors: + print("DEBUG: domain has pending error be updated:", domain) + update_last_error(domain, pending_errors[domain]) + del pending_errors[domain] + + except BaseException as e: + print("ERROR: failed SQL query:", domain, e) sys.exit(255) else: # NOISY-DEBUG: print("DEBUG: Updating nodeinfo for domain:", domain) @@ -394,20 +427,20 @@ def get_mastodon_blocks(domain: str) -> dict: } translations = { - "Silenced instances": "Silenced servers", - "Suspended instances": "Suspended servers", - "Gesperrte Server": "Suspended servers", - "Gefilterte Medien": "Filtered media", - "Stummgeschaltete Server": "Silenced servers", - "停止済みのサーバー": "Suspended servers", + "Silenced instances" : "Silenced servers", + "Suspended instances" : "Suspended servers", + "Gesperrte Server" : "Suspended servers", + "Gefilterte Medien" : "Filtered media", + "Stummgeschaltete Server" : "Silenced servers", + "停止済みのサーバー" : "Suspended servers", "メディアを拒否しているサーバー": "Filtered media", - "サイレンス済みのサーバー": "Silenced servers", - "שרתים מושעים": "Suspended servers", - "מדיה מסוננת": "Filtered media", - "שרתים מוגבלים": "Silenced servers", - "Serveurs suspendus": "Suspended servers", - "Médias filtrés": "Filtered media", - "Serveurs limités": "Silenced servers", + "サイレンス済みのサーバー" : "Silenced servers", + "שרתים מושעים" : "Suspended servers", + "מדיה מסוננת" : "Filtered media", + "שרתים מוגבלים" : "Silenced servers", + "Serveurs suspendus" : "Suspended servers", + "Médias filtrés" : "Filtered media", + "Serveurs limités" : "Silenced servers", } try: @@ -478,7 +511,7 @@ def get_misskey_blocks(domain: str) -> dict: # NOISY-DEBUG: print("DEBUG: Fetching misskey blocks from domain:", domain) blocks = { "suspended": [], - "blocked": [] + "blocked" : [] } try: diff --git a/fetch_instances.py b/fetch_instances.py index 0fb8a45..41d1d77 100644 --- a/fetch_instances.py +++ b/fetch_instances.py @@ -5,13 +5,7 @@ import time import fba def fetch_instances(domain: str, origin: str): - # NOISY-DEBUG: print("DEBUG: Fetching instances for domain:", domain, origin) - peerlist = fba.get_peers(domain) - - if (peerlist is None): - print("ERROR: Cannot fetch peers:", domain) - return - + # NOISY-DEBUG: print("DEBUG: domain,origin:", domain, origin) fba.cursor.execute( "SELECT domain FROM instances WHERE domain = ? LIMIT 1", [domain] ) @@ -20,6 +14,13 @@ def fetch_instances(domain: str, origin: str): # NOISY-DEBUG: print("DEBUG: Adding new domain:", domain, origin) fba.add_instance(domain, origin, sys.argv[0]) + # NOISY-DEBUG: print("DEBUG: Fetching instances for domain:", domain, origin) + peerlist = fba.get_peers(domain) + + if (peerlist is None): + print("ERROR: Cannot fetch peers:", domain) + return + print(f"INFO: Checking {len(peerlist)} instances from {domain} ...") for instance in peerlist: instance = instance.lower() @@ -58,12 +59,14 @@ fba.cursor.execute( "SELECT domain FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial', 'bookwyrm', 'takahe') AND (last_nodeinfo IS NULL OR last_nodeinfo < ?) ORDER BY rowid DESC", [time.time() - fba.config["recheck_instance"]] ) -for instance in fba.cursor.fetchall(): - if fba.is_blacklisted(instance[0]): - # NOISY-DEBUG: print("DEBUG: domain is blacklisted:", instance) +for row in fba.cursor.fetchall(): + domain = row[0] + # NOISY-DEBUG: print("DEBUG: domain:", domain) + if fba.is_blacklisted(domain): + # NOISY-DEBUG: print("DEBUG: domain is blacklisted:", domain) continue - print("INFO: Fetching instances for instance:", instance[0]) - fetch_instances(instance[0], None) + print("INFO: Fetching instances for instance:", domain) + fetch_instances(domain, None) fba.connection.close() -- 2.39.5