From 4d2950eb3b780cd7eb8ae712388d76830555baad Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Mon, 22 May 2023 03:35:28 +0200 Subject: [PATCH] Continued: - fixed bad handling of errors during nodeinfo discovery --- fba.py | 110 ++++++++++++++++++++++++------------------------ fetch_blocks.py | 3 ++ 2 files changed, 58 insertions(+), 55 deletions(-) diff --git a/fba.py b/fba.py index 03800f2..6d20d6e 100644 --- a/fba.py +++ b/fba.py @@ -28,23 +28,18 @@ connection = sqlite3.connect("blocks.db") cursor = connection.cursor() def is_blacklisted(domain: str) -> bool: - # NOISY-DEBUG: print("DEBUG: Checking blacklist for domain:", domain) blacklisted = False for peer in blacklist: - # NOISY-DEBUG: print("DEBUG: domain,peer:", domain, peer) if peer in domain: blacklisted = True - # NOISY-DEBUG: print("DEBUG: blacklisted:", blacklisted) return blacklisted def get_hash(domain: str) -> str: - # NOISY-DEBUG: print("DEBUG: Calculating hash for domain:", domain) return sha256(domain.encode("utf-8")).hexdigest() def update_last_blocked(domain: str): # NOISY-DEBUG: print("DEBUG: Updating last_blocked for domain", domain) - try: cursor.execute("UPDATE instances SET last_blocked = ?, last_updated = ? WHERE domain = ?", [ time.time(), @@ -109,6 +104,7 @@ def get_peers(domain: str) -> list: def post_json_api(domain: str, path: str, data: str) -> list: # NOISY-DEBUG: print("DEBUG: Sending POST to domain,path,data:", domain, path, data) + json = {} try: res = reqto.post(f"https://{domain}{path}", data=data, headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])) @@ -137,60 +133,61 @@ def fetch_nodeinfo(domain: str) -> list: f"https://{domain}/api/v1/instance" ] - json = None + json = {} for request in requests: - # NOISY-DEBUG: print("DEBUG: Fetching request:", request) - res = reqto.get(request, headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])) - - # NOISY-DEBUG: print("DEBUG: res.ok,res.json[]:", res.ok, type(res.json())) - if res.ok and res.json() is not None: - # NOISY-DEBUG: print("DEBUG: Success:", request) - json = res.json() - break - elif not res.ok or res.status_code >= 400: - # NOISY-DEBUG: print("DEBUG: Failed fetching nodeinfo from domain:", domain) - update_last_error(domain, res) - continue + try: + # NOISY-DEBUG: print("DEBUG: Fetching request:", request) + res = reqto.get(request, headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])) + + # NOISY-DEBUG: print("DEBUG: res.ok,res.json[]:", res.ok, type(res.json())) + if res.ok and res.json() is not None: + # NOISY-DEBUG: print("DEBUG: Success:", request) + json = res.json() + break + elif not res.ok or res.status_code >= 400: + # NOISY-DEBUG: print("DEBUG: Failed fetching nodeinfo from domain:", domain) + update_last_error(domain, res) + continue + + except: + # NOISY-DEBUG: print("DEBUG: Cannot fetch API request:", request) + pass if json is None: print("WARNING: Failed fetching nodeinfo from domain:", domain) - # NOISY-DEBUG: print("DEBUG: Updating last_nodeinfo for domain:", domain) - update_last_nodeinfo(domain) - - # NOISY-DEBUG: print("DEBUG: Returning json():", len(json)) + # NOISY-DEBUG: print("DEBUG: Returning json[]:", type(json)) return json def determine_software(domain: str) -> str: # NOISY-DEBUG: print("DEBUG: Determining software for domain:", domain) software = None - try: - json = fetch_nodeinfo(domain) - - # NOISY-DEBUG: print("DEBUG: json():", len(json)) - software = tidyup(json["software"]["name"]) - - # NOISY-DEBUG: print("DEBUG: BEFORE software:", software) - if software in ["akkoma", "rebased"]: - # NOISY-DEBUG: print("DEBUG: Setting pleroma:", domain, software) - software = "pleroma" - elif software in ["hometown", "ecko"]: - # NOISY-DEBUG: print("DEBUG: Setting mastodon:", domain, software) - software = "mastodon" - elif software in ["calckey", "groundpolis", "foundkey", "cherrypick"]: - # NOISY-DEBUG: print("DEBUG: Setting misskey:", domain, software) - software = "misskey" - elif software.find("/") > 0: - print("WARNING: Spliting of path:", software) - software = software.split("/")[-1]; - - if software == "": - print("WARNING: tidyup() left no software name behind:", domain) - software = None - # NOISY-DEBUG: print("DEBUG: AFTER software:", software) - except: - print("WARNING: Could not determine software type:", domain) + json = fetch_nodeinfo(domain) + if len(json) == 0: + print("DEBUG: Could not determine software type:", domain) + return None + + # NOISY-DEBUG: print("DEBUG: json():", len(json)) + software = tidyup(json["software"]["name"]) + + # NOISY-DEBUG: print("DEBUG: tidyup software:", software) + if software in ["akkoma", "rebased"]: + # NOISY-DEBUG: print("DEBUG: Setting pleroma:", domain, software) + software = "pleroma" + elif software in ["hometown", "ecko"]: + # NOISY-DEBUG: print("DEBUG: Setting mastodon:", domain, software) + software = "mastodon" + elif software in ["calckey", "groundpolis", "foundkey", "cherrypick"]: + # NOISY-DEBUG: print("DEBUG: Setting misskey:", domain, software) + software = "misskey" + elif software.find("/") > 0: + print("WARNING: Spliting of path:", software) + software = software.split("/")[-1]; + + if software == "": + print("WARNING: tidyup() left no software name behind:", domain) + software = None # NOISY-DEBUG: print("DEBUG: Returning domain,software:", domain, software) return software @@ -266,6 +263,9 @@ def add_instance(domain: str, origin: str, originator: str): print("WARNING: Bad origin name:", origin) raise + software = determine_software(domain) + # NOISY-DEBUG: print("DEBUG: Determined software:", software) + print(f"--- Adding new instance {domain} (origin: {origin})") try: cursor.execute( @@ -275,7 +275,7 @@ def add_instance(domain: str, origin: str, originator: str): origin, originator, get_hash(domain), - determine_software(domain), + software, time.time() ), ) @@ -283,6 +283,9 @@ def add_instance(domain: str, origin: str, originator: str): except: print("ERROR: failed SQL query:", domain) sys.exit(255) + else: + # NOISY-DEBUG: print("DEBUG: Updating nodeinfo for domain:", domain) + update_last_nodeinfo(domain) def send_bot_post(instance: str, blocks: dict): message = instance + " has blocked the following instances:\n\n" @@ -316,9 +319,9 @@ def get_mastodon_blocks(domain: str) -> dict: # NOISY-DEBUG: print("DEBUG: Fetching mastodon blocks from domain:", domain) blocks = { "Suspended servers": [], - "Filtered media": [], - "Limited servers": [], - "Silenced servers": [], + "Filtered media" : [], + "Limited servers" : [], + "Silenced servers" : [], } translations = { @@ -518,8 +521,6 @@ def get_misskey_blocks(domain: str) -> dict: return {} def tidyup(string: str) -> str: - # NOISY-DEBUG: print("DEBUG: BEFORE string:", string) - # some retards put their blocks in variable case string = string.lower().strip() @@ -538,5 +539,4 @@ def tidyup(string: str) -> str: # the biggest retards of them all try to block individual users string = re.sub("(.+)\@", "", string) - # NOISY-DEBUG: print("DEBUG: AFTER string:", string) return string diff --git a/fetch_blocks.py b/fetch_blocks.py index 704176e..9420689 100644 --- a/fetch_blocks.py +++ b/fetch_blocks.py @@ -30,6 +30,9 @@ for blocker, software in fba.cursor.fetchall(): print("WARNING: Could not fetch nodeinfo from blocker:", blocker) continue + print("DEBUG: Updating nodeinfo:", blocker) + fba.update_last_nodeinfo(blocker) + federation = json["metadata"]["federation"] if "enabled" in federation: -- 2.39.5