From 9d02437d2618037d219f312f81a528f2e5fb19f6 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Sat, 20 May 2023 10:14:56 +0200 Subject: [PATCH] Continued: - rewrote fetch_nodeinfo() to test all URLs --- fba.py | 67 ++++++++++++++++++++++++++++++++++------------------------ 1 file changed, 39 insertions(+), 28 deletions(-) diff --git a/fba.py b/fba.py index c8e2ed0..8c90f77 100644 --- a/fba.py +++ b/fba.py @@ -6,6 +6,7 @@ import re import sqlite3 import json import sys +import time with open("config.json") as f: config = json.loads(f.read()) @@ -56,36 +57,39 @@ def post_json_api(domain: str, path: str, data: str) -> list: return doc def fetch_nodeinfo(domain: str) -> list: - print("DEBUG: Fetching nodeinfo from domain:", domain) - json = None - try: - print("DEBUG: Fetching 2.1.json from domain:", domain) - res = reqto.get(f"https://{domain}/nodeinfo/2.1.json", headers=headers, timeout=5) + # NOISY-DEBUG: print("DEBUG: Fetching nodeinfo from domain:", domain) - if res.status_code == 404 or "text/html" in res.headers["content-type"]: - print("DEBUG: Fetching 2.0 from domain:", domain) - res = reqto.get(f"https://{domain}/nodeinfo/2.0", headers=headers, timeout=5) + requests = [ + f"https://{domain}/nodeinfo/2.1.json", + f"https://{domain}/nodeinfo/2.0", + f"https://{domain}/nodeinfo/2.0.json", + f"https://{domain}/nodeinfo/2.1", + f"https://{domain}/api/v1/instance" + ] - if res.status_code == 404 or "text/html" in res.headers["content-type"]: - print("DEBUG: Fetching 2.0.json from domain:", domain) - res = reqto.get(f"https://{domain}/nodeinfo/2.0.json", headers=headers, timeout=5) + json = None + for request in requests: + # NOISY-DEBUG: print("DEBUG: Fetching request:", request) - if res.ok and "text/html" in res.headers["content-type"]: - print("DEBUG: Fetching 2.1 from domain:", domain) - res = reqto.get(f"https://{domain}/nodeinfo/2.1", headers=headers, timeout=5) + try: + res = reqto.get(request, headers=headers, timeout=5) - if res.status_code == 404 or "text/html" in res.headers["content-type"]: - print("DEBUG: Fetching /api/v1/instance from domain:", domain) - res = reqto.get(f"https://{domain}/api/v1/instance", headers=headers, timeout=5) + # NOISY-DEBUG: print("DEBUG: res.ok,res.json[]:", res.ok, type(res.json())) + if res.ok and res.json() is not None: + # NOISY-DEBUG: print("DEBUG: Success:", request) + json = res.json() - print("DEBUG: domain,res.ok,res.status_code:", domain, res.ok, res.status_code) - if res.ok: - json = res.json() + except: + # NOISY-DEBUG: print("DEBUG: Failed fetching nodeinfo from domain:", domain) + continue - except: + if json is not None: + break + + if json is None: print("WARNING: Failed fetching nodeinfo from domain:", domain) - print("DEBUG: Returning json():", len(json)) + # NOISY-DEBUG: print("DEBUG: Returning json():", len(json)) return json def determine_software(domain: str) -> str: @@ -95,12 +99,16 @@ def determine_software(domain: str) -> str: # NOISY-DEBUG: print("DEBUG: json():", len(json)) if json["software"]["name"] in ["akkoma", "rebased"]: + # NOISY-DEBUG: print("DEBUG: Setting pleroma:", domain, json["software"]["name"]) software = "pleroma" elif json["software"]["name"] in ["hometown", "ecko"]: + # NOISY-DEBUG: print("DEBUG: Setting mastodon:", domain, json["software"]["name"]) software = "mastodon" elif json["software"]["name"] in ["calckey", "groundpolis", "foundkey", "cherrypick"]: + # NOISY-DEBUG: print("DEBUG: Setting misskey:", domain, json["software"]["name"]) software = "misskey" else: + # NOISY-DEBUG: print("DEBUG: Using name:", domain, json["software"]["name"]) software = json["software"]["name"] # NOISY-DEBUG: print("DEBUG: Returning domain,software:", domain, software) @@ -120,7 +128,7 @@ def update_block_reason(reason: str, blocker: str, blocked: str, block_level: st ) except: - print("ERROR: failed SQL query") + print("ERROR: failed SQL query:", reason, blocker, blocked, block_level) sys.exit(255) def update_last_seen(last_seen: int, blocker: str, blocked: str, block_level: str): @@ -137,10 +145,11 @@ def update_last_seen(last_seen: int, blocker: str, blocked: str, block_level: st ) except: - print("ERROR: failed SQL query") + print("ERROR: failed SQL query:", last_seen, blocker, blocked, block_level) sys.exit(255) def block_instance(blocker: str, blocked: str, reason: str, block_level: str, first_added: int, last_seen: int): + # NOISY-DEBUG: print("DEBUG: blocker,blocked,reason,block_level,first_added,last_seen:", blocker, blocked, reason, block_level, first_added, last_seen) if blocker.find("@") > 0: print("WARNING: Bad blocker:", blocker) raise @@ -163,10 +172,11 @@ def block_instance(blocker: str, blocked: str, reason: str, block_level: str, fi ) except: - print("ERROR: failed SQL query") + print("ERROR: failed SQL query:", blocker, blocked, reason, block_level, first_added, last_seen) sys.exit(255) def add_instance(domain: str): + # NOISY-DEBUG: print("DEBUG: domain:", domain) if domain.find("@") > 0: print("WARNING: Bad domain name:", domain) raise @@ -174,16 +184,17 @@ def add_instance(domain: str): print("--- Adding new instance:", domain) try: c.execute( - "INSERT INTO instances SELECT ?, ?, ?", + "INSERT INTO instances SELECT ?, ?, ?, ?", ( domain, get_hash(domain), - determine_software(domain) + determine_software(domain), + time.time() ), ) except: - print("ERROR: failed SQL query") + print("ERROR: failed SQL query:", domain) sys.exit(255) def send_bot_post(instance: str, blocks: dict): -- 2.39.5