From 3b97ae4ffc2c29d73f8afa16f11d830ed69b72f6 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Sun, 11 Jun 2023 15:20:43 +0200 Subject: [PATCH] Continued: - removed another broad except block - added missing variable initialization - there is no 'domain', aka. instance here - fixed bare-except, you can use 'if foo in bar' instead --- fba/commands.py | 2 - fba/federation.py | 1 + fba/instances.py | 8 +- fba/networks/pleroma.py | 296 ++++++++++++++++++++-------------------- 4 files changed, 150 insertions(+), 157 deletions(-) diff --git a/fba/commands.py b/fba/commands.py index 28d1412..f84ce6f 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -71,11 +71,9 @@ def fetch_bkali(args: argparse.Namespace) -> int: # DEBUG: print(f"DEBUG: fetched[]='{type(fetched)}'") if "error_message" in fetched: print(f"WARNING: post_json_api() for 'gql.api.bka.li' returned error message: {fetched['error_message']}") - instances.update_last_error(domain, fetched) return 100 elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]: print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}") - instances.update_last_error(domain, fetched["json"]["error"]["message"]) return 101 rows = fetched["json"] diff --git a/fba/federation.py b/fba/federation.py index abee665..ee839c7 100644 --- a/fba/federation.py +++ b/fba/federation.py @@ -193,6 +193,7 @@ def fetch_nodeinfo(domain: str, path: str = None) -> dict: # No CSRF by default, you don't have to add network.api_headers by yourself here headers = tuple() + data = dict() try: # DEBUG: print(f"DEBUG: Checking CSRF for domain='{domain}'") diff --git a/fba/instances.py b/fba/instances.py index 8eb1496..039661d 100644 --- a/fba/instances.py +++ b/fba/instances.py @@ -129,13 +129,11 @@ def update_data(domain: str): # DEBUG: print("DEBUG: Committing changes ...") fba.connection.commit() - # DEBUG: print("DEBUG: Deleting _pending for domain:", domain) + # DEBUG: print(f"DEBUG: Deleting _pending for domain='{domain}'") for key in _pending: - try: - # DEBUG: print("DEBUG: Deleting key:", key) + # DEBUG: print(f"DEBUG: domain='{domain}',key='{key}'") + if domain in _pending[key]: del _pending[key][domain] - except: - pass except BaseException as exception: print(f"ERROR: failed SQL query: domain='{domain}',sql_string='{sql_string}',exception[{type(exception)}]:'{str(exception)}'") diff --git a/fba/networks/pleroma.py b/fba/networks/pleroma.py index e15ea00..c1082ad 100644 --- a/fba/networks/pleroma.py +++ b/fba/networks/pleroma.py @@ -39,165 +39,161 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): elif nodeinfo_url == "": raise ValueError("Parameter 'nodeinfo_url' is empty") - try: - # Blocks - blockdict = list() - rows = federation.fetch_nodeinfo(domain, nodeinfo_url) - - if rows is None: - print("WARNING: Could not fetch nodeinfo from domain:", domain) - return - elif "metadata" not in rows: - print(f"WARNING: rows()={len(rows)} does not have key 'metadata', domain='{domain}'") - return - elif "federation" not in rows["metadata"]: - print(f"WARNING: rows()={len(rows['metadata'])} does not have key 'federation', domain='{domain}'") - return - - # DEBUG: print("DEBUG: Updating nodeinfo:", domain) - instances.update_last_nodeinfo(domain) - - data = rows["metadata"]["federation"] - - if "enabled" in data: - # DEBUG: print("DEBUG: Instance has no block list to analyze:", domain) - return - - if "mrf_simple" in data: - for block_level, blocklist in ( - {**data["mrf_simple"], - **{"quarantined_instances": data["quarantined_instances"]}} - ).items(): - # DEBUG: print("DEBUG: block_level, blocklist():", block_level, len(blocklist)) - block_level = tidyup.domain(block_level) - # DEBUG: print("DEBUG: BEFORE block_level:", block_level) - - if block_level == "": - print("WARNING: block_level is now empty!") + # Blocks + blockdict = list() + rows = federation.fetch_nodeinfo(domain, nodeinfo_url) + + if rows is None: + print("WARNING: Could not fetch nodeinfo from domain:", domain) + return + elif "metadata" not in rows: + print(f"WARNING: rows()={len(rows)} does not have key 'metadata', domain='{domain}'") + return + elif "federation" not in rows["metadata"]: + print(f"WARNING: rows()={len(rows['metadata'])} does not have key 'federation', domain='{domain}'") + return + + # DEBUG: print("DEBUG: Updating nodeinfo:", domain) + instances.update_last_nodeinfo(domain) + + data = rows["metadata"]["federation"] + + if "enabled" in data: + # DEBUG: print("DEBUG: Instance has no block list to analyze:", domain) + return + + if "mrf_simple" in data: + for block_level, blocklist in ( + {**data["mrf_simple"], + **{"quarantined_instances": data["quarantined_instances"]}} + ).items(): + # DEBUG: print("DEBUG: block_level, blocklist():", block_level, len(blocklist)) + block_level = tidyup.domain(block_level) + # DEBUG: print("DEBUG: BEFORE block_level:", block_level) + + if block_level == "": + print("WARNING: block_level is now empty!") + continue + + # DEBUG: print(f"DEBUG: Checking {len(blocklist)} entries from domain='{domain}',block_level='{block_level}' ...") + for blocked in blocklist: + # DEBUG: print("DEBUG: BEFORE blocked:", blocked) + blocked = tidyup.domain(blocked) + # DEBUG: print("DEBUG: AFTER blocked:", blocked) + + if blocked == "": + print("WARNING: blocked is empty after tidyup.domain():", domain, block_level) continue - - # DEBUG: print(f"DEBUG: Checking {len(blocklist)} entries from domain='{domain}',block_level='{block_level}' ...") - for blocked in blocklist: - # DEBUG: print("DEBUG: BEFORE blocked:", blocked) - blocked = tidyup.domain(blocked) - # DEBUG: print("DEBUG: AFTER blocked:", blocked) - - if blocked == "": - print("WARNING: blocked is empty after tidyup.domain():", domain, block_level) - continue - elif blacklist.is_blacklisted(blocked): - # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!") - continue - elif blocked.count("*") > 1: - # -ACK!-oma also started obscuring domains without hash - fba.cursor.execute( - "SELECT domain, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")] - ) - searchres = fba.cursor.fetchone() - - print(f"DEBUG: searchres[]='{type(searchres)}'") - if searchres is None: - print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!") - continue - - blocked = searchres[0] - nodeinfo_url = searchres[1] - # DEBUG: print("DEBUG: Looked up domain:", blocked) - elif not validators.domain(blocked): - print(f"WARNING: blocked='{blocked}',software='pleroma' is not a valid domain name - skipped!") + elif blacklist.is_blacklisted(blocked): + # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!") + continue + elif blocked.count("*") > 1: + # -ACK!-oma also started obscuring domains without hash + fba.cursor.execute( + "SELECT domain, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")] + ) + searchres = fba.cursor.fetchone() + + print(f"DEBUG: searchres[]='{type(searchres)}'") + if searchres is None: + print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!") continue - # DEBUG: print("DEBUG: Looking up instance by domain:", blocked) - if not validators.domain(blocked): - print(f"WARNING: blocked='{blocked}',software='pleroma' is not a valid domain name - skipped!") - continue - elif not instances.is_registered(blocked): - # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'") - instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url) - - if not blocks.is_instance_blocked(domain, blocked, block_level): - # DEBUG: print("DEBUG: Blocking:", domain, blocked, block_level) - blocks.add_instance(domain, blocked, "unknown", block_level) - - if block_level == "reject": - # DEBUG: print("DEBUG: Adding to blockdict:", blocked) - blockdict.append( - { - "blocked": blocked, - "reason" : None - }) + blocked = searchres[0] + nodeinfo_url = searchres[1] + # DEBUG: print("DEBUG: Looked up domain:", blocked) + elif not validators.domain(blocked): + print(f"WARNING: blocked='{blocked}',software='pleroma' is not a valid domain name - skipped!") + continue + + # DEBUG: print("DEBUG: Looking up instance by domain:", blocked) + if not validators.domain(blocked): + print(f"WARNING: blocked='{blocked}',software='pleroma' is not a valid domain name - skipped!") + continue + elif not instances.is_registered(blocked): + # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'") + instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url) + + if not blocks.is_instance_blocked(domain, blocked, block_level): + # DEBUG: print("DEBUG: Blocking:", domain, blocked, block_level) + blocks.add_instance(domain, blocked, "unknown", block_level) + + if block_level == "reject": + # DEBUG: print("DEBUG: Adding to blockdict:", blocked) + blockdict.append({ + "blocked": blocked, + "reason" : None + }) else: # DEBUG: print(f"DEBUG: Updating block last seen for domain='{domain}',blocked='{blocked}' ...") blocks.update_last_seen(domain, blocked, block_level) - # DEBUG: print("DEBUG: Committing changes ...") - fba.connection.commit() - - # Reasons - if "mrf_simple_info" in data: - # DEBUG: print("DEBUG: Found mrf_simple_info:", domain) - for block_level, info in ( - {**data["mrf_simple_info"], - **(data["quarantined_instances_info"] - if "quarantined_instances_info" in data - else {})} - ).items(): - # DEBUG: print("DEBUG: block_level, info.items():", block_level, len(info.items())) - block_level = tidyup.domain(block_level) - # DEBUG: print("DEBUG: BEFORE block_level:", block_level) - - if block_level == "": - print("WARNING: block_level is now empty!") + # DEBUG: print("DEBUG: Committing changes ...") + fba.connection.commit() + + # Reasons + if "mrf_simple_info" in data: + # DEBUG: print("DEBUG: Found mrf_simple_info:", domain) + for block_level, info in ( + {**data["mrf_simple_info"], + **(data["quarantined_instances_info"] + if "quarantined_instances_info" in data + else {})} + ).items(): + # DEBUG: print("DEBUG: block_level, info.items():", block_level, len(info.items())) + block_level = tidyup.domain(block_level) + # DEBUG: print("DEBUG: BEFORE block_level:", block_level) + + if block_level == "": + print("WARNING: block_level is now empty!") + continue + + # DEBUG: print(f"DEBUG: Checking {len(info.items())} entries from domain='{domain}',software='pleroma',block_level='{block_level}' ...") + for blocked, reason in info.items(): + # DEBUG: print(f"DEBUG: blocked='{blocked}',reason='{reason}' - BEFORE!") + blocked = tidyup.domain(blocked) + reason = tidyup.reason(reason) if reason is not None and reason != "" else None + # DEBUG: print(f"DEBUG: blocked='{blocked}',reason='{reason}' - AFTER!") + + if blocked == "": + print("WARNING: blocked is empty after tidyup.domain():", domain, block_level) continue + elif not validators.domain(blocked): + print(f"WARNING: blocked='{blocked}',software='pleroma' is not a valid domain name - skipped!") + continue + elif blacklist.is_blacklisted(blocked): + # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!") + continue + elif blocked.count("*") > 1: + # same domain guess as above, but for reasons field + fba.cursor.execute( + "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")] + ) + searchres = fba.cursor.fetchone() + + # DEBUG: print(f"DEBUG: searchres[]='{type(searchres)}'") + if searchres is None: + print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!") + continue - # DEBUG: print(f"DEBUG: Checking {len(info.items())} entries from domain='{domain}',software='pleroma',block_level='{block_level}' ...") - for blocked, reason in info.items(): - # DEBUG: print(f"DEBUG: blocked='{blocked}',reason='{reason}' - BEFORE!") - blocked = tidyup.domain(blocked) - reason = tidyup.reason(reason) if reason is not None and reason != "" else None - # DEBUG: print(f"DEBUG: blocked='{blocked}',reason='{reason}' - AFTER!") + blocked = searchres[0] + origin = searchres[1] + nodeinfo_url = searchres[2] - if blocked == "": - print("WARNING: blocked is empty after tidyup.domain():", domain, block_level) - continue - elif not validators.domain(blocked): - print(f"WARNING: blocked='{blocked}',software='pleroma' is not a valid domain name - skipped!") - continue - elif blacklist.is_blacklisted(blocked): - # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!") - continue - elif blocked.count("*") > 1: - # same domain guess as above, but for reasons field - fba.cursor.execute( - "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")] - ) - searchres = fba.cursor.fetchone() - - # DEBUG: print(f"DEBUG: searchres[]='{type(searchres)}'") - if searchres is None: - print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!") - continue - - blocked = searchres[0] - origin = searchres[1] - nodeinfo_url = searchres[2] - - # DEBUG: print("DEBUG: Looking up instance by domain:", blocked) - if not instances.is_registered(blocked): - # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'") - instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url) - - # DEBUG: print("DEBUG: Updating block reason:", domain, blocked, reason["reason"]) - blocks.update_reason(reason["reason"], domain, blocked, block_level) - - # DEBUG: print(f"DEBUG: blockdict()={len(blockdict)}") - for entry in blockdict: - if entry["blocked"] == blocked: - # DEBUG: print("DEBUG: Updating entry reason:", blocked) - entry["reason"] = reason["reason"] - - fba.connection.commit() - except BaseException as exception: - print(f"ERROR: domain='{domain}',software='pleroma',exception[{type(exception)}]:'{str(exception)}'") + # DEBUG: print("DEBUG: Looking up instance by domain:", blocked) + if not instances.is_registered(blocked): + # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'") + instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url) + + # DEBUG: print("DEBUG: Updating block reason:", domain, blocked, reason["reason"]) + blocks.update_reason(reason["reason"], domain, blocked, block_level) + + # DEBUG: print(f"DEBUG: blockdict()={len(blockdict)}") + for entry in blockdict: + if entry["blocked"] == blocked: + # DEBUG: print("DEBUG: Updating entry reason:", blocked) + entry["reason"] = reason["reason"] + + fba.connection.commit() # DEBUG: print("DEBUG: EXIT!") -- 2.39.5