From 59986e3d2613e5c7a696fb146cbedc379903c027 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Thu, 8 Jun 2023 07:38:33 +0200 Subject: [PATCH] Continued: - moved and renamed fba.add_instance() to instances.add() --- fba/commands.py | 2 +- fba/fba.py | 64 ++-------------------------------- fba/federation/gotosocial.py | 34 +++++++++---------- fba/federation/mastodon.py | 4 +-- fba/federation/pleroma.py | 4 +-- fba/instances.py | 66 ++++++++++++++++++++++++++++++++++++ 6 files changed, 90 insertions(+), 84 deletions(-) diff --git a/fba/commands.py b/fba/commands.py index 4f6839b..514da07 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -220,7 +220,7 @@ def fetch_blocks(args: argparse.Namespace): continue elif not fba.is_instance_registered(blocked): # DEBUG: print("DEBUG: Hash wasn't found, adding:", blocked, blocker) - fba.add_instance(blocked, blocker, inspect.currentframe().f_code.co_name, nodeinfo_url) + instances.add(blocked, blocker, inspect.currentframe().f_code.co_name, nodeinfo_url) if not blocks.is_instance_blocked(blocker, blocked, block_level): blocks.add_instance(blocker, blocked, reason, block_level) diff --git a/fba/fba.py b/fba/fba.py index 95176a9..2d11a69 100644 --- a/fba/fba.py +++ b/fba/fba.py @@ -103,7 +103,7 @@ def fetch_instances(domain: str, origin: str, software: str, script: str, path: if not is_instance_registered(domain): # DEBUG: print("DEBUG: Adding new domain:", domain, origin) - add_instance(domain, origin, script, path) + instances.add(domain, origin, script, path) # DEBUG: print("DEBUG: Fetching instances for domain:", domain, software) peerlist = get_peers(domain, software) @@ -139,7 +139,7 @@ def fetch_instances(domain: str, origin: str, software: str, script: str, path: try: if not is_instance_registered(instance): # DEBUG: print("DEBUG: Adding new instance:", instance, domain) - add_instance(instance, domain, script) + instances.add(instance, domain, script) except BaseException as e: print(f"ERROR: instance='{instance}',exception[{type(e)}]:'{str(e)}'") continue @@ -773,66 +773,6 @@ def is_instance_registered(domain: str) -> bool: # DEBUG: print(f"DEBUG: registered='{registered}' - EXIT!") return registered -def add_instance(domain: str, origin: str, originator: str, path: str = None): - # DEBUG: print(f"DEBUG: domain='{domain}',origin='{origin}',originator='{originator}',path='{path}' - CALLED!") - if type(domain) != str: - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError(f"Parameter 'domain' is empty") - elif type(origin) != str and origin != None: - raise ValueError(f"origin[]={type(origin)} is not 'str'") - elif type(originator) != str: - raise ValueError(f"originator[]={type(originator)} is not 'str'") - elif originator == "": - raise ValueError(f"originator cannot be empty") - elif not validators.domain(domain.split("/")[0]): - raise ValueError(f"Bad domain name='{domain}'") - elif origin is not None and not validators.domain(origin.split("/")[0]): - raise ValueError(f"Bad origin name='{origin}'") - elif blacklist.is_blacklisted(domain): - raise Exception(f"domain='{domain}' is blacklisted, but method invoked") - - # DEBUG: print("DEBUG: domain,origin,originator,path:", domain, origin, originator, path) - software = determine_software(domain, path) - # DEBUG: print("DEBUG: Determined software:", software) - - print(f"INFO: Adding instance domain='{domain}' (origin='{origin}',software='{software}')") - try: - cursor.execute( - "INSERT INTO instances (domain, origin, originator, hash, software, first_seen) VALUES (?, ?, ?, ?, ?, ?)", - ( - domain, - origin, - originator, - get_hash(domain), - software, - time.time() - ), - ) - - cache.set_sub_key("is_registered", domain, True) - - if instances.has_pending_instance_data(domain): - # DEBUG: print(f"DEBUG: domain='{domain}' has pending nodeinfo being updated ...") - instances.set("last_status_code" , domain, None) - instances.set("last_error_details", domain, None) - instances.update_instance_data(domain) - remove_pending_error(domain) - - if domain in pending_errors: - # DEBUG: print("DEBUG: domain has pending error being updated:", domain) - update_last_error(domain, pending_errors[domain]) - remove_pending_error(domain) - - except BaseException as e: - print(f"ERROR: failed SQL query: domain='{domain}',exception[{type(e)}]:'{str(e)}'") - sys.exit(255) - else: - # DEBUG: print("DEBUG: Updating nodeinfo for domain:", domain) - update_last_nodeinfo(domain) - - # DEBUG: print("DEBUG: EXIT!") - def send_bot_post(instance: str, blocklist: dict): # DEBUG: print(f"DEBUG: instance={instance},blocklist()={len(blocklist)} - CALLED!") if type(domain) != str: diff --git a/fba/federation/gotosocial.py b/fba/federation/gotosocial.py index 606b091..4008110 100644 --- a/fba/federation/gotosocial.py +++ b/fba/federation/gotosocial.py @@ -22,7 +22,7 @@ from fba import blocks from fba import config from fba import fba -def fetch_blocks(blocker: str, origin: str, nodeinfo_url: str): +def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): print(f"DEBUG: domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}' - CALLED!") if type(domain) != str: raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") @@ -39,14 +39,14 @@ def fetch_blocks(blocker: str, origin: str, nodeinfo_url: str): try: # Blocks - federation = fba.get_response(blocker, f"{fba.get_peers_url}?filter=suspended", fba.api_headers, (config.get("connection_timeout"), config.get("read_timeout"))).json() + federation = fba.get_response(domain, f"{fba.get_peers_url}?filter=suspended", fba.api_headers, (config.get("connection_timeout"), config.get("read_timeout"))).json() if (federation == None): - print("WARNING: No valid response:", blocker); + print("WARNING: No valid response:", domain); elif "error" in federation: print("WARNING: API returned error:", federation["error"]) else: - print(f"INFO: Checking {len(federation)} entries from blocker='{blocker}',software='{software}' ...") + print(f"INFO: Checking {len(federation)} entries from domain='{domain}',software='gotosocial' ...") for peer in federation: blocked = peer["domain"].lower() # DEBUG: print("DEBUG: BEFORE blocked:", blocked) @@ -54,7 +54,7 @@ def fetch_blocks(blocker: str, origin: str, nodeinfo_url: str): # DEBUG: print("DEBUG: AFTER blocked:", blocked) if blocked == "": - print("WARNING: blocked is empty:", blocker) + print("WARNING: blocked is empty:", domain) continue elif blacklist.is_blacklisted(blocked): # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!") @@ -74,32 +74,32 @@ def fetch_blocks(blocker: str, origin: str, nodeinfo_url: str): origin = searchres[1] nodeinfo_url = searchres[2] elif not validators.domain(blocked): - print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!") + print(f"WARNING: blocked='{blocked}',software='gotosocial' is not a valid domain name - skipped!") continue # DEBUG: print("DEBUG: Looking up instance by domain:", blocked) if not validators.domain(blocked): - print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!") + print(f"WARNING: blocked='{blocked}',software='gotosocial' is not a valid domain name - skipped!") continue elif not fba.is_instance_registered(blocked): - # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., blocker='{blocker}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'") - fba.add_instance(blocked, blocker, inspect.currentframe().f_code.co_name, nodeinfo_url) + # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'") + instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url) - if not blocks.is_instance_blocked(blocker, blocked, "reject"): - # DEBUG: print(f"DEBUG: blocker='{blocker}' is blocking '{blocked}' for unknown reason at this point") - blocks.add_instance(blocker, blocked, "unknown", "reject") + if not blocks.is_instance_blocked(domain, blocked, "reject"): + # DEBUG: print(f"DEBUG: domain='{domain}' is blocking '{blocked}' for unknown reason at this point") + blocks.add_instance(domain, blocked, "unknown", "reject") blockdict.append({ "blocked": blocked, "reason" : None }) else: - # DEBUG: print(f"DEBUG: Updating block last seen for blocker='{blocker}',blocked='{blocked}' ...") - blocks.update_last_seen(blocker, blocked, "reject") + # DEBUG: print(f"DEBUG: Updating block last seen for domain='{domain}',blocked='{blocked}' ...") + blocks.update_last_seen(domain, blocked, "reject") if "public_comment" in peer: - # DEBUG: print("DEBUG: Updating block reason:", blocker, blocked, peer["public_comment"]) - blocks.update_reason(peer["public_comment"], blocker, blocked, "reject") + # DEBUG: print("DEBUG: Updating block reason:", domain, blocked, peer["public_comment"]) + blocks.update_reason(peer["public_comment"], domain, blocked, "reject") for entry in blockdict: if entry["blocked"] == blocked: @@ -109,6 +109,6 @@ def fetch_blocks(blocker: str, origin: str, nodeinfo_url: str): # DEBUG: print("DEBUG: Committing changes ...") fba.connection.commit() except Exception as e: - print(f"ERROR: blocker='{blocker}',software='{software}',exception[{type(e)}]:'{str(e)}'") + print(f"ERROR: domain='{domain}',software='gotosocial',exception[{type(e)}]:'{str(e)}'") # DEBUG: print("DEBUG: EXIT!") diff --git a/fba/federation/mastodon.py b/fba/federation/mastodon.py index 3d0fe8b..e0c180d 100644 --- a/fba/federation/mastodon.py +++ b/fba/federation/mastodon.py @@ -219,7 +219,7 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): continue elif not fba.is_instance_registered(blocked): # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'") - fba.add_instance(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url) + instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url) elif not validators.domain(blocked): print(f"WARNING: blocked='{blocked}',software='mastodon' is not a valid domain name - skipped!") continue @@ -230,7 +230,7 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): continue elif not fba.is_instance_registered(blocked): # DEBUG: print("DEBUG: Hash wasn't found, adding:", blocked, domain) - fba.add_instance(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url) + instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url) blocking = blocked if blocked.count("*") <= 1 else blocked_hash # DEBUG: print(f"DEBUG: blocking='{blocking}',blocked='{blocked}',blocked_hash='{blocked_hash}'") diff --git a/fba/federation/pleroma.py b/fba/federation/pleroma.py index ce9895e..52c0f00 100644 --- a/fba/federation/pleroma.py +++ b/fba/federation/pleroma.py @@ -110,7 +110,7 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): continue elif not fba.is_instance_registered(blocked): # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'") - fba.add_instance(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url) + instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url) if not blocks.is_instance_blocked(domain, blocked, block_level): # DEBUG: print("DEBUG: Blocking:", domain, blocked, block_level) @@ -183,7 +183,7 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): continue elif not fba.is_instance_registered(blocked): # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'") - fba.add_instance(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url) + instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url) # DEBUG: print("DEBUG: Updating block reason:", domain, blocked, reason["reason"]) blocks.update_reason(reason["reason"], domain, blocked, block_level) diff --git a/fba/instances.py b/fba/instances.py index 77c4ded..8818920 100644 --- a/fba/instances.py +++ b/fba/instances.py @@ -166,3 +166,69 @@ def update_last_blocked(domain: str): update_instance_data(domain) # DEBUG: print("DEBUG: EXIT!") + +def add(domain: str, origin: str, originator: str, path: str = None): + # DEBUG: print(f"DEBUG: domain='{domain}',origin='{origin}',originator='{originator}',path='{path}' - CALLED!") + if type(domain) != str: + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError(f"Parameter 'domain' is empty") + elif type(origin) != str and origin != None: + raise ValueError(f"origin[]={type(origin)} is not 'str'") + elif type(originator) != str: + raise ValueError(f"originator[]={type(originator)} is not 'str'") + elif originator == "": + raise ValueError(f"originator cannot be empty") + elif not validators.domain(domain.split("/")[0]): + raise ValueError(f"Bad domain name='{domain}'") + elif origin is not None and not validators.domain(origin.split("/")[0]): + raise ValueError(f"Bad origin name='{origin}'") + elif blacklist.is_blacklisted(domain): + raise Exception(f"domain='{domain}' is blacklisted, but method invoked") + + # DEBUG: print("DEBUG: domain,origin,originator,path:", domain, origin, originator, path) + if domain.find("/profile") > 0: + # Need to cut off /profile/* part + domain = domain.split("/profile/")[0] + if instances.is_registered(domain): + raise Exception(f"WARNING: After removing /profile/ domain='{domain}' is already registered - EXIT!") + + software = determine_software(domain, path) + # DEBUG: print("DEBUG: Determined software:", software) + + print(f"INFO: Adding instance domain='{domain}' (origin='{origin}',software='{software}')") + try: + cursor.execute( + "INSERT INTO instances (domain, origin, originator, hash, software, first_seen) VALUES (?, ?, ?, ?, ?, ?)", + ( + domain, + origin, + originator, + get_hash(domain), + software, + time.time() + ), + ) + + cache.set_sub_key("is_registered", domain, True) + + if instances.has_pending_instance_data(domain): + # DEBUG: print(f"DEBUG: domain='{domain}' has pending nodeinfo being updated ...") + instances.set("last_status_code" , domain, None) + instances.set("last_error_details", domain, None) + instances.update_instance_data(domain) + remove_pending_error(domain) + + if domain in pending_errors: + # DEBUG: print("DEBUG: domain has pending error being updated:", domain) + update_last_error(domain, pending_errors[domain]) + remove_pending_error(domain) + + except BaseException as e: + print(f"ERROR: failed SQL query: domain='{domain}',exception[{type(e)}]:'{str(e)}'") + sys.exit(255) + else: + # DEBUG: print("DEBUG: Updating nodeinfo for domain:", domain) + update_last_nodeinfo(domain) + + # DEBUG: print("DEBUG: EXIT!") -- 2.39.5