From 65e7ded0da2d597d118450be6730458327c0c7ea Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Tue, 13 Jun 2023 04:18:29 +0200 Subject: [PATCH] Continued: - removed some wrapper functions - update instance data even when the connection didn't work or some error status code (>= 400) came --- fba/commands.py | 6 +++++- fba/federation.py | 9 +++++---- fba/models/instances.py | 32 -------------------------------- fba/networks/friendica.py | 4 ---- fba/networks/lemmy.py | 3 --- fba/networks/misskey.py | 6 ------ fba/networks/peertube.py | 5 +---- 7 files changed, 11 insertions(+), 54 deletions(-) diff --git a/fba/commands.py b/fba/commands.py index 63c185f..7a64081 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -169,7 +169,7 @@ def fetch_blocks(args: argparse.Namespace): continue # DEBUG: print(f"DEBUG: blocker='{blocker}'") - instances.update_last_blocked(blocker) + instances.set_data("last_blocked", blocker, time.time()) if software == "pleroma": print(f"INFO: blocker='{blocker}',software='{software}'") @@ -267,6 +267,10 @@ def fetch_blocks(args: argparse.Namespace): else: print("WARNING: Unknown software:", blocker, software) + if instances.has_pending(blocker): + # DEBUG: print(f"DEBUG: Invoking instances.update_data({blocker}) ...") + instances.update_data(blocker) + if config.get("bot_enabled") and len(blockdict) > 0: network.send_bot_post(blocker, blockdict) diff --git a/fba/federation.py b/fba/federation.py index 9e0922d..118c45d 100644 --- a/fba/federation.py +++ b/fba/federation.py @@ -14,6 +14,7 @@ # along with this program. If not, see . import bs4 +import time import validators from fba import blacklist @@ -54,7 +55,7 @@ def fetch_instances(domain: str, origin: str, software: str, command: str, path: raise ValueError(f"Parameter origin[]='{type(origin)}' is not 'str'") elif software is None: # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") - instances.update_last_instance_fetch(domain) + instances.set_data("last_instance_fetch", domain, time.time()) # DEBUG: print(f"DEBUG: software for domain='{domain}' is not set, determining ...") software = determine_software(domain, path) @@ -72,6 +73,9 @@ def fetch_instances(domain: str, origin: str, software: str, command: str, path: # DEBUG: print("DEBUG: Adding new domain:", domain, origin) instances.add(domain, origin, command, path) + # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") + instances.set_data("last_instance_fetch", domain, time.time()) + # DEBUG: print("DEBUG: Fetching instances for domain:", domain, software) peerlist = fetch_peers(domain, software) @@ -184,9 +188,6 @@ def fetch_peers(domain: str, software: str) -> list: # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'") instances.set_data("total_peers", domain, len(peers)) - # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") - instances.update_last_instance_fetch(domain) - # DEBUG: print("DEBUG: Returning peers[]:", type(peers)) return peers diff --git a/fba/models/instances.py b/fba/models/instances.py index 99e61ec..91c4722 100644 --- a/fba/models/instances.py +++ b/fba/models/instances.py @@ -146,38 +146,6 @@ def update_data(domain: str): # DEBUG: print("DEBUG: EXIT!") -def update_last_instance_fetch(domain: str): - # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") - if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") - elif domain == "": - raise ValueError("Parameter 'domain' is empty") - - # DEBUG: print("DEBUG: Updating last_instance_fetch for domain:", domain) - set_data("last_instance_fetch", domain, time.time()) - - # Running pending updated - # DEBUG: print(f"DEBUG: Invoking update_data({domain}) ...") - update_data(domain) - - # DEBUG: print("DEBUG: EXIT!") - -def update_last_blocked(domain: str): - # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") - if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") - elif domain == "": - raise ValueError("Parameter 'domain' is empty") - - # DEBUG: print("DEBUG: Updating last_blocked for domain", domain) - set_data("last_blocked", domain, time.time()) - - # Running pending updated - # DEBUG: print(f"DEBUG: Invoking update_data({domain}) ...") - update_data(domain) - - # DEBUG: print("DEBUG: EXIT!") - def add(domain: str, origin: str, command: str, path: str = None): # DEBUG: print(f"DEBUG: domain='{domain}',origin='{origin}',command='{command}',path='{path}' - CALLED!") if not isinstance(domain, str): diff --git a/fba/networks/friendica.py b/fba/networks/friendica.py index 6941dab..aef3f23 100644 --- a/fba/networks/friendica.py +++ b/fba/networks/friendica.py @@ -50,10 +50,6 @@ def fetch_blocks(domain: str) -> dict: except network.exceptions as exception: print(f"WARNING: Exception '{type(exception)}' during fetching instances (friendica) from domain='{domain}'") instances.update_last_error(domain, exception) - - if instances.has_pending(domain): - instances.update_data(domain) - return dict() # Prevents exceptions: diff --git a/fba/networks/lemmy.py b/fba/networks/lemmy.py index 8a0a962..df6b6bc 100644 --- a/fba/networks/lemmy.py +++ b/fba/networks/lemmy.py @@ -67,8 +67,5 @@ def fetch_peers(domain: str) -> list: # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'") instances.set_data("total_peers", domain, len(peers)) - # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") - instances.update_last_instance_fetch(domain) - # DEBUG: print("DEBUG: Returning peers[]:", type(peers)) return peers diff --git a/fba/networks/misskey.py b/fba/networks/misskey.py index 1dedde7..e1e7360 100644 --- a/fba/networks/misskey.py +++ b/fba/networks/misskey.py @@ -119,9 +119,6 @@ def fetch_peers(domain: str) -> list: # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'") instances.set_data("total_peers", domain, len(peers)) - # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") - instances.update_last_instance_fetch(domain) - # DEBUG: print(f"DEBUG: Returning peers[]='{type(peers)}'") return peers @@ -286,9 +283,6 @@ def fetch_blocks(domain: str) -> dict: offset = 0 break - # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") - instances.update_last_instance_fetch(domain) - # DEBUG: print(f"DEBUG: Returning for domain='{domain}',blocked()={len(blocklist['blocked'])},suspended()={len(blocklist['suspended'])}") return { "reject" : blocklist["blocked"], diff --git a/fba/networks/peertube.py b/fba/networks/peertube.py index 793fb45..b6115d5 100644 --- a/fba/networks/peertube.py +++ b/fba/networks/peertube.py @@ -53,7 +53,7 @@ def fetch_peers(domain: str) -> list: print(f"DEBUG: data['{type(data)}']='{data}'") if "error_message" not in data: - print("DEBUG: Success, data[json]:", len(data["json"])) + print(f"DEBUG: Success, data[json]()={len(data['json'])}") if "data" in data["json"]: rows = data["json"]["data"] @@ -76,8 +76,5 @@ def fetch_peers(domain: str) -> list: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'") instances.set_data("total_peers", domain, len(peers)) - print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") - instances.update_last_instance_fetch(domain) - print(f"DEBUG: Returning peers[]='{type(peers)}'") return peers -- 2.39.5