From f534a8cbbbdbd4cdcf6a9bcf749c8cc35e03738d Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Tue, 6 Jun 2023 21:55:39 +0200 Subject: [PATCH] Continued: - moved functions for table 'blocks' to own module - renamed some of them - fixed missing reference to fba.api_headers --- fba/__init__.py | 1 + fba/blocks.py | 170 ++++++++++++++++++++++++++++++++++++ fba/commands.py | 17 ++-- fba/config.py | 2 +- fba/fba.py | 189 ++++++++-------------------------------- fba/instances.py | 4 +- fba/network/lemmy.py | 6 +- fba/network/mastodon.py | 37 ++++---- fba/network/misskey.py | 4 +- fba/network/peertube.py | 2 +- fba/network/pleroma.py | 35 ++++---- 11 files changed, 255 insertions(+), 212 deletions(-) create mode 100644 fba/blocks.py diff --git a/fba/__init__.py b/fba/__init__.py index 0a14a1a..5694777 100644 --- a/fba/__init__.py +++ b/fba/__init__.py @@ -1,4 +1,5 @@ __all__ = [ + 'blocks', 'boot', 'cache', 'commands', diff --git a/fba/blocks.py b/fba/blocks.py new file mode 100644 index 0000000..1a4c077 --- /dev/null +++ b/fba/blocks.py @@ -0,0 +1,170 @@ +# Fedi API Block - An aggregator for fetching blocking data from fediverse nodes +# Copyright (C) 2023 Free Software Foundation +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import sys +import time +import validators + +from fba import fba + +def update_reason(reason: str, blocker: str, blocked: str, block_level: str): + # DEBUG: print(f"DEBUG: reason='{reason}',blocker={blocker},blocked={blocked},block_level={block_level} - CALLED!") + if type(reason) != str and reason != None: + raise ValueError(f"Parameter reason[]='{type(reason)}' is not 'str'") + elif type(blocker) != str: + raise ValueError(f"Parameter blocker[]='{type(blocker)}' is not 'str'") + elif blocker == "": + raise ValueError("Parameter 'blocker' is empty") + elif type(blocked) != str: + raise ValueError(f"Parameter blocked[]='{type(blocked)}' is not 'str'") + elif blocked == "": + raise ValueError("Parameter 'blocked' is empty") + elif type(block_level) != str: + raise ValueError(f"Parameter block_level[]='{type(block_level)}' is not 'str'") + elif block_level == "": + raise ValueError("Parameter 'block_level' is empty") + + # DEBUG: print("DEBUG: Updating block reason:", reason, blocker, blocked, block_level) + try: + fba.cursor.execute( + "UPDATE blocks SET reason = ?, last_seen = ? WHERE blocker = ? AND blocked = ? AND block_level = ? AND reason IN ('','unknown') LIMIT 1", + ( + reason, + time.time(), + blocker, + blocked, + block_level + ), + ) + + # DEBUG: print(f"DEBUG: fba.cursor.rowcount={fba.cursor.rowcount}") + if fba.cursor.rowcount == 0: + # DEBUG: print(f"DEBUG: Did not update any rows: blocker='{blocker}',blocked='{blocked}',block_level='{block_level}',reason='{reason}' - EXIT!") + return + + except BaseException as e: + print(f"ERROR: failed SQL query: reason='{reason}',blocker='{blocker}',blocked='{blocked}',block_level='{block_level}',exception[{type(e)}]:'{str(e)}'") + sys.exit(255) + + # DEBUG: print("DEBUG: EXIT!") + +def update_last_seen(blocker: str, blocked: str, block_level: str): + # DEBUG: print("DEBUG: Updating last_seen for:", blocker, blocked, block_level) + if type(blocker) != str: + raise ValueError(f"Parameter blocker[]='{type(blocker)}' is not 'str'") + elif blocker == "": + raise ValueError("Parameter 'blocker' is empty") + elif type(blocked) != str: + raise ValueError(f"Parameter blocked[]='{type(blocked)}' is not 'str'") + elif blocked == "": + raise ValueError("Parameter 'blocked' is empty") + elif type(block_level) != str: + raise ValueError(f"Parameter block_level[]='{type(block_level)}' is not 'str'") + elif block_level == "": + raise ValueError("Parameter 'block_level' is empty") + + try: + fba.cursor.execute( + "UPDATE blocks SET last_seen = ? WHERE blocker = ? AND blocked = ? AND block_level = ? LIMIT 1", + ( + time.time(), + blocker, + blocked, + block_level + ) + ) + + # DEBUG: print(f"DEBUG: fba.cursor.rowcount={fba.cursor.rowcount}") + if fba.cursor.rowcount == 0: + # DEBUG: print(f"DEBUG: Did not update any rows: blocker='{blocker}',blocked='{blocked}',block_level='{block_level}' - EXIT!") + return + + except BaseException as e: + print(f"ERROR: failed SQL query: blocker='{blocker}',blocked='{blocked}',block_level='{block_level}',exception[{type(e)}]:'{str(e)}'") + sys.exit(255) + + # DEBUG: print("DEBUG: EXIT!") + +def is_instance_blocked(blocker: str, blocked: str, block_level: str) -> bool: + # DEBUG: print(f"DEBUG: blocker={blocker},blocked={blocked},block_level={block_level} - CALLED!") + if type(blocker) != str: + raise ValueError(f"Parameter blocker[]={type(blocker)} is not of type 'str'") + elif blocker == "": + raise ValueError("Parameter 'blocker' is empty") + elif type(blocked) != str: + raise ValueError(f"Parameter blocked[]={type(blocked)} is not of type 'str'") + elif blocked == "": + raise ValueError("Parameter 'blocked' is empty") + elif type(block_level) != str: + raise ValueError(f"Parameter block_level[]={type(block_level)} is not of type 'str'") + elif block_level == "": + raise ValueError("Parameter 'block_level' is empty") + + fba.cursor.execute( + "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ? LIMIT 1", + ( + blocker, + blocked, + block_level + ), + ) + + is_blocked = fba.cursor.fetchone() != None + + # DEBUG: print(f"DEBUG: is_blocked='{is_blocked}' - EXIT!") + return is_blocked + +def add_instance(blocker: str, blocked: str, reason: str, block_level: str): + # DEBUG: print("DEBUG: blocker,blocked,reason,block_level:", blocker, blocked, reason, block_level) + if type(blocker) != str: + raise ValueError(f"Parameter blocker[]={type(blocker)} is not 'str'") + elif blocker == "": + raise ValueError(f"Parameter 'blocker' is empty") + elif not validators.domain(blocker.split("/")[0]): + raise ValueError(f"Bad blocker='{blocker}'") + elif type(blocked) != str: + raise ValueError(f"Parameter blocked[]={type(blocked)} is not 'str'") + elif blocked == "": + raise ValueError(f"Parameter 'blocked' is empty") + elif not validators.domain(blocked.split("/")[0]): + raise ValueError(f"Bad blocked='{blocked}'") + elif fba.is_blacklisted(blocker): + raise Exception(f"blocker='{blocker}' is blacklisted but function invoked") + elif fba.is_blacklisted(blocked): + raise Exception(f"blocked='{blocked}' is blacklisted but function invoked") + + if reason != None: + # Maybe needs cleaning + reason = fba.tidyup_reason(reason) + + print(f"INFO: New block: blocker='{blocker}',blocked='{blocked}', reason='{reason}', block_level='{block_level}'") + try: + fba.cursor.execute( + "INSERT INTO blocks (blocker, blocked, reason, block_level, first_seen, last_seen) VALUES(?, ?, ?, ?, ?, ?)", + ( + blocker, + blocked, + reason, + block_level, + time.time(), + time.time() + ), + ) + except BaseException as e: + print(f"ERROR: failed SQL query: blocker='{blocker}',blocked='{blocked}',reason='{reason}',block_level='{block_level}',exception[{type(e)}]:'{str(e)}'") + sys.exit(255) + + # DEBUG: print("DEBUG: EXIT!") diff --git a/fba/commands.py b/fba/commands.py index ce413ce..da62402 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -26,6 +26,7 @@ import sys import time import validators +from fba import blocks from fba import boot from fba import config from fba import fba @@ -218,8 +219,8 @@ def fetch_blocks(args: argparse.Namespace): # DEBUG: print("DEBUG: Hash wasn't found, adding:", blocked, blocker) fba.add_instance(blocked, blocker, inspect.currentframe().f_code.co_name, nodeinfo_url) - if not fba.is_instance_blocked(blocker, blocked, block_level): - fba.block_instance(blocker, blocked, reason, block_level) + if not blocks.is_instance_blocked(blocker, blocked, block_level): + blocks.add_instance(blocker, blocked, reason, block_level) if block_level == "reject": blockdict.append({ @@ -229,7 +230,7 @@ def fetch_blocks(args: argparse.Namespace): else: # DEBUG: print(f"DEBUG: Updating block last seen and reason for blocker='{blocker}',blocked='{blocked}' ...") fba.update_last_seen(blocker, blocked, block_level) - fba.update_block_reason(reason, blocker, blocked, block_level) + blocks.update_reason(reason, blocker, blocked, block_level) # DEBUG: print("DEBUG: Committing changes ...") fba.connection.commit() @@ -285,9 +286,9 @@ def fetch_blocks(args: argparse.Namespace): # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., blocker='{blocker}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'") fba.add_instance(blocked, blocker, inspect.currentframe().f_code.co_name, nodeinfo_url) - if not fba.is_instance_blocked(blocker, blocked, "reject"): + if not blocks.is_instance_blocked(blocker, blocked, "reject"): # DEBUG: print(f"DEBUG: blocker='{blocker}' is blocking '{blocked}' for unknown reason at this point") - fba.block_instance(blocker, blocked, "unknown", "reject") + blocks.add_instance(blocker, blocked, "unknown", "reject") blockdict.append({ "blocked": blocked, @@ -299,7 +300,7 @@ def fetch_blocks(args: argparse.Namespace): if "public_comment" in peer: # DEBUG: print("DEBUG: Updating block reason:", blocker, blocked, peer["public_comment"]) - fba.update_block_reason(peer["public_comment"], blocker, blocked, "reject") + blocks.update_reason(peer["public_comment"], blocker, blocked, "reject") for entry in blockdict: if entry["blocked"] == blocked: @@ -360,9 +361,9 @@ def fetch_cs(args: argparse.Namespace): print(f"INFO: Fetching instances from domain='{row['domain']}' ...") fba.fetch_instances(row["domain"], None, None, inspect.currentframe().f_code.co_name) - if not fba.is_instance_blocked('chaos.social', row["domain"], block_level): + if not blocks.is_instance_blocked('chaos.social', row["domain"], block_level): # DEBUG: print(f"DEBUG: domain='{row['domain']}',block_level='{block_level}' blocked by chaos.social, adding ...") - fba.block_instance('chaos.social', row["domain"], row["reason"], block_level) + blocks.add_instance('chaos.social', row["domain"], row["reason"], block_level) # DEBUG: print("DEBUG: Committing changes ...") fba.connection.commit() diff --git a/fba/config.py b/fba/config.py index dc5dd22..b488ef2 100644 --- a/fba/config.py +++ b/fba/config.py @@ -24,7 +24,7 @@ def get(key: str) -> any: if type(key) != str: raise ValueError(f"Parameter key[]='{type(key)}' is not 'str'") elif key == "": - raise ValueError("Parameter 'key' cannot be empty") + raise ValueError("Parameter 'key' is empty") elif not key in _config: raise KeyError(f"key='{key}' does not exist in _config array") diff --git a/fba/fba.py b/fba/fba.py index 16c07d1..1cc62a4 100644 --- a/fba/fba.py +++ b/fba/fba.py @@ -109,13 +109,13 @@ def fetch_instances(domain: str, origin: str, software: str, script: str, path: if type(domain) != str: raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") + raise ValueError(f"Parameter 'domain' is empty") elif type(origin) != str and origin != None: raise ValueError(f"Parameter origin[]={type(origin)} is not 'str'") elif type(script) != str: raise ValueError(f"Parameter script[]={type(script)} is not 'str'") elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") + raise ValueError(f"Parameter 'domain' is empty") if not is_instance_registered(domain): # DEBUG: print("DEBUG: Adding new domain:", domain, origin) @@ -304,7 +304,7 @@ def is_blacklisted(domain: str) -> bool: if type(domain) != str: raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") + raise ValueError(f"Parameter 'domain' is empty") blacklisted = False for peer in blacklist: @@ -317,7 +317,7 @@ def remove_pending_error(domain: str): if type(domain) != str: raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") + raise ValueError(f"Parameter 'domain' is empty") try: # Prevent updating any pending errors, nodeinfo was found @@ -332,7 +332,7 @@ def get_hash(domain: str) -> str: if type(domain) != str: raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") + raise ValueError(f"Parameter 'domain' is empty") return hashlib.sha256(domain.encode("utf-8")).hexdigest() @@ -340,7 +340,7 @@ def update_last_blocked(domain: str): if type(domain) != str: raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") + raise ValueError(f"Parameter 'domain' is empty") # DEBUG: print("DEBUG: Updating last_blocked for domain", domain) instances.set("last_blocked", domain, time.time()) @@ -356,7 +356,7 @@ def log_error(domain: str, response: requests.models.Response): if type(domain) != str: raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") + raise ValueError(f"Parameter 'domain' is empty") try: # DEBUG: print("DEBUG: BEFORE response[]:", type(response)) @@ -392,7 +392,7 @@ def update_last_error(domain: str, response: requests.models.Response): if type(domain) != str: raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") + raise ValueError(f"Parameter 'domain' is empty") # DEBUG: print("DEBUG: BEFORE response[]:", type(response)) if isinstance(response, BaseException) or isinstance(response, json.decoder.JSONDecodeError): @@ -421,7 +421,7 @@ def update_last_instance_fetch(domain: str): if type(domain) != str: raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") + raise ValueError(f"Parameter 'domain' is empty") # DEBUG: print("DEBUG: Updating last_instance_fetch for domain:", domain) instances.set("last_instance_fetch", domain, time.time()) @@ -437,7 +437,7 @@ def update_last_nodeinfo(domain: str): if type(domain) != str: raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") + raise ValueError(f"Parameter 'domain' is empty") # DEBUG: print("DEBUG: Updating last_nodeinfo for domain:", domain) instances.set("last_nodeinfo", domain, time.time()) @@ -454,7 +454,7 @@ def get_peers(domain: str, software: str) -> list: if type(domain) != str: raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") + raise ValueError(f"Parameter 'domain' is empty") elif type(software) != str and software != None: raise ValueError(f"software[]={type(software)} is not 'str'") @@ -516,7 +516,7 @@ def post_json_api(domain: str, path: str, parameter: str, extra_headers: dict = if type(domain) != str: raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") + raise ValueError(f"Parameter 'domain' is empty") elif type(path) != str: raise ValueError(f"path[]={type(path)} is not 'str'") elif path == "": @@ -551,7 +551,7 @@ def fetch_nodeinfo(domain: str, path: str = None) -> list: if type(domain) != str: raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") + raise ValueError(f"Parameter 'domain' is empty") elif type(path) != str and path != None: raise ValueError(f"Parameter path[]={type(path)} is not 'str'") @@ -610,7 +610,7 @@ def fetch_wellknown_nodeinfo(domain: str) -> list: if type(domain) != str: raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") + raise ValueError(f"Parameter 'domain' is empty") # DEBUG: print("DEBUG: Fetching .well-known info for domain:", domain) data = {} @@ -656,11 +656,11 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str: if type(domain) != str: raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") + raise ValueError(f"Parameter 'domain' is empty") elif type(path) != str: raise ValueError(f"path[]={type(path)} is not 'str'") elif path == "": - raise ValueError(f"Parameter 'domain' cannot be empty") + raise ValueError(f"Parameter 'domain' is empty") # DEBUG: print(f"DEBUG: domain='{domain}',path='{path}' - CALLED!") software = None @@ -727,7 +727,7 @@ def determine_software(domain: str, path: str = None) -> str: if type(domain) != str: raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") + raise ValueError(f"Parameter 'domain' is empty") elif type(path) != str and path != None: raise ValueError(f"Parameter path[]={type(path)} is not 'str'") @@ -807,142 +807,12 @@ def determine_software(domain: str, path: str = None) -> str: # DEBUG: print("DEBUG: Returning domain,software:", domain, software) return software -def update_block_reason(reason: str, blocker: str, blocked: str, block_level: str): - # DEBUG: print(f"DEBUG: reason='{reason}',blocker={blocker},blocked={blocked},block_level={block_level} - CALLED!") - if type(reason) != str and reason != None: - raise ValueError(f"Parameter reason[]='{type(reason)}' is not 'str'") - elif type(blocker) != str: - raise ValueError(f"Parameter blocker[]='{type(blocker)}' is not 'str'") - elif type(blocked) != str: - raise ValueError(f"Parameter blocked[]='{type(blocked)}' is not 'str'") - elif type(block_level) != str: - raise ValueError(f"Parameter block_level[]='{type(block_level)}' is not 'str'") - - # DEBUG: print("DEBUG: Updating block reason:", reason, blocker, blocked, block_level) - try: - cursor.execute( - "UPDATE blocks SET reason = ?, last_seen = ? WHERE blocker = ? AND blocked = ? AND block_level = ? AND reason IN ('','unknown') LIMIT 1", - ( - reason, - time.time(), - blocker, - blocked, - block_level - ), - ) - - # DEBUG: print(f"DEBUG: cursor.rowcount={cursor.rowcount}") - if cursor.rowcount == 0: - # DEBUG: print(f"DEBUG: Did not update any rows: blocker='{blocker}',blocked='{blocked}',block_level='{block_level}',reason='{reason}' - EXIT!") - return - - except BaseException as e: - print(f"ERROR: failed SQL query: reason='{reason}',blocker='{blocker}',blocked='{blocked}',block_level='{block_level}',exception[{type(e)}]:'{str(e)}'") - sys.exit(255) - - # DEBUG: print("DEBUG: EXIT!") - -def update_last_seen(blocker: str, blocked: str, block_level: str): - # DEBUG: print("DEBUG: Updating last_seen for:", blocker, blocked, block_level) - try: - cursor.execute( - "UPDATE blocks SET last_seen = ? WHERE blocker = ? AND blocked = ? AND block_level = ? LIMIT 1", - ( - time.time(), - blocker, - blocked, - block_level - ) - ) - - # DEBUG: print(f"DEBUG: cursor.rowcount={cursor.rowcount}") - if cursor.rowcount == 0: - # DEBUG: print(f"DEBUG: Did not update any rows: blocker='{blocker}',blocked='{blocked}',block_level='{block_level}' - EXIT!") - return - - except BaseException as e: - print(f"ERROR: failed SQL query: blocker='{blocker}',blocked='{blocked}',block_level='{block_level}',exception[{type(e)}]:'{str(e)}'") - sys.exit(255) - - # DEBUG: print("DEBUG: EXIT!") - -def is_instance_blocked(blocker: str, blocked: str, block_level: str) -> bool: - # DEBUG: print(f"DEBUG: blocker={blocker},blocked={blocked},block_level={block_level} - CALLED!") - if type(blocker) != str: - raise ValueError(f"Parameter blocker[]={type(blocker)} is not of type 'str'") - elif blocker == "": - raise ValueError("Parameter 'blocker' cannot be empty") - elif type(blocked) != str: - raise ValueError(f"Parameter blocked[]={type(blocked)} is not of type 'str'") - elif blocked == "": - raise ValueError("Parameter 'blocked' cannot be empty") - elif type(block_level) != str: - raise ValueError(f"Parameter block_level[]={type(block_level)} is not of type 'str'") - elif block_level == "": - raise ValueError("Parameter 'block_level' cannot be empty") - - cursor.execute( - "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ? LIMIT 1", - ( - blocker, - blocked, - block_level - ), - ) - - is_blocked = cursor.fetchone() != None - - # DEBUG: print(f"DEBUG: is_blocked='{is_blocked}' - EXIT!") - return is_blocked - -def block_instance(blocker: str, blocked: str, reason: str, block_level: str): - # DEBUG: print("DEBUG: blocker,blocked,reason,block_level:", blocker, blocked, reason, block_level) - if type(blocker) != str: - raise ValueError(f"Parameter blocker[]={type(blocker)} is not 'str'") - elif blocker == "": - raise ValueError(f"Parameter 'blocker' cannot be empty") - elif not validators.domain(blocker.split("/")[0]): - raise ValueError(f"Bad blocker='{blocker}'") - elif type(blocked) != str: - raise ValueError(f"Parameter blocked[]={type(blocked)} is not 'str'") - elif blocked == "": - raise ValueError(f"Parameter 'blocked' cannot be empty") - elif not validators.domain(blocked.split("/")[0]): - raise ValueError(f"Bad blocked='{blocked}'") - elif is_blacklisted(blocker): - raise Exception(f"blocker='{blocker}' is blacklisted but function invoked") - elif is_blacklisted(blocked): - raise Exception(f"blocked='{blocked}' is blacklisted but function invoked") - - if reason != None: - # Maybe needs cleaning - reason = tidyup_reason(reason) - - print(f"INFO: New block: blocker='{blocker}',blocked='{blocked}', reason='{reason}', block_level='{block_level}'") - try: - cursor.execute( - "INSERT INTO blocks (blocker, blocked, reason, block_level, first_seen, last_seen) VALUES(?, ?, ?, ?, ?, ?)", - ( - blocker, - blocked, - reason, - block_level, - time.time(), - time.time() - ), - ) - except BaseException as e: - print(f"ERROR: failed SQL query: blocker='{blocker}',blocked='{blocked}',reason='{reason}',block_level='{block_level}',exception[{type(e)}]:'{str(e)}'") - sys.exit(255) - - # DEBUG: print("DEBUG: EXIT!") - def is_instance_registered(domain: str) -> bool: # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") if type(domain) != str: raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") + raise ValueError(f"Parameter 'domain' is empty") # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") if not cache.key_exists("is_registered"): @@ -967,7 +837,7 @@ def add_instance(domain: str, origin: str, originator: str, path: str = None): if type(domain) != str: raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") + raise ValueError(f"Parameter 'domain' is empty") elif type(origin) != str and origin != None: raise ValueError(f"origin[]={type(origin)} is not 'str'") elif type(originator) != str: @@ -1024,6 +894,13 @@ def add_instance(domain: str, origin: str, originator: str, path: str = None): def send_bot_post(instance: str, blocks: dict): # DEBUG: print(f"DEBUG: instance={instance},blocks()={len(blocks)} - CALLED!") + if type(domain) != str: + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") + elif domain == "": + raise ValueError("Parameter 'domain' is empty") + elif type(blocks) != dict: + raise ValueError(f"Parameter blocks[]='{type(blocks)}' is not 'dict'") + message = instance + " has blocked the following instances:\n\n" truncated = False @@ -1031,7 +908,9 @@ def send_bot_post(instance: str, blocks: dict): truncated = True blocks = blocks[0 : 19] + # DEBUG: print(f"DEBUG: blocks()={len(blocks)}") for block in blocks: + # DEBUG: print(f"DEBUG: block['{type(block)}']={block}") if block["reason"] == None or block["reason"] == '': message = message + block["blocked"] + " with unspecified reason\n" else: @@ -1063,7 +942,7 @@ def fetch_friendica_blocks(domain: str) -> dict: if type(domain) != str: raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") + raise ValueError(f"Parameter 'domain' is empty") # DEBUG: print("DEBUG: Fetching friendica blocks from domain:", domain) blocks = [] @@ -1102,7 +981,7 @@ def fetch_misskey_blocks(domain: str) -> dict: if type(domain) != str: raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") + raise ValueError(f"Parameter 'domain' is empty") # DEBUG: print("DEBUG: Fetching misskey blocks from domain:", domain) blocks = { @@ -1302,11 +1181,11 @@ def get_response(domain: str, path: str, headers: dict, timeout: list) -> reques if type(domain) != str: raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": - raise ValueError("Parameter 'domain' cannot be empty") + raise ValueError("Parameter 'domain' is empty") elif type(path) != str: raise ValueError(f"Parameter path[]='{type(path)}' is not 'str'") elif path == "": - raise ValueError("Parameter 'path' cannot be empty") + raise ValueError("Parameter 'path' is empty") try: # DEBUG: print(f"DEBUG: Sending request to '{domain}{path}' ...") @@ -1330,7 +1209,7 @@ def has_key(keys: list, search: str, value: any) -> bool: elif type(search) != str: raise ValueError(f"Parameter search[]='{type(search)}' is not 'str'") elif search == "": - raise ValueError("Parameter 'search' cannot be empty") + raise ValueError("Parameter 'search' is empty") has = False # DEBUG: print(f"DEBUG: Checking keys()={len(keys)} ...") @@ -1404,7 +1283,7 @@ def get_url(url: str, headers: dict, timeout: list) -> requests.models.Response: if type(url) != str: raise ValueError(f"Parameter url[]='{type(url)}' is not 'str'") elif url == "": - raise ValueError("Parameter 'url' cannot be empty") + raise ValueError("Parameter 'url' is empty") # DEBUG: print(f"DEBUG: Parsing url='{url}'") components = urlparse(url) diff --git a/fba/instances.py b/fba/instances.py index be8cd71..9f1606a 100644 --- a/fba/instances.py +++ b/fba/instances.py @@ -69,7 +69,7 @@ def has_pending_instance_data(domain: str) -> bool: if type(domain) != str: raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") + raise ValueError(f"Parameter 'domain' is empty") has_pending = False for key in _pending: @@ -86,7 +86,7 @@ def update_instance_data(domain: str): if type(domain) != str: raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") + raise ValueError(f"Parameter 'domain' is empty") elif not has_pending_instance_data(domain): raise Exception(f"Domain '{domain}' has no pending instance data, but function invoked") diff --git a/fba/network/lemmy.py b/fba/network/lemmy.py index 05c8df0..c75e6ae 100644 --- a/fba/network/lemmy.py +++ b/fba/network/lemmy.py @@ -25,14 +25,12 @@ def get_peers(domain: str) -> list: if type(domain) != str: raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") - elif type(software) != str and software != None: - raise ValueError(f"software[]={type(software)} is not 'str'") + raise ValueError(f"Parameter 'domain' is empty") # DEBUG: print(f"DEBUG: domain='{domain}' is Lemmy, fetching JSON ...") peers = list() try: - response = fba.get_response(domain, "/api/v3/site", api_headers, (config.get("connection_timeout"), config.get("read_timeout"))) + response = fba.get_response(domain, "/api/v3/site", fba.api_headers, (config.get("connection_timeout"), config.get("read_timeout"))) data = fba.json_from_response(response) diff --git a/fba/network/mastodon.py b/fba/network/mastodon.py index 89bded8..3de8a57 100644 --- a/fba/network/mastodon.py +++ b/fba/network/mastodon.py @@ -17,6 +17,7 @@ import bs4 import validators +from fba import blocks from fba import config from fba import fba @@ -51,7 +52,7 @@ def fetch_blocks_from_about(domain: str) -> dict: if type(domain) != str: raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") + raise ValueError(f"Parameter 'domain' is empty") # DEBUG: print("DEBUG: Fetching mastodon blocks from domain:", domain) blocks = { @@ -101,24 +102,20 @@ def fetch_blocks_from_about(domain: str) -> dict: "followers_only": blocks["Limited servers"] + blocks["Silenced servers"], } -def fetch_blocks(domain: str, software: str, origin: str, nodeinfo_url: str): - print(f"DEBUG: domain='{domain}',software='{software}',origin='{origin}',nodeinfo_url='{nodeinfo_url}' - CALLED!") +def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): + print(f"DEBUG: domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}' - CALLED!") if type(domain) != str: raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") - elif type(software) != str: - raise ValueError(f"Parameter software[]={type(software)} is not 'str'") - elif software == "": - raise ValueError(f"Parameter 'software' cannot be empty") + raise ValueError(f"Parameter 'domain' is empty") elif type(origin) != str and origin != None: raise ValueError(f"Parameter origin[]={type(origin)} is not 'str'") elif origin == "": - raise ValueError(f"Parameter 'origin' cannot be empty") + raise ValueError(f"Parameter 'origin' is empty") elif type(nodeinfo_url) != str: raise ValueError(f"Parameter nodeinfo_url[]={type(nodeinfo_url)} is not 'str'") elif nodeinfo_url == "": - raise ValueError(f"Parameter 'nodeinfo_url' cannot be empty") + raise ValueError(f"Parameter 'nodeinfo_url' is empty") try: # json endpoint for newer mastodongs @@ -148,7 +145,7 @@ def fetch_blocks(domain: str, software: str, origin: str, nodeinfo_url: str): # DEBUG: print("DEBUG: Querying API domain_blocks:", domain) blocks = fba.get_response(domain, "/api/v1/instance/domain_blocks", reqheaders, (config.get("connection_timeout"), config.get("read_timeout"))).json() - print(f"INFO: Checking {len(blocks)} entries from domain='{domain}',software='{software}' ...") + print(f"INFO: Checking {len(blocks)} entries from domain='{domain}',software='mastodon' ...") for block in blocks: entry = { 'domain': block['domain'], @@ -176,7 +173,7 @@ def fetch_blocks(domain: str, software: str, origin: str, nodeinfo_url: str): # DEBUG: print(f"DEBUG: Failed, trying mastodon-specific fetches: domain='{domain}',exception[{type(e)}]={str(e)}") json = fetch_blocks_from_about(domain) - print(f"INFO: Checking {len(json.items())} entries from domain='{domain}',software='{software}' ...") + print(f"INFO: Checking {len(json.items())} entries from domain='{domain}',software='mastodon' ...") for block_level, blocks in json.items(): # DEBUG: print("DEBUG: domain,block_level,blocks():", domain, block_level, len(blocks)) block_level = fba.tidyup_domain(block_level) @@ -186,7 +183,7 @@ def fetch_blocks(domain: str, software: str, origin: str, nodeinfo_url: str): print("WARNING: block_level is empty, domain:", domain) continue - # DEBUG: print(f"DEBUG: Checking {len(blocks)} entries from domain='{domain}',software='{software}',block_level='{block_level}' ...") + # DEBUG: print(f"DEBUG: Checking {len(blocks)} entries from domain='{domain}',software='mastodon',block_level='{block_level}' ...") for block in blocks: blocked, blocked_hash, reason = block.values() # DEBUG: print("DEBUG: blocked,hash,reason:", blocked, blocked_hash, reason) @@ -217,18 +214,18 @@ def fetch_blocks(domain: str, software: str, origin: str, nodeinfo_url: str): # DEBUG: print("DEBUG: Looking up instance by domain:", blocked) if not validators.domain(blocked): - print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!") + print(f"WARNING: blocked='{blocked}',software='mastodon' is not a valid domain name - skipped!") continue elif not fba.is_instance_registered(blocked): # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'") fba.add_instance(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url) elif not validators.domain(blocked): - print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!") + print(f"WARNING: blocked='{blocked}',software='mastodon' is not a valid domain name - skipped!") continue # DEBUG: print("DEBUG: Looking up instance by domain:", blocked) if not validators.domain(blocked): - print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!") + print(f"WARNING: blocked='{blocked}',software='mastodon' is not a valid domain name - skipped!") continue elif not fba.is_instance_registered(blocked): # DEBUG: print("DEBUG: Hash wasn't found, adding:", blocked, domain) @@ -237,9 +234,9 @@ def fetch_blocks(domain: str, software: str, origin: str, nodeinfo_url: str): blocking = blocked if blocked.count("*") <= 1 else blocked_hash # DEBUG: print(f"DEBUG: blocking='{blocking}',blocked='{blocked}',blocked_hash='{blocked_hash}'") - if not fba.is_instance_blocked(domain, blocked, block_level): + if not blocks.is_instance_blocked(domain, blocked, block_level): # DEBUG: print("DEBUG: Blocking:", domain, blocked, block_level) - fba.block_instance(domain, blocking, reason, block_level) + blocks.add_instance(domain, blocking, reason, block_level) if block_level == "reject": blockdict.append({ @@ -249,11 +246,11 @@ def fetch_blocks(domain: str, software: str, origin: str, nodeinfo_url: str): else: # DEBUG: print(f"DEBUG: Updating block last seen and reason for domain='{domain}',blocking='{blocking}' ...") fba.update_last_seen(domain, blocking, block_level) - fba.update_block_reason(reason, domain, blocking, block_level) + blocks.update_reason(reason, domain, blocking, block_level) # DEBUG: print("DEBUG: Committing changes ...") fba.connection.commit() except Exception as e: - print(f"ERROR: domain='{domain}',software='{software}',exception[{type(e)}]:'{str(e)}'") + print(f"ERROR: domain='{domain}',software='mastodon',exception[{type(e)}]:'{str(e)}'") # DEBUG: print("DEBUG: EXIT!") diff --git a/fba/network/misskey.py b/fba/network/misskey.py index 5a90d37..3444db7 100644 --- a/fba/network/misskey.py +++ b/fba/network/misskey.py @@ -25,7 +25,7 @@ def get_peers(domain: str) -> list: if type(domain) != str: raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") + raise ValueError(f"Parameter 'domain' is empty") # DEBUG: print(f"DEBUG: domain='{domain}' is misskey, sending API POST request ...") peers = list() @@ -82,7 +82,7 @@ def get_peers(domain: str) -> list: elif type(row["host"]) != str: print(f"WARNING: row[host][]={type(row['host'])} is not 'str'") continue - elif is_blacklisted(row["host"]): + elif fba.is_blacklisted(row["host"]): # DEBUG: print(f"DEBUG: row[host]='{row['host']}' is blacklisted. domain='{domain}'") continue elif row["host"] in peers: diff --git a/fba/network/peertube.py b/fba/network/peertube.py index 1906abc..36d9fc4 100644 --- a/fba/network/peertube.py +++ b/fba/network/peertube.py @@ -23,7 +23,7 @@ def get_peers(domain: str) -> list: if type(domain) != str: raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") + raise ValueError(f"Parameter 'domain' is empty") # DEBUG: print(f"DEBUG: domain='{domain}' is a PeerTube, fetching JSON ...") peers = list() diff --git a/fba/network/pleroma.py b/fba/network/pleroma.py index 5cfc5ba..a6a41b5 100644 --- a/fba/network/pleroma.py +++ b/fba/network/pleroma.py @@ -17,26 +17,23 @@ import inspect import validators +from fba import blocks from fba import fba -def fetch_blocks(domain: str, software: str, origin: str, nodeinfo_url: str): - print(f"DEBUG: domain='{domain}',software='{software}',origin='{origin}',nodeinfo_url='{nodeinfo_url}' - CALLED!") +def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): + print(f"DEBUG: domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}' - CALLED!") if type(domain) != str: raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") - elif type(software) != str: - raise ValueError(f"Parameter software[]={type(software)} is not 'str'") - elif software == "": - raise ValueError(f"Parameter 'software' cannot be empty") + raise ValueError(f"Parameter 'domain' is empty") elif type(origin) != str and origin != None: raise ValueError(f"Parameter origin[]={type(origin)} is not 'str'") elif origin == "": - raise ValueError(f"Parameter 'origin' cannot be empty") + raise ValueError(f"Parameter 'origin' is empty") elif type(nodeinfo_url) != str: raise ValueError(f"Parameter nodeinfo_url[]={type(nodeinfo_url)} is not 'str'") elif nodeinfo_url == "": - raise ValueError(f"Parameter 'nodeinfo_url' cannot be empty") + raise ValueError(f"Parameter 'nodeinfo_url' is empty") try: # Blocks @@ -75,7 +72,7 @@ def fetch_blocks(domain: str, software: str, origin: str, nodeinfo_url: str): print("WARNING: block_level is now empty!") continue - # DEBUG: print(f"DEBUG: Checking {len(blocks)} entries from domain='{domain}',software='{software}',block_level='{block_level}' ...") + # DEBUG: print(f"DEBUG: Checking {len(blocks)} entries from domain='{domain}',block_level='{block_level}' ...") for blocked in blocks: # DEBUG: print("DEBUG: BEFORE blocked:", blocked) blocked = fba.tidyup_domain(blocked) @@ -103,20 +100,20 @@ def fetch_blocks(domain: str, software: str, origin: str, nodeinfo_url: str): nodeinfo_url = searchres[1] # DEBUG: print("DEBUG: Looked up domain:", blocked) elif not validators.domain(blocked): - print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!") + print(f"WARNING: blocked='{blocked}',software='pleroma' is not a valid domain name - skipped!") continue # DEBUG: print("DEBUG: Looking up instance by domain:", blocked) if not validators.domain(blocked): - print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!") + print(f"WARNING: blocked='{blocked}',software='pleroma' is not a valid domain name - skipped!") continue elif not fba.is_instance_registered(blocked): # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'") fba.add_instance(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url) - if not fba.is_instance_blocked(domain, blocked, block_level): + if not blocks.is_instance_blocked(domain, blocked, block_level): # DEBUG: print("DEBUG: Blocking:", domain, blocked, block_level) - fba.block_instance(domain, blocked, "unknown", block_level) + blocks.add_instance(domain, blocked, "unknown", block_level) if block_level == "reject": # DEBUG: print("DEBUG: Adding to blockdict:", blocked) @@ -149,7 +146,7 @@ def fetch_blocks(domain: str, software: str, origin: str, nodeinfo_url: str): print("WARNING: block_level is now empty!") continue - # DEBUG: print(f"DEBUG: Checking {len(info.items())} entries from domain='{domain}',software='{software}',block_level='{block_level}' ...") + # DEBUG: print(f"DEBUG: Checking {len(info.items())} entries from domain='{domain}',software='pleroma',block_level='{block_level}' ...") for blocked, reason in info.items(): # DEBUG: print("DEBUG: BEFORE blocked:", blocked) blocked = fba.tidyup_domain(blocked) @@ -176,19 +173,19 @@ def fetch_blocks(domain: str, software: str, origin: str, nodeinfo_url: str): origin = searchres[1] nodeinfo_url = searchres[2] elif not validators.domain(blocked): - print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!") + print(f"WARNING: blocked='{blocked}',software='pleroma' is not a valid domain name - skipped!") continue # DEBUG: print("DEBUG: Looking up instance by domain:", blocked) if not validators.domain(blocked): - print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!") + print(f"WARNING: blocked='{blocked}',software='pleroma' is not a valid domain name - skipped!") continue elif not fba.is_instance_registered(blocked): # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'") fba.add_instance(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url) # DEBUG: print("DEBUG: Updating block reason:", domain, blocked, reason["reason"]) - fba.update_block_reason(reason["reason"], domain, blocked, block_level) + blocks.update_reason(reason["reason"], domain, blocked, block_level) # DEBUG: print(f"DEBUG: blockdict()={count(blockdict)") for entry in blockdict: @@ -198,6 +195,6 @@ def fetch_blocks(domain: str, software: str, origin: str, nodeinfo_url: str): fba.connection.commit() except Exception as e: - print(f"ERROR: domain='{domain}',software='{software}',exception[{type(e)}]:'{str(e)}'") + print(f"ERROR: domain='{domain}',software='pleroma',exception[{type(e)}]:'{str(e)}'") # DEBUG: print("DEBUG: EXIT!") -- 2.39.5