From: Roland Häder Date: Sun, 11 Jun 2023 11:26:17 +0000 (+0200) Subject: Continued: X-Git-Url: https://git.mxchange.org/?a=commitdiff_plain;h=96f9cee8f4bdd14be4e0f58636a9d8980c880293;p=fba.git Continued: - if dict 'error' cannot be handled (their keys) then raise an exception - put all type(foo) into single quotes - commented out debug line --- diff --git a/fba/blacklist.py b/fba/blacklist.py index f71f735..0877db4 100644 --- a/fba/blacklist.py +++ b/fba/blacklist.py @@ -35,7 +35,7 @@ blacklist = [ def is_blacklisted(domain: str) -> bool: # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") diff --git a/fba/blocks.py b/fba/blocks.py index ed4a32d..28ae2bc 100644 --- a/fba/blocks.py +++ b/fba/blocks.py @@ -103,15 +103,15 @@ def update_last_seen(blocker: str, blocked: str, block_level: str): def is_instance_blocked(blocker: str, blocked: str, block_level: str) -> bool: # DEBUG: print(f"DEBUG: blocker={blocker},blocked={blocked},block_level={block_level} - CALLED!") if not isinstance(blocker, str): - raise ValueError(f"Parameter blocker[]={type(blocker)} is not of type 'str'") + raise ValueError(f"Parameter blocker[]='{type(blocker)}' is not of type 'str'") elif blocker == "": raise ValueError("Parameter 'blocker' is empty") elif not isinstance(blocked, str): - raise ValueError(f"Parameter blocked[]={type(blocked)} is not of type 'str'") + raise ValueError(f"Parameter blocked[]='{type(blocked)}' is not of type 'str'") elif blocked == "": raise ValueError("Parameter 'blocked' is empty") elif not isinstance(block_level, str): - raise ValueError(f"Parameter block_level[]={type(block_level)} is not of type 'str'") + raise ValueError(f"Parameter block_level[]='{type(block_level)}' is not of type 'str'") elif block_level == "": raise ValueError("Parameter 'block_level' is empty") @@ -132,15 +132,19 @@ def is_instance_blocked(blocker: str, blocked: str, block_level: str) -> bool: def add_instance(blocker: str, blocked: str, reason: str, block_level: str): # DEBUG: print("DEBUG: blocker,blocked,reason,block_level:", blocker, blocked, reason, block_level) if not isinstance(blocker, str): - raise ValueError(f"Parameter blocker[]={type(blocker)} is not 'str'") + raise ValueError(f"Parameter blocker[]='{type(blocker)}' is not 'str'") elif blocker == "": raise ValueError("Parameter 'blocker' is empty") elif not validators.domain(blocker.split("/")[0]): raise ValueError(f"Bad blocker='{blocker}'") elif not isinstance(blocked, str): - raise ValueError(f"Parameter blocked[]={type(blocked)} is not 'str'") + raise ValueError(f"Parameter blocked[]='{type(blocked)}' is not 'str'") elif blocked == "": raise ValueError("Parameter 'blocked' is empty") + elif not isinstance(block_level, str): + raise ValueError(f"Parameter block_level[]='{type(block_level)}' is not of type 'str'") + elif block_level == "": + raise ValueError("Parameter 'block_level' is empty") elif not validators.domain(blocked.split("/")[0]): raise ValueError(f"Bad blocked='{blocked}'") elif blacklist.is_blacklisted(blocker): diff --git a/fba/cache.py b/fba/cache.py index 8645781..cfc5278 100644 --- a/fba/cache.py +++ b/fba/cache.py @@ -23,7 +23,7 @@ def key_exists(key: str) -> bool: return key in _cache def set_all(key: str, rows: list, value: any): - # DEBUG: print(f"DEBUG: key='{key}',rows()={len(rows)},value[]={type(value)} - CALLED!") + # DEBUG: print(f"DEBUG: key='{key}',rows()={len(rows)},value[]='{type(value)}' - CALLED!") if not isinstance(key, str): raise ValueError("Parameter key[]='{type(key)}' is not 'str'") elif not key_exists(key): diff --git a/fba/commands.py b/fba/commands.py index 078bc4f..e746dee 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -63,7 +63,7 @@ def check_instance(args: argparse.Namespace) -> int: return status def fetch_bkali(args: argparse.Namespace): - # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!") + # DEBUG: print(f"DEBUG: args[]='{type(args)}' - CALLED!") domains = list() try: fetched = network.post_json_api("gql.api.bka.li", "/v1/graphql", json.dumps({ @@ -116,7 +116,7 @@ def fetch_bkali(args: argparse.Namespace): # DEBUG: print("DEBUG: EXIT!") def fetch_blocks(args: argparse.Namespace): - # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!") + # DEBUG: print(f"DEBUG: args[]='{type(args)}' - CALLED!") if args.domain is not None and args.domain != "": # DEBUG: print(f"DEBUG: args.domain='{args.domain}' - checking ...") if not validators.domain(args.domain): @@ -266,7 +266,7 @@ def fetch_blocks(args: argparse.Namespace): # DEBUG: print("DEBUG: EXIT!") def fetch_cs(args: argparse.Namespace): - # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!") + # DEBUG: print(f"DEBUG: args[]='{type(args)}' - CALLED!") extensions = [ 'extra', 'abbr', @@ -293,17 +293,17 @@ def fetch_cs(args: argparse.Namespace): } raw = fba.fetch_url("https://raw.githubusercontent.com/chaossocial/meta/master/federation.md", network.web_headers, (config.get("connection_timeout"), config.get("read_timeout"))).text - # DEBUG: print(f"DEBUG: raw()={len(raw)}[]={type(raw)}") + # DEBUG: print(f"DEBUG: raw()={len(raw)}[]='{type(raw)}'") doc = bs4.BeautifulSoup(markdown.markdown(raw, extensions=extensions), features='html.parser') - # DEBUG: print(f"DEBUG: doc()={len(doc)}[]={type(doc)}") + # DEBUG: print(f"DEBUG: doc()={len(doc)}[]='{type(doc)}'") silenced = doc.find("h2", {"id": "silenced-instances"}).findNext("table").find("tbody") - # DEBUG: print(f"DEBUG: silenced[]={type(silenced)}") + # DEBUG: print(f"DEBUG: silenced[]='{type(silenced)}'") domains["silenced"] = domains["silenced"] + federation.find_domains(silenced) blocked = doc.find("h2", {"id": "blocked-instances"}).findNext("table").find("tbody") - # DEBUG: print(f"DEBUG: blocked[]={type(blocked)}") + # DEBUG: print(f"DEBUG: blocked[]='{type(blocked)}'") domains["reject"] = domains["reject"] + federation.find_domains(blocked) # DEBUG: print(f"DEBUG: domains()={len(domains)}") @@ -334,7 +334,7 @@ def fetch_cs(args: argparse.Namespace): # DEBUG: print("DEBUG: EXIT!") def fetch_fba_rss(args: argparse.Namespace): - # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!") + # DEBUG: print(f"DEBUG: args[]='{type(args)}' - CALLED!") domains = list() print(f"INFO: Fetch FBA-specific RSS args.feed='{args.feed}' ...") @@ -345,7 +345,7 @@ def fetch_fba_rss(args: argparse.Namespace): # DEBUG: print(f"DEBUG: Parsing RSS feed ({len(response.text)} Bytes) ...") rss = atoma.parse_rss_bytes(response.content) - # DEBUG: print(f"DEBUG: rss[]={type(rss)}") + # DEBUG: print(f"DEBUG: rss[]='{type(rss)}'") for item in rss.items: # DEBUG: print(f"DEBUG: item={item}") domain = item.link.split("=")[1] @@ -379,7 +379,7 @@ def fetch_fba_rss(args: argparse.Namespace): # DEBUG: print("DEBUG: EXIT!") def fetch_fbabot_atom(args: argparse.Namespace): - # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!") + # DEBUG: print(f"DEBUG: args[]='{type(args)}' - CALLED!") feed = "https://ryona.agency/users/fba/feed.atom" domains = list() @@ -392,11 +392,11 @@ def fetch_fbabot_atom(args: argparse.Namespace): # DEBUG: print(f"DEBUG: Parsing ATOM feed ({len(response.text)} Bytes) ...") atom = atoma.parse_atom_bytes(response.content) - # DEBUG: print(f"DEBUG: atom[]={type(atom)}") + # DEBUG: print(f"DEBUG: atom[]='{type(atom)}'") for entry in atom.entries: - # DEBUG: print(f"DEBUG: entry[]={type(entry)}") + # DEBUG: print(f"DEBUG: entry[]='{type(entry)}'") doc = bs4.BeautifulSoup(entry.content.value, "html.parser") - # DEBUG: print(f"DEBUG: doc[]={type(doc)}") + # DEBUG: print(f"DEBUG: doc[]='{type(doc)}'") for element in doc.findAll("a"): for href in element["href"].split(","): # DEBUG: print(f"DEBUG: href[{type(href)}]={href}") @@ -432,7 +432,7 @@ def fetch_fbabot_atom(args: argparse.Namespace): # DEBUG: print("DEBUG: EXIT!") def fetch_instances(args: argparse.Namespace): - # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!") + # DEBUG: print(f"DEBUG: args[]='{type(args)}' - CALLED!") locking.acquire() # Initial fetch @@ -471,7 +471,7 @@ def fetch_instances(args: argparse.Namespace): # DEBUG: print("DEBUG: EXIT!") def fetch_federater(args: argparse.Namespace): - # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!") + # DEBUG: print(f"DEBUG: args[]='{type(args)}' - CALLED!") locking.acquire() # Fetch this URL @@ -482,7 +482,7 @@ def fetch_federater(args: argparse.Namespace): ## DEBUG: print(f"DEBUG: response.content={response.content}") reader = csv.DictReader(response.content.decode('utf-8').splitlines(), dialect='unix') #, fieldnames='domain,severity,reject_media,reject_reports,public_comment,obfuscate' - # DEBUG: print(f"DEBUG: reader[]={type(reader)}") + # DEBUG: print(f"DEBUG: reader[]='{type(reader)}'") for row in reader: if not validators.domain(row["#domain"]): print(f"WARNING: domain='{row['#domain']}' is not a valid domain - skipped!") diff --git a/fba/fba.py b/fba/fba.py index 9ea0db0..4ae3c51 100644 --- a/fba/fba.py +++ b/fba/fba.py @@ -80,7 +80,7 @@ def remove_version(software: str) -> str: # Run match() match = pattern.match(version) - # DEBUG: print(f"DEBUG: match[]={type(match)}") + # DEBUG: print(f"DEBUG: match[]='{type(match)}'") if isinstance(match, re.Match): # DEBUG: print(f"DEBUG: version='{version}' is matching pattern='{pattern}'") break @@ -170,7 +170,7 @@ def strip_until(software: str, until: str) -> str: def get_hash(domain: str) -> str: if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") @@ -179,7 +179,7 @@ def get_hash(domain: str) -> str: def log_error(domain: str, error: dict): # DEBUG: print("DEBUG: domain,error[]:", domain, type(error)) if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") elif config.get("write_error_log").lower() != "true": diff --git a/fba/federation.py b/fba/federation.py index 9f1e009..5b4c4ba 100644 --- a/fba/federation.py +++ b/fba/federation.py @@ -44,19 +44,19 @@ nodeinfo_identifier = [ def fetch_instances(domain: str, origin: str, software: str, script: str, path: str = None): # DEBUG: print(f"DEBUG: domain='{domain}',origin='{origin}',software='{software}',path='{path}' - CALLED!") if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") elif not isinstance(origin, str) and origin is not None: - raise ValueError(f"Parameter origin[]={type(origin)} is not 'str'") + raise ValueError(f"Parameter origin[]='{type(origin)}' is not 'str'") elif software is None: # DEBUG: print(f"DEBUG: software for domain='{domain}' is not set, determining ...") software = determine_software(domain, path) # DEBUG: print(f"DEBUG: Determined software='{software}' for domain='{domain}'") elif not isinstance(software, str): - raise ValueError(f"Parameter software[]={type(software)} is not 'str'") + raise ValueError(f"Parameter software[]='{type(software)}' is not 'str'") elif not isinstance(script, str): - raise ValueError(f"Parameter script[]={type(script)} is not 'str'") + raise ValueError(f"Parameter script[]='{type(script)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") @@ -104,11 +104,11 @@ def fetch_instances(domain: str, origin: str, software: str, script: str, path: def fetch_peers(domain: str, software: str) -> list: # DEBUG: print(f"DEBUG: domain({len(domain)})={domain},software={software} - CALLED!") if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") elif not isinstance(software, str) and software is not None: - raise ValueError(f"software[]={type(software)} is not 'str'") + raise ValueError(f"software[]='{type(software)}' is not 'str'") if software == "misskey": # DEBUG: print(f"DEBUG: Invoking misskey.fetch_peers({domain}) ...") @@ -176,11 +176,11 @@ def fetch_peers(domain: str, software: str) -> list: def fetch_nodeinfo(domain: str, path: str = None) -> list: # DEBUG: print(f"DEBUG: domain='{domain}',path={path} - CALLED!") if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") elif not isinstance(path, str) and path is not None: - raise ValueError(f"Parameter path[]={type(path)} is not 'str'") + raise ValueError(f"Parameter path[]='{type(path)}' is not 'str'") # DEBUG: print(f"DEBUG: Fetching nodeinfo from domain='{domain}' ...") nodeinfo = fetch_wellknown_nodeinfo(domain) @@ -236,7 +236,7 @@ def fetch_nodeinfo(domain: str, path: str = None) -> list: def fetch_wellknown_nodeinfo(domain: str) -> list: # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") @@ -290,11 +290,11 @@ def fetch_wellknown_nodeinfo(domain: str) -> list: def fetch_generator_from_path(domain: str, path: str = "/") -> str: # DEBUG: print(f"DEBUG: domain({len(domain)})={domain},path={path} - CALLED!") if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") elif not isinstance(path, str): - raise ValueError(f"path[]={type(path)} is not 'str'") + raise ValueError(f"path[]='{type(path)}' is not 'str'") elif path == "": raise ValueError("Parameter 'path' is empty") @@ -325,7 +325,7 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str: print(f"INFO: domain='{domain}' has og:site_name='{software}'") instances.set_data("detection_mode", domain, "SITE_NAME") - # DEBUG: print(f"DEBUG: software[]={type(software)}") + # DEBUG: print(f"DEBUG: software[]='{type(software)}'") if isinstance(software, str) and software == "": # DEBUG: print(f"DEBUG: Corrected empty string to None for software of domain='{domain}'") software = None @@ -333,7 +333,7 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str: # DEBUG: print(f"DEBUG: software='{software}' may contain a version number, domain='{domain}', removing it ...") software = fba.remove_version(software) - # DEBUG: print(f"DEBUG: software[]={type(software)}") + # DEBUG: print(f"DEBUG: software[]='{type(software)}'") if isinstance(software, str) and " powered by " in software: # DEBUG: print(f"DEBUG: software='{software}' has 'powered by' in it") software = fba.remove_version(fba.strip_powered_by(software)) @@ -353,11 +353,11 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str: def determine_software(domain: str, path: str = None) -> str: # DEBUG: print(f"DEBUG: domain({len(domain)})={domain},path={path} - CALLED!") if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") elif not isinstance(path, str) and path is not None: - raise ValueError(f"Parameter path[]={type(path)} is not 'str'") + raise ValueError(f"Parameter path[]='{type(path)}' is not 'str'") # DEBUG: print("DEBUG: Determining software for domain,path:", domain, path) software = None @@ -414,12 +414,12 @@ def determine_software(domain: str, path: str = None) -> str: # DEBUG: print(f"DEBUG: software='{software}' has ' see ' in it") software = fba.strip_until(software, " see ") - # DEBUG: print(f"DEBUG: software[]={type(software)}") + # DEBUG: print(f"DEBUG: software[]='{type(software)}'") if software == "": print("WARNING: tidyup.domain() left no software name behind:", domain) software = None - # DEBUG: print(f"DEBUG: software[]={type(software)}") + # DEBUG: print(f"DEBUG: software[]='{type(software)}'") if str(software) == "": # DEBUG: print(f"DEBUG: software for '{domain}' was not detected, trying generator ...") software = fetch_generator_from_path(domain) @@ -427,7 +427,7 @@ def determine_software(domain: str, path: str = None) -> str: # DEBUG: print(f"DEBUG: software='{software}' may contain a version number, domain='{domain}', removing it ...") software = fba.remove_version(software) - # DEBUG: print(f"DEBUG: software[]={type(software)}") + # DEBUG: print(f"DEBUG: software[]='{type(software)}'") if isinstance(software, str) and "powered by" in software: # DEBUG: print(f"DEBUG: software='{software}' has 'powered by' in it") software = fba.remove_version(fba.strip_powered_by(software)) @@ -436,15 +436,15 @@ def determine_software(domain: str, path: str = None) -> str: return software def find_domains(tag: bs4.element.Tag) -> list: - # DEBUG: print(f"DEBUG: tag[]={type(tag)} - CALLED!") + # DEBUG: print(f"DEBUG: tag[]='{type(tag)}' - CALLED!") if not isinstance(tag, bs4.element.Tag): - raise ValueError(f"Parameter tag[]={type(tag)} is not type of bs4.element.Tag") + raise ValueError(f"Parameter tag[]='{type(tag)}' is not type of bs4.element.Tag") elif len(tag.select("tr")) == 0: raise KeyError("No table rows found in table!") domains = list() for element in tag.select("tr"): - # DEBUG: print(f"DEBUG: element[]={type(element)}") + # DEBUG: print(f"DEBUG: element[]='{type(element)}'") if not element.find("td"): # DEBUG: print("DEBUG: Skipping element, no found") continue diff --git a/fba/helpers/tidyup.py b/fba/helpers/tidyup.py index d2d322f..e5f8eab 100644 --- a/fba/helpers/tidyup.py +++ b/fba/helpers/tidyup.py @@ -18,7 +18,7 @@ import re def reason(string: str) -> str: # DEBUG: print(f"DEBUG: string='{string}' - CALLED!") if not isinstance(string, str): - raise ValueError(f"Parameter string[]={type(string)} is not 'str'") + raise ValueError(f"Parameter string[]='{type(string)}' is not 'str'") # Strip string string = string.strip() @@ -32,7 +32,7 @@ def reason(string: str) -> str: def domain(string: str) -> str: # DEBUG: print(f"DEBUG: string='{string}' - CALLED!") if not isinstance(string, str): - raise ValueError(f"Parameter string[]={type(string)} is not 'str'") + raise ValueError(f"Parameter string[]='{type(string)}' is not 'str'") # All lower-case and strip spaces out + last dot string = string.lower().strip().rstrip(".") diff --git a/fba/instances.py b/fba/instances.py index 7e3d201..595dd03 100644 --- a/fba/instances.py +++ b/fba/instances.py @@ -74,7 +74,7 @@ def set_data(key: str, domain: str, value: any): def has_pending_instance_data(domain: str) -> bool: # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") @@ -91,7 +91,7 @@ def has_pending_instance_data(domain: str) -> bool: def update_data(domain: str): # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") elif not has_pending_instance_data(domain): @@ -146,7 +146,7 @@ def update_data(domain: str): def update_last_instance_fetch(domain: str): # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") @@ -161,7 +161,7 @@ def update_last_instance_fetch(domain: str): def update_last_blocked(domain: str): if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") @@ -177,15 +177,15 @@ def update_last_blocked(domain: str): def add(domain: str, origin: str, command: str, path: str = None): # DEBUG: print(f"DEBUG: domain='{domain}',origin='{origin}',command='{command}',path='{path}' - CALLED!") if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") elif not isinstance(origin, str) and origin is not None: - raise ValueError(f"origin[]={type(origin)} is not 'str'") + raise ValueError(f"origin[]='{type(origin)}' is not 'str'") elif origin == "": raise ValueError("Parameter 'origin' is empty") elif not isinstance(command, str): - raise ValueError(f"command[]={type(command)} is not 'str'") + raise ValueError(f"command[]='{type(command)}' is not 'str'") elif command == "": raise ValueError("Parameter 'command' is empty") elif not validators.domain(domain.split("/")[0]): @@ -241,7 +241,7 @@ def add(domain: str, origin: str, command: str, path: str = None): def update_last_nodeinfo(domain: str): # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") @@ -258,7 +258,7 @@ def update_last_nodeinfo(domain: str): def update_last_error(domain: str, error: dict): # DEBUG: print("DEBUG: domain,error[]:", domain, type(error)) if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") @@ -275,10 +275,15 @@ def update_last_error(domain: str, error: dict): # DEBUG: print(f"DEBUG: Setting last_error_details='{error.reason}'") set_data("last_status_code" , domain, error.status_code) set_data("last_error_details", domain, error.reason) - else: + elif "status_code" in error and "error_message" in error: # DEBUG: print(f"DEBUG: Setting last_error_details='{error['error_message']}'") set_data("last_status_code" , domain, error["status_code"]) set_data("last_error_details", domain, error["error_message"]) + elif "json" in error and "error" in error["json"]: + set_data("last_status_code" , domain, error["status_code"]) + set_data("last_error_details", domain, error["json"]["error"]) + else: + raise KeyError(f"Cannot handle keys in error[{type(error)}]='{error}'") # Running pending updated # DEBUG: print(f"DEBUG: Invoking update_data({domain}) ...") @@ -291,7 +296,7 @@ def update_last_error(domain: str, error: dict): def is_registered(domain: str) -> bool: # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") diff --git a/fba/network.py b/fba/network.py index 759e8e7..281afb2 100644 --- a/fba/network.py +++ b/fba/network.py @@ -44,17 +44,17 @@ exceptions = ( def post_json_api(domain: str, path: str, data: str, headers: dict = {}) -> dict: # DEBUG: print(f"DEBUG: domain='{domain}',path='{path}',data='{data}',headers()={len(headers)} - CALLED!") if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") elif not isinstance(path, str): - raise ValueError(f"path[]={type(path)} is not 'str'") + raise ValueError(f"path[]='{type(path)}' is not 'str'") elif path == "": raise ValueError("Parameter 'path' cannot be empty") elif not isinstance(data, str): - raise ValueError(f"data[]={type(data)} is not 'str'") + raise ValueError(f"data[]='{type(data)}' is not 'str'") elif not isinstance(headers, dict): - raise ValueError(f"headers[]={type(headers)} is not 'list'") + raise ValueError(f"headers[]='{type(headers)}' is not 'list'") json_reply = { "status_code": 200, @@ -93,7 +93,7 @@ def fetch_api_url(url: str, timeout: tuple) -> dict: if not isinstance(url, str): raise ValueError(f"Parameter url[]='{type(url)}' is not 'str'") elif not isinstance(timeout, tuple): - raise ValueError(f"timeout[]={type(timeout)} is not 'tuple'") + raise ValueError(f"timeout[]='{type(timeout)}' is not 'tuple'") json_reply = { "status_code": 200, @@ -123,17 +123,17 @@ def fetch_api_url(url: str, timeout: tuple) -> dict: def get_json_api(domain: str, path: str, headers: dict, timeout: tuple) -> dict: # DEBUG: print(f"DEBUG: domain='{domain}',path='{path}',timeout()={len(timeout)} - CALLED!") if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") elif not isinstance(path, str): - raise ValueError(f"path[]={type(path)} is not 'str'") + raise ValueError(f"path[]='{type(path)}' is not 'str'") elif path == "": raise ValueError("Parameter 'path' cannot be empty") elif not isinstance(headers, dict): - raise ValueError(f"headers[]={type(headers)} is not 'list'") + raise ValueError(f"headers[]='{type(headers)}' is not 'list'") elif not isinstance(timeout, tuple): - raise ValueError(f"timeout[]={type(timeout)} is not 'tuple'") + raise ValueError(f"timeout[]='{type(timeout)}' is not 'tuple'") json_reply = { "status_code": 200, @@ -222,9 +222,9 @@ def fetch_response(domain: str, path: str, headers: dict, timeout: tuple) -> req elif path == "": raise ValueError("Parameter 'path' is empty") elif not isinstance(headers, dict): - raise ValueError(f"headers[]={type(headers)} is not 'dict'") + raise ValueError(f"headers[]='{type(headers)}' is not 'dict'") elif not isinstance(timeout, tuple): - raise ValueError(f"timeout[]={type(timeout)} is not 'tuple'") + raise ValueError(f"timeout[]='{type(timeout)}' is not 'tuple'") try: # DEBUG: print(f"DEBUG: Sending GET request to '{domain}{path}' ...") @@ -243,7 +243,7 @@ def fetch_response(domain: str, path: str, headers: dict, timeout: tuple) -> req return response def json_from_response(response: requests.models.Response) -> list: - # DEBUG: print(f"DEBUG: response[]={type(response)} - CALLED!") + # DEBUG: print(f"DEBUG: response[]='{type(response)}' - CALLED!") if not isinstance(response, requests.models.Response): raise ValueError(f"Parameter response[]='{type(response)}' is not type of 'Response'") @@ -255,5 +255,5 @@ def json_from_response(response: requests.models.Response) -> list: except json.decoder.JSONDecodeError: pass - # DEBUG: print(f"DEBUG: data[]={type(data)} - EXIT!") + # DEBUG: print(f"DEBUG: data[]='{type(data)}' - EXIT!") return data diff --git a/fba/networks/friendica.py b/fba/networks/friendica.py index 2b88863..0fac01a 100644 --- a/fba/networks/friendica.py +++ b/fba/networks/friendica.py @@ -24,7 +24,7 @@ from fba.helpers import tidyup def fetch_blocks(domain: str) -> dict: # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") diff --git a/fba/networks/lemmy.py b/fba/networks/lemmy.py index 9b283c0..82078be 100644 --- a/fba/networks/lemmy.py +++ b/fba/networks/lemmy.py @@ -23,7 +23,7 @@ from fba import network def fetch_peers(domain: str) -> list: # DEBUG: print(f"DEBUG: domain({len(domain)})={domain},software='lemmy' - CALLED!") if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") diff --git a/fba/networks/mastodon.py b/fba/networks/mastodon.py index 2ce6331..7f6a0d5 100644 --- a/fba/networks/mastodon.py +++ b/fba/networks/mastodon.py @@ -59,7 +59,7 @@ language_mapping = { def fetch_blocks_from_about(domain: str) -> dict: # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") @@ -119,15 +119,15 @@ def fetch_blocks_from_about(domain: str) -> dict: def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): # DEBUG: print(f"DEBUG: domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}' - CALLED!") if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") elif not isinstance(origin, str) and origin is not None: - raise ValueError(f"Parameter origin[]={type(origin)} is not 'str'") + raise ValueError(f"Parameter origin[]='{type(origin)}' is not 'str'") elif origin == "": raise ValueError("Parameter 'origin' is empty") elif not isinstance(nodeinfo_url, str): - raise ValueError(f"Parameter nodeinfo_url[]={type(nodeinfo_url)} is not 'str'") + raise ValueError(f"Parameter nodeinfo_url[]='{type(nodeinfo_url)}' is not 'str'") elif nodeinfo_url == "": raise ValueError("Parameter 'nodeinfo_url' is empty") @@ -163,6 +163,11 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): if "error_message" in data: print(f"WARNING: Was not able to fetch domain_blocks from domain='{domain}': status_code='{data['status_code']}',error_message='{data['error_message']}'") instances.update_last_error(domain, data) + return + elif "json" in data and "error" in data["json"]: + print(f"WARNING: JSON API returned error message: '{data['json']['error']}'") + instances.update_last_error(domain, data) + return else: # Getting blocklist blocklist = data["json"] @@ -171,6 +176,7 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): print(f"INFO: Checking {len(blocklist)} entries from domain='{domain}',software='mastodon' ...") for block in blocklist: # Map block -> entry + # DEBUG: print(f"DEBUG: block[{type(block)}]='{block}'") entry = { "domain": block["domain"], "hash" : block["digest"], diff --git a/fba/networks/misskey.py b/fba/networks/misskey.py index f797912..6b24628 100644 --- a/fba/networks/misskey.py +++ b/fba/networks/misskey.py @@ -30,7 +30,7 @@ from fba.helpers import tidyup def fetch_peers(domain: str) -> list: # DEBUG: print(f"DEBUG: domain({len(domain)})={domain} - CALLED!") if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") @@ -67,7 +67,7 @@ def fetch_peers(domain: str) -> list: }), headers) # Check records - # DEBUG: print(f"DEBUG: fetched[]={type(fetched)}") + # DEBUG: print(f"DEBUG: fetched[]='{type(fetched)}'") if "error_message" in fetched: print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}") instances.update_last_error(domain, fetched) @@ -91,14 +91,14 @@ def fetch_peers(domain: str) -> list: offset = offset + step already = 0 - # DEBUG: print(f"DEBUG: rows({len(rows)})[]={type(rows)}") + # DEBUG: print(f"DEBUG: rows({len(rows)})[]='{type(rows)}'") for row in rows: # DEBUG: print(f"DEBUG: row()={len(row)}") if not "host" in row: print(f"WARNING: row()={len(row)} does not contain key 'host': {row},domain='{domain}'") continue elif not isinstance(row["host"], str): - print(f"WARNING: row[host][]={type(row['host'])} is not 'str'") + print(f"WARNING: row[host][]='{type(row['host'])}' is not 'str'") continue elif blacklist.is_blacklisted(row["host"]): # DEBUG: print(f"DEBUG: row[host]='{row['host']}' is blacklisted. domain='{domain}'") @@ -127,7 +127,7 @@ def fetch_peers(domain: str) -> list: def fetch_blocks(domain: str) -> dict: # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") @@ -171,7 +171,7 @@ def fetch_blocks(domain: str) -> dict: "offset" : offset - 1 }), headers) - # DEBUG: print(f"DEBUG: fetched[]={type(fetched)}") + # DEBUG: print(f"DEBUG: fetched[]='{type(fetched)}'") if "error_message" in fetched: print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}") instances.update_last_error(domain, fetched) @@ -240,7 +240,7 @@ def fetch_blocks(domain: str) -> dict: "offset" : offset - 1 }), headers) - # DEBUG: print(f"DEBUG: fetched[]={type(fetched)}") + # DEBUG: print(f"DEBUG: fetched[]='{type(fetched)}'") if "error_message" in fetched: print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}") instances.update_last_error(domain, fetched) diff --git a/fba/networks/peertube.py b/fba/networks/peertube.py index 44c8c07..5eb7946 100644 --- a/fba/networks/peertube.py +++ b/fba/networks/peertube.py @@ -22,7 +22,7 @@ from fba import network def fetch_peers(domain: str) -> list: print(f"DEBUG: domain({len(domain)})={domain},software='peertube' - CALLED!") if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") diff --git a/fba/networks/pleroma.py b/fba/networks/pleroma.py index 8941b8a..e15ea00 100644 --- a/fba/networks/pleroma.py +++ b/fba/networks/pleroma.py @@ -27,15 +27,15 @@ from fba.helpers import tidyup def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): # DEBUG: print(f"DEBUG: domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}' - CALLED!") if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") elif not isinstance(origin, str) and origin is not None: - raise ValueError(f"Parameter origin[]={type(origin)} is not 'str'") + raise ValueError(f"Parameter origin[]='{type(origin)}' is not 'str'") elif origin == "": raise ValueError("Parameter 'origin' is empty") elif not isinstance(nodeinfo_url, str): - raise ValueError(f"Parameter nodeinfo_url[]={type(nodeinfo_url)} is not 'str'") + raise ValueError(f"Parameter nodeinfo_url[]='{type(nodeinfo_url)}' is not 'str'") elif nodeinfo_url == "": raise ValueError("Parameter 'nodeinfo_url' is empty")