From: Roland Häder Date: Wed, 21 Jun 2023 17:56:35 +0000 (+0200) Subject: Continued: X-Git-Url: https://git.mxchange.org/?a=commitdiff_plain;h=b7cc4315e607c1dc0d4e9473dbd98add2644ec5d;p=fba.git Continued: - also rewrote these logger messages to lazy '%' way - fixed some unused-variable/import (pylint) - fixed some syntax errors --- diff --git a/fba/commands.py b/fba/commands.py index d1cf4c5..0057fa4 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -79,10 +79,10 @@ def fetch_bkali(args: argparse.Namespace) -> int: logger.debug("fetched[]='%s'", type(fetched)) if "error_message" in fetched: - logger.warning(f"post_json_api() for 'gql.api.bka.li' returned error message: {fetched['error_message']}") + logger.warning("post_json_api() for 'gql.api.bka.li' returned error message='%s", fetched['error_message']) return 100 elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]: - logger.warning(f"post_json_api() returned error: {fetched['error']['message']}") + logger.warning("post_json_api() returned error: '%s", fetched['error']['message']) return 101 rows = fetched["json"] @@ -127,7 +127,7 @@ def fetch_bkali(args: argparse.Namespace) -> int: logger.debug("Invoking cookies.clear(%s) ...", domain) cookies.clear(domain) except network.exceptions as exception: - logger.warning(f"Exception '{type(exception)}' during fetching instances (fetch_bkali) from domain='{domain}'") + logger.warning("Exception '%s' during fetching instances (fetch_bkali) from domain='%s'", type(exception), domain) instances.set_last_error(domain, exception) logger.debug("Success - EXIT!") @@ -138,13 +138,13 @@ def fetch_blocks(args: argparse.Namespace): if args.domain is not None and args.domain != "": logger.debug(f"args.domain='{args.domain}' - checking ...") if not validators.domain(args.domain): - logger.warning(f"domain='{args.domain}' is not valid.") + logger.warning("args.domain='%s' is not valid.", args.domain) return elif blacklist.is_blacklisted(args.domain): - logger.warning(f"domain='{args.domain}' is blacklisted, won't check it!") + logger.warning("args.domain='%s' is blacklisted, won't check it!", args.domain) return elif not instances.is_registered(args.domain): - logger.warning(f"domain='{args.domain}' is not registered, please run ./utils.py fetch_instances {args.domain} first.") + logger.warning("args.domain='%s' is not registered, please run ./utils.py fetch_instances '%s' first.", args.domain, args.domain) return locking.acquire() @@ -181,8 +181,8 @@ def fetch_blocks(args: argparse.Namespace): elif nodeinfo_url is None or nodeinfo_url == "": logger.debug(f"blocker='{blocker}',software='{software}' has empty nodeinfo_url") continue - elif blacklist.is_blacklisted(blocker): - logger.warning(f"blocker='{blocker}' is blacklisted now!") + elif not utils.is_domain_wanted(blocker): + logger.warning("blocker='%s' is not wanted - SKIPPED!", blocker) continue logger.debug(f"blocker='{blocker}'") @@ -235,7 +235,7 @@ def fetch_blocks(args: argparse.Namespace): logger.debug("row[]='%s'", type(row)) if row is None: - logger.warning(f"Cannot deobsfucate blocked='{blocked}',blocker='{blocker}',software='{software}' - SKIPPED!") + logger.warning("Cannot deobsfucate blocked='%s',blocker='%s',software='%s' - SKIPPED!", blocked, blocker, software) continue blocked = row[0] @@ -247,7 +247,7 @@ def fetch_blocks(args: argparse.Namespace): logger.debug("row[]='%s'", type(row)) if row is None: - logger.warning(f"Cannot deobsfucate blocked='{blocked}',blocker='{blocker}',software='{software}' - SKIPPED!") + logger.warning("Cannot deobsfucate blocked='%s',blocker='%s',software='%s' - SKIPPED!", blocked, blocker, software) continue blocked = row[0] @@ -343,12 +343,12 @@ def fetch_observer(args: argparse.Namespace): try: logger.debug(f"Fetching table data for software='{software}' ...") raw = utils.fetch_url(f"https://fediverse.observer/app/views/tabledata.php?software={software}", network.web_headers, (config.get("connection_timeout"), config.get("read_timeout"))).text - logger.debug(f"raw[{type(raw)}]()={len(raw)}") + logger.debug("raw[%s]()=%d", type(raw), len(raw)) doc = bs4.BeautifulSoup(raw, features='html.parser') logger.debug("doc[]='%s'", type(doc)) except network.exceptions as exception: - logger.warning(f"Cannot fetch software='{software}' from fediverse.observer: '{type(exception)}'") + logger.warning("Cannot fetch software='%s' from fediverse.observer: '%s'", software, type(exception)) continue items = doc.findAll("a", {"class": "url"}) @@ -405,13 +405,13 @@ def fetch_todon_wiki(args: argparse.Namespace): if not instances.is_registered(blocked): try: - logger.info("Fetching instances from domain='%s' ...", row['domain']) + logger.info("Fetching instances from domain='%s' ...", blocked) federation.fetch_instances(blocked, 'chaos.social', None, inspect.currentframe().f_code.co_name) - logger.debug("Invoking cookies.clear(%s) ...", row['domain']) + logger.debug("Invoking cookies.clear(%s) ...", blocked) cookies.clear(blocked) except network.exceptions as exception: - logger.warning(f"Exception '{type(exception)}' during fetching instances (fetch_cs) from domain='{row['domain']}'") + logger.warning("Exception '%s' during fetching instances (fetch_cs) from blocked='%s'", type(exception), blocked) instances.set_last_error(blocked, exception) if blocks.is_instance_blocked("todon.eu", blocked, block_level): @@ -478,17 +478,17 @@ def fetch_cs(args: argparse.Namespace): logger.debug(f"row='{row}'") if not instances.is_registered(row["domain"]): try: - logger.info("Fetching instances from domain='%s' ...", row['domain']) + logger.info("Fetching instances from domain='%s' ...", row["domain"]) federation.fetch_instances(row["domain"], 'chaos.social', None, inspect.currentframe().f_code.co_name) - logger.debug("Invoking cookies.clear(%s) ...", row['domain']) + logger.debug("Invoking cookies.clear(%s) ...", row["domain"]) cookies.clear(row["domain"]) except network.exceptions as exception: - logger.warning(f"Exception '{type(exception)}' during fetching instances (fetch_cs) from domain='{row['domain']}'") + logger.warning("Exception '%s' during fetching instances (fetch_cs) from row[domain]='%s'", type(exception), row["domain"]) instances.set_last_error(row["domain"], exception) if not blocks.is_instance_blocked('chaos.social', row["domain"], block_level): - logger.debug(f"domain='{row['domain']}',block_level='{block_level}' blocked by chaos.social, adding ...") + logger.debug("domain='%s',block_level='%s' blocked by chaos.social, adding ...", row["domain"], block_level) blocks.add_instance('chaos.social', row["domain"], row["reason"], block_level) logger.debug("Invoking commit() ...") @@ -539,7 +539,7 @@ def fetch_fba_rss(args: argparse.Namespace): logger.debug("Invoking cookies.clear(%s) ...", domain) cookies.clear(domain) except network.exceptions as exception: - logger.warning(f"Exception '{type(exception)}' during fetching instances (fetch_fba_rss) from domain='{domain}'") + logger.warning("Exception '%s' during fetching instances (fetch_fba_rss) from domain='%s'", type(exception), domain) instances.set_last_error(domain, exception) logger.debug("EXIT!") @@ -595,7 +595,7 @@ def fetch_fbabot_atom(args: argparse.Namespace): logger.debug("Invoking cookies.clear(%s) ...", domain) cookies.clear(domain) except network.exceptions as exception: - logger.warning(f"Exception '{type(exception)}' during fetching instances (fetch_fbabot_atom) from domain='{domain}'") + logger.warning("Exception '%s' during fetching instances (fetch_fbabot_atom) from domain='%s'", type(exception), domain) instances.set_last_error(domain, exception) logger.debug("EXIT!") @@ -612,7 +612,7 @@ def fetch_instances(args: argparse.Namespace) -> int: logger.debug(f"Invoking cookies.clear({args.domain}) ...") cookies.clear(args.domain) except network.exceptions as exception: - logger.warning(f"Exception '{type(exception)}' during fetching instances (fetch_instances) from args.domain='{args.domain}'") + logger.warning("Exception '%s' during fetching instances (fetch_instances) from args.domain='%s'", type(exception), args.domain) instances.set_last_error(args.domain, exception) return 100 @@ -641,7 +641,7 @@ def fetch_instances(args: argparse.Namespace) -> int: logger.debug(f"Invoking cookies.clear({row[0]}) ...") cookies.clear(row[0]) except network.exceptions as exception: - logger.warning(f"Exception '{type(exception)}' during fetching instances (fetch_instances) from domain='{row[0]}'") + logger.warning("Exception '%s' during fetching instances (fetch_instances) from row[0]='%s'", type(exception), row[0]) instances.set_last_error(row[0], exception) logger.debug("Success - EXIT!") diff --git a/fba/helpers/cache.py b/fba/helpers/cache.py index 97d5a7d..a58d173 100644 --- a/fba/helpers/cache.py +++ b/fba/helpers/cache.py @@ -41,7 +41,7 @@ def set_all(key: str, rows: list, value: any): logger.debug(f"Setting key='{key}',sub[{type(sub)}]='{sub}',value[]='{type(value)}'") _cache[key][sub[0]] = value else: - logger.warning(f"Unsupported type sub[]='{type(sub)}'") + logger.warning("Unsupported type sub[]='%s'", type(sub)) logger.debug("EXIT!") diff --git a/fba/helpers/version.py b/fba/helpers/version.py index 55d65ad..f15ef36 100644 --- a/fba/helpers/version.py +++ b/fba/helpers/version.py @@ -32,9 +32,9 @@ patterns = [ ] def remove(software: str) -> str: - logger.debug(f"software='{software}' - CALLED!") + logger.debug("software='%s' - CALLED!", software) if "." not in software and " " not in software: - logger.warning(f"software='{software}' does not contain a version number.") + logger.warning("software='%s' does not contain a version number.", software) return software temp = software @@ -70,7 +70,7 @@ def remove(software: str) -> str: logger.debug(f"version[{type(version)}]='{version}',match='{match}'") if not isinstance(match, re.Match): - logger.warning(f"version='{version}' does not match regex, leaving software='{software}' untouched.") + logger.warning("version='%s' does not match regex, leaving software='%s' untouched.", version, software) return software logger.debug(f"Found valid version number: '{version}', removing it ...") @@ -82,17 +82,17 @@ def remove(software: str) -> str: logger.debug(f"software='{software}' contains word ' version'") software = strip_until(software, " version") - logger.debug(f"software='{software}' - EXIT!") + logger.debug("software='%s' - EXIT!", software) return software def strip_powered_by(software: str) -> str: - logger.debug(f"software='{software}' - CALLED!") + logger.debug("software='%s' - CALLED!", software) if not isinstance(software, str): raise ValueError(f"Parameter software[]='{type(software)}' is not 'str'") elif software == "": raise ValueError("Parameter 'software' is empty") elif "powered by" not in software: - logger.warning(f"Cannot find 'powered by' in software='{software}'!") + logger.warning("Cannot find 'powered by' in software='%s'!", software) return software start = software.find("powered by ") @@ -103,17 +103,17 @@ def strip_powered_by(software: str) -> str: software = strip_until(software, " - ") - logger.debug(f"software='{software}' - EXIT!") + logger.debug("software='%s' - EXIT!", software) return software def strip_hosted_on(software: str) -> str: - logger.debug(f"software='{software}' - CALLED!") + logger.debug("software='%s' - CALLED!", software) if not isinstance(software, str): raise ValueError(f"Parameter software[]='{type(software)}' is not 'str'") elif software == "": raise ValueError("Parameter 'software' is empty") elif "hosted on" not in software: - logger.warning(f"Cannot find 'hosted on' in '{software}'!") + logger.warning("Cannot find 'hosted on' in sofware='%s'!", software) return software end = software.find("hosted on ") @@ -124,7 +124,7 @@ def strip_hosted_on(software: str) -> str: software = strip_until(software, " - ") - logger.debug(f"software='{software}' - EXIT!") + logger.debug("software='%s' - EXIT!", software) return software def strip_until(software: str, until: str) -> str: @@ -138,7 +138,7 @@ def strip_until(software: str, until: str) -> str: elif until == "": raise ValueError("Parameter 'until' is empty") elif not until in software: - logger.warning(f"Cannot find '{until}' in '{software}'!") + logger.warning("Cannot find until='%s' in software='%s'!", until, software) return software # Next, strip until part @@ -148,5 +148,5 @@ def strip_until(software: str, until: str) -> str: if end > 0: software = software[0:end].strip() - logger.debug(f"software='{software}' - EXIT!") + logger.debug("software='%s' - EXIT!", software) return software diff --git a/fba/http/federation.py b/fba/http/federation.py index 7968384..8cfc02a 100644 --- a/fba/http/federation.py +++ b/fba/http/federation.py @@ -23,7 +23,6 @@ import validators from fba import csrf from fba import utils -from fba.helpers import blacklist from fba.helpers import config from fba.helpers import tidyup from fba.helpers import version @@ -76,8 +75,8 @@ def fetch_instances(domain: str, origin: str, software: str, command: str, path: try: software = determine_software(domain, path) except network.exceptions as exception: - logger.debug(f"Exception '{type(exception)}' during determining software type") - pass + logger.warning("Exception '%s' during determining software type", type(exception)) + instances.set_last_error(domain, exception) logger.debug(f"Determined software='{software}' for domain='{domain}'") elif not isinstance(software, str): @@ -381,7 +380,7 @@ def fetch_wellknown_nodeinfo(domain: str) -> dict: return data def fetch_generator_from_path(domain: str, path: str = "/") -> str: - logger.debug(f"domain({len(domain)})='{domain}',path='{path}' - CALLED!") + logger.debug("domain(%d)='%s',path='%s' - CALLED!", len(domain), domain, path) if not isinstance(domain, str): raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": @@ -455,11 +454,11 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str: logger.debug(f"software='{software}' has ' see ' in it") software = version.strip_until(software, " see ") - logger.debug(f"software='{software}' - EXIT!") + logger.debug("software='%s' - EXIT!", software) return software def determine_software(domain: str, path: str = None) -> str: - logger.debug(f"domain({len(domain)})='{domain}',path='{path}' - CALLED!") + logger.debug("domain(%d)='%s',path='%s' - CALLED!", len(domain), domain, path) if not isinstance(domain, str): raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": @@ -508,7 +507,7 @@ def determine_software(domain: str, path: str = None) -> str: logger.debug("Returning None - EXIT!") return None - sofware = tidyup.domain(software) + software = tidyup.domain(software) logger.debug("sofware after tidyup.domain():", software) if software in ["akkoma", "rebased", "akkounfucked", "ched"]: diff --git a/fba/models/instances.py b/fba/models/instances.py index 4238b25..d3f7f43 100644 --- a/fba/models/instances.py +++ b/fba/models/instances.py @@ -223,14 +223,14 @@ def add(domain: str, origin: str, command: str, path: str = None, software: str logger.debug("domain,origin,command,path:", domain, origin, command, path) software = federation.determine_software(domain, path) except network.exceptions as exception: - logger.warning(f"Exception '{type(exception)}' during determining software type") + logger.warning("Exception '%s' during determining software type, domain='%s'", type(exception), domain) set_last_error(domain, exception) logger.debug("Determined software:", software) if software == "lemmy" and domain.find("/c/") > 0: domain = domain.split("/c/")[0] if is_registered(domain): - logger.warning(f"domain='{domain}' already registered after cutting off user part. - EXIT!") + logger.warning("domain='%s' already registered after cutting off user part. - EXIT!", domain) return logger.info("Adding instance domain='%s' (origin='%s',software='%s')", domain, origin, software) diff --git a/fba/networks/friendica.py b/fba/networks/friendica.py index 3714119..39b81d3 100644 --- a/fba/networks/friendica.py +++ b/fba/networks/friendica.py @@ -21,7 +21,6 @@ import validators from fba import utils -from fba.helpers import blacklist from fba.helpers import config from fba.helpers import tidyup diff --git a/fba/networks/lemmy.py b/fba/networks/lemmy.py index 12c40d3..a26fe15 100644 --- a/fba/networks/lemmy.py +++ b/fba/networks/lemmy.py @@ -24,7 +24,6 @@ from fba import csrf from fba import database from fba import utils -from fba.helpers import blacklist from fba.helpers import config from fba.helpers import tidyup @@ -155,14 +154,6 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): try: # json endpoint for newer mastodongs found_blocks = list() - blocklist = list() - - rows = { - "reject" : [], - "media_removal" : [], - "followers_only": [], - "report_removal": [], - } logger.debug(f"Fetching /instances from domain='{domain}'") response = network.fetch_response( diff --git a/fba/utils.py b/fba/utils.py index 7850b39..d98a8e1 100644 --- a/fba/utils.py +++ b/fba/utils.py @@ -116,7 +116,7 @@ def process_domain(domain: str, blocker: str, command: str) -> bool: logger.debug(f"row[{type(row)}]='{row}'") if row is None: - logger.warning(f"Cannot de-obfucate domain='{domain}' - SKIPPED!") + logger.warning("Cannot de-obfucate domain='%s' - SKIPPED!", domain) return False logger.debug(f"domain='{domain}' de-obscured to '{row[0]}'") @@ -127,7 +127,7 @@ def process_domain(domain: str, blocker: str, command: str) -> bool: logger.debug(f"row[{type(row)}]='{row}'") if row is None: - logger.warning(f"Cannot de-obfucate domain='{domain}' - SKIPPED!") + logger.warning("Cannot de-obfucate domain='%s' - SKIPPED!", domain) return False logger.debug(f"domain='{domain}' de-obscured to '{row[0]}'") @@ -149,7 +149,7 @@ def process_domain(domain: str, blocker: str, command: str) -> bool: logger.debug("Invoking cookies.clear(%s) ...", domain) cookies.clear(domain) except network.exceptions as exception: - logger.warning(f"Exception '{type(exception)}' during fetching instances (fetch_oliphant) from domain='{domain}'") + logger.warning("Exception '%s' during fetching instances (fetch_oliphant) from domain='%s'", type(exception), domain) instances.set_last_error(domain, exception) logger.debug(f"processed='{processed}' - EXIT!")