From a436e35a18dc0a69f7d04f326865943fd331c719 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Sat, 7 Oct 2023 14:42:50 +0200 Subject: [PATCH] Continued: - moved fba/deprecated.py to deprecated/commands.py - fixed many issues pylint has reported --- fba/deprecated.py => deprecated/commands.py | 0 fba/commands.py | 28 ++++++++++----------- fba/helpers/processing.py | 12 ++++----- fba/http/federation.py | 4 +-- fba/http/network.py | 2 +- fba/models/instances.py | 4 +-- fba/networks/mastodon.py | 9 +++---- fba/networks/misskey.py | 4 +-- 8 files changed, 31 insertions(+), 32 deletions(-) rename fba/deprecated.py => deprecated/commands.py (100%) diff --git a/fba/deprecated.py b/deprecated/commands.py similarity index 100% rename from fba/deprecated.py rename to deprecated/commands.py diff --git a/fba/commands.py b/fba/commands.py index 5589af7..2142bcc 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -383,7 +383,7 @@ def fetch_blocks(args: argparse.Namespace) -> int: obfuscated = obfuscated + 1 # Some friendica servers also obscure domains without hash - row = instances.deobfuscate("*", block["blocked"], block["hash"] if "hash" in block else None) + row = instances.deobfuscate("*", block["blocked"], block["digest"] if "digest" in block else None) logger.debug("row[]='%s'", type(row)) if row is None: @@ -400,7 +400,7 @@ def fetch_blocks(args: argparse.Namespace) -> int: obfuscated = obfuscated + 1 # Some obscure them with question marks, not sure if that's dependent on version or not - row = instances.deobfuscate("?", block["blocked"], block["hash"] if "hash" in block else None) + row = instances.deobfuscate("?", block["blocked"], block["digest"] if "digest" in block else None) logger.debug("row[]='%s'", type(row)) if row is None: @@ -541,7 +541,7 @@ def fetch_observer(args: argparse.Namespace) -> int: for item in items: logger.debug("item[]='%s'", type(item)) domain = item.decode_contents() - domain = tidyup.domain(domain) if domain != None and domain != "" else None + domain = tidyup.domain(domain) if domain not in [None, ""] else None logger.debug("domain='%s' - AFTER!", domain) if domain is None or domain == "": @@ -799,7 +799,7 @@ def fetch_fba_rss(args: argparse.Namespace) -> int: for item in rss.items: logger.debug("item[%s]='%s'", type(item), item) domain = item.link.split("=")[1] - domain = tidyup.domain(domain) if domain != None and domain != "" else None + domain = tidyup.domain(domain) if domain not in[None, ""] else None logger.debug("domain='%s' - AFTER!", domain) if domain is None or domain == "": @@ -883,7 +883,7 @@ def fetch_fbabot_atom(args: argparse.Namespace) -> int: logger.debug("element[]='%s'", type(element)) for href in element["href"].split(","): logger.debug("href[%s]='%s' - BEFORE!", type(href), href) - domain = tidyup.domain(href) if href != None and href != "" else None + domain = tidyup.domain(href) if href not in [None, ""] else None logger.debug("domain='%s' - AFTER!", domain) if domain is None or domain == "": @@ -1078,7 +1078,7 @@ def fetch_txt(args: argparse.Namespace) -> int: logger.info("Processing %d domains ...", len(domains)) for domain in domains: logger.debug("domain='%s' - BEFORE!", domain) - domain = tidyup.domain(domain) if domain != None and domain != "" else None + domain = tidyup.domain(domain) if domain not in[None, ""] else None logger.debug("domain='%s' - AFTER!", domain) if domain is None or domain == "": @@ -1134,7 +1134,7 @@ def fetch_fedipact(args: argparse.Namespace) -> int: logger.info("Checking %d row(s) ...", len(rows)) for row in rows: logger.debug("row[]='%s'", type(row)) - domain = tidyup.domain(row.contents[0]) if row.contents[0] != None and row.contents[0] != "" else None + domain = tidyup.domain(row.contents[0]) if row.contents[0] not in [None, ""] else None logger.debug("domain='%s' - AFTER!", domain) if domain is None or domain == "": @@ -1331,7 +1331,7 @@ def recheck_obfuscation(args: argparse.Namespace) -> int: elif block["blocked"].find("*") >= 0 or block["blocked"].find("?") >= 0: logger.debug("block='%s' is obfuscated.", block["blocked"]) obfuscated = obfuscated + 1 - blocked = utils.deobfuscate(block["blocked"], row["domain"], block["hash"] if "hash" in block else None) + blocked = utils.deobfuscate(block["blocked"], row["domain"], block["digest"] if "digest" in block else None) elif not domain_helper.is_wanted(block["blocked"]): logger.debug("blocked='%s' is not wanted - SKIPPED!", block["blocked"]) continue @@ -1432,7 +1432,7 @@ def fetch_fedilist(args: argparse.Namespace) -> int: continue logger.debug("row[hostname]='%s' - BEFORE!", row["hostname"]) - domain = tidyup.domain(row["hostname"]) if row["hostname"] != None and row["hostname"] != "" else None + domain = tidyup.domain(row["hostname"]) if row["hostname"] not in [None, ""] else None logger.debug("domain='%s' - AFTER!", domain) if domain is None or domain == "": @@ -1572,7 +1572,7 @@ def fetch_instances_social(args: argparse.Namespace) -> int: logger.info("Checking %d row(s) ...", len(rows)) for row in rows: logger.debug("row[]='%s'", type(row)) - domain = tidyup.domain(row["name"]) if row["name"] != None and row["name"] != "" else None + domain = tidyup.domain(row["name"]) if row["name"] not in [None, ""] else None logger.debug("domain='%s' - AFTER!", domain) if domain is None and domain == "": @@ -1644,7 +1644,7 @@ def fetch_relays(args: argparse.Namespace) -> int: instances.set_last_instance_fetch(row["domain"]) instances.update(row["domain"]) continue - elif not "json" in raw: + elif "json" not in raw: logger.warning("raw()=%d does not contain key 'json' in response - SKIPPED!", len(raw)) continue elif not "metadata" in raw["json"]: @@ -1700,7 +1700,7 @@ def fetch_relays(args: argparse.Namespace) -> int: continue logger.debug("domain='%s' - BEFORE!", domain) - domain = tidyup.domain(domain) if domain != None and domain != "" else None + domain = tidyup.domain(domain) if domain not in[None, ""] else None logger.debug("domain='%s' - AFTER!", domain) if domain is None or domain == "": @@ -1741,7 +1741,7 @@ def fetch_relays(args: argparse.Namespace) -> int: domain = components.netloc.lower().split(":")[0] logger.debug("domain='%s' - BEFORE!", domain) - domain = tidyup.domain(domain) if domain != None and domain != "" else None + domain = tidyup.domain(domain) if domain not in[None, ""] else None logger.debug("domain='%s' - AFTER!", domain) if domain is None or domain == "": @@ -1764,7 +1764,7 @@ def fetch_relays(args: argparse.Namespace) -> int: logger.debug("Checking %d peer(s) row[domain]='%s' ...", len(raw["json"]["metadata"]["peers"]), row["domain"]) for domain in raw["json"]["metadata"]["peers"]: logger.debug("domain='%s' - BEFORE!", domain) - domain = tidyup.domain(domain) if domain != None and domain != "" else None + domain = tidyup.domain(domain) if domain not in[None, ""] else None logger.debug("domain='%s' - AFTER!", domain) if domain is None or domain == "": diff --git a/fba/helpers/processing.py b/fba/helpers/processing.py index 0face7f..d702807 100644 --- a/fba/helpers/processing.py +++ b/fba/helpers/processing.py @@ -147,11 +147,11 @@ def csv_block(blocker: str, url: str, command: str): reject_media = reject_reports = False if "#domain" in row: - domain = tidyup.domain(row["#domain"]) if row["#domain"] != None and row["#domain"] != "" else None + domain = tidyup.domain(row["#domain"]) if row["#domain"] not in [None, ""] else None elif "domain" in row: - domain = tidyup.domain(row["domain"]) if row["domain"] != None and row["domain"] != "" else None + domain = tidyup.domain(row["domain"]) if row["domain"] not in [None, ""] else None elif "Domain" in row: - domain = tidyup.domain(row["Domain"]) if row["Domain"] != None and row["Domain"] != "" else None + domain = tidyup.domain(row["Domain"]) if row["Domain"] not in [None, ""] else None else: logger.warning("row='%s' does not contain domain column - SKIPPED!", row) continue @@ -165,11 +165,11 @@ def csv_block(blocker: str, url: str, command: str): severity = "reject" if "reason" in row: - reason = tidyup.reason(row["reason"]) if row["reason"] != None and row["reason"] != "" else None + reason = tidyup.reason(row["reason"]) if row["reason"] not in [None, ""] else None elif "comment" in row: - reason = tidyup.reason(row["comment"]) if row["comment"] != None and row["comment"] != "" else None + reason = tidyup.reason(row["comment"]) if row["comment"] not in [None, ""] else None elif "Comment" in row: - reason = tidyup.reason(row["Comment"]) if row["Comment"] != None and row["Comment"] != "" else None + reason = tidyup.reason(row["Comment"]) if row["Comment"] not in [None, ""] else None else: logger.debug("row='%s' has no reason/comment key provided", row) diff --git a/fba/http/federation.py b/fba/http/federation.py index c18d543..46c1da6 100644 --- a/fba/http/federation.py +++ b/fba/http/federation.py @@ -87,7 +87,7 @@ def fetch_instances(domain: str, origin: str, software: str, command: str, path: logger.debug("software='%s'", software) if software_helper.is_relay(software): logger.debug("software='%s' is a relay software - EXIT!", software) - return list() + return logger.debug("Updating last_instance_fetch for domain='%s' ...", domain) instances.set_last_instance_fetch(domain) @@ -618,7 +618,7 @@ def fetch_blocks(domain: str) -> list: blocklist.append({ "blocker" : domain, "blocked" : block["domain"], - "hash" : block["digest"] if "digest" in block else None, + "digest" : block["digest"] if "digest" in block else None, "reason" : reason, "block_level": blocks.alias_block_level(block["severity"]), }) diff --git a/fba/http/network.py b/fba/http/network.py index 982a1b8..3b5542e 100644 --- a/fba/http/network.py +++ b/fba/http/network.py @@ -15,10 +15,10 @@ # along with this program. If not, see . import logging +import time import reqto import requests -import time import urllib3 from fba import utils diff --git a/fba/models/instances.py b/fba/models/instances.py index 4a10f8b..f8e28b3 100644 --- a/fba/models/instances.py +++ b/fba/models/instances.py @@ -410,9 +410,9 @@ def set_last_response_time(domain: str, response_time: float): domain_helper.raise_on(domain) if not isinstance(response_time, float): - raise ValueException(f"response_time[]='{type(response_time)}' is not of type 'float'") + raise ValueError(f"response_time[]='{type(response_time)}' is not of type 'float'") elif response_time < 0: - raise ValueException(f"response_time={response_time} is below zero") + raise ValueError(f"response_time={response_time} is below zero") # Set timestamp _set_data("last_response_time", domain, response_time) diff --git a/fba/networks/mastodon.py b/fba/networks/mastodon.py index 88fd35f..85cd5c7 100644 --- a/fba/networks/mastodon.py +++ b/fba/networks/mastodon.py @@ -23,7 +23,6 @@ from fba.helpers import config from fba.helpers import domain as domain_helper from fba.helpers import tidyup -from fba.http import federation from fba.http import network from fba.models import blocks @@ -116,7 +115,7 @@ def fetch_blocks_from_about(domain: str) -> dict: # replaced find_next_siblings with find_all_next to account for instances that e.g. hide lists in dropdown menu for line in header.find_all_next("table")[0].find_all("tr")[1:]: domain = line.find("span").text - hash = line.find("span")["title"][9:] + digest = line.find("span")["title"][9:] reason = line.find_all("td")[1].text logger.debug("domain='%s',reason='%s' - BEFORE!", domain, reason) @@ -128,10 +127,10 @@ def fetch_blocks_from_about(domain: str) -> dict: logger.warning("domain='%s' is empty,line='%s' - SKIPPED!", domain, line) continue - logger.debug("Appending domain='%s',hash='%s',reason='%s' to blocklist header_text='%s' ...", domain, hash, reason, blocklist) + logger.debug("Appending domain='%s',digest='%s',reason='%s' to blocklist header_text='%s' ...", domain, digest, reason, blocklist) blocklist[header_text].append({ "domain": domain, - "hash" : hash, + "digest": digest, "reason": reason, }) else: @@ -188,7 +187,7 @@ def fetch_blocks(domain: str) -> list: blocklist.append({ "blocker" : domain, "blocked" : block["domain"], - "hash" : block["digest"] if "digest" in block else None, + "digest" : block["digest"] if "digest" in block else None, "reason" : reason, "block_level": blocks.alias_block_level(block["severity"]), }) diff --git a/fba/networks/misskey.py b/fba/networks/misskey.py index 7217b86..b57c252 100644 --- a/fba/networks/misskey.py +++ b/fba/networks/misskey.py @@ -203,8 +203,8 @@ def fetch_blocks(domain: str) -> list: logger.warning("instance(%d)='%s' has no key 'host' - SKIPPED!", len(instance), instance) continue elif instance["host"] is None or instance["host"] == "": - logger.debug("instance[host]='%s' is None or empty - SKIPPED!", instance["host"]) - continue + logger.debug("instance[host]='%s' is None or empty - SKIPPED!", instance["host"]) + continue logger.debug("instance[host]='%s' - BEFORE!", instance["host"]) blocked = tidyup.domain(instance["host"]) -- 2.39.5