From dbbb41dc9aae1607165ed190df4ccdfd8108cbdc Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Thu, 31 Aug 2023 07:53:16 +0200 Subject: [PATCH] Continued: - during fetching block lists or peers, filter out already blacklisted entries which will keep the list variable's memory usage smaller --- fba/networks/friendica.py | 3 +++ fba/networks/lemmy.py | 8 ++++++++ fba/networks/mastodon.py | 3 +++ fba/networks/misskey.py | 17 +++++++++++++++-- fba/networks/peertube.py | 9 +++++++-- fba/networks/pleroma.py | 9 +++++++++ 6 files changed, 45 insertions(+), 4 deletions(-) diff --git a/fba/networks/friendica.py b/fba/networks/friendica.py index 046bf90..f30f126 100644 --- a/fba/networks/friendica.py +++ b/fba/networks/friendica.py @@ -85,6 +85,9 @@ def fetch_blocks(domain: str) -> list: if blocked == "": logger.debug("line[]='%s' returned empty blocked domain - SKIPPED!", type(line)) continue + elif not domain_helper.is_wanted(blocked): + logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked) + continue logger.debug("Appending blocked='%s',reason='%s'", blocked, reason) blocklist.append({ diff --git a/fba/networks/lemmy.py b/fba/networks/lemmy.py index 07a854a..efc85de 100644 --- a/fba/networks/lemmy.py +++ b/fba/networks/lemmy.py @@ -206,6 +206,9 @@ def fetch_blocks(domain: str) -> list: if blocked == "": logger.warning("blocked='%s' is empty after tidyup.domain() - SKIPPED!", tag.contents[0]) continue + elif not domain_helper.is_wanted(blocked): + logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked) + continue logger.debug("Appending blocker='%s',blocked='%s',block_level='reject' ...", domain, blocked) blocklist.append({ @@ -267,6 +270,9 @@ def fetch_instances(domain: str, origin: str) -> list: if peer == "": logger.debug("peer is empty - SKIPPED!") continue + elif not domain_helper.is_wanted(peer): + logger.debug("peer='%s' is not wanted - SKIPPED!", peer) + continue elif peer in peers: logger.debug("peer='%s' already added - SKIPPED!", peer) continue @@ -364,6 +370,8 @@ def parse_script(doc: bs4.BeautifulSoup, only: str = None) -> list: if peer == "": logger.debug("peer is empty - SKIPPED!") continue + elif not domain_helper.is_wanted(peer): + logger.debug("peer='%s' is not wanted - SKIPPED!", peer) elif peer in peers: logger.debug("peer='%s' already added - SKIPPED!", peer) continue diff --git a/fba/networks/mastodon.py b/fba/networks/mastodon.py index bea34c9..cda3cc4 100644 --- a/fba/networks/mastodon.py +++ b/fba/networks/mastodon.py @@ -159,6 +159,9 @@ def fetch_blocks(domain: str) -> list: elif "domain" not in block: logger.warning("block()=%d does not contain element 'domain' - SKIPPED!", len(block)) continue + elif not domain_helper.is_wanted(block["domain"]): + logger.debug("block[domain]='%s' is not wanted - SKIPPED!", block["domain"]) + continue elif "severity" not in block: logger.warning("block()=%d does not contain element 'severity' - SKIPPED!", len(block)) continue diff --git a/fba/networks/misskey.py b/fba/networks/misskey.py index e9750da..b21db1e 100644 --- a/fba/networks/misskey.py +++ b/fba/networks/misskey.py @@ -108,6 +108,9 @@ def fetch_peers(domain: str) -> list: elif row["host"] in peers: logger.debug("Not adding row[host]='%s', already found - SKIPPED!", row['host']) continue + elif not domain_helper.is_wanted(row["host"]): + logger.debug("row[host]='%s' is not wanted - SKIPPED!", row["host"]) + continue logger.debug("Adding peer: row[host]='%s'", row['host']) added = added + 1 @@ -197,7 +200,12 @@ def fetch_blocks(domain: str) -> list: # Is it there? logger.debug("instance[]='%s'", type(instance)) blocked = tidyup.domain(instance["host"]) - if "isSuspended" in instance and instance["isSuspended"] and not dict_helper.has_key(blocklist, "blocked", blocked): + + logger.debug("blocked='%s'", blocked) + if not domain_helper.is_wanted(blocked): + logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked) + continue + elif "isSuspended" in instance and instance["isSuspended"] and not dict_helper.has_key(blocklist, "blocked", blocked): count = count + 1 logger.debug("Appending blocker='%s',blocked='%s',block_level='suspended'", domain, blocked) blocklist.append({ @@ -267,7 +275,12 @@ def fetch_blocks(domain: str) -> list: # Is it there? logger.debug("instance[]='%s'", type(instance)) blocked = tidyup.domain(instance["host"]) - if "isBlocked" in instance and instance["isBlocked"] and not dict_helper.has_key(blocklist, "blocked", blocked): + + logger.debug("blocked='%s'", blocked) + if not domain_helper.is_wanted(blocked): + logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked) + continue + elif "isBlocked" in instance and instance["isBlocked"] and not dict_helper.has_key(blocklist, "blocked", blocked): count = count + 1 logger.debug("Appending blocker='%s',blocked='%s',block_level='reject'", domain, blocked) blocklist.append({ diff --git a/fba/networks/peertube.py b/fba/networks/peertube.py index 58bf115..fd188d3 100644 --- a/fba/networks/peertube.py +++ b/fba/networks/peertube.py @@ -60,7 +60,7 @@ def fetch_peers(domain: str) -> list: logger.debug("data[]='%s'", type(data)) if "error_message" not in data: - logger.debug("Success, data[json]()=%d", len(data['json'])) + logger.debug("Success, data[json]()=%d", len(data["json"])) instances.set_success(domain) if "data" in data["json"]: @@ -72,7 +72,12 @@ def fetch_peers(domain: str) -> list: for mode2 in ["follower", "following"]: logger.debug("mode=%s,mode2='%s'", mode, mode2) if mode2 in record and "host" in record[mode2] and record[mode2]["host"] != domain: - logger.debug("Found mode2='%s',host='%s', adding ...", mode2, record[mode2]['host']) + logger.debug("record[%s][host]='%s'", mode2, record[mode2]["host"]) + if not domain_helper.is_wanted(record[mode2]["host"]): + logger.debug("record[%s][host]='%s' is not wanted - SKIPPED!", mode2, record[mode2]["host"]) + continue + + logger.debug("Found mode2='%s',host='%s', adding ...", mode2, record[mode2]["host"]) peers.append(record[mode2]["host"]) if len(rows) < 100: diff --git a/fba/networks/pleroma.py b/fba/networks/pleroma.py index 73720f7..ebd5b78 100644 --- a/fba/networks/pleroma.py +++ b/fba/networks/pleroma.py @@ -135,6 +135,9 @@ def fetch_blocks(domain: str) -> list: blocked = utils.deobfuscate(blocked, domain) logger.debug("blocked='%s' - DEOBFUSCATED!", blocked) + if not domain_helper.is_wanted(blocked): + logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked) + continue logger.debug("Appending blocker='%s',blocked='%s',block_level='%s' ...", domain, blocked, block_level) blockdict.append({ @@ -157,6 +160,9 @@ def fetch_blocks(domain: str) -> list: if blocked == "": logger.warning("blocked is empty after tidyup.domain(): domain='%s',block_level='%s'", domain, block_level) continue + elif not domain_helper.is_wanted(blocked): + logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked) + continue logger.debug("Appending blocker='%s',blocked='%s',block_level='%s' ...", domain, blocked, block_level) blockdict.append({ @@ -365,6 +371,9 @@ def fetch_blocks_from_about(domain: str) -> dict: if blocked is None or blocked == "": logger.debug("domain='%s',block_level='%s': blocked is empty - SKIPPED!", domain, block_level) continue + elif not domain_helper.is_wanted(blocked): + logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked) + continue logger.debug("Appending block_level='%s',blocked='%s',reason='%s' ...", block_level, blocked, reason) blocklist[block_level].append({ -- 2.39.5