Continued:

- during fetching block lists or peers, filter out already blacklisted
  entries which will keep the list variable's memory usage smaller

diff --git a/fba/networks/friendica.py b/fba/networks/friendica.py
index 046bf909d0cfd5480f0c8cc830281754ee59c71f..f30f1263492318af0007716460e3398d91c43d70 100644 (file)
--- a/fba/networks/friendica.py
+++ b/fba/networks/friendica.py
@@ -85,6 +85,9 @@ def fetch_blocks(domain: str) -> list:
         if blocked == "":
             logger.debug("line[]='%s' returned empty blocked domain - SKIPPED!", type(line))
             continue
+        elif not domain_helper.is_wanted(blocked):
+            logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
+            continue
 
         logger.debug("Appending blocked='%s',reason='%s'", blocked, reason)
         blocklist.append({

diff --git a/fba/networks/lemmy.py b/fba/networks/lemmy.py
index 07a854a1593c28953714f343fb17c5c876f25f29..efc85de953927d325a9a76f6f84d14dde05752ff 100644 (file)
--- a/fba/networks/lemmy.py
+++ b/fba/networks/lemmy.py
@@ -206,6 +206,9 @@ def fetch_blocks(domain: str) -> list:
                 if blocked == "":
                     logger.warning("blocked='%s' is empty after tidyup.domain() - SKIPPED!", tag.contents[0])
                     continue
+                elif not domain_helper.is_wanted(blocked):
+                    logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
+                    continue
 
                 logger.debug("Appending blocker='%s',blocked='%s',block_level='reject' ...", domain, blocked)
                 blocklist.append({
@@ -267,6 +270,9 @@ def fetch_instances(domain: str, origin: str) -> list:
                         if peer == "":
                             logger.debug("peer is empty - SKIPPED!")
                             continue
+                        elif not domain_helper.is_wanted(peer):
+                            logger.debug("peer='%s' is not wanted - SKIPPED!", peer)
+                            continue
                         elif peer in peers:
                             logger.debug("peer='%s' already added - SKIPPED!", peer)
                             continue
@@ -364,6 +370,8 @@ def parse_script(doc: bs4.BeautifulSoup, only: str = None) -> list:
                 if peer == "":
                     logger.debug("peer is empty - SKIPPED!")
                     continue
+                elif not domain_helper.is_wanted(peer):
+                    logger.debug("peer='%s' is not wanted - SKIPPED!", peer)
                 elif peer in peers:
                     logger.debug("peer='%s' already added - SKIPPED!", peer)
                     continue

diff --git a/fba/networks/mastodon.py b/fba/networks/mastodon.py
index bea34c9a75c9c293cd3b208e2c12f33e9efcf148..cda3cc4786b328cc2e20307de4b9721da831dbd8 100644 (file)
--- a/fba/networks/mastodon.py
+++ b/fba/networks/mastodon.py
@@ -159,6 +159,9 @@ def fetch_blocks(domain: str) -> list:
             elif "domain" not in block:
                 logger.warning("block()=%d does not contain element 'domain' - SKIPPED!", len(block))
                 continue
+            elif not domain_helper.is_wanted(block["domain"]):
+                logger.debug("block[domain]='%s' is not wanted - SKIPPED!", block["domain"])
+                continue
             elif "severity" not in block:
                 logger.warning("block()=%d does not contain element 'severity' - SKIPPED!", len(block))
                 continue

diff --git a/fba/networks/misskey.py b/fba/networks/misskey.py
index e9750daf698a98903506ebd305c8cb74241fa610..b21db1ea524b282fce2e8e3e1cf498617f2089b1 100644 (file)
--- a/fba/networks/misskey.py
+++ b/fba/networks/misskey.py
@@ -108,6 +108,9 @@ def fetch_peers(domain: str) -> list:
             elif row["host"] in peers:
                 logger.debug("Not adding row[host]='%s', already found - SKIPPED!", row['host'])
                 continue
+            elif not domain_helper.is_wanted(row["host"]):
+                logger.debug("row[host]='%s' is not wanted - SKIPPED!", row["host"])
+                continue
 
             logger.debug("Adding peer: row[host]='%s'", row['host'])
             added = added + 1
@@ -197,7 +200,12 @@ def fetch_blocks(domain: str) -> list:
                 # Is it there?
                 logger.debug("instance[]='%s'", type(instance))
                 blocked = tidyup.domain(instance["host"])
-                if "isSuspended" in instance and instance["isSuspended"] and not dict_helper.has_key(blocklist, "blocked", blocked):
+
+                logger.debug("blocked='%s'", blocked)
+                if not domain_helper.is_wanted(blocked):
+                    logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
+                    continue
+                elif "isSuspended" in instance and instance["isSuspended"] and not dict_helper.has_key(blocklist, "blocked", blocked):
                     count = count + 1
                     logger.debug("Appending blocker='%s',blocked='%s',block_level='suspended'", domain, blocked)
                     blocklist.append({
@@ -267,7 +275,12 @@ def fetch_blocks(domain: str) -> list:
                 # Is it there?
                 logger.debug("instance[]='%s'", type(instance))
                 blocked = tidyup.domain(instance["host"])
-                if "isBlocked" in instance and instance["isBlocked"] and not dict_helper.has_key(blocklist, "blocked", blocked):
+
+                logger.debug("blocked='%s'", blocked)
+                if not domain_helper.is_wanted(blocked):
+                    logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
+                    continue
+                elif "isBlocked" in instance and instance["isBlocked"] and not dict_helper.has_key(blocklist, "blocked", blocked):
                     count = count + 1
                     logger.debug("Appending blocker='%s',blocked='%s',block_level='reject'", domain, blocked)
                     blocklist.append({

diff --git a/fba/networks/peertube.py b/fba/networks/peertube.py
index 58bf11583a75f7e459f46d13122421c094ff35bc..fd188d39148806a15e4a2195c361bd24e763f3bf 100644 (file)
--- a/fba/networks/peertube.py
+++ b/fba/networks/peertube.py
@@ -60,7 +60,7 @@ def fetch_peers(domain: str) -> list:
 
             logger.debug("data[]='%s'", type(data))
             if "error_message" not in data:
-                logger.debug("Success, data[json]()=%d", len(data['json']))
+                logger.debug("Success, data[json]()=%d", len(data["json"]))
                 instances.set_success(domain)
 
                 if "data" in data["json"]:
@@ -72,7 +72,12 @@ def fetch_peers(domain: str) -> list:
                         for mode2 in ["follower", "following"]:
                             logger.debug("mode=%s,mode2='%s'", mode, mode2)
                             if mode2 in record and "host" in record[mode2] and record[mode2]["host"] != domain:
-                                logger.debug("Found mode2='%s',host='%s', adding ...", mode2, record[mode2]['host'])
+                                logger.debug("record[%s][host]='%s'", mode2, record[mode2]["host"])
+                                if not domain_helper.is_wanted(record[mode2]["host"]):
+                                    logger.debug("record[%s][host]='%s' is not wanted - SKIPPED!", mode2, record[mode2]["host"])
+                                    continue
+
+                                logger.debug("Found mode2='%s',host='%s', adding ...", mode2, record[mode2]["host"])
                                 peers.append(record[mode2]["host"])
 
                     if len(rows) < 100:

diff --git a/fba/networks/pleroma.py b/fba/networks/pleroma.py
index 73720f7ee6eef8def87b8973716b4cc14b771344..ebd5b789c1b16f4ec9e892fbdb08a984b3b844ff 100644 (file)
--- a/fba/networks/pleroma.py
+++ b/fba/networks/pleroma.py
@@ -135,6 +135,9 @@ def fetch_blocks(domain: str) -> list:
                     blocked = utils.deobfuscate(blocked, domain)
 
                     logger.debug("blocked='%s' - DEOBFUSCATED!", blocked)
+                    if not domain_helper.is_wanted(blocked):
+                        logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
+                        continue
 
                     logger.debug("Appending blocker='%s',blocked='%s',block_level='%s' ...", domain, blocked, block_level)
                     blockdict.append({
@@ -157,6 +160,9 @@ def fetch_blocks(domain: str) -> list:
             if blocked == "":
                 logger.warning("blocked is empty after tidyup.domain(): domain='%s',block_level='%s'", domain, block_level)
                 continue
+            elif not domain_helper.is_wanted(blocked):
+                logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
+                continue
 
             logger.debug("Appending blocker='%s',blocked='%s',block_level='%s' ...", domain, blocked, block_level)
             blockdict.append({
@@ -365,6 +371,9 @@ def fetch_blocks_from_about(domain: str) -> dict:
                 if blocked is None or blocked == "":
                     logger.debug("domain='%s',block_level='%s': blocked is empty - SKIPPED!", domain, block_level)
                     continue
+                elif not domain_helper.is_wanted(blocked):
+                    logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
+                    continue
 
                 logger.debug("Appending block_level='%s',blocked='%s',reason='%s' ...", block_level, blocked, reason)
                 blocklist[block_level].append({