X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;f=fba%2Fnetworks%2Fpleroma.py;h=5a52f9cd58e823cdb37497d01489a13d8888dad2;hb=bd36d58fadcf377a7982103ca4d7e5c4376ef463;hp=390b01e064c0f93bfe7195a39f779a57922c89c0;hpb=ec05e15ffd8d63c41c01f3fa342ca71a247f2409;p=fba.git

diff --git a/fba/networks/pleroma.py b/fba/networks/pleroma.py
index 390b01e..5a52f9c 100644
--- a/fba/networks/pleroma.py
+++ b/fba/networks/pleroma.py
@@ -14,7 +14,6 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-import inspect
 import logging
 
 import bs4
@@ -22,13 +21,12 @@ import bs4
 from fba import database
 from fba import utils
 
-from fba.helpers import blacklist
 from fba.helpers import config
 from fba.helpers import domain as domain_helper
 from fba.helpers import tidyup
 
-from fba.http import federation
 from fba.http import network
+from fba.http import nodeinfo
 
 from fba.models import blocks
 from fba.models import instances
@@ -39,47 +37,67 @@ logger = logging.getLogger(__name__)
 # Language mapping X -> English
 language_mapping = {
     # English -> English
-    "Reject": "Suspended servers",
+    "filtered media"   : "filtered_media",
+    "limited servers"  : "followers_only",
+    "followers-only"   : "followers_only",
+    "media removal"    : "media_removal",
+    "media_removal"    : "media_removal",
+    "media force-set as sensitive": "media_nsfw",
+    "nsfw"             : "media_nsfw",
+    "reject"           : "reject",
+    "suspended servers": "reject",
+    "silenced servers" : "silenced",
+    "removal from \"the whole known network\" timeline": "federated_timeline_removal",
 }
 
-def fetch_blocks(domain: str, origin: str, nodeinfo_url: str):
-    logger.debug(f"domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}' - CALLED!")
+def fetch_blocks(domain: str, nodeinfo_url: str) -> list:
+    logger.debug("domain='%s',nodeinfo_url='%s' - CALLED!", domain, nodeinfo_url)
     domain_helper.raise_on(domain)
-    if not isinstance(origin, str) and origin is not None:
-        raise ValueError(f"Parameter origin[]='{type(origin)}' is not 'str'")
-    elif origin == "":
-        raise ValueError("Parameter 'origin' is empty")
-    elif not isinstance(nodeinfo_url, str):
-        raise ValueError(f"Parameter nodeinfo_url[]='{type(nodeinfo_url)}' is not 'str'")
+
+    if not isinstance(nodeinfo_url, str):
+        raise ValueError(f"Parameter nodeinfo_url[]='{type(nodeinfo_url)}' is not of type 'str'")
     elif nodeinfo_url == "":
         raise ValueError("Parameter 'nodeinfo_url' is empty")
 
-    # @TODO Unused blockdict
     blockdict = list()
     rows = None
     try:
-        logger.debug(f"Fetching nodeinfo: domain='{domain}',nodeinfo_url='{nodeinfo_url}'")
-        rows = federation.fetch_nodeinfo(domain, nodeinfo_url)
+        logger.debug("Fetching nodeinfo: domain='%s',nodeinfo_url='%s'", domain, nodeinfo_url)
+        rows = nodeinfo.fetch_nodeinfo(domain, nodeinfo_url)
+
+        if "error_message" in rows:
+            logger.warning("Error message '%s' during fetching nodeinfo for domain='%s',nodeinfo_url='%s'", rows["error_message"], domain, nodeinfo_url)
+            instances.set_last_error(domain, rows)
+
+            logger.debug("Returning empty list ... - EXIT!")
+            return list()
+        elif "exception" in rows:
+            logger.warning("Exception '%s' during fetching nodeinfo for domain='%s',nodeinfo_url='%s' - EXIT!", type(rows["exception"]), domain, nodeinfo_url)
+            return list()
+        elif "json" in rows:
+            logger.debug("rows[json] found for domain='%s',nodeinfo_url='%s'", domain, nodeinfo_url)
+            rows = rows["json"]
+
     except network.exceptions as exception:
         logger.warning("Exception '%s' during fetching nodeinfo from domain='%s'", type(exception), domain)
         instances.set_last_error(domain, exception)
 
     if rows is None:
-        logger.warning("Could not fetch nodeinfo from domain:", domain)
-        return
+        logger.warning("Could not fetch nodeinfo from domain='%s' - EXIT!", domain)
+        return list()
     elif "metadata" not in rows:
-        logger.warning("rows()=%d does not have key 'metadata', domain='%s'", len(rows), domain)
-        return
+        logger.warning("rows()=%d does not have key 'metadata', domain='%s' - EXIT!", len(rows), domain)
+        return list()
     elif "federation" not in rows["metadata"]:
-        logger.warning("rows()=%d does not have key 'federation', domain='%s'", len(rows['metadata']), domain)
-        return
+        logger.warning("rows()=%d does not have key 'federation', domain='%s' - EXIT!", len(rows["metadata"]), domain)
+        return list()
 
     data = rows["metadata"]["federation"]
     found = False
 
     logger.debug("data[]='%s'", type(data))
     if "mrf_simple" in data:
-        logger.debug("Found mrf_simple:", domain)
+        logger.debug("Found mrf_simple in API response from domain='%s'", domain)
         found = True
         for block_level, blocklist in (
             {
@@ -100,6 +118,8 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str):
                 logger.debug("domain='%s' skipping block_level='accept'", domain)
                 continue
 
+            block_level = blocks.alias_block_level(block_level)
+
             logger.debug("Checking %d entries from domain='%s',block_level='%s' ...", len(blocklist), domain, block_level)
             if len(blocklist) > 0:
                 for blocked in blocklist:
@@ -108,66 +128,28 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str):
                     logger.debug("blocked='%s' - AFTER!", blocked)
 
                     if blocked == "":
-                        logger.warning("blocked is empty after tidyup.domain():", domain, block_level)
-                        continue
-                    elif blocked.endswith(".arpa"):
-                        logger.debug("blocked='%s' is a reverse IP address - SKIPPED!", blocked)
+                        logger.warning("blocked is empty after tidyup.domain(): domain='%s',block_level='%s' - SKIPPED!", domain, block_level)
                         continue
-                    elif blocked.endswith(".tld"):
-                        logger.debug("blocked='%s' is a fake domain - SKIPPED!", blocked)
+                    elif not domain_helper.is_wanted(blocked):
+                        logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
                         continue
-                    elif blocked.count("*") > 0:
-                        # Obscured domain name with no hash
-                        row = instances.deobscure("*", blocked)
-
-                        logger.debug("row[]='%s'", type(row))
-                        if row is None:
-                            logger.warning("Cannot deobsfucate blocked='%s',domain='%s',origin='%s' - SKIPPED!", blocked, domain, origin)
-                            continue
-
-                        logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0])
-                        blocked      = row[0]
-                        origin       = row[1]
-                        nodeinfo_url = row[2]
-                    elif blocked.count("?") > 0:
-                        # Obscured domain name with no hash
-                        row = instances.deobscure("?", blocked)
-
-                        logger.debug("row[]='%s'", type(row))
-                        if row is None:
-                            logger.warning("Cannot deobsfucate blocked='%s',domain='%s',origin='%s' - SKIPPED!", blocked, domain, origin)
-                            continue
-
-                        logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0])
-                        blocked      = row[0]
-                        origin       = row[1]
-                        nodeinfo_url = row[2]
-
-                    logger.debug("blocked='%s'", blocked)
-                    if not utils.is_domain_wanted(blocked):
+
+                    logger.debug("Invoking utils.deobfuscate(%s, %s) ...", blocked, domain)
+                    blocked = utils.deobfuscate(blocked, domain)
+
+                    logger.debug("blocked='%s' - DEOBFUSCATED!", blocked)
+                    if not domain_helper.is_wanted(blocked):
                         logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
                         continue
-                    elif not instances.is_registered(blocked):
-                        # Commit changes
-                        logger.debug("Invoking commit() ...")
-                        database.connection.commit()
-
-                        logger.debug("Domain blocked='%s' wasn't found, adding ..., domain='%s',origin='%s',nodeinfo_url='%s'", blocked, domain, origin, nodeinfo_url)
-                        instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url)
-
-                    if not blocks.is_instance_blocked(domain, blocked, block_level):
-                        logger.debug("Blocking domain='%s',blocked='%s',block_level='%s' ...", domain, blocked, block_level)
-                        blocks.add_instance(domain, blocked, None, block_level)
-
-                        if block_level == "reject":
-                            logger.debug("Appending blocked='%s' ...", blocked)
-                            blockdict.append({
-                                "blocked": blocked,
-                                "reason" : None
-                            })
-                    else:
-                        logger.debug("Updating block last seen for domain='%s',blocked='%s',block_level='%s' ...", domain, blocked, block_level)
-                        blocks.update_last_seen(domain, blocked, block_level)
+
+                    logger.debug("Appending blocker='%s',blocked='%s',block_level='%s' ...", domain, blocked, block_level)
+                    blockdict.append({
+                        "blocker"    : domain,
+                        "blocked"    : blocked,
+                        "reason"     : None,
+                        "block_level": block_level,
+                    })
+
     elif "quarantined_instances" in data:
         logger.debug("Found 'quarantined_instances' in JSON response: domain='%s'", domain)
         found = True
@@ -179,66 +161,28 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str):
             logger.debug("blocked='%s' - AFTER!", blocked)
 
             if blocked == "":
-                logger.warning("blocked is empty after tidyup.domain():", domain, block_level)
-                continue
-            elif blocked.endswith(".arpa"):
-                logger.debug("blocked='%s' is a reverse IP address - SKIPPED!", blocked)
+                logger.warning("blocked is empty after tidyup.domain(): domain='%s',block_level='%s'", domain, block_level)
                 continue
-            elif blocked.endswith(".tld"):
-                logger.debug("blocked='%s' is a fake domain - SKIPPED!", blocked)
+            elif not domain_helper.is_wanted(blocked):
+                logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
                 continue
-            elif blocked.count("*") > 0:
-                # Obscured domain name with no hash
-                row = instances.deobscure("*", blocked)
 
-                logger.debug("row[]='%s'", type(row))
-                if row is None:
-                    logger.warning("Cannot deobsfucate blocked='%s',domain='%s',origin='%s' - SKIPPED!", blocked, domain, origin)
-                    continue
-
-                logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0])
-                blocked      = row[0]
-                origin       = row[1]
-                nodeinfo_url = row[2]
-            elif blocked.count("?") > 0:
-                # Obscured domain name with no hash
-                row = instances.deobscure("?", blocked)
-
-                logger.debug("row[]='%s'", type(row))
-                if row is None:
-                    logger.warning("Cannot deobsfucate blocked='%s',domain='%s',origin='%s' - SKIPPED!", blocked, domain, origin)
-                    continue
-
-                logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0])
-                blocked      = row[0]
-                origin       = row[1]
-                nodeinfo_url = row[2]
+            logger.debug("Invoking utils.deobfuscate(%s, %s) ...", blocked, domain)
+            blocked = utils.deobfuscate(blocked, domain)
 
-            logger.debug("blocked='%s' - DEOBSFUCATED!", blocked)
-            if not utils.is_domain_wanted(blocked):
+            logger.debug("blocked='%s' - DEOBFUSCATED!", blocked)
+            if not domain_helper.is_wanted(blocked):
                 logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
                 continue
-            elif not instances.is_registered(blocked):
-                # Commit changes
-                logger.debug("Invoking commit() ...")
-                database.connection.commit()
 
-                logger.debug("Domain blocked='%s' wasn't found, adding ..., domain='%s',origin='%s',nodeinfo_url='{nodeinfo_url}'", blocked, domain, origin)
-                instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url)
+            logger.debug("Appending blocker='%s',blocked='%s',block_level='%s' ...", domain, blocked, block_level)
+            blockdict.append({
+                "blocker"    : domain,
+                "blocked"    : blocked,
+                "reason"     : None,
+                "block_level": block_level,
+            })
 
-            if not blocks.is_instance_blocked(domain, blocked, block_level):
-                logger.debug("Blocking domain='%s',blocked='%s',block_level='%s' ...", domain, blocked, block_level)
-                blocks.add_instance(domain, blocked, None, block_level)
-
-                if block_level == "reject":
-                    logger.debug("Appending blocked='%s' ...", blocked)
-                    blockdict.append({
-                        "blocked": blocked,
-                        "reason" : None
-                    })
-            else:
-                logger.debug("Updating block last seen for domain='%s',blocked='%s',block_level='%s' ...", domain, blocked, block_level)
-                blocks.update_last_seen(domain, blocked, block_level)
     else:
         logger.warning("Cannot find 'mrf_simple' or 'quarantined_instances' in JSON reply: domain='%s'", domain)
 
@@ -263,9 +207,11 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str):
                 logger.warning("block_level is now empty!")
                 continue
             elif block_level == "accept":
-                logger.debug("domain='%s' skipping block_level='accept'", domain)
+                logger.debug("domain='%s': Skipping block_level='%s' ...", domain, block_level)
                 continue
 
+            block_level = blocks.alias_block_level(block_level)
+
             logger.debug("Checking %d entries from domain='%s',block_level='%s' ...", len(info.items()), domain, block_level)
             for blocked, reason in info.items():
                 logger.debug("blocked='%s',reason[%s]='%s' - BEFORE!", blocked, type(reason), reason)
@@ -277,58 +223,29 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str):
                     reason = tidyup.reason(reason)
                 elif isinstance(reason, dict) and "reason" in reason:
                     logger.debug("reason[] is a dict")
-                    reason = tidyup.reason(reason["reason"])
+                    reason = tidyup.reason(reason["reason"]) if isinstance(reason["reason"], str) else None
                 elif reason is not None:
                     raise ValueError(f"Cannot handle reason[]='{type(reason)}'")
 
                 logger.debug("blocked='%s',reason='%s' - AFTER!", blocked, reason)
 
                 if blocked == "":
-                    logger.warning("blocked is empty after tidyup.domain():", domain, block_level)
+                    logger.warning("blocked is empty after tidyup.domain(): domain='%s',block_level='%s'", domain, block_level)
                     continue
-                elif blocked.count("*") > 0:
-                    # Obscured domain name with no hash
-                    row = instances.deobscure("*", blocked)
-
-                    logger.debug("row[]='%s'", type(row))
-                    if row is None:
-                        logger.warning("Cannot deobsfucate blocked='%s',domain='%s',origin='%s' - SKIPPED!", blocked, domain, origin)
-                        continue
-
-                    logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0])
-                    blocked      = row[0]
-                    origin       = row[1]
-                    nodeinfo_url = row[2]
-                elif blocked.count("?") > 0:
-                    # Obscured domain name with no hash
-                    row = instances.deobscure("?", blocked)
-
-                    logger.debug("row[]='%s'", type(row))
-                    if row is None:
-                        logger.warning("Cannot deobsfucate blocked='%s',domain='%s',origin='%s' - SKIPPED!", blocked, domain, origin)
-                        continue
-
-                    logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0])
-                    blocked      = row[0]
-                    origin       = row[1]
-                    nodeinfo_url = row[2]
-
-                logger.debug("blocked='%s' - DEOBSFUCATED!", blocked)
-                if not utils.is_domain_wanted(blocked):
+                elif not domain_helper.is_wanted(blocked):
                     logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
                     continue
-                elif not instances.is_registered(blocked):
-                    logger.debug("Domain blocked='%s' wasn't found, adding ..., domain='%s',origin='%s',nodeinfo_url='%s'", blocked, domain, origin, nodeinfo_url)
-                    instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url)
 
-                logger.debug("Updating block reason: reason='%s',domain='%s',blocked='%s',block_level='%s'", reason, domain, blocked, block_level)
-                blocks.update_reason(reason, domain, blocked, block_level)
+                logger.debug("Invoking utils.deobfuscate(%s, %s) ...", blocked, domain)
+                blocked = utils.deobfuscate(blocked, domain)
+                logger.debug("blocked='%s' - DEOBFUSCATED!", blocked)
 
                 logger.debug("Checking %d blockdict records ...", len(blockdict))
-                for entry in blockdict:
-                    if entry["blocked"] == blocked:
-                        logger.debug("Updating entry reason: blocked='%s',reason='%s'", blocked, reason)
-                        entry["reason"] = reason
+                for block in blockdict:
+                    logger.debug("block[blocked]='%s',blocked='%s'", block["blocked"], blocked)
+                    if block["blocked"] == blocked:
+                        logger.debug("Updating reason='%s' for blocker='%s'", reason, block["blocked"])
+                        block["reason"] = reason
 
     elif "quarantined_instances_info" in data and "quarantined_instances" in data["quarantined_instances_info"]:
         logger.debug("Found 'quarantined_instances_info' in JSON response: domain='%s'", domain)
@@ -339,62 +256,34 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str):
         rows = data["quarantined_instances_info"]["quarantined_instances"]
         for blocked in rows:
             logger.debug("blocked='%s' - BEFORE!", blocked)
+            reason = tidyup.reason(rows[blocked]["reason"])
             blocked = tidyup.domain(blocked)
-            logger.debug("blocked='%s' - AFTER!", blocked)
+            logger.debug("blocked='%s',reason='%s' - AFTER!", blocked, reason)
 
             if blocked not in rows or "reason" not in rows[blocked]:
                 logger.warning("Cannot find blocked='%s' in rows()=%d,domain='%s' - BREAK!", blocked, len(rows), domain)
                 break
-
-            reason = rows[blocked]["reason"]
-            logger.debug("reason='%s'", reason)
-
-            if blocked == "":
-                logger.warning("blocked is empty after tidyup.domain():", domain, block_level)
+            elif blocked == "":
+                logger.warning("blocked is empty after tidyup.domain(): domain='%s',block_level='%s'", domain, block_level)
+                continue
+            elif not domain_helper.is_wanted(blocked):
+                logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
                 continue
-            elif blocked.count("*") > 0:
-                # Obscured domain name with no hash
-                row = instances.deobscure("*", blocked)
-
-                logger.debug("row[]='%s'", type(row))
-                if row is None:
-                    logger.warning("Cannot deobsfucate blocked='%s',domain='%s',origin='%s' - SKIPPED!", blocked, domain, origin)
-                    continue
-
-                logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0])
-                blocked      = row[0]
-                origin       = row[1]
-                nodeinfo_url = row[2]
-            elif blocked.count("?") > 0:
-                # Obscured domain name with no hash
-                row = instances.deobscure("?", blocked)
-
-                logger.debug("row[]='%s'", type(row))
-                if row is None:
-                    logger.warning("Cannot deobsfucate blocked='%s',domain='%s',origin='%s' - SKIPPED!", blocked, domain, origin)
-                    continue
 
-                logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0])
-                blocked      = row[0]
-                origin       = row[1]
-                nodeinfo_url = row[2]
+            logger.debug("Invoking utils.deobfuscate(%s, %s) ...", blocked, domain)
+            blocked = utils.deobfuscate(blocked, domain)
 
-            logger.debug("blocked='%s'", blocked)
-            if not utils.is_domain_wanted(blocked):
+            logger.debug("blocked='%s' - DEOBFUSCATED!", blocked)
+            if not domain_helper.is_wanted(blocked):
                 logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
                 continue
-            elif not instances.is_registered(blocked):
-                logger.debug("Domain blocked='%s' wasn't found, adding ..., domain='%s',origin='%s',nodeinfo_url='%s'", blocked, domain, origin, nodeinfo_url)
-                instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url)
-
-            logger.debug("Updating block reason: reason='%s',domain='%s',blocked='%s',block_level='%s'", reason, domain, blocked, block_level)
-            blocks.update_reason(reason, domain, blocked, block_level)
 
             logger.debug("Checking %d blockdict records ...", len(blockdict))
-            for entry in blockdict:
-                if entry["blocked"] == blocked:
-                    logger.debug("Updating entry reason: blocked='%s',reason='%s'", blocked, reason)
-                    entry["reason"] = reason
+            for block in blockdict:
+                logger.debug("block[blocked]='%s',blocked='%s'", block["blocked"], blocked)
+                if block["blocked"] == blocked:
+                    logger.debug("Updating reason='%s' for blocker='%s'", reason, block["blocked"])
+                    block["reason"] = reason
     else:
         logger.warning("Cannot find 'mrf_simple_info' or 'quarantined_instances_info' in JSON reply: domain='%s'", domain)
 
@@ -404,77 +293,34 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str):
 
         logger.debug("blocklist()=%d", len(blocklist))
         if len(blocklist) > 0:
-            logger.info("Checking %d record(s) ...", len(blocklist))
+            logger.info("Checking %d different blocklists ...", len(blocklist))
             for block_level in blocklist:
                 logger.debug("block_level='%s'", block_level)
-
                 rows = blocklist[block_level]
+
                 logger.debug("rows[%s]()=%d'", type(rows), len(rows))
-                for record in rows:
-                    logger.debug("record[]='%s'", type(record))
-                    blocked = tidyup.domain(record["blocked"])
-                    reason  = tidyup.reason(record["reason"])
-                    logger.debug("blocked='%s',reason='%s' - AFTER!", blocked, reason)
+                for block in rows:
+                    logger.debug("Invoking utils.deobfuscate(%s, %s) ...", block["blocked"], domain)
+                    block["blocked"] = utils.deobfuscate(block["blocked"], domain)
 
-                    if blocked == "":
-                        logger.warning("blocked is empty after tidyup.domain():", domain, block_level)
-                        continue
-                    elif blocked.count("*") > 0:
-                        # Obscured domain name with no hash
-                        row = instances.deobscure("*", blocked)
-
-                        logger.debug("row[]='%s'", type(row))
-                        if row is None:
-                            logger.warning("Cannot deobsfucate blocked='%s',domain='%s',origin='%s' - SKIPPED!", blocked, domain, origin)
-                            continue
-
-                        logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0])
-                        blocked      = row[0]
-                        origin       = row[1]
-                        nodeinfo_url = row[2]
-                    elif blocked.count("?") > 0:
-                        # Obscured domain name with no hash
-                        row = instances.deobscure("?", blocked)
-
-                        logger.debug("row[]='%s'", type(row))
-                        if row is None:
-                            logger.warning("Cannot deobsfucate blocked='%s',domain='%s',origin='%s' - SKIPPED!", blocked, domain, origin)
-                            continue
-
-                        logger.debug("blocked='%s' de-obscured to '%s'", blocked, row[0])
-                        blocked      = row[0]
-                        origin       = row[1]
-                        nodeinfo_url = row[2]
-
-                    logger.debug("blocked='%s' - DEOBSFUCATED!", blocked)
-                    if not utils.is_domain_wanted(blocked):
-                        logger.warning("blocked='%s' is not wanted - SKIPPED!", blocked)
+                    logger.debug("block[blocked]='%s' - DEOBFUSCATED!", block["blocked"])
+                    if not domain_helper.is_wanted(block["blocked"]):
+                        logger.debug("block[blocked]='%s' is not wanted - SKIPPED!", block["blocked"])
                         continue
-                    elif not instances.is_registered(blocked):
-                        logger.debug("Domain blocked='%s' wasn't found, adding ..., domain='%s',origin='%s',nodeinfo_url='%s'", blocked, domain, origin, nodeinfo_url)
-                        instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url)
-
-                    if not blocks.is_instance_blocked(domain, blocked, block_level):
-                        logger.debug("Blocking domain='%s',blocked='%s', block_level='%s' ...", domain, blocked, block_level)
-                        blocks.add_instance(domain, blocked, reason, block_level)
-
-                        if block_level == "reject":
-                            logger.debug("Appending blocked='%s' ...", blocked)
-                            blockdict.append({
-                                "blocked": blocked,
-                                "reason" : reason
-                            })
-                    else:
-                        logger.debug("Updating block last seen for domain='%s',blocked='%s',block_level='%s' ...", domain, blocked, block_level)
-                        blocks.update_reason(reason, domain, blocked, block_level)
 
-    logger.debug("Invoking commit() ...")
-    database.connection.commit()
+                    logger.debug("Appending blocker='%s',block[blocked]='%s',block[reason]='%s',block_level='%s' ...",domain, block["blocked"], block["reason"], block_level)
+                    blockdict.append({
+                        "blocker"    : domain,
+                        "blocked"    : block["blocked"],
+                        "reason"     : block["reason"],
+                        "block_level": block_level,
+                    })
 
-    logger.debug("EXIT!")
+    logger.debug("blockdict()=%d - EXIT!", len(blockdict))
+    return blockdict
 
 def fetch_blocks_from_about(domain: str) -> dict:
-    logger.debug("domain(%d)='%s' - CALLED!", len(domain), domain)
+    logger.debug("domain='%s' - CALLED!", domain)
     domain_helper.raise_on(domain)
 
     logger.debug("Fetching mastodon blocks from domain='%s'", domain)
@@ -492,7 +338,7 @@ def fetch_blocks_from_about(domain: str) -> dict:
                 (config.get("connection_timeout"), config.get("read_timeout"))
             )
 
-            logger.debug("response.ok='%s',response.status_code='%d',response.text()=%d", response.ok, response.status_code, len(response.text))
+            logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text))
             if not response.ok or response.text.strip() == "":
                 logger.warning("path='%s' does not exist on domain='%s' - SKIPPED!", path, domain)
                 continue
@@ -509,48 +355,60 @@ def fetch_blocks_from_about(domain: str) -> dict:
                 break
 
         except network.exceptions as exception:
-            logger.warning("Cannot fetch from domain:", domain, exception)
+            logger.warning("Cannot fetch from domain='%s',exception[%s]='%s'", domain, type(exception), str(exception))
             instances.set_last_error(domain, exception)
             break
 
     blocklist = {
-        "Suspended servers": [],
-        "Filtered media"   : [],
-        "Limited servers"  : [],
-        "Silenced servers" : [],
+        "reject"        : [],
+        "filtered_media": [],
+        "followers_only": [],
+        "silenced"      : [],
+        "media_nsfw"    : [],
+        "media_removal" : [],
+        "federated_timeline_removal": [],
     }
 
     logger.debug("doc[]='%s'", type(doc))
     if doc is None:
         logger.warning("Cannot fetch any /about pages for domain='%s' - EXIT!", domain)
-        return blocklist
+        return list()
+
+    headers = doc.find_all("h2")
+
+    logger.debug("headers[]='%s'", type(headers))
+    if headers is None:
+        logger.warning("Cannot fetch any /about pages for domain='%s' - EXIT!", domain)
+        return list()
 
-    for header in doc.find_all("h2"):
-        header_text = tidyup.reason(header.text)
+    logger.info("Checking %d headers ...", len(headers))
+    for header in headers:
+        logger.debug("header[%s]='%s'", type(header), header)
+        block_level = tidyup.reason(header.text).lower()
 
-        logger.debug("header_text='%s' - BEFORE!", header_text)
-        if header_text in language_mapping:
-            logger.debug("header_text='%s' - FOUND!", header_text)
-            header_text = language_mapping[header_text]
+        logger.debug("block_level='%s' - BEFORE!", block_level)
+        if block_level in language_mapping:
+            logger.debug("block_level='%s' - FOUND!", block_level)
+            block_level = language_mapping[block_level].lower()
         else:
-            logger.warning("header_text='%s' not found in language mapping table", header_text)
+            logger.warning("block_level='%s' not found in language mapping table", block_level)
 
-        logger.debug("header_text='%s - AFTER!'", header_text)
-        if header_text in blocklist or header_text.lower() in blocklist:
+        logger.debug("block_level='%s - AFTER!'", block_level)
+        if block_level in blocklist:
             # replaced find_next_siblings with find_all_next to account for instances that e.g. hide lists in dropdown menu
-            logger.debug("Found header_text='%s', importing domain blocks ...", header_text)
+            logger.debug("Found block_level='%s', importing domain blocks ...", block_level)
             for line in header.find_next("table").find_all("tr")[1:]:
                 logger.debug("line[]='%s'", type(line))
-                blocklist[header_text].append({
-                    "blocked": tidyup.domain(line.find_all("td")[0].text),
-                    "reason" : tidyup.reason(line.find_all("td")[1].text),
+                blocked = tidyup.domain(line.find_all("td")[0].text)
+                reason = tidyup.reason(line.find_all("td")[1].text)
+
+                logger.debug("Appending block_level='%s',blocked='%s',reason='%s' ...", block_level, blocked, reason)
+                blocklist[block_level].append({
+                    "blocked": blocked,
+                    "reason" : reason,
                 })
         else:
-            logger.warning("header_text='%s' not found in blocklist()=%d", header_text, len(blocklist))
+            logger.warning("block_level='%s' not found in blocklist()=%d", block_level, len(blocklist))
 
     logger.debug("Returning blocklist for domain='%s' - EXIT!", domain)
-    return {
-        "reject"        : blocklist["Suspended servers"],
-        "media_removal" : blocklist["Filtered media"],
-        "followers_only": blocklist["Limited servers"] + blocklist["Silenced servers"],
-    }
+    return blocklist