Continued:

- typical for oliphant members: Hide their own blocklist and then handle it
  over to Codeberg repository of oliphant
- this helper now has a simple function to check if the provided domain should
  be excluded

diff --git a/fba/commands.py b/fba/commands.py
index fb363fe6ea528cc506024621360683003ccbf62f..78ef33a6d739a2cf0724bd8a00aa01d3ccbe7d10 100644 (file)
--- a/fba/commands.py
+++ b/fba/commands.py
@@ -34,6 +34,7 @@ from fba import database
 from fba import utils
 
 from fba.helpers import blacklist
+from fba.helpers import blocklists
 from fba.helpers import config
 from fba.helpers import cookies
 from fba.helpers import dicts as dict_helper
@@ -320,7 +321,7 @@ def fetch_blocks(args: argparse.Namespace) -> int:
 
         blocking = list()
 
-        if blocker != "chaos.social":
+        if federation.is_excluded_blocklist(blocker):
             logger.debug("blocker='%s',software='%s'", blocker, software)
             if software == "pleroma":
                 logger.info("blocker='%s',software='%s'", blocker, software)
@@ -985,57 +986,10 @@ def fetch_oliphant(args: argparse.Namespace) -> int:
     # Base URL
     base_url = f"https://{source_domain}/oliphant/blocklists/raw/branch/main/blocklists"
 
-    # URLs to fetch
-    blocklists = (
-        {
-            "blocker": "artisan.chat",
-            "csv_url": "mastodon/artisan.chat.csv",
-        },{
-            "blocker": "mastodon.art",
-            "csv_url": "mastodon/mastodon.art.csv",
-        },{
-            "blocker": "pleroma.envs.net",
-            "csv_url": "mastodon/pleroma.envs.net.csv",
-        },{
-            "blocker": "oliphant.social",
-            "csv_url": "mastodon/_unified_tier3_blocklist.csv",
-        },{
-            "blocker": "mastodon.online",
-            "csv_url": "mastodon/mastodon.online.csv",
-        },{
-            "blocker": "mastodon.social",
-            "csv_url": "mastodon/mastodon.social.csv",
-        },{
-            "blocker": "mastodon.social",
-            "csv_url": "other/missing-tier0-mastodon.social.csv",
-        },{
-            "blocker": "rage.love",
-            "csv_url": "mastodon/rage.love.csv",
-        },{
-            "blocker": "sunny.garden",
-            "csv_url": "mastodon/sunny.garden.csv",
-        },{
-            "blocker": "sunny.garden",
-            "csv_url": "mastodon/gardenfence.csv",
-        },{
-            "blocker": "solarpunk.moe",
-            "csv_url": "mastodon/solarpunk.moe.csv",
-        },{
-            "blocker": "toot.wales",
-            "csv_url": "mastodon/toot.wales.csv",
-        },{
-            "blocker": "union.place",
-            "csv_url": "mastodon/union.place.csv",
-        },{
-            "blocker": "oliphant.social",
-            "csv_url": "mastodon/birdsite.csv",
-        }
-    )
-
     domains = list()
 
-    logger.debug("Downloading %d files ...", len(blocklists))
-    for block in blocklists:
+    logger.debug("Downloading %d files ...", len(blocklists.oliphant_blocklists))
+    for block in blocklists.oliphant_blocklists:
         # Is domain given and not equal blocker?
         if isinstance(args.domain, str) and args.domain != block["blocker"]:
             logger.debug("Skipping blocker='%s', not matching args.domain='%s'", block["blocker"], args.domain)
@@ -1141,7 +1095,7 @@ def fetch_oliphant(args: argparse.Namespace) -> int:
                 processing.block(block["blocker"], domain, None, "reject_reports")
 
         logger.debug("block[blocker]='%s'", block["blocker"])
-        if block["blocker"] != "chaos.social":
+        if blocklists.is_excluded(block["blocker"]):
             logger.debug("Invoking instances.set_total_blocks(%s, domains()=%d) ...", block["blocker"], len(domains))
             instances.set_total_blocks(block["blocker"], domains)
 
@@ -1586,7 +1540,7 @@ def recheck_obfuscation(args: argparse.Namespace) -> int:
 
         logger.debug("row[domain]='%s'", row["domain"])
         # chaos.social requires special care ...
-        if row["domain"] != "chaos.social":
+        if blocklists.is_excluded(row["domain"]):
             logger.debug("Invoking instances.set_total_blocks(%s, %d) ...", row["domain"], len(blocking))
             instances.set_last_blocked(row["domain"])
             instances.set_total_blocks(row["domain"], blocking)

diff --git a/fba/helpers/__init__.py b/fba/helpers/__init__.py
index ce9f5f8b1a8b30a6cbcaa6caff070fc3010de33a..efcd204fd7f165438498e445a1b340a82ea9eb85 100644 (file)
--- a/fba/helpers/__init__.py
+++ b/fba/helpers/__init__.py
@@ -15,6 +15,7 @@
 
 __all__ = [
     'blacklist',
+    'blocklists',
     'cache',
     'config',
     'cookies',

diff --git a/fba/helpers/blocklists.py b/fba/helpers/blocklists.py
new file mode 100644 (file)
index 0000000..a060a4e
--- /dev/null
+++ b/fba/helpers/blocklists.py
@@ -0,0 +1,91 @@
+# Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
+# Copyright (C) 2023 Free Software Foundation
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import logging
+
+from fba.helpers import domain as domain_helper
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+#logger.setLevel(logging.DEBUG)
+
+# URLs to fetch
+oliphant_blocklists = (
+    {
+        "blocker": "artisan.chat",
+        "csv_url": "mastodon/artisan.chat.csv",
+    },{
+        "blocker": "colorid.es",
+        "csv_url": "mastodon/colorid.es.csv",
+    },{
+        "blocker": "indiepocalypse.social",
+        "csv_url": "mastodon/indiepocalypse.social.csv",
+    },{
+        "blocker": "mastodon.art",
+        "csv_url": "mastodon/mastodon.art.csv",
+    },{
+        "blocker": "pleroma.envs.net",
+        "csv_url": "mastodon/pleroma.envs.net.csv",
+    },{
+        "blocker": "oliphant.social",
+        "csv_url": "mastodon/_unified_tier3_blocklist.csv",
+    },{
+        "blocker": "mastodon.online",
+        "csv_url": "mastodon/mastodon.online.csv",
+    },{
+        "blocker": "mastodon.social",
+        "csv_url": "mastodon/mastodon.social.csv",
+    },{
+        "blocker": "mastodon.social",
+        "csv_url": "other/missing-tier0-mastodon.social.csv",
+    },{
+        "blocker": "rage.love",
+        "csv_url": "mastodon/rage.love.csv",
+    },{
+        "blocker": "sunny.garden",
+        "csv_url": "mastodon/sunny.garden.csv",
+    },{
+        "blocker": "sunny.garden",
+        "csv_url": "mastodon/gardenfence.csv",
+    },{
+        "blocker": "solarpunk.moe",
+        "csv_url": "mastodon/solarpunk.moe.csv",
+    },{
+        "blocker": "toot.wales",
+        "csv_url": "mastodon/toot.wales.csv",
+    },{
+        "blocker": "union.place",
+        "csv_url": "mastodon/union.place.csv",
+    },{
+        "blocker": "oliphant.social",
+        "csv_url": "mastodon/birdsite.csv",
+    }
+)
+
+def is_excluded(domain: str) -> bool:
+    logger.debug("domain='%s' - CALLED!")
+    domain_helper.raise_on(domain)
+
+    # Default is not excluded
+    excluded = False
+    for row in oliphant_blocklists:
+        logger.debug("row[blocker]='%s',domain='%s'", row["blocker"], domain)
+        if row["blocker"] == domain:
+            excluded = True
+            break
+
+    logger.debug("excluded='%s' - EXIT!")
+    return excluded

diff --git a/fba/http/federation.py b/fba/http/federation.py
index 071c702379bf615b9a09673bc5a1911d3719dcbd..75f006b29df666659f425c491a7f81ef10136947 100644 (file)
--- a/fba/http/federation.py
+++ b/fba/http/federation.py
@@ -39,6 +39,7 @@ from fba.networks import lemmy
 from fba.networks import misskey
 from fba.networks import peertube
 
+# Depth counter, being raised and lowered
 _DEPTH = 0
 
 logging.basicConfig(level=logging.INFO)