From: Roland Häder Date: Tue, 15 Aug 2023 17:12:09 +0000 (+0200) Subject: Continued: X-Git-Url: https://git.mxchange.org/?a=commitdiff_plain;h=ef6c409f16267882c7755c8347b49113c378811f;p=fba.git Continued: - typical for oliphant members: Hide their own blocklist and then handle it over to Codeberg repository of oliphant - this helper now has a simple function to check if the provided domain should be excluded --- diff --git a/fba/commands.py b/fba/commands.py index fb363fe..78ef33a 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -34,6 +34,7 @@ from fba import database from fba import utils from fba.helpers import blacklist +from fba.helpers import blocklists from fba.helpers import config from fba.helpers import cookies from fba.helpers import dicts as dict_helper @@ -320,7 +321,7 @@ def fetch_blocks(args: argparse.Namespace) -> int: blocking = list() - if blocker != "chaos.social": + if federation.is_excluded_blocklist(blocker): logger.debug("blocker='%s',software='%s'", blocker, software) if software == "pleroma": logger.info("blocker='%s',software='%s'", blocker, software) @@ -985,57 +986,10 @@ def fetch_oliphant(args: argparse.Namespace) -> int: # Base URL base_url = f"https://{source_domain}/oliphant/blocklists/raw/branch/main/blocklists" - # URLs to fetch - blocklists = ( - { - "blocker": "artisan.chat", - "csv_url": "mastodon/artisan.chat.csv", - },{ - "blocker": "mastodon.art", - "csv_url": "mastodon/mastodon.art.csv", - },{ - "blocker": "pleroma.envs.net", - "csv_url": "mastodon/pleroma.envs.net.csv", - },{ - "blocker": "oliphant.social", - "csv_url": "mastodon/_unified_tier3_blocklist.csv", - },{ - "blocker": "mastodon.online", - "csv_url": "mastodon/mastodon.online.csv", - },{ - "blocker": "mastodon.social", - "csv_url": "mastodon/mastodon.social.csv", - },{ - "blocker": "mastodon.social", - "csv_url": "other/missing-tier0-mastodon.social.csv", - },{ - "blocker": "rage.love", - "csv_url": "mastodon/rage.love.csv", - },{ - "blocker": "sunny.garden", - "csv_url": "mastodon/sunny.garden.csv", - },{ - "blocker": "sunny.garden", - "csv_url": "mastodon/gardenfence.csv", - },{ - "blocker": "solarpunk.moe", - "csv_url": "mastodon/solarpunk.moe.csv", - },{ - "blocker": "toot.wales", - "csv_url": "mastodon/toot.wales.csv", - },{ - "blocker": "union.place", - "csv_url": "mastodon/union.place.csv", - },{ - "blocker": "oliphant.social", - "csv_url": "mastodon/birdsite.csv", - } - ) - domains = list() - logger.debug("Downloading %d files ...", len(blocklists)) - for block in blocklists: + logger.debug("Downloading %d files ...", len(blocklists.oliphant_blocklists)) + for block in blocklists.oliphant_blocklists: # Is domain given and not equal blocker? if isinstance(args.domain, str) and args.domain != block["blocker"]: logger.debug("Skipping blocker='%s', not matching args.domain='%s'", block["blocker"], args.domain) @@ -1141,7 +1095,7 @@ def fetch_oliphant(args: argparse.Namespace) -> int: processing.block(block["blocker"], domain, None, "reject_reports") logger.debug("block[blocker]='%s'", block["blocker"]) - if block["blocker"] != "chaos.social": + if blocklists.is_excluded(block["blocker"]): logger.debug("Invoking instances.set_total_blocks(%s, domains()=%d) ...", block["blocker"], len(domains)) instances.set_total_blocks(block["blocker"], domains) @@ -1586,7 +1540,7 @@ def recheck_obfuscation(args: argparse.Namespace) -> int: logger.debug("row[domain]='%s'", row["domain"]) # chaos.social requires special care ... - if row["domain"] != "chaos.social": + if blocklists.is_excluded(row["domain"]): logger.debug("Invoking instances.set_total_blocks(%s, %d) ...", row["domain"], len(blocking)) instances.set_last_blocked(row["domain"]) instances.set_total_blocks(row["domain"], blocking) diff --git a/fba/helpers/__init__.py b/fba/helpers/__init__.py index ce9f5f8..efcd204 100644 --- a/fba/helpers/__init__.py +++ b/fba/helpers/__init__.py @@ -15,6 +15,7 @@ __all__ = [ 'blacklist', + 'blocklists', 'cache', 'config', 'cookies', diff --git a/fba/helpers/blocklists.py b/fba/helpers/blocklists.py new file mode 100644 index 0000000..a060a4e --- /dev/null +++ b/fba/helpers/blocklists.py @@ -0,0 +1,91 @@ +# Fedi API Block - An aggregator for fetching blocking data from fediverse nodes +# Copyright (C) 2023 Free Software Foundation +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import logging + +from fba.helpers import domain as domain_helper + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) +#logger.setLevel(logging.DEBUG) + +# URLs to fetch +oliphant_blocklists = ( + { + "blocker": "artisan.chat", + "csv_url": "mastodon/artisan.chat.csv", + },{ + "blocker": "colorid.es", + "csv_url": "mastodon/colorid.es.csv", + },{ + "blocker": "indiepocalypse.social", + "csv_url": "mastodon/indiepocalypse.social.csv", + },{ + "blocker": "mastodon.art", + "csv_url": "mastodon/mastodon.art.csv", + },{ + "blocker": "pleroma.envs.net", + "csv_url": "mastodon/pleroma.envs.net.csv", + },{ + "blocker": "oliphant.social", + "csv_url": "mastodon/_unified_tier3_blocklist.csv", + },{ + "blocker": "mastodon.online", + "csv_url": "mastodon/mastodon.online.csv", + },{ + "blocker": "mastodon.social", + "csv_url": "mastodon/mastodon.social.csv", + },{ + "blocker": "mastodon.social", + "csv_url": "other/missing-tier0-mastodon.social.csv", + },{ + "blocker": "rage.love", + "csv_url": "mastodon/rage.love.csv", + },{ + "blocker": "sunny.garden", + "csv_url": "mastodon/sunny.garden.csv", + },{ + "blocker": "sunny.garden", + "csv_url": "mastodon/gardenfence.csv", + },{ + "blocker": "solarpunk.moe", + "csv_url": "mastodon/solarpunk.moe.csv", + },{ + "blocker": "toot.wales", + "csv_url": "mastodon/toot.wales.csv", + },{ + "blocker": "union.place", + "csv_url": "mastodon/union.place.csv", + },{ + "blocker": "oliphant.social", + "csv_url": "mastodon/birdsite.csv", + } +) + +def is_excluded(domain: str) -> bool: + logger.debug("domain='%s' - CALLED!") + domain_helper.raise_on(domain) + + # Default is not excluded + excluded = False + for row in oliphant_blocklists: + logger.debug("row[blocker]='%s',domain='%s'", row["blocker"], domain) + if row["blocker"] == domain: + excluded = True + break + + logger.debug("excluded='%s' - EXIT!") + return excluded diff --git a/fba/http/federation.py b/fba/http/federation.py index 071c702..75f006b 100644 --- a/fba/http/federation.py +++ b/fba/http/federation.py @@ -39,6 +39,7 @@ from fba.networks import lemmy from fba.networks import misskey from fba.networks import peertube +# Depth counter, being raised and lowered _DEPTH = 0 logging.basicConfig(level=logging.INFO)