From: Roland Häder Date: Mon, 28 Aug 2023 14:30:09 +0000 (+0200) Subject: Continued: X-Git-Url: https://git.mxchange.org/?a=commitdiff_plain;h=d8a406eb5b96cb8161af7ec88604f4b0d526ed4e;p=fba.git Continued: - some misskey instances may have no nodeinfo URL, e.g. they got only detected through APP_NAME method - still they may provide a blocklist - it is now rewritten that first a generic "/api/v1/instance/domain_blocks" is fetched and if it fails, a software-specific attempt is done --- diff --git a/fba/__init__.py b/fba/__init__.py index 4f6529a..9746ec1 100644 --- a/fba/__init__.py +++ b/fba/__init__.py @@ -19,7 +19,6 @@ __all__ = [ # Main packages: 'boot', 'commands', - 'csrf', 'database', 'utils', # Sub packages: diff --git a/fba/commands.py b/fba/commands.py index 9775189..19702ae 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -29,7 +29,6 @@ import markdown import reqto import validators -from fba import csrf from fba import database from fba import utils @@ -44,6 +43,7 @@ from fba.helpers import processing from fba.helpers import software as software_helper from fba.helpers import tidyup +from fba.http import csrf from fba.http import federation from fba.http import network @@ -308,10 +308,7 @@ def fetch_blocks(args: argparse.Namespace) -> int: for blocker, software, origin, nodeinfo_url in rows: logger.debug("blocker='%s',software='%s',origin='%s',nodeinfo_url='%s'", blocker, software, origin, nodeinfo_url) - if nodeinfo_url is None: - logger.debug("blocker='%s',software='%s' has no nodeinfo_url set - SKIPPED!", blocker, software) - continue - elif not domain_helper.is_wanted(blocker): + if not domain_helper.is_wanted(blocker): logger.warning("blocker='%s' is not wanted - SKIPPED!", blocker) continue @@ -319,11 +316,16 @@ def fetch_blocks(args: argparse.Namespace) -> int: instances.set_last_blocked(blocker) instances.set_has_obfuscation(blocker, False) - blocking = list() - # c.s isn't part of oliphant's "hidden" blocklists - logger.debug("blocker='%s'", blocker) - if blocker != "chaos.social" and not blocklists.is_excluded(blocker): + if blocker == "chaos.social" or blocklists.has(blocker): + logger.debug("Skipping blocker='%s', run ./fba.py fetch_cs or fetch_oliphant instead!", blocker) + continue + + logger.debug("Invoking federation.fetch_blocks(%s) ...", blocker) + blocking = federation.fetch_blocks(blocker) + + logger.debug("blocking()=%d,nodeinfo_url='%s'", len(blocking), nodeinfo_url) + if len(blocking) == 0: logger.debug("blocker='%s',software='%s'", blocker, software) if software == "pleroma": logger.info("blocker='%s',software='%s'", blocker, software) @@ -348,10 +350,8 @@ def fetch_blocks(args: argparse.Namespace) -> int: else: logger.warning("Unknown software: blocker='%s',software='%s'", blocker, software) - logger.debug("Invoking instances.set_total_blocks(%s, %d) ...", blocker, len(blocking)) - instances.set_total_blocks(blocker, blocking) - else: - logger.debug("Skipping blocker='%s', run ./fba.py fetch_cs or fetch_oliphant instead!", blocker) + logger.debug("Invoking instances.set_total_blocks(%s, %d) ...", blocker, len(blocking)) + instances.set_total_blocks(blocker, blocking) logger.info("Checking %d entries from blocker='%s',software='%s' ...", len(blocking), blocker, software) blockdict = list() @@ -1108,7 +1108,7 @@ def fetch_oliphant(args: argparse.Namespace) -> int: processing.block(block["blocker"], domain, None, "reject_reports") logger.debug("block[blocker]='%s'", block["blocker"]) - if not blocklists.is_excluded(block["blocker"]): + if not blocklists.has(block["blocker"]): logger.debug("Invoking instances.set_total_blocks(%s, domains()=%d) ...", block["blocker"], len(domains)) instances.set_total_blocks(block["blocker"], domains) @@ -1532,28 +1532,32 @@ def recheck_obfuscation(args: argparse.Namespace) -> int: logger.debug("row[domain]='%s' has been recently checked, args.force[]='%s' - SKIPPED!", row["domain"], type(args.force)) continue - blocking = list() - if row["software"] == "pleroma": - logger.debug("domain='%s',software='%s'", row["domain"], row["software"]) - blocking = pleroma.fetch_blocks(row["domain"], row["nodeinfo_url"]) - elif row["software"] == "mastodon": - logger.debug("domain='%s',software='%s'", row["domain"], row["software"]) - blocking = mastodon.fetch_blocks(row["domain"], row["nodeinfo_url"]) - elif row["software"] == "lemmy": - logger.debug("domain='%s',software='%s'", row["domain"], row["software"]) - blocking = lemmy.fetch_blocks(row["domain"], row["nodeinfo_url"]) - elif row["software"] == "friendica": - logger.debug("domain='%s',software='%s'", row["domain"], row["software"]) - blocking = friendica.fetch_blocks(row["domain"]) - elif row["software"] == "misskey": - logger.debug("domain='%s',software='%s'", row["domain"], row["software"]) - blocking = misskey.fetch_blocks(row["domain"]) - else: - logger.warning("Unknown software: domain='%s',software='%s'", row["domain"], row["software"]) + logger.debug("Invoking federation.fetch_blocks(%s) ...", row["domain"]) + blocking = federation.fetch_blocks(row["domain"]) + + logger.debug("blocking()=%d", len(blocking)) + if len(blocking) == 0: + if row["software"] == "pleroma": + logger.debug("domain='%s',software='%s'", row["domain"], row["software"]) + blocking = pleroma.fetch_blocks(row["domain"], row["nodeinfo_url"]) + elif row["software"] == "mastodon": + logger.debug("domain='%s',software='%s'", row["domain"], row["software"]) + blocking = mastodon.fetch_blocks(row["domain"], row["nodeinfo_url"]) + elif row["software"] == "lemmy": + logger.debug("domain='%s',software='%s'", row["domain"], row["software"]) + blocking = lemmy.fetch_blocks(row["domain"], row["nodeinfo_url"]) + elif row["software"] == "friendica": + logger.debug("domain='%s',software='%s'", row["domain"], row["software"]) + blocking = friendica.fetch_blocks(row["domain"]) + elif row["software"] == "misskey": + logger.debug("domain='%s',software='%s'", row["domain"], row["software"]) + blocking = misskey.fetch_blocks(row["domain"]) + else: + logger.warning("Unknown software: domain='%s',software='%s'", row["domain"], row["software"]) # c.s isn't part of oliphant's "hidden" blocklists logger.debug("row[domain]='%s'", row["domain"]) - if row["domain"] != "chaos.social" and not blocklists.is_excluded(row["domain"]): + if row["domain"] != "chaos.social" and not blocklists.has(row["domain"]): logger.debug("Invoking instances.set_total_blocks(%s, %d) ...", row["domain"], len(blocking)) instances.set_last_blocked(row["domain"]) instances.set_total_blocks(row["domain"], blocking) diff --git a/fba/csrf.py b/fba/csrf.py deleted file mode 100644 index eadbc0e..0000000 --- a/fba/csrf.py +++ /dev/null @@ -1,77 +0,0 @@ -# Fedi API Block - An aggregator for fetching blocking data from fediverse nodes -# Copyright (C) 2023 Free Software Foundation -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published -# by the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -import logging - -import bs4 -import reqto -import requests - -from fba.helpers import config -from fba.helpers import cookies -from fba.helpers import domain as domain_helper - -from fba.http import network - -from fba.models import instances - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - -def determine(domain: str, headers: dict) -> dict: - logger.debug("domain='%s',headers()=%d - CALLED!", domain, len(headers)) - domain_helper.raise_on(domain) - - if not isinstance(headers, dict): - raise ValueError(f"Parameter headers[]='{type(headers)}' is not of type 'dict'") - - # Default headers with no CSRF - reqheaders = headers - - # Fetch / to check for meta tag indicating csrf - logger.debug("Fetching / from domain='%s' for CSRF check ...", domain) - response = reqto.get( - f"https://{domain}/", - headers=network.web_headers, - timeout=(config.get("connection_timeout"), config.get("read_timeout")) - ) - - logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text)) - if response.ok and response.status_code == 200 and response.text.strip() != "" and response.text.find(" 0 and domain_helper.is_in_url(domain, response.url): - # Save cookies - logger.debug("Parsing response.text()=%d Bytes ...", len(response.text)) - cookies.store(domain, response.cookies.get_dict()) - - # Parse text - meta = bs4.BeautifulSoup( - response.text, - "html.parser" - ) - logger.debug("meta[]='%s'", type(meta)) - tag = meta.find("meta", attrs={"name": "csrf-token"}) - - logger.debug("tag[%s]='%s'", type(tag), tag) - if tag is not None: - logger.debug("Adding CSRF token='%s' for domain='%s'", tag["content"], domain) - reqheaders["X-CSRF-Token"] = tag["content"] - elif not domain_helper.is_in_url(domain, response.url): - logger.warning("domain='%s' doesn't match with response.url='%s', maybe redirect to other domain?", domain, response.url) - message = f"Redirect from domain='{domain}' to response.url='{response.url}'" - instances.set_last_error(domain, message) - raise requests.exceptions.TooManyRedirects(message) - - logger.debug("reqheaders()=%d - EXIT!", len(reqheaders)) - return reqheaders diff --git a/fba/helpers/blocklists.py b/fba/helpers/blocklists.py index 309f8a6..b6d018b 100644 --- a/fba/helpers/blocklists.py +++ b/fba/helpers/blocklists.py @@ -75,18 +75,18 @@ oliphant_blocklists = ( } ) -def is_excluded(domain: str) -> bool: +def has(domain: str) -> bool: logger.debug("domain='%s' - CALLED!") domain_helper.raise_on(domain) - # Default is not excluded - excluded = False + # Default is not found + found = False for row in oliphant_blocklists: logger.debug("row[blocker]='%s',domain='%s'", row["blocker"], domain) if row["blocker"] == domain: - excluded = True - logger.debug("domain='%s' is excluded from regular fetch_blocks command - BREAK!", domain) + found = True + logger.debug("domain='%s' is found and excluded from regular fetch_blocks command - BREAK!", domain) break - logger.debug("excluded='%s' - EXIT!") - return excluded + logger.debug("found='%s' - EXIT!", found) + return found diff --git a/fba/http/__init__.py b/fba/http/__init__.py index fcfdfd9..a011e29 100644 --- a/fba/http/__init__.py +++ b/fba/http/__init__.py @@ -14,6 +14,7 @@ # along with this program. If not, see . __all__ = [ + 'csrf', 'federation', 'network', 'nodeinfo', diff --git a/fba/http/csrf.py b/fba/http/csrf.py new file mode 100644 index 0000000..eadbc0e --- /dev/null +++ b/fba/http/csrf.py @@ -0,0 +1,77 @@ +# Fedi API Block - An aggregator for fetching blocking data from fediverse nodes +# Copyright (C) 2023 Free Software Foundation +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import logging + +import bs4 +import reqto +import requests + +from fba.helpers import config +from fba.helpers import cookies +from fba.helpers import domain as domain_helper + +from fba.http import network + +from fba.models import instances + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +def determine(domain: str, headers: dict) -> dict: + logger.debug("domain='%s',headers()=%d - CALLED!", domain, len(headers)) + domain_helper.raise_on(domain) + + if not isinstance(headers, dict): + raise ValueError(f"Parameter headers[]='{type(headers)}' is not of type 'dict'") + + # Default headers with no CSRF + reqheaders = headers + + # Fetch / to check for meta tag indicating csrf + logger.debug("Fetching / from domain='%s' for CSRF check ...", domain) + response = reqto.get( + f"https://{domain}/", + headers=network.web_headers, + timeout=(config.get("connection_timeout"), config.get("read_timeout")) + ) + + logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text)) + if response.ok and response.status_code == 200 and response.text.strip() != "" and response.text.find(" 0 and domain_helper.is_in_url(domain, response.url): + # Save cookies + logger.debug("Parsing response.text()=%d Bytes ...", len(response.text)) + cookies.store(domain, response.cookies.get_dict()) + + # Parse text + meta = bs4.BeautifulSoup( + response.text, + "html.parser" + ) + logger.debug("meta[]='%s'", type(meta)) + tag = meta.find("meta", attrs={"name": "csrf-token"}) + + logger.debug("tag[%s]='%s'", type(tag), tag) + if tag is not None: + logger.debug("Adding CSRF token='%s' for domain='%s'", tag["content"], domain) + reqheaders["X-CSRF-Token"] = tag["content"] + elif not domain_helper.is_in_url(domain, response.url): + logger.warning("domain='%s' doesn't match with response.url='%s', maybe redirect to other domain?", domain, response.url) + message = f"Redirect from domain='{domain}' to response.url='{response.url}'" + instances.set_last_error(domain, message) + raise requests.exceptions.TooManyRedirects(message) + + logger.debug("reqheaders()=%d - EXIT!", len(reqheaders)) + return reqheaders diff --git a/fba/http/federation.py b/fba/http/federation.py index 7a6e7e7..c7960d6 100644 --- a/fba/http/federation.py +++ b/fba/http/federation.py @@ -21,8 +21,6 @@ import bs4 import requests import validators -from fba import csrf - from fba.helpers import config from fba.helpers import cookies from fba.helpers import domain as domain_helper @@ -30,9 +28,11 @@ from fba.helpers import software as software_helper from fba.helpers import tidyup from fba.helpers import version +from fba.http import csrf from fba.http import network from fba.http import nodeinfo +from fba.models import blocks from fba.models import instances from fba.networks import lemmy @@ -530,3 +530,92 @@ def add_peers(rows: dict) -> list: logger.debug("peers()=%d - EXIT!", len(peers)) return peers + +def fetch_blocks(domain: str) -> list: + logger.debug("domain='%s' - CALLED!", domain) + domain_helper.raise_on(domain) + + # Init block list + blocklist = list() + + # No CSRF by default, you don't have to add network.api_headers by yourself here + headers = tuple() + + try: + logger.debug("Checking CSRF for domain='%s'", domain) + headers = csrf.determine(domain, dict()) + except network.exceptions as exception: + logger.warning("Exception '%s' during checking CSRF (fetch_blocks,%s)", type(exception), __name__) + instances.set_last_error(domain, exception) + + logger.debug("Returning empty list ... - EXIT!") + return list() + + try: + # json endpoint for newer mastodongs + logger.debug("Querying API domain_blocks: domain='%s'", domain) + data = network.get_json_api( + domain, + "/api/v1/instance/domain_blocks", + headers, + (config.get("connection_timeout"), config.get("read_timeout")) + ) + rows = list() + + logger.debug("data[]='%s'", type(data)) + if "error_message" in data: + logger.debug("Was not able to fetch domain_blocks from domain='%s': status_code=%d,error_message='%s'", domain, data['status_code'], data['error_message']) + instances.set_last_error(domain, data) + return blocklist + elif "json" in data and "error" in data["json"]: + logger.warning("JSON API returned error message: '%s'", data["json"]["error"]) + instances.set_last_error(domain, data) + return blocklist + else: + # Getting blocklist + rows = data["json"] + + logger.debug("Marking domain='%s' as successfully handled ...", domain) + instances.set_success(domain) + + logger.debug("rows[%s]()=%d", type(rows), len(rows)) + if len(rows) > 0: + logger.debug("Checking %d entries from domain='%s' ...", len(rows), domain) + for block in rows: + # Check type + logger.debug("block[]='%s'", type(block)) + if not isinstance(block, dict): + logger.debug("block[]='%s' is of type 'dict' - SKIPPED!", type(block)) + continue + elif "domain" not in block: + logger.warning("block()=%d does not contain element 'domain' - SKIPPED!", len(block)) + continue + elif "severity" not in block: + logger.warning("block()=%d does not contain element 'severity' - SKIPPED!", len(block)) + continue + elif block["severity"] in ["accept", "accepted"]: + logger.debug("block[domain]='%s' has unwanted severity level '%s' - SKIPPED!", block["domain"], block["severity"]) + continue + elif "digest" in block and not validators.hashes.sha256(block["digest"]): + logger.warning("block[domain]='%s' has invalid block[digest]='%s' - SKIPPED!", block["domain"], block["digest"]) + continue + + reason = tidyup.reason(block["comment"]) if "comment" in block and block["comment"] is not None and block["comment"] != "" else None + + logger.debug("Appending blocker='%s',blocked='%s',reason='%s',block_level='%s'", domain, block["domain"], reason, block["severity"]) + blocklist.append({ + "blocker" : domain, + "blocked" : block["domain"], + "hash" : block["digest"] if "digest" in block else None, + "reason" : reason, + "block_level": blocks.alias_block_level(block["severity"]), + }) + else: + logger.debug("domain='%s' has no block list", domain) + + except network.exceptions as exception: + logger.warning("domain='%s',exception[%s]='%s'", domain, type(exception), str(exception)) + instances.set_last_error(domain, exception) + + logger.debug("blocklist()=%d - EXIT!", len(blocklist)) + return blocklist diff --git a/fba/http/nodeinfo.py b/fba/http/nodeinfo.py index c804a59..c578a18 100644 --- a/fba/http/nodeinfo.py +++ b/fba/http/nodeinfo.py @@ -17,11 +17,10 @@ import logging from urllib.parse import urlparse import validators -from fba import csrf - from fba.helpers import config from fba.helpers import domain as domain_helper +from fba.http import csrf from fba.http import network from fba.models import instances diff --git a/fba/networks/lemmy.py b/fba/networks/lemmy.py index 31f3019..ae45e75 100644 --- a/fba/networks/lemmy.py +++ b/fba/networks/lemmy.py @@ -19,12 +19,11 @@ import logging import bs4 -from fba import csrf - from fba.helpers import config from fba.helpers import domain as domain_helper from fba.helpers import tidyup +from fba.http import csrf from fba.http import federation from fba.http import network diff --git a/fba/networks/mastodon.py b/fba/networks/mastodon.py index 77e77c8..ecb682d 100644 --- a/fba/networks/mastodon.py +++ b/fba/networks/mastodon.py @@ -19,12 +19,11 @@ import validators import bs4 -from fba import csrf - from fba.helpers import config from fba.helpers import domain as domain_helper from fba.helpers import tidyup +from fba.http import federation from fba.http import network from fba.models import blocks @@ -137,92 +136,50 @@ def fetch_blocks(domain: str, nodeinfo_url: str) -> list: elif nodeinfo_url == "": raise ValueError("Parameter 'nodeinfo_url' is empty") - # Init block list blocklist = list() - # No CSRF by default, you don't have to add network.api_headers by yourself here - headers = tuple() - - try: - logger.debug("Checking CSRF for domain='%s'", domain) - headers = csrf.determine(domain, dict()) - except network.exceptions as exception: - logger.warning("Exception '%s' during checking CSRF (fetch_blocks,%s)", type(exception), __name__) - instances.set_last_error(domain, exception) - - logger.debug("Returning empty list ... - EXIT!") - return list() - - try: - # json endpoint for newer mastodongs - logger.debug("Querying API domain_blocks: domain='%s'", domain) - data = network.get_json_api( - domain, - "/api/v1/instance/domain_blocks", - headers, - (config.get("connection_timeout"), config.get("read_timeout")) - ) - rows = list() - - logger.debug("data[]='%s'", type(data)) - if "error_message" in data: - logger.debug("Was not able to fetch domain_blocks from domain='%s': status_code=%d,error_message='%s'", domain, data['status_code'], data['error_message']) - instances.set_last_error(domain, data) - return blocklist - elif "json" in data and "error" in data["json"]: - logger.warning("JSON API returned error message: '%s'", data["json"]["error"]) - instances.set_last_error(domain, data) - return blocklist - else: - # Getting blocklist - rows = data["json"] - - logger.debug("Marking domain='%s' as successfully handled ...", domain) - instances.set_success(domain) - - logger.debug("rows[%s]()=%d", type(rows), len(rows)) - if len(rows) == 0: - logger.debug("domain='%s' has returned zero rows, trying /about/more page ...", domain) - rows = fetch_blocks_from_about(domain) - - logger.debug("rows[%s]()=%d", type(rows), len(rows)) - if len(rows) > 0: - logger.debug("Checking %d entries from domain='%s' ...", len(rows), domain) - for block in rows: - # Check type - logger.debug("block[]='%s'", type(block)) - if not isinstance(block, dict): - logger.debug("block[]='%s' is of type 'dict' - SKIPPED!", type(block)) - continue - elif "domain" not in block: - logger.warning("block()=%d does not contain element 'domain' - SKIPPED!", len(block)) - continue - elif "severity" not in block: - logger.warning("block()=%d does not contain element 'severity' - SKIPPED!", len(block)) - continue - elif block["severity"] in ["accept", "accepted"]: - logger.debug("block[domain]='%s' has unwanted severity level '%s' - SKIPPED!", block["domain"], block["severity"]) - continue - elif "digest" in block and not validators.hashes.sha256(block["digest"]): - logger.warning("block[domain]='%s' has invalid block[digest]='%s' - SKIPPED!", block["domain"], block["digest"]) - continue - - reason = tidyup.reason(block["comment"]) if "comment" in block and block["comment"] is not None and block["comment"] != "" else None - - logger.debug("Appending blocker='%s',blocked='%s',reason='%s',block_level='%s'", domain, block["domain"], reason, block["severity"]) - blocklist.append({ - "blocker" : domain, - "blocked" : block["domain"], - "hash" : block["digest"] if "digest" in block else None, - "reason" : reason, - "block_level": blocks.alias_block_level(block["severity"]), - }) - else: - logger.debug("domain='%s' has no block list", domain) - - except network.exceptions as exception: - logger.warning("domain='%s',exception[%s]='%s'", domain, type(exception), str(exception)) - instances.set_last_error(domain, exception) + logger.debug("Invoking federation.fetch_blocks(%s) ...", domain) + rows = federation.fetch_blocks(domain) + + logger.debug("rows[%s]()=%d", type(rows), len(rows)) + if len(rows) == 0: + logger.debug("domain='%s' has returned zero rows, trying /about/more page ...", domain) + rows = fetch_blocks_from_about(domain) + + logger.debug("rows[%s]()=%d", type(rows), len(rows)) + if len(rows) > 0: + logger.debug("Checking %d entries from domain='%s' ...", len(rows), domain) + for block in rows: + # Check type + logger.debug("block[]='%s'", type(block)) + if not isinstance(block, dict): + logger.debug("block[]='%s' is of type 'dict' - SKIPPED!", type(block)) + continue + elif "domain" not in block: + logger.warning("block()=%d does not contain element 'domain' - SKIPPED!", len(block)) + continue + elif "severity" not in block: + logger.warning("block()=%d does not contain element 'severity' - SKIPPED!", len(block)) + continue + elif block["severity"] in ["accept", "accepted"]: + logger.debug("block[domain]='%s' has unwanted severity level '%s' - SKIPPED!", block["domain"], block["severity"]) + continue + elif "digest" in block and not validators.hashes.sha256(block["digest"]): + logger.warning("block[domain]='%s' has invalid block[digest]='%s' - SKIPPED!", block["domain"], block["digest"]) + continue + + reason = tidyup.reason(block["comment"]) if "comment" in block and block["comment"] is not None and block["comment"] != "" else None + + logger.debug("Appending blocker='%s',blocked='%s',reason='%s',block_level='%s'", domain, block["domain"], reason, block["severity"]) + blocklist.append({ + "blocker" : domain, + "blocked" : block["domain"], + "hash" : block["digest"] if "digest" in block else None, + "reason" : reason, + "block_level": blocks.alias_block_level(block["severity"]), + }) + else: + logger.debug("domain='%s' has no block list", domain) logger.debug("blocklist()=%d - EXIT!", len(blocklist)) return blocklist diff --git a/fba/networks/misskey.py b/fba/networks/misskey.py index 7583a9e..fd8b234 100644 --- a/fba/networks/misskey.py +++ b/fba/networks/misskey.py @@ -17,13 +17,12 @@ import json import logging -from fba import csrf - from fba.helpers import config from fba.helpers import dicts as dict_helper from fba.helpers import domain as domain_helper from fba.helpers import tidyup +from fba.http import csrf from fba.http import network from fba.models import instances diff --git a/fba/networks/peertube.py b/fba/networks/peertube.py index 758c8dd..58bf115 100644 --- a/fba/networks/peertube.py +++ b/fba/networks/peertube.py @@ -16,11 +16,10 @@ import logging -from fba import csrf - from fba.helpers import config from fba.helpers import domain as domain_helper +from fba.http import csrf from fba.http import network from fba.models import instances