From: Roland Häder Date: Fri, 9 Jun 2023 17:52:11 +0000 (+0200) Subject: Continued: X-Git-Url: https://git.mxchange.org/?a=commitdiff_plain;h=f248d55bfebb9cd7885486637e89ad0b1960899a;p=fba.git Continued: - renamed fba/federation/ -> fba/networks/ to make room for ... - added fba.federation.py module which has generic functions for federation - introduced fba/locking.py module - renamed aqcuire_lock() -> aqcuire() - added locking.release() - introduced fba/helpers/dict.py and tidyup.py - renamed tidyu_*() to tidyup.*() - this all-in-all was an attempt to fix cyclic imports, still some are left --- diff --git a/fba/__init__.py b/fba/__init__.py index 01c1bf6..40310f9 100644 --- a/fba/__init__.py +++ b/fba/__init__.py @@ -1,3 +1,18 @@ +# Copyright (C) 2023 Free Software Foundation +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + __all__ = [ 'blacklist', 'blocks', @@ -8,6 +23,9 @@ __all__ = [ 'csrf', 'federation', 'fba', + 'helpers', 'instances', + 'locking', 'network', + 'networks', ] diff --git a/fba/blocks.py b/fba/blocks.py index 50ceec8..ed4a32d 100644 --- a/fba/blocks.py +++ b/fba/blocks.py @@ -20,6 +20,7 @@ import validators from fba import blacklist from fba import fba +from fba.helpers import tidyup def update_reason(reason: str, blocker: str, blocked: str, block_level: str): # DEBUG: print(f"DEBUG: reason='{reason}',blocker={blocker},blocked={blocked},block_level={block_level} - CALLED!") @@ -149,7 +150,7 @@ def add_instance(blocker: str, blocked: str, reason: str, block_level: str): if reason is not None: # Maybe needs cleaning - reason = fba.tidyup_reason(reason) + reason = tidyup.reason(reason) print(f"INFO: New block: blocker='{blocker}',blocked='{blocked}', reason='{reason}', block_level='{block_level}'") try: diff --git a/fba/boot.py b/fba/boot.py index 91c2880..0ff07f3 100644 --- a/fba/boot.py +++ b/fba/boot.py @@ -15,30 +15,13 @@ # along with this program. If not, see . import argparse -import os -import sys -import tempfile -import zc.lockfile from fba import commands from fba import fba +from fba import locking -# Lock file -lockfile = tempfile.gettempdir() + '/fba.lock' -LOCK = None _PARSER = None -def acquire_lock(): - global LOCK - try: - # DEBUG: print(f"DEBUG: Acquiring lock: '{lockfile}'") - LOCK = zc.lockfile.LockFile(lockfile) - # DEBUG: print("DEBUG: Lock obtained.") - - except zc.lockfile.LockError: - print(f"ERROR: Cannot aquire lock: '{lockfile}'") - sys.exit(100) - def init_parser(): # DEBUG: # DEBUG: print("DEBUG: init_parser(): CALLED!") global _PARSER @@ -129,11 +112,5 @@ def run_command(): def shutdown(): # DEBUG: print("DEBUG: Closing database connection ...") fba.connection.close() - - if LOCK is not None: - # DEBUG: print("DEBUG: Releasing lock ...") - LOCK.close() - # DEBUG: print(f"DEBUG: Deleting lockfile='{lockfile}' ...") - os.remove(lockfile) - + locking.release() # DEBUG: print("DEBUG: Shutdown completed.") diff --git a/fba/commands.py b/fba/commands.py index d911be6..ab601c9 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -28,13 +28,19 @@ import validators from fba import blacklist from fba import blocks -from fba import boot from fba import config +from fba import federation from fba import fba from fba import instances +from fba import locking from fba import network -from fba.federation import * +from fba.helpers import tidyup + +from fba.networks import friendica +from fba.networks import mastodon +from fba.networks import misskey +from fba.networks import pleroma def check_instance(args: argparse.Namespace) -> int: # DEBUG: print(f"DEBUG: args.domain='{args.domain}' - CALLED!") @@ -65,9 +71,9 @@ def fetch_bkali(args: argparse.Namespace): # DEBUG: print(f"DEBUG: fetched({len(fetched)})[]='{type(fetched)}'") if len(fetched) == 0: raise Exception("WARNING: Returned no records") - elif not "data" in fetched: + elif "data" not in fetched: raise Exception(f"WARNING: fetched()={len(fetched)} does not contain key 'data'") - elif not "nodeinfo" in fetched["data"]: + elif "nodeinfo" not in fetched["data"]: raise Exception(f"WARNING: fetched()={len(fetched['data'])} does not contain key 'nodeinfo'") for entry in fetched["data"]["nodeinfo"]: @@ -94,12 +100,12 @@ def fetch_bkali(args: argparse.Namespace): # DEBUG: print(f"DEBUG: domains()={len(domains)}") if len(domains) > 0: - boot.acquire_lock() + locking.acquire() print(f"INFO: Adding {len(domains)} new instances ...") for domain in domains: print(f"INFO: Fetching instances from domain='{domain}' ...") - fba.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) + federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) # DEBUG: print("DEBUG: EXIT!") @@ -117,7 +123,7 @@ def fetch_blocks(args: argparse.Namespace): print(f"WARNING: domain='{args.domain}' is not registered, please run ./fba.py fetch_instances {args.domain} first.") return - boot.acquire_lock() + locking.acquire() if args.domain is not None and args.domain != "": # Re-check single domain @@ -135,7 +141,7 @@ def fetch_blocks(args: argparse.Namespace): for blocker, software, origin, nodeinfo_url in rows: # DEBUG: print("DEBUG: BEFORE blocker,software,origin,nodeinfo_url:", blocker, software, origin, nodeinfo_url) blockdict = list() - blocker = fba.tidyup_domain(blocker) + blocker = tidyup.domain(blocker) # DEBUG: print("DEBUG: AFTER blocker,software:", blocker, software) if blocker == "": @@ -164,7 +170,7 @@ def fetch_blocks(args: argparse.Namespace): print(f"INFO: Checking {len(rows.items())} entries from blocker='{blocker}',software='{software}' ...") for block_level, blocklist in rows.items(): # DEBUG: print("DEBUG: blocker,block_level,blocklist():", blocker, block_level, len(blocklist)) - block_level = fba.tidyup_domain(block_level) + block_level = tidyup.domain(block_level) # DEBUG: print("DEBUG: AFTER-block_level:", block_level) if block_level == "": print("WARNING: block_level is empty, blocker:", blocker) @@ -174,8 +180,8 @@ def fetch_blocks(args: argparse.Namespace): for block in blocklist: blocked, reason = block.values() # DEBUG: print(f"DEBUG: blocked='{blocked}',reason='{reason}' - BEFORE!") - blocked = fba.tidyup_domain(blocked) - reason = fba.tidyup_reason(reason) if reason is not None and reason != "" else None + blocked = tidyup.domain(blocked) + reason = tidyup.reason(reason) if reason is not None and reason != "" else None # DEBUG: print(f"DEBUG: blocked='{blocked}',reason='{reason}' - AFTER!") if blocked == "": @@ -286,15 +292,15 @@ def fetch_cs(args: argparse.Namespace): # DEBUG: print(f"DEBUG: doc()={len(doc)}[]={type(doc)}") silenced = doc.find("h2", {"id": "silenced-instances"}).findNext("table").find("tbody") # DEBUG: print(f"DEBUG: silenced[]={type(silenced)}") - domains["silenced"] = domains["silenced"] + fba.find_domains(silenced) + domains["silenced"] = domains["silenced"] + federation.find_domains(silenced) blocked = doc.find("h2", {"id": "blocked-instances"}).findNext("table").find("tbody") # DEBUG: print(f"DEBUG: blocked[]={type(blocked)}") - domains["reject"] = domains["reject"] + fba.find_domains(blocked) + domains["reject"] = domains["reject"] + federation.find_domains(blocked) # DEBUG: print(f"DEBUG: domains()={len(domains)}") if len(domains) > 0: - boot.acquire_lock() + locking.acquire() print(f"INFO: Adding {len(domains)} new instances ...") for block_level in domains: @@ -308,7 +314,7 @@ def fetch_cs(args: argparse.Namespace): if not instances.is_registered(row["domain"]): print(f"INFO: Fetching instances from domain='{row['domain']}' ...") - fba.fetch_instances(row["domain"], 'chaos.social', None, inspect.currentframe().f_code.co_name) + federation.fetch_instances(row["domain"], 'chaos.social', None, inspect.currentframe().f_code.co_name) # DEBUG: print("DEBUG: Committing changes ...") fba.connection.commit() @@ -347,12 +353,12 @@ def fetch_fba_rss(args: argparse.Namespace): # DEBUG: print(f"DEBUG: domains()={len(domains)}") if len(domains) > 0: - boot.acquire_lock() + locking.acquire() print(f"INFO: Adding {len(domains)} new instances ...") for domain in domains: print(f"INFO: Fetching instances from domain='{domain}' ...") - fba.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) + federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) # DEBUG: print("DEBUG: EXIT!") @@ -378,7 +384,7 @@ def fetch_fbabot_atom(args: argparse.Namespace): for element in doc.findAll("a"): for href in element["href"].split(","): # DEBUG: print(f"DEBUG: href[{type(href)}]={href}") - domain = fba.tidyup_domain(href) + domain = tidyup.domain(href) # DEBUG: print(f"DEBUG: domain='{domain}'") if blacklist.is_blacklisted(domain): @@ -396,21 +402,21 @@ def fetch_fbabot_atom(args: argparse.Namespace): # DEBUG: print(f"DEBUG: domains({len(domains)})={domains}") if len(domains) > 0: - boot.acquire_lock() + locking.acquire() print(f"INFO: Adding {len(domains)} new instances ...") for domain in domains: print(f"INFO: Fetching instances from domain='{domain}' ...") - fba.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) + federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) # DEBUG: print("DEBUG: EXIT!") def fetch_instances(args: argparse.Namespace): # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!") - boot.acquire_lock() + locking.acquire() # Initial fetch - fba.fetch_instances(args.domain, None, None, inspect.currentframe().f_code.co_name) + federation.fetch_instances(args.domain, None, None, inspect.currentframe().f_code.co_name) if args.single: # DEBUG: print("DEBUG: Not fetching more instances - EXIT!") @@ -430,13 +436,13 @@ def fetch_instances(args: argparse.Namespace): continue print(f"INFO: Fetching instances for instance '{row[0]}' ('{row[2]}') of origin='{row[1]}',nodeinfo_url='{row[3]}'") - fba.fetch_instances(row[0], row[1], row[2], inspect.currentframe().f_code.co_name, row[3]) + federation.fetch_instances(row[0], row[1], row[2], inspect.currentframe().f_code.co_name, row[3]) # DEBUG: print("DEBUG: EXIT!") def fetch_federater(args: argparse.Namespace): # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!") - boot.acquire_lock() + locking.acquire() # Fetch this URL response = fba.fetch_url("https://github.com/federater/blocks_recommended/raw/main/federater.csv", network.web_headers, (config.get("connection_timeout"), config.get("read_timeout"))) @@ -459,6 +465,6 @@ def fetch_federater(args: argparse.Namespace): continue print(f"INFO: Fetching instances for instane='{row['#domain']}' ...") - fba.fetch_instances(row["#domain"], None, None, inspect.currentframe().f_code.co_name) + federation.fetch_instances(row["#domain"], None, None, inspect.currentframe().f_code.co_name) # DEBUG: print("DEBUG: EXIT!") diff --git a/fba/csrf.py b/fba/csrf.py index caaed38..fb0bec3 100644 --- a/fba/csrf.py +++ b/fba/csrf.py @@ -22,7 +22,7 @@ from fba import config from fba import network def determine(domain: str, headers: dict) -> dict: - # DEBUG: print(f"DEBUG: domain='{domain}',headers()={len(headers)} - CALLED!") + print(f"DEBUG: domain='{domain}',headers()={len(headers)} - CALLED!") if not isinstance(domain, str): raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": @@ -35,32 +35,31 @@ def determine(domain: str, headers: dict) -> dict: try: # Fetch / to check for meta tag indicating csrf - # DEBUG: print(f"DEBUG: Fetching / from domain='{domain}' for CSRF check ...") + print(f"DEBUG: Fetching / from domain='{domain}' for CSRF check ...") response = reqto.get( f"https://{domain}/", headers=network.web_headers, timeout=(config.get("connection_timeout"), config.get("read_timeout")) ) - # DEBUG: print(f"DEBUG: response.ok='{response.ok}',response.status_code={response.status_code},response.text()={len(response.text)}") + print(f"DEBUG: response.ok='{response.ok}',response.status_code={response.status_code},response.text()={len(response.text)}") if response.ok and len(response.text) > 0: meta = bs4.BeautifulSoup( response.text, "html.parser" ) - # DEBUG: print(f"DEBUG: meta[]='{type(meta)}'") + print(f"DEBUG: meta[]='{type(meta)}'") tag = meta.find("meta", attrs={"name": "csrf-token"}) - # DEBUG: print(f"DEBUG: tag={tag}") + print(f"DEBUG: tag={tag}") csrf = tag["content"] - # DEBUG: print(f"DEBUG: Adding CSRF token='{csrf}' for domain='{domain}'") - - reqheaders = {**headers, **{"X-CSRF-Token": csrf}} + print(f"DEBUG: Adding CSRF token='{csrf}' for domain='{domain}'") + reqheaders["X-CSRF-Token"] = csrf except BaseException as exception: - # DEBUG: print(f"DEBUG: No CSRF token found, using normal headers: domain='{domain}',exception[{type(exception)}]={exception}") + print(f"DEBUG: No CSRF token found, using normal headers: domain='{domain}',exception[{type(exception)}]={exception}") pass - # DEBUG: print(f"DEBUG: reqheaders()={len(reqheaders)} - EXIT!") + print(f"DEBUG: reqheaders()={len(reqheaders)} - EXIT!") return reqheaders diff --git a/fba/fba.py b/fba/fba.py index 2552466..a89d20f 100644 --- a/fba/fba.py +++ b/fba/fba.py @@ -22,35 +22,15 @@ import time from urllib.parse import urlparse -import bs4 import requests -import validators -from fba import blacklist from fba import config -from fba import instances from fba import network -from fba.federation import lemmy -from fba.federation import misskey -from fba.federation import peertube - # Array with pending errors needed to be written to database pending_errors = { } -# "rel" identifiers (no real URLs) -nodeinfo_identifier = [ - "https://nodeinfo.diaspora.software/ns/schema/2.1", - "https://nodeinfo.diaspora.software/ns/schema/2.0", - "https://nodeinfo.diaspora.software/ns/schema/1.1", - "https://nodeinfo.diaspora.software/ns/schema/1.0", - "http://nodeinfo.diaspora.software/ns/schema/2.1", - "http://nodeinfo.diaspora.software/ns/schema/2.0", - "http://nodeinfo.diaspora.software/ns/schema/1.1", - "http://nodeinfo.diaspora.software/ns/schema/1.0", -] - # Connect to database connection = sqlite3.connect("blocks.db") cursor = connection.cursor() @@ -70,97 +50,11 @@ patterns = [ ##### Other functions ##### def is_primitive(var: any) -> bool: - # DEBUG: print(f"DEBUG: var[]='{type(var)}' - CALLED!") + print(f"DEBUG: var[]='{type(var)}' - CALLED!") return type(var) in {int, str, float, bool} or var is None -def fetch_instances(domain: str, origin: str, software: str, script: str, path: str = None): - # DEBUG: print(f"DEBUG: domain='{domain}',origin='{origin}',software='{software}',path='{path}' - CALLED!") - if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError("Parameter 'domain' is empty") - elif not isinstance(origin, str) and origin is not None: - raise ValueError(f"Parameter origin[]={type(origin)} is not 'str'") - elif software is None: - # DEBUG: print(f"DEBUG: software for domain='{domain}' is not set, determining ...") - software = determine_software(domain, path) - # DEBUG: print(f"DEBUG: Determined software='{software}' for domain='{domain}'") - elif not isinstance(software, str): - raise ValueError(f"Parameter software[]={type(software)} is not 'str'") - elif not isinstance(script, str): - raise ValueError(f"Parameter script[]={type(script)} is not 'str'") - elif domain == "": - raise ValueError("Parameter 'domain' is empty") - - if not instances.is_registered(domain): - # DEBUG: print("DEBUG: Adding new domain:", domain, origin) - instances.add(domain, origin, script, path) - - # DEBUG: print("DEBUG: Fetching instances for domain:", domain, software) - peerlist = fetch_peers(domain, software) - - if peerlist is None: - print("ERROR: Cannot fetch peers:", domain) - return - elif instances.has_pending_instance_data(domain): - # DEBUG: print(f"DEBUG: domain='{domain}' has pending nodeinfo data, flushing ...") - instances.update_data(domain) - - print(f"INFO: Checking {len(peerlist)} instances from {domain} ...") - for instance in peerlist: - if instance is None: - # Skip "None" types as tidup() cannot parse them - continue - - # DEBUG: print(f"DEBUG: instance='{instance}' - BEFORE") - instance = tidyup_domain(instance) - # DEBUG: print(f"DEBUG: instance='{instance}' - AFTER") - - if instance == "": - print("WARNING: Empty instance after tidyup_domain(), domain:", domain) - continue - elif not validators.domain(instance.split("/")[0]): - print(f"WARNING: Bad instance='{instance}' from domain='{domain}',origin='{origin}',software='{software}'") - continue - elif blacklist.is_blacklisted(instance): - # DEBUG: print("DEBUG: instance is blacklisted:", instance) - continue - - # DEBUG: print("DEBUG: Handling instance:", instance) - try: - if not instances.is_registered(instance): - # DEBUG: print("DEBUG: Adding new instance:", instance, domain) - instances.add(instance, domain, script) - except BaseException as exception: - print(f"ERROR: instance='{instance}',exception[{type(exception)}]:'{str(exception)}'") - continue - - # DEBUG: print("DEBUG: EXIT!") - -def add_peers(rows: dict) -> list: - # DEBUG: print(f"DEBUG: rows()={len(rows)} - CALLED!") - peers = list() - for key in ["linked", "allowed", "blocked"]: - # DEBUG: print(f"DEBUG: Checking key='{key}'") - if key in rows and rows[key] is not None: - # DEBUG: print(f"DEBUG: Adding {len(rows[key])} peer(s) to peers list ...") - for peer in rows[key]: - # DEBUG: print(f"DEBUG: peer='{peer}' - BEFORE!") - peer = tidyup_domain(peer) - - # DEBUG: print(f"DEBUG: peer='{peer}' - AFTER!") - if blacklist.is_blacklisted(peer): - # DEBUG: print(f"DEBUG: peer='{peer}' is blacklisted, skipped!") - continue - - # DEBUG: print(f"DEBUG: Adding peer='{peer}' ...") - peers.append(peer) - - # DEBUG: print(f"DEBUG: peers()={len(peers)} - EXIT!") - return peers - def remove_version(software: str) -> str: - # DEBUG: print(f"DEBUG: software='{software}' - CALLED!") + print(f"DEBUG: software='{software}' - CALLED!") if not "." in software and " " not in software: print(f"WARNING: software='{software}' does not contain a version number.") return software @@ -173,7 +67,7 @@ def remove_version(software: str) -> str: elif " - " in software: temp = software.split(" - ")[0] - # DEBUG: print(f"DEBUG: software='{software}'") + print(f"DEBUG: software='{software}'") version = None if " " in software: version = temp.split(" ")[-1] @@ -182,81 +76,81 @@ def remove_version(software: str) -> str: elif "-" in software: version = temp.split("-")[-1] else: - # DEBUG: print(f"DEBUG: Was not able to find common seperator, returning untouched software='{software}'") + print(f"DEBUG: Was not able to find common seperator, returning untouched software='{software}'") return software match = None - # DEBUG: print(f"DEBUG: Checking {len(patterns)} patterns ...") + print(f"DEBUG: Checking {len(patterns)} patterns ...") for pattern in patterns: # Run match() match = pattern.match(version) - # DEBUG: print(f"DEBUG: match[]={type(match)}") + print(f"DEBUG: match[]={type(match)}") if isinstance(match, re.Match): - # DEBUG: print(f"DEBUG: version='{version}' is matching pattern='{pattern}'") + print(f"DEBUG: version='{version}' is matching pattern='{pattern}'") break - # DEBUG: print(f"DEBUG: version[{type(version)}]='{version}',match='{match}'") + print(f"DEBUG: version[{type(version)}]='{version}',match='{match}'") if not isinstance(match, re.Match): print(f"WARNING: version='{version}' does not match regex, leaving software='{software}' untouched.") return software - # DEBUG: print(f"DEBUG: Found valid version number: '{version}', removing it ...") + print(f"DEBUG: Found valid version number: '{version}', removing it ...") end = len(temp) - len(version) - 1 - # DEBUG: print(f"DEBUG: end[{type(end)}]={end}") + print(f"DEBUG: end[{type(end)}]={end}") software = temp[0:end].strip() if " version" in software: - # DEBUG: print(f"DEBUG: software='{software}' contains word ' version'") + print(f"DEBUG: software='{software}' contains word ' version'") software = strip_until(software, " version") - # DEBUG: print(f"DEBUG: software='{software}' - EXIT!") + print(f"DEBUG: software='{software}' - EXIT!") return software def strip_powered_by(software: str) -> str: - # DEBUG: print(f"DEBUG: software='{software}' - CALLED!") + print(f"DEBUG: software='{software}' - CALLED!") if not isinstance(software, str): raise ValueError(f"Parameter software[]='{type(software)}' is not 'str'") elif software == "": raise ValueError("Parameter 'software' is empty") - elif not "powered by" in software: + elif "powered by" not in software: print(f"WARNING: Cannot find 'powered by' in software='{software}'!") return software start = software.find("powered by ") - # DEBUG: print(f"DEBUG: start[{type(start)}]='{start}'") + print(f"DEBUG: start[{type(start)}]='{start}'") software = software[start + 11:].strip() - # DEBUG: print(f"DEBUG: software='{software}'") + print(f"DEBUG: software='{software}'") software = strip_until(software, " - ") - # DEBUG: print(f"DEBUG: software='{software}' - EXIT!") + print(f"DEBUG: software='{software}' - EXIT!") return software def strip_hosted_on(software: str) -> str: - # DEBUG: print(f"DEBUG: software='{software}' - CALLED!") + print(f"DEBUG: software='{software}' - CALLED!") if not isinstance(software, str): raise ValueError(f"Parameter software[]='{type(software)}' is not 'str'") elif software == "": raise ValueError("Parameter 'software' is empty") - elif not "hosted on" in software: + elif "hosted on" not in software: print(f"WARNING: Cannot find 'hosted on' in '{software}'!") return software end = software.find("hosted on ") - # DEBUG: print(f"DEBUG: end[{type(end)}]='{end}'") + print(f"DEBUG: end[{type(end)}]='{end}'") software = software[0, end].strip() - # DEBUG: print(f"DEBUG: software='{software}'") + print(f"DEBUG: software='{software}'") software = strip_until(software, " - ") - # DEBUG: print(f"DEBUG: software='{software}' - EXIT!") + print(f"DEBUG: software='{software}' - EXIT!") return software def strip_until(software: str, until: str) -> str: - # DEBUG: print(f"DEBUG: software='{software}',until='{until}' - CALLED!") + print(f"DEBUG: software='{software}',until='{until}' - CALLED!") if not isinstance(software, str): raise ValueError(f"Parameter software[]='{type(software)}' is not 'str'") elif software == "": @@ -272,11 +166,11 @@ def strip_until(software: str, until: str) -> str: # Next, strip until part end = software.find(until) - # DEBUG: print(f"DEBUG: end[{type(end)}]='{end}'") + print(f"DEBUG: end[{type(end)}]='{end}'") if end > 0: software = software[0:end].strip() - # DEBUG: print(f"DEBUG: software='{software}' - EXIT!") + print(f"DEBUG: software='{software}' - EXIT!") return software def remove_pending_error(domain: str): @@ -292,7 +186,7 @@ def remove_pending_error(domain: str): except: pass - # DEBUG: print("DEBUG: EXIT!") + print("DEBUG: EXIT!") def get_hash(domain: str) -> str: if not isinstance(domain, str): @@ -303,21 +197,21 @@ def get_hash(domain: str) -> str: return hashlib.sha256(domain.encode("utf-8")).hexdigest() def log_error(domain: str, response: requests.models.Response): - # DEBUG: print("DEBUG: domain,response[]:", domain, type(response)) + print("DEBUG: domain,response[]:", domain, type(response)) if not isinstance(domain, str): raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") elif config.get("write_error_log").lower() != "true": - # DEBUG: print(f"DEBUG: Writing to error_log is disabled in configuruation file - EXIT!") + print(f"DEBUG: Writing to error_log is disabled in configuruation file - EXIT!") return try: - # DEBUG: print("DEBUG: BEFORE response[]:", type(response)) + print("DEBUG: BEFORE response[]:", type(response)) if isinstance(response, BaseException) or isinstance(response, json.decoder.JSONDecodeError): response = f"response[{type(response)}]='{str(response)}'" - # DEBUG: print("DEBUG: AFTER response[]:", type(response)) + print("DEBUG: AFTER response[]:", type(response)) if isinstance(response, str): cursor.execute("INSERT INTO error_log (domain, error_code, error_message, created) VALUES (?, 999, ?, ?)",[ domain, @@ -333,452 +227,16 @@ def log_error(domain: str, response: requests.models.Response): ]) # Cleanup old entries - # DEBUG: print(f"DEBUG: Purging old records (distance: {config.get('error_log_cleanup')})") + print(f"DEBUG: Purging old records (distance: {config.get('error_log_cleanup')})") cursor.execute("DELETE FROM error_log WHERE created < ?", [time.time() - config.get("error_log_cleanup")]) except BaseException as exception: print(f"ERROR: failed SQL query: domain='{domain}',exception[{type(exception)}]:'{str(exception)}'") sys.exit(255) - # DEBUG: print("DEBUG: EXIT!") - -def fetch_peers(domain: str, software: str) -> list: - # DEBUG: print(f"DEBUG: domain({len(domain)})={domain},software={software} - CALLED!") - if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError("Parameter 'domain' is empty") - elif not isinstance(software, str) and software is not None: - raise ValueError(f"software[]={type(software)} is not 'str'") - - if software == "misskey": - # DEBUG: print(f"DEBUG: Invoking misskey.fetch_peers({domain}) ...") - return misskey.fetch_peers(domain) - elif software == "lemmy": - # DEBUG: print(f"DEBUG: Invoking lemmy.fetch_peers({domain}) ...") - return lemmy.fetch_peers(domain) - elif software == "peertube": - # DEBUG: print(f"DEBUG: Invoking peertube.fetch_peers({domain}) ...") - return peertube.fetch_peers(domain) - - # DEBUG: print(f"DEBUG: Fetching peers from '{domain}',software='{software}' ...") - peers = list() - response = network.fetch_response(domain, "/api/v1/instance/peers", network.api_headers, (config.get("connection_timeout"), config.get("read_timeout"))) - # DEBUG: print(f"DEBUG: response[]='{type(response)}'") - - data = json_from_response(response) - # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'") - - if not response.ok or response.status_code >= 400: - # DEBUG: print("DEBUG: Was not able to fetch peers, trying alternative ...") - response = network.fetch_response(domain, "/api/v3/site", network.api_headers, (config.get("connection_timeout"), config.get("read_timeout"))) - - data = json_from_response(response) - # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'") - if not response.ok or response.status_code >= 400: - print("WARNING: Could not reach any JSON API:", domain) - instances.update_last_error(domain, response) - elif response.ok and isinstance(data, list): - # DEBUG: print(f"DEBUG: domain='{domain}' returned a list: '{data}'") - sys.exit(255) - elif "federated_instances" in data: - # DEBUG: print(f"DEBUG: Found federated_instances for domain='{domain}'") - peers = peers + add_peers(data["federated_instances"]) - # DEBUG: print("DEBUG: Added instance(s) to peers") - else: - print("WARNING: JSON response does not contain 'federated_instances':", domain) - instances.update_last_error(domain, response) - else: - # DEBUG: print("DEBUG: Querying API was successful:", domain, len(data)) - peers = data - - # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'") - instances.set_data("total_peers", domain, len(peers)) - - # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") - instances.update_last_instance_fetch(domain) - - # DEBUG: print("DEBUG: Returning peers[]:", type(peers)) - return peers - -def fetch_nodeinfo(domain: str, path: str = None) -> list: - # DEBUG: print(f"DEBUG: domain='{domain}',path={path} - CALLED!") - if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError("Parameter 'domain' is empty") - elif not isinstance(path, str) and path is not None: - raise ValueError(f"Parameter path[]={type(path)} is not 'str'") - - # DEBUG: print(f"DEBUG: Fetching nodeinfo from domain='{domain}' ...") - nodeinfo = fetch_wellknown_nodeinfo(domain) - - # DEBUG: print(f"DEBUG: nodeinfo({len(nodeinfo)})={nodeinfo}") - if len(nodeinfo) > 0: - # DEBUG: print("DEBUG: nodeinfo()={len(nodeinfo))} - EXIT!") - return nodeinfo - - request_paths = [ - "/nodeinfo/2.1.json", - "/nodeinfo/2.1", - "/nodeinfo/2.0.json", - "/nodeinfo/2.0", - "/nodeinfo/1.0", - "/api/v1/instance" - ] - - data = {} - for request in request_paths: - if path is not None and path != "" and path != f"https://{domain}{path}": - # DEBUG: print(f"DEBUG: path='{path}' does not match request='{request}' - SKIPPED!") - continue - - # DEBUG: print(f"DEBUG: Fetching request='{request}' from domain='{domain}' ...") - response = network.fetch_response(domain, request, network.api_headers, (config.get("nodeinfo_connection_timeout"), config.get("nodeinfo_read_timeout"))) - - data = json_from_response(response) - # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'") - if response.ok and isinstance(data, dict): - # DEBUG: print("DEBUG: Success:", request) - instances.set_data("detection_mode", domain, "STATIC_CHECK") - instances.set_data("nodeinfo_url" , domain, request) - break - elif response.ok and isinstance(data, list): - print(f"UNSUPPORTED: domain='{domain}' returned a list: '{data}'") - sys.exit(255) - elif not response.ok or response.status_code >= 400: - print("WARNING: Failed fetching nodeinfo from domain:", domain) - instances.update_last_error(domain, response) - continue - - # DEBUG: print(f"DEBUG: data()={len(data)} - EXIT!") - return data - -def fetch_wellknown_nodeinfo(domain: str) -> list: - # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") - if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError("Parameter 'domain' is empty") - - # DEBUG: print("DEBUG: Fetching .well-known info for domain:", domain) - data = {} - - response = network.fetch_response(domain, "/.well-known/nodeinfo", network.api_headers, (config.get("nodeinfo_connection_timeout"), config.get("nodeinfo_read_timeout"))) - - data = json_from_response(response) - # DEBUG: print("DEBUG: domain,response.ok,data[]:", domain, response.ok, type(data)) - if response.ok and isinstance(data, dict): - nodeinfo = data - # DEBUG: print("DEBUG: Found entries:", len(nodeinfo), domain) - if "links" in nodeinfo: - # DEBUG: print("DEBUG: Found links in nodeinfo():", len(nodeinfo["links"])) - for link in nodeinfo["links"]: - # DEBUG: print("DEBUG: rel,href:", link["rel"], link["href"]) - if link["rel"] in nodeinfo_identifier: - # DEBUG: print("DEBUG: Fetching nodeinfo from:", link["href"]) - response = fetch_url(link["href"], network.api_headers, (config.get("connection_timeout"), config.get("read_timeout"))) - - data = json_from_response(response) - # DEBUG: print("DEBUG: href,response.ok,response.status_code:", link["href"], response.ok, response.status_code) - if response.ok and isinstance(data, dict): - # DEBUG: print("DEBUG: Found JSON nodeinfo():", len(data)) - instances.set_data("detection_mode", domain, "AUTO_DISCOVERY") - instances.set_data("nodeinfo_url" , domain, link["href"]) - break - else: - print("WARNING: Unknown 'rel' value:", domain, link["rel"]) - else: - print("WARNING: nodeinfo does not contain 'links':", domain) - - # DEBUG: print("DEBUG: Returning data[]:", type(data)) - return data - -def fetch_generator_from_path(domain: str, path: str = "/") -> str: - # DEBUG: print(f"DEBUG: domain({len(domain)})={domain},path={path} - CALLED!") - if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError("Parameter 'domain' is empty") - elif not isinstance(path, str): - raise ValueError(f"path[]={type(path)} is not 'str'") - elif path == "": - raise ValueError("Parameter 'path' is empty") - - # DEBUG: print(f"DEBUG: domain='{domain}',path='{path}' - CALLED!") - software = None - - # DEBUG: print(f"DEBUG: Fetching path='{path}' from '{domain}' ...") - response = network.fetch_response(domain, path, network.web_headers, (config.get("connection_timeout"), config.get("read_timeout"))) - - # DEBUG: print("DEBUG: domain,response.ok,response.status_code,response.text[]:", domain, response.ok, response.status_code, type(response.text)) - if response.ok and response.status_code < 300 and len(response.text) > 0: - # DEBUG: print("DEBUG: Search for :", domain) - doc = bs4.BeautifulSoup(response.text, "html.parser") - - # DEBUG: print("DEBUG: doc[]:", type(doc)) - generator = doc.find("meta", {"name": "generator"}) - site_name = doc.find("meta", {"property": "og:site_name"}) - - # DEBUG: print(f"DEBUG: generator='{generator}',site_name='{site_name}'") - if isinstance(generator, bs4.element.Tag): - # DEBUG: print("DEBUG: Found generator meta tag:", domain) - software = tidyup_domain(generator.get("content")) - print(f"INFO: domain='{domain}' is generated by '{software}'") - instances.set_data("detection_mode", domain, "GENERATOR") - remove_pending_error(domain) - elif isinstance(site_name, bs4.element.Tag): - # DEBUG: print("DEBUG: Found property=og:site_name:", domain) - sofware = tidyup_domain(site_name.get("content")) - print(f"INFO: domain='{domain}' has og:site_name='{software}'") - instances.set_data("detection_mode", domain, "SITE_NAME") - remove_pending_error(domain) - - # DEBUG: print(f"DEBUG: software[]={type(software)}") - if isinstance(software, str) and software == "": - # DEBUG: print(f"DEBUG: Corrected empty string to None for software of domain='{domain}'") - software = None - elif isinstance(software, str) and ("." in software or " " in software): - # DEBUG: print(f"DEBUG: software='{software}' may contain a version number, domain='{domain}', removing it ...") - software = remove_version(software) - - # DEBUG: print(f"DEBUG: software[]={type(software)}") - if isinstance(software, str) and " powered by " in software: - # DEBUG: print(f"DEBUG: software='{software}' has 'powered by' in it") - software = remove_version(strip_powered_by(software)) - elif isinstance(software, str) and " hosted on " in software: - # DEBUG: print(f"DEBUG: software='{software}' has 'hosted on' in it") - software = remove_version(strip_hosted_on(software)) - elif isinstance(software, str) and " by " in software: - # DEBUG: print(f"DEBUG: software='{software}' has ' by ' in it") - software = strip_until(software, " by ") - elif isinstance(software, str) and " see " in software: - # DEBUG: print(f"DEBUG: software='{software}' has ' see ' in it") - software = strip_until(software, " see ") - - # DEBUG: print(f"DEBUG: software='{software}' - EXIT!") - return software - -def determine_software(domain: str, path: str = None) -> str: - # DEBUG: print(f"DEBUG: domain({len(domain)})={domain},path={path} - CALLED!") - if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError("Parameter 'domain' is empty") - elif not isinstance(path, str) and path is not None: - raise ValueError(f"Parameter path[]={type(path)} is not 'str'") - - # DEBUG: print("DEBUG: Determining software for domain,path:", domain, path) - software = None - - # DEBUG: print(f"DEBUG: Fetching nodeinfo from '{domain}' ...") - data = fetch_nodeinfo(domain, path) - - # DEBUG: print("DEBUG: data[]:", type(data)) - if not isinstance(data, dict) or len(data) == 0: - # DEBUG: print("DEBUG: Could not determine software type:", domain) - return fetch_generator_from_path(domain) - - # DEBUG: print("DEBUG: data():", len(data), data) - if "status" in data and data["status"] == "error" and "message" in data: - print("WARNING: JSON response is an error:", data["message"]) - instances.update_last_error(domain, data["message"]) - return fetch_generator_from_path(domain) - elif "message" in data: - print("WARNING: JSON response contains only a message:", data["message"]) - instances.update_last_error(domain, data["message"]) - return fetch_generator_from_path(domain) - elif "software" not in data or "name" not in data["software"]: - # DEBUG: print(f"DEBUG: JSON response from domain='{domain}' does not include [software][name], fetching / ...") - software = fetch_generator_from_path(domain) - - # DEBUG: print(f"DEBUG: Generator for domain='{domain}' is: {software}, EXIT!") - return software - - software = tidyup_domain(data["software"]["name"]) - - # DEBUG: print("DEBUG: sofware after tidyup_domain():", software) - if software in ["akkoma", "rebased"]: - # DEBUG: print("DEBUG: Setting pleroma:", domain, software) - software = "pleroma" - elif software in ["hometown", "ecko"]: - # DEBUG: print("DEBUG: Setting mastodon:", domain, software) - software = "mastodon" - elif software in ["calckey", "groundpolis", "foundkey", "cherrypick", "meisskey"]: - # DEBUG: print("DEBUG: Setting misskey:", domain, software) - software = "misskey" - elif software.find("/") > 0: - print("WARNING: Spliting of slash:", software) - software = tidyup_domain(software.split("/")[-1]) - elif software.find("|") > 0: - print("WARNING: Spliting of pipe:", software) - software = tidyup_domain(software.split("|")[0]) - elif "powered by" in software: - # DEBUG: print(f"DEBUG: software='{software}' has 'powered by' in it") - software = strip_powered_by(software) - elif isinstance(software, str) and " by " in software: - # DEBUG: print(f"DEBUG: software='{software}' has ' by ' in it") - software = strip_until(software, " by ") - elif isinstance(software, str) and " see " in software: - # DEBUG: print(f"DEBUG: software='{software}' has ' see ' in it") - software = strip_until(software, " see ") - - # DEBUG: print(f"DEBUG: software[]={type(software)}") - if software == "": - print("WARNING: tidyup_domain() left no software name behind:", domain) - software = None - - # DEBUG: print(f"DEBUG: software[]={type(software)}") - if str(software) == "": - # DEBUG: print(f"DEBUG: software for '{domain}' was not detected, trying generator ...") - software = fetch_generator_from_path(domain) - elif len(str(software)) > 0 and ("." in software or " " in software): - # DEBUG: print(f"DEBUG: software='{software}' may contain a version number, domain='{domain}', removing it ...") - software = remove_version(software) - - # DEBUG: print(f"DEBUG: software[]={type(software)}") - if isinstance(software, str) and "powered by" in software: - # DEBUG: print(f"DEBUG: software='{software}' has 'powered by' in it") - software = remove_version(strip_powered_by(software)) - - # DEBUG: print("DEBUG: Returning domain,software:", domain, software) - return software - -def tidyup_reason(reason: str) -> str: - # DEBUG: print(f"DEBUG: reason='{reason}' - CALLED!") - if not isinstance(reason, str): - raise ValueError(f"Parameter reason[]={type(reason)} is not 'str'") - - # Strip string - reason = reason.strip() - - # Replace â with " - reason = re.sub("â", "\"", reason) - - # DEBUG: print(f"DEBUG: reason='{reason}' - EXIT!") - return reason - -def tidyup_domain(domain: str) -> str: - # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") - if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - - # All lower-case and strip spaces out + last dot - domain = domain.lower().strip().rstrip(".") - - # No port number - domain = re.sub("\:\d+$", "", domain) - - # No protocol, sometimes without the slashes - domain = re.sub("^https?\:(\/*)", "", domain) - - # No trailing slash - domain = re.sub("\/$", "", domain) - - # No @ sign - domain = re.sub("^\@", "", domain) - - # No individual users in block lists - domain = re.sub("(.+)\@", "", domain) - if domain.find("/profile/"): - domain = domain.split("/profile/")[0] - elif domain.find("/users/"): - domain = domain.split("/users/")[0] - - # DEBUG: print(f"DEBUG: domain='{domain}' - EXIT!") - return domain - -def json_from_response(response: requests.models.Response) -> list: - # DEBUG: print(f"DEBUG: response[]={type(response)} - CALLED!") - if not isinstance(response, requests.models.Response): - raise ValueError(f"Parameter response[]='{type(response)}' is not type of 'Response'") - - data = list() - if response.text.strip() != "": - # DEBUG: print(f"DEBUG: response.text()={len(response.text)} is not empty, invoking response.json() ...") - try: - data = response.json() - except json.decoder.JSONDecodeError: - pass - - # DEBUG: print(f"DEBUG: data[]={type(data)} - EXIT!") - return data - -def has_key(lists: list, key: str, value: any) -> bool: - # DEBUG: print(f"DEBUG: lists()={len(lists)},key='{key}',value[]='{type(value)}' - CALLED!") - if not isinstance(lists, list): - raise ValueError(f"Parameter lists[]='{type(lists)}' is not 'list'") - elif not isinstance(key, str): - raise ValueError(f"Parameter key[]='{type(key)}' is not 'str'") - elif key == "": - raise ValueError("Parameter 'key' is empty") - - has = False - # DEBUG: print(f"DEBUG: Checking lists()={len(lists)} ...") - for row in lists: - # DEBUG: print(f"DEBUG: row['{type(row)}']={row}") - if not isinstance(row, dict): - raise ValueError(f"row[]='{type(row)}' is not 'dict'") - elif not key in row: - raise KeyError(f"Cannot find key='{key}'") - elif row[key] == value: - has = True - break - - # DEBUG: print(f"DEBUG: has={has} - EXIT!") - return has - -def find_domains(tag: bs4.element.Tag) -> list: - # DEBUG: print(f"DEBUG: tag[]={type(tag)} - CALLED!") - if not isinstance(tag, bs4.element.Tag): - raise ValueError(f"Parameter tag[]={type(tag)} is not type of bs4.element.Tag") - elif len(tag.select("tr")) == 0: - raise KeyError("No table rows found in table!") - - domains = list() - for element in tag.select("tr"): - # DEBUG: print(f"DEBUG: element[]={type(element)}") - if not element.find("td"): - # DEBUG: print("DEBUG: Skipping element, no found") - continue - - domain = tidyup_domain(element.find("td").text) - reason = tidyup_reason(element.findAll("td")[1].text) - - # DEBUG: print(f"DEBUG: domain='{domain}',reason='{reason}'") - - if blacklist.is_blacklisted(domain): - print(f"WARNING: domain='{domain}' is blacklisted - skipped!") - continue - elif domain == "gab.com/.ai, develop.gab.com": - # DEBUG: print("DEBUG: Multiple domains detected in one row") - domains.append({ - "domain": "gab.com", - "reason": reason, - }) - domains.append({ - "domain": "gab.ai", - "reason": reason, - }) - domains.append({ - "domain": "develop.gab.com", - "reason": reason, - }) - continue - elif not validators.domain(domain): - print(f"WARNING: domain='{domain}' is not a valid domain - skipped!") - continue - - # DEBUG: print(f"DEBUG: Adding domain='{domain}' ...") - domains.append({ - "domain": domain, - "reason": reason, - }) - - # DEBUG: print(f"DEBUG: domains()={len(domains)} - EXIT!") - return domains + print("DEBUG: EXIT!") def fetch_url(url: str, headers: dict, timeout: tuple) -> requests.models.Response: - # DEBUG: print(f"DEBUG: url='{url}',headers()={len(headers)},timeout={timeout} - CALLED!") + print(f"DEBUG: url='{url}',headers()={len(headers)},timeout={timeout} - CALLED!") if not isinstance(url, str): raise ValueError(f"Parameter url[]='{type(url)}' is not 'str'") elif url == "": @@ -788,15 +246,15 @@ def fetch_url(url: str, headers: dict, timeout: tuple) -> requests.models.Respon elif not isinstance(timeout, tuple): raise ValueError(f"Parameter timeout[]='{type(timeout)}' is not 'tuple'") - # DEBUG: print(f"DEBUG: Parsing url='{url}'") + print(f"DEBUG: Parsing url='{url}'") components = urlparse(url) # Invoke other function, avoid trailing ? - # DEBUG: print(f"DEBUG: components[{type(components)}]={components}") + print(f"DEBUG: components[{type(components)}]={components}") if components.query != "": response = network.fetch_response(components.hostname, f"{components.path}?{components.query}", headers, timeout) else: response = network.fetch_response(components.hostname, f"{components.path}", headers, timeout) - # DEBUG: print(f"DEBUG: response[]='{type(response)}' - EXXIT!") + print(f"DEBUG: response[]='{type(response)}' - EXXIT!") return response diff --git a/fba/federation.py b/fba/federation.py new file mode 100644 index 0000000..b7e86a2 --- /dev/null +++ b/fba/federation.py @@ -0,0 +1,478 @@ +# Copyright (C) 2023 Free Software Foundation +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import sys + +import bs4 +import validators + +from fba import blacklist +from fba import config +from fba import fba +from fba import instances +from fba import network + +from fba.helpers import tidyup + +from fba.networks import lemmy +from fba.networks import misskey +from fba.networks import peertube + +# "rel" identifiers (no real URLs) +nodeinfo_identifier = [ + "https://nodeinfo.diaspora.software/ns/schema/2.1", + "https://nodeinfo.diaspora.software/ns/schema/2.0", + "https://nodeinfo.diaspora.software/ns/schema/1.1", + "https://nodeinfo.diaspora.software/ns/schema/1.0", + "http://nodeinfo.diaspora.software/ns/schema/2.1", + "http://nodeinfo.diaspora.software/ns/schema/2.0", + "http://nodeinfo.diaspora.software/ns/schema/1.1", + "http://nodeinfo.diaspora.software/ns/schema/1.0", +] + +def fetch_instances(domain: str, origin: str, software: str, script: str, path: str = None): + print(f"DEBUG: domain='{domain}',origin='{origin}',software='{software}',path='{path}' - CALLED!") + if not isinstance(domain, str): + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError("Parameter 'domain' is empty") + elif not isinstance(origin, str) and origin is not None: + raise ValueError(f"Parameter origin[]={type(origin)} is not 'str'") + elif software is None: + print(f"DEBUG: software for domain='{domain}' is not set, determining ...") + software = determine_software(domain, path) + print(f"DEBUG: Determined software='{software}' for domain='{domain}'") + elif not isinstance(software, str): + raise ValueError(f"Parameter software[]={type(software)} is not 'str'") + elif not isinstance(script, str): + raise ValueError(f"Parameter script[]={type(script)} is not 'str'") + elif domain == "": + raise ValueError("Parameter 'domain' is empty") + + if not instances.is_registered(domain): + print("DEBUG: Adding new domain:", domain, origin) + instances.add(domain, origin, script, path) + + print("DEBUG: Fetching instances for domain:", domain, software) + peerlist = fetch_peers(domain, software) + + if peerlist is None: + print("ERROR: Cannot fetch peers:", domain) + return + elif instances.has_pending_instance_data(domain): + print(f"DEBUG: domain='{domain}' has pending nodeinfo data, flushing ...") + instances.update_data(domain) + + print(f"INFO: Checking {len(peerlist)} instances from {domain} ...") + for instance in peerlist: + if instance is None: + # Skip "None" types as tidup() cannot parse them + continue + + print(f"DEBUG: instance='{instance}' - BEFORE") + instance = tidyup.domain(instance) + print(f"DEBUG: instance='{instance}' - AFTER") + + if instance == "": + print("WARNING: Empty instance after tidyup.domain(), domain:", domain) + continue + elif not validators.domain(instance.split("/")[0]): + print(f"WARNING: Bad instance='{instance}' from domain='{domain}',origin='{origin}',software='{software}'") + continue + elif blacklist.is_blacklisted(instance): + print("DEBUG: instance is blacklisted:", instance) + continue + + print("DEBUG: Handling instance:", instance) + try: + if not instances.is_registered(instance): + print("DEBUG: Adding new instance:", instance, domain) + instances.add(instance, domain, script) + except BaseException as exception: + print(f"ERROR: instance='{instance}',exception[{type(exception)}]:'{str(exception)}'") + continue + + print("DEBUG: EXIT!") + +def fetch_peers(domain: str, software: str) -> list: + print(f"DEBUG: domain({len(domain)})={domain},software={software} - CALLED!") + if not isinstance(domain, str): + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError("Parameter 'domain' is empty") + elif not isinstance(software, str) and software is not None: + raise ValueError(f"software[]={type(software)} is not 'str'") + + if software == "misskey": + print(f"DEBUG: Invoking misskey.fetch_peers({domain}) ...") + return misskey.fetch_peers(domain) + elif software == "lemmy": + print(f"DEBUG: Invoking lemmy.fetch_peers({domain}) ...") + return lemmy.fetch_peers(domain) + elif software == "peertube": + print(f"DEBUG: Invoking peertube.fetch_peers({domain}) ...") + return peertube.fetch_peers(domain) + + print(f"DEBUG: Fetching peers from '{domain}',software='{software}' ...") + peers = list() + response = network.fetch_response(domain, "/api/v1/instance/peers", network.api_headers, (config.get("connection_timeout"), config.get("read_timeout"))) + print(f"DEBUG: response[]='{type(response)}'") + + data = network.json_from_response(response) + print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'") + + if not response.ok or response.status_code >= 400: + print("DEBUG: Was not able to fetch peers, trying alternative ...") + response = network.fetch_response(domain, "/api/v3/site", network.api_headers, (config.get("connection_timeout"), config.get("read_timeout"))) + + data = network.json_from_response(response) + print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'") + if not response.ok or response.status_code >= 400: + print("WARNING: Could not reach any JSON API:", domain) + instances.update_last_error(domain, response) + elif response.ok and isinstance(data, list): + print(f"DEBUG: domain='{domain}' returned a list: '{data}'") + sys.exit(255) + elif "federated_instances" in data: + print(f"DEBUG: Found federated_instances for domain='{domain}'") + peers = peers + add_peers(data["federated_instances"]) + print("DEBUG: Added instance(s) to peers") + else: + print("WARNING: JSON response does not contain 'federated_instances':", domain) + instances.update_last_error(domain, response) + else: + print("DEBUG: Querying API was successful:", domain, len(data)) + peers = data + + print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'") + instances.set_data("total_peers", domain, len(peers)) + + print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") + instances.update_last_instance_fetch(domain) + + print("DEBUG: Returning peers[]:", type(peers)) + return peers + +def fetch_nodeinfo(domain: str, path: str = None) -> list: + print(f"DEBUG: domain='{domain}',path={path} - CALLED!") + if not isinstance(domain, str): + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError("Parameter 'domain' is empty") + elif not isinstance(path, str) and path is not None: + raise ValueError(f"Parameter path[]={type(path)} is not 'str'") + + print(f"DEBUG: Fetching nodeinfo from domain='{domain}' ...") + nodeinfo = fetch_wellknown_nodeinfo(domain) + + print(f"DEBUG: nodeinfo({len(nodeinfo)})={nodeinfo}") + if len(nodeinfo) > 0: + print("DEBUG: nodeinfo()={len(nodeinfo))} - EXIT!") + return nodeinfo + + request_paths = [ + "/nodeinfo/2.1.json", + "/nodeinfo/2.1", + "/nodeinfo/2.0.json", + "/nodeinfo/2.0", + "/nodeinfo/1.0", + "/api/v1/instance" + ] + + for request in request_paths: + if path is not None and path != "" and path != request: + print(f"DEBUG: path='{path}' does not match request='{request}' - SKIPPED!") + continue + + print(f"DEBUG: Fetching request='{request}' from domain='{domain}' ...") + response = network.fetch_response(domain, request, network.api_headers, (config.get("nodeinfo_connection_timeout"), config.get("nodeinfo_read_timeout"))) + + data = network.json_from_response(response) + print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'") + if response.ok and isinstance(data, dict): + print("DEBUG: Success:", request) + instances.set_data("detection_mode", domain, "STATIC_CHECK") + instances.set_data("nodeinfo_url" , domain, request) + break + elif response.ok and isinstance(data, list): + print(f"UNSUPPORTED: domain='{domain}' returned a list: '{data}'") + sys.exit(255) + elif not response.ok or response.status_code >= 400: + print("WARNING: Failed fetching nodeinfo from domain:", domain) + instances.update_last_error(domain, response) + continue + + print(f"DEBUG: data()={len(data)} - EXIT!") + return data + +def fetch_wellknown_nodeinfo(domain: str) -> list: + print(f"DEBUG: domain='{domain}' - CALLED!") + if not isinstance(domain, str): + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError("Parameter 'domain' is empty") + + print("DEBUG: Fetching .well-known info for domain:", domain) + response = network.fetch_response(domain, "/.well-known/nodeinfo", network.api_headers, (config.get("nodeinfo_connection_timeout"), config.get("nodeinfo_read_timeout"))) + + data = network.json_from_response(response) + print("DEBUG: domain,response.ok,data[]:", domain, response.ok, type(data)) + if response.ok and isinstance(data, dict): + nodeinfo = data + print("DEBUG: Found entries:", len(nodeinfo), domain) + if "links" in nodeinfo: + print("DEBUG: Found links in nodeinfo():", len(nodeinfo["links"])) + for link in nodeinfo["links"]: + print("DEBUG: rel,href:", link["rel"], link["href"]) + if link["rel"] in nodeinfo_identifier: + print("DEBUG: Fetching nodeinfo from:", link["href"]) + response = fba.fetch_url(link["href"], network.api_headers, (config.get("connection_timeout"), config.get("read_timeout"))) + + data = network.json_from_response(response) + print("DEBUG: href,response.ok,response.status_code:", link["href"], response.ok, response.status_code) + if response.ok and isinstance(data, dict): + print("DEBUG: Found JSON nodeinfo():", len(data)) + instances.set_data("detection_mode", domain, "AUTO_DISCOVERY") + instances.set_data("nodeinfo_url" , domain, link["href"]) + break + else: + print("WARNING: Unknown 'rel' value:", domain, link["rel"]) + else: + print("WARNING: nodeinfo does not contain 'links':", domain) + + print("DEBUG: Returning data[]:", type(data)) + return data + +def fetch_generator_from_path(domain: str, path: str = "/") -> str: + print(f"DEBUG: domain({len(domain)})={domain},path={path} - CALLED!") + if not isinstance(domain, str): + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError("Parameter 'domain' is empty") + elif not isinstance(path, str): + raise ValueError(f"path[]={type(path)} is not 'str'") + elif path == "": + raise ValueError("Parameter 'path' is empty") + + print(f"DEBUG: domain='{domain}',path='{path}' - CALLED!") + software = None + + print(f"DEBUG: Fetching path='{path}' from '{domain}' ...") + response = network.fetch_response(domain, path, network.web_headers, (config.get("connection_timeout"), config.get("read_timeout"))) + + print("DEBUG: domain,response.ok,response.status_code,response.text[]:", domain, response.ok, response.status_code, type(response.text)) + if response.ok and response.status_code < 300 and len(response.text) > 0: + print("DEBUG: Search for :", domain) + doc = bs4.BeautifulSoup(response.text, "html.parser") + + print("DEBUG: doc[]:", type(doc)) + generator = doc.find("meta", {"name": "generator"}) + site_name = doc.find("meta", {"property": "og:site_name"}) + + print(f"DEBUG: generator='{generator}',site_name='{site_name}'") + if isinstance(generator, bs4.element.Tag): + print("DEBUG: Found generator meta tag:", domain) + software = tidyup.domain(generator.get("content")) + print(f"INFO: domain='{domain}' is generated by '{software}'") + instances.set_data("detection_mode", domain, "GENERATOR") + fba.remove_pending_error(domain) + elif isinstance(site_name, bs4.element.Tag): + print("DEBUG: Found property=og:site_name:", domain) + sofware = tidyup.domain(site_name.get("content")) + print(f"INFO: domain='{domain}' has og:site_name='{software}'") + instances.set_data("detection_mode", domain, "SITE_NAME") + fba.remove_pending_error(domain) + + print(f"DEBUG: software[]={type(software)}") + if isinstance(software, str) and software == "": + print(f"DEBUG: Corrected empty string to None for software of domain='{domain}'") + software = None + elif isinstance(software, str) and ("." in software or " " in software): + print(f"DEBUG: software='{software}' may contain a version number, domain='{domain}', removing it ...") + software = fba.remove_version(software) + + print(f"DEBUG: software[]={type(software)}") + if isinstance(software, str) and " powered by " in software: + print(f"DEBUG: software='{software}' has 'powered by' in it") + software = fba.remove_version(fba.strip_powered_by(software)) + elif isinstance(software, str) and " hosted on " in software: + print(f"DEBUG: software='{software}' has 'hosted on' in it") + software = fba.remove_version(fba.strip_hosted_on(software)) + elif isinstance(software, str) and " by " in software: + print(f"DEBUG: software='{software}' has ' by ' in it") + software = fba.strip_until(software, " by ") + elif isinstance(software, str) and " see " in software: + print(f"DEBUG: software='{software}' has ' see ' in it") + software = fba.strip_until(software, " see ") + + print(f"DEBUG: software='{software}' - EXIT!") + return software + +def determine_software(domain: str, path: str = None) -> str: + print(f"DEBUG: domain({len(domain)})={domain},path={path} - CALLED!") + if not isinstance(domain, str): + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError("Parameter 'domain' is empty") + elif not isinstance(path, str) and path is not None: + raise ValueError(f"Parameter path[]={type(path)} is not 'str'") + + print("DEBUG: Determining software for domain,path:", domain, path) + software = None + + print(f"DEBUG: Fetching nodeinfo from '{domain}' ...") + data = fetch_nodeinfo(domain, path) + + print("DEBUG: data[]:", type(data)) + if not isinstance(data, dict) or len(data) == 0: + print("DEBUG: Could not determine software type:", domain) + return fetch_generator_from_path(domain) + + print("DEBUG: data():", len(data), data) + if "status" in data and data["status"] == "error" and "message" in data: + print("WARNING: JSON response is an error:", data["message"]) + instances.update_last_error(domain, data["message"]) + return fetch_generator_from_path(domain) + elif "message" in data: + print("WARNING: JSON response contains only a message:", data["message"]) + instances.update_last_error(domain, data["message"]) + return fetch_generator_from_path(domain) + elif "software" not in data or "name" not in data["software"]: + print(f"DEBUG: JSON response from domain='{domain}' does not include [software][name], fetching / ...") + software = fetch_generator_from_path(domain) + + print(f"DEBUG: Generator for domain='{domain}' is: {software}, EXIT!") + return software + + software = tidyup.domain(data["software"]["name"]) + + print("DEBUG: sofware after tidyup.domain():", software) + if software in ["akkoma", "rebased"]: + print("DEBUG: Setting pleroma:", domain, software) + software = "pleroma" + elif software in ["hometown", "ecko"]: + print("DEBUG: Setting mastodon:", domain, software) + software = "mastodon" + elif software in ["calckey", "groundpolis", "foundkey", "cherrypick", "meisskey"]: + print("DEBUG: Setting misskey:", domain, software) + software = "misskey" + elif software.find("/") > 0: + print("WARNING: Spliting of slash:", software) + software = tidyup.domain(software.split("/")[-1]) + elif software.find("|") > 0: + print("WARNING: Spliting of pipe:", software) + software = tidyup.domain(software.split("|")[0]) + elif "powered by" in software: + print(f"DEBUG: software='{software}' has 'powered by' in it") + software = fba.strip_powered_by(software) + elif isinstance(software, str) and " by " in software: + print(f"DEBUG: software='{software}' has ' by ' in it") + software = fba.strip_until(software, " by ") + elif isinstance(software, str) and " see " in software: + print(f"DEBUG: software='{software}' has ' see ' in it") + software = fba.strip_until(software, " see ") + + print(f"DEBUG: software[]={type(software)}") + if software == "": + print("WARNING: tidyup.domain() left no software name behind:", domain) + software = None + + print(f"DEBUG: software[]={type(software)}") + if str(software) == "": + print(f"DEBUG: software for '{domain}' was not detected, trying generator ...") + software = fetch_generator_from_path(domain) + elif len(str(software)) > 0 and ("." in software or " " in software): + print(f"DEBUG: software='{software}' may contain a version number, domain='{domain}', removing it ...") + software = fba.remove_version(software) + + print(f"DEBUG: software[]={type(software)}") + if isinstance(software, str) and "powered by" in software: + print(f"DEBUG: software='{software}' has 'powered by' in it") + software = fba.remove_version(fba.strip_powered_by(software)) + + print("DEBUG: Returning domain,software:", domain, software) + return software + +def find_domains(tag: bs4.element.Tag) -> list: + print(f"DEBUG: tag[]={type(tag)} - CALLED!") + if not isinstance(tag, bs4.element.Tag): + raise ValueError(f"Parameter tag[]={type(tag)} is not type of bs4.element.Tag") + elif len(tag.select("tr")) == 0: + raise KeyError("No table rows found in table!") + + domains = list() + for element in tag.select("tr"): + print(f"DEBUG: element[]={type(element)}") + if not element.find("td"): + print("DEBUG: Skipping element, no found") + continue + + domain = tidyup.domain(element.find("td").text) + reason = tidyup.reason(element.findAll("td")[1].text) + + print(f"DEBUG: domain='{domain}',reason='{reason}'") + + if blacklist.is_blacklisted(domain): + print(f"WARNING: domain='{domain}' is blacklisted - skipped!") + continue + elif domain == "gab.com/.ai, develop.gab.com": + print("DEBUG: Multiple domains detected in one row") + domains.append({ + "domain": "gab.com", + "reason": reason, + }) + domains.append({ + "domain": "gab.ai", + "reason": reason, + }) + domains.append({ + "domain": "develop.gab.com", + "reason": reason, + }) + continue + elif not validators.domain(domain): + print(f"WARNING: domain='{domain}' is not a valid domain - skipped!") + continue + + print(f"DEBUG: Adding domain='{domain}' ...") + domains.append({ + "domain": domain, + "reason": reason, + }) + + print(f"DEBUG: domains()={len(domains)} - EXIT!") + return domains + +def add_peers(rows: dict) -> list: + # DEBUG: print(f"DEBUG: rows()={len(rows)} - CALLED!") + peers = list() + for key in ["linked", "allowed", "blocked"]: + # DEBUG: print(f"DEBUG: Checking key='{key}'") + if key in rows and rows[key] is not None: + # DEBUG: print(f"DEBUG: Adding {len(rows[key])} peer(s) to peers list ...") + for peer in rows[key]: + # DEBUG: print(f"DEBUG: peer='{peer}' - BEFORE!") + peer = tidyup.domain(peer) + + # DEBUG: print(f"DEBUG: peer='{peer}' - AFTER!") + if blacklist.is_blacklisted(peer): + # DEBUG: print(f"DEBUG: peer='{peer}' is blacklisted, skipped!") + continue + + # DEBUG: print(f"DEBUG: Adding peer='{peer}' ...") + peers.append(peer) + + # DEBUG: print(f"DEBUG: peers()={len(peers)} - EXIT!") + return peers diff --git a/fba/federation/__init__.py b/fba/federation/__init__.py deleted file mode 100644 index af23ef3..0000000 --- a/fba/federation/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -__all__ = [ - 'friendica', - 'lemmy', - 'mastodon', - 'misskey', - 'peertube', - 'pleroma', -] diff --git a/fba/federation/friendica.py b/fba/federation/friendica.py deleted file mode 100644 index 4dd72fb..0000000 --- a/fba/federation/friendica.py +++ /dev/null @@ -1,72 +0,0 @@ -# Fedi API Block - An aggregator for fetching blocking data from fediverse nodes -# Copyright (C) 2023 Free Software Foundation -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published -# by the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -import bs4 - -from fba import config -from fba import fba -from fba import instances -from fba import network - -def fetch_blocks(domain: str) -> dict: - # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") - if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError("Parameter 'domain' is empty") - - # DEBUG: print("DEBUG: Fetching friendica blocks from domain:", domain) - blocked = list() - - doc = bs4.BeautifulSoup( - network.fetch_response( - domain, - "/friendica", - network.web_headers, - (config.get("connection_timeout"), config.get("read_timeout")) - ).text, - "html.parser", - ) - print(f"DEBUG: doc[]='{type(doc)}'") - - blocklist = doc.find(id="about_blocklist") - - # Prevents exceptions: - if blocklist is None: - # DEBUG: print("DEBUG: Instance has no block list:", domain) - return {} - - table = blocklist.find("table") - - # DEBUG: print(f"DEBUG: table[]='{type(table)}'") - if table.find("tbody"): - rows = table.find("tbody").find_all("tr") - else: - rows = table.find_all("tr") - - # DEBUG: print(f"DEBUG: Found rows()={len(rows)}") - for line in rows: - # DEBUG: print(f"DEBUG: line='{line}'") - blocked.append({ - "domain": fba.tidyup_domain(line.find_all("td")[0].text), - "reason": fba.tidyup_reason(line.find_all("td")[1].text) - }) - # DEBUG: print("DEBUG: Next!") - - # DEBUG: print("DEBUG: Returning blocklist() for domain:", domain, len(blocklist)) - return { - "reject": blocked - } diff --git a/fba/federation/lemmy.py b/fba/federation/lemmy.py deleted file mode 100644 index 8e07a83..0000000 --- a/fba/federation/lemmy.py +++ /dev/null @@ -1,65 +0,0 @@ -# Fedi API Block - An aggregator for fetching blocking data from fediverse nodes -# Copyright (C) 2023 Free Software Foundation -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published -# by the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -from fba import config -from fba import fba -from fba import instances -from fba import network - -def fetch_peers(domain: str) -> list: - # DEBUG: print(f"DEBUG: domain({len(domain)})={domain},software='lemmy' - CALLED!") - if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError("Parameter 'domain' is empty") - - peers = list() - try: - # DEBUG: print(f"DEBUG: domain='{domain}' is Lemmy, fetching JSON ...") - response = network.fetch_response( - domain, - "/api/v3/site", - network.api_headers, - (config.get("connection_timeout"), config.get("read_timeout")) - ) - - data = fba.json_from_response(response) - - # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code='{response.status_code}',data[]='{type(data)}'") - if not response.ok or response.status_code >= 400: - print("WARNING: Could not reach any JSON API:", domain) - instances.update_last_error(domain, response) - elif response.ok and isinstance(data, list): - print(f"UNSUPPORTED: domain='{domain}' returned a list: '{data}'") - elif "federated_instances" in data: - # DEBUG: print(f"DEBUG: Found federated_instances for domain='{domain}'") - peers = peers + fba.add_peers(data["federated_instances"]) - # DEBUG: print("DEBUG: Added instance(s) to peers") - else: - print("WARNING: JSON response does not contain 'federated_instances':", domain) - instances.update_last_error(domain, response) - - except BaseException as exception: - print(f"WARNING: Exception during fetching JSON: domain='{domain}',exception[{type(exception)}]:'{str(exception)}'") - - # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'") - instances.set_data("total_peers", domain, len(peers)) - - # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") - instances.update_last_instance_fetch(domain) - - # DEBUG: print("DEBUG: Returning peers[]:", type(peers)) - return peers diff --git a/fba/federation/mastodon.py b/fba/federation/mastodon.py deleted file mode 100644 index f55a4cd..0000000 --- a/fba/federation/mastodon.py +++ /dev/null @@ -1,262 +0,0 @@ -# Fedi API Block - An aggregator for fetching blocking data from fediverse nodes -# Copyright (C) 2023 Free Software Foundation -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published -# by the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -import inspect - -import bs4 -import validators - -from fba import blacklist -from fba import blocks -from fba import config -from fba import fba -from fba import instances -from fba import network - -language_mapping = { - # English -> English - "Silenced instances" : "Silenced servers", - "Suspended instances" : "Suspended servers", - "Limited instances" : "Limited servers", - "Filtered media" : "Filtered media", - # Mappuing German -> English - "Gesperrte Server" : "Suspended servers", - "Gefilterte Medien" : "Filtered media", - "Stummgeschaltete Server" : "Silenced servers", - # Japanese -> English - "停止済みのサーバー" : "Suspended servers", - "制限中のサーバー" : "Limited servers", - "メディアを拒否しているサーバー": "Filtered media", - "サイレンス済みのサーバー" : "Silenced servers", - # ??? -> English - "שרתים מושעים" : "Suspended servers", - "מדיה מסוננת" : "Filtered media", - "שרתים מוגבלים" : "Silenced servers", - # French -> English - "Serveurs suspendus" : "Suspended servers", - "Médias filtrés" : "Filtered media", - "Serveurs limités" : "Limited servers", - "Serveurs modérés" : "Limited servers", -} - -def fetch_blocks_from_about(domain: str) -> dict: - # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") - if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError("Parameter 'domain' is empty") - - # DEBUG: print("DEBUG: Fetching mastodon blocks from domain:", domain) - blocklist = { - "Suspended servers": [], - "Filtered media" : [], - "Limited servers" : [], - "Silenced servers" : [], - } - - try: - doc = bs4.BeautifulSoup( - network.fetch_response( - domain, - "/about/more", - network.web_headers, - (config.get("connection_timeout"), config.get("read_timeout")) - ).text, - "html.parser", - ) - except BaseException as exception: - print("ERROR: Cannot fetch from domain:", domain, exception) - instances.update_last_error(domain, exception) - return {} - - for header in doc.find_all("h3"): - header_text = fba.tidyup_reason(header.text) - - # DEBUG: print(f"DEBUG: header_text='{header_text}'") - if header_text in language_mapping: - # DEBUG: print(f"DEBUG: header_text='{header_text}'") - header_text = language_mapping[header_text] - else: - print(f"WARNING: header_text='{header_text}' not found in language mapping table") - - if header_text in blocklist or header_text.lower() in blocklist: - # replaced find_next_siblings with find_all_next to account for instances that e.g. hide lists in dropdown menu - for line in header.find_all_next("table")[0].find_all("tr")[1:]: - blocklist[header_text].append( - { - "domain": fba.tidyup_domain(line.find("span").text), - "hash" : fba.tidyup_domain(line.find("span")["title"][9:]), - "reason": fba.tidyup_reason(line.find_all("td")[1].text), - } - ) - else: - print(f"WARNING: header_text='{header_text}' not found in blocklist()={len(blocklist)}") - - # DEBUG: print("DEBUG: Returning blocklist for domain:", domain) - return { - "reject" : blocklist["Suspended servers"], - "media_removal" : blocklist["Filtered media"], - "followers_only": blocklist["Limited servers"] + blocklist["Silenced servers"], - } - -def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): - # DEBUG: print(f"DEBUG: domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}' - CALLED!") - if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError("Parameter 'domain' is empty") - elif not isinstance(origin, str) and origin is not None: - raise ValueError(f"Parameter origin[]={type(origin)} is not 'str'") - elif origin == "": - raise ValueError("Parameter 'origin' is empty") - elif not isinstance(nodeinfo_url, str): - raise ValueError(f"Parameter nodeinfo_url[]={type(nodeinfo_url)} is not 'str'") - elif nodeinfo_url == "": - raise ValueError("Parameter 'nodeinfo_url' is empty") - - try: - # json endpoint for newer mastodongs - blockdict = list() - try: - rows = { - "reject" : [], - "media_removal" : [], - "followers_only": [], - "report_removal": [], - } - - # DEBUG: print("DEBUG: Querying API domain_blocks:", domain) - response = network.fetch_response( - domain, - "/api/v1/instance/domain_blocks", - network.api_headers, - (config.get("connection_timeout"), config.get("read_timeout")) - ) - - # DEBUG: print(f"DEBUG: response[]='{type(response)}'") - blocklist = fba.json_from_response(response) - - print(f"INFO: Checking {len(blocklist)} entries from domain='{domain}',software='mastodon' ...") - for block in blocklist: - entry = { - 'domain': block['domain'], - 'hash' : block['digest'], - 'reason': block['comment'] - } - - # DEBUG: print("DEBUG: severity,domain,hash,comment:", block['severity'], block['domain'], block['digest'], block['comment']) - if block['severity'] == 'suspend': - # DEBUG: print(f"DEBUG: Adding entry='{entry}' with severity='{block['severity']}' ...") - rows['reject'].append(entry) - elif block['severity'] == 'silence': - # DEBUG: print(f"DEBUG: Adding entry='{entry}' with severity='{block['severity']}' ...") - rows['followers_only'].append(entry) - elif block['severity'] == 'reject_media': - # DEBUG: print(f"DEBUG: Adding entry='{entry}' with severity='{block['severity']}' ...") - rows['media_removal'].append(entry) - elif block['severity'] == 'reject_reports': - # DEBUG: print(f"DEBUG: Adding entry='{entry}' with severity='{block['severity']}' ...") - rows['report_removal'].append(entry) - else: - print("WARNING: Unknown severity:", block['severity'], block['domain']) - - except BaseException as exception: - # DEBUG: print(f"DEBUG: Failed, trying mastodon-specific fetches: domain='{domain}',exception[{type(exception)}]={str(exception)}") - rows = fetch_blocks_from_about(domain) - - print(f"INFO: Checking {len(rows.items())} entries from domain='{domain}',software='mastodon' ...") - for block_level, blocklist in rows.items(): - # DEBUG: print("DEBUG: domain,block_level,blocklist():", domain, block_level, len(blocklist)) - block_level = fba.tidyup_domain(block_level) - - # DEBUG: print("DEBUG: AFTER-block_level:", block_level) - if block_level == "": - print("WARNING: block_level is empty, domain:", domain) - continue - - # DEBUG: print(f"DEBUG: Checking {len(blocklist)} entries from domain='{domain}',software='mastodon',block_level='{block_level}' ...") - for block in blocklist: - # DEBUG: print(f"DEBUG: block[]='{type(block)}'") - blocked, blocked_hash, reason = block.values() - # DEBUG: print(f"DEBUG: blocked='{blocked}',blocked_hash='{blocked_hash}',reason='{reason}':") - blocked = fba.tidyup_domain(blocked) - reason = fba.tidyup_reason(reason) if reason is not None and reason != "" else None - # DEBUG: print(f"DEBUG: blocked='{blocked}',reason='{reason}' - AFTER!") - - if blocked == "": - print("WARNING: blocked is empty:", domain) - continue - elif blacklist.is_blacklisted(blocked): - # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!") - continue - elif blocked.count("*") > 0: - # Doing the hash search for instance names as well to tidy up DB - fba.cursor.execute( - "SELECT domain, origin, nodeinfo_url FROM instances WHERE hash = ? LIMIT 1", [blocked_hash] - ) - searchres = fba.cursor.fetchone() - - if searchres is None: - print(f"WARNING: Cannot deobsfucate blocked='{blocked}',blocked_hash='{blocked_hash}' - SKIPPED!") - continue - - # DEBUG: print("DEBUG: Updating domain: ", searchres[0]) - blocked = searchres[0] - origin = searchres[1] - nodeinfo_url = searchres[2] - - # DEBUG: print("DEBUG: Looking up instance by domain:", blocked) - if not validators.domain(blocked): - print(f"WARNING: blocked='{blocked}',software='mastodon' is not a valid domain name - skipped!") - continue - elif not instances.is_registered(blocked): - # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'") - instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url) - elif not validators.domain(blocked): - print(f"WARNING: blocked='{blocked}',software='mastodon' is not a valid domain name - skipped!") - continue - - # DEBUG: print("DEBUG: Looking up instance by domain:", blocked) - if not validators.domain(blocked): - print(f"WARNING: blocked='{blocked}',software='mastodon' is not a valid domain name - skipped!") - continue - elif not instances.is_registered(blocked): - # DEBUG: print("DEBUG: Hash wasn't found, adding:", blocked, domain) - instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url) - - blocking = blocked if blocked.count("*") <= 1 else blocked_hash - # DEBUG: print(f"DEBUG: blocking='{blocking}',blocked='{blocked}',blocked_hash='{blocked_hash}'") - - if not blocks.is_instance_blocked(domain, blocked, block_level): - # DEBUG: print("DEBUG: Blocking:", domain, blocked, block_level) - blocks.add_instance(domain, blocking, reason, block_level) - - if block_level == "reject": - blockdict.append({ - "blocked": blocked, - "reason" : reason - }) - else: - # DEBUG: print(f"DEBUG: Updating block last seen and reason for domain='{domain}',blocking='{blocking}' ...") - blocks.update_last_seen(domain, blocking, block_level) - blocks.update_reason(reason, domain, blocking, block_level) - - # DEBUG: print("DEBUG: Committing changes ...") - fba.connection.commit() - except BaseException as exception: - print(f"ERROR: domain='{domain}',software='mastodon',exception[{type(exception)}]:'{str(exception)}'") - - # DEBUG: print("DEBUG: EXIT!") diff --git a/fba/federation/misskey.py b/fba/federation/misskey.py deleted file mode 100644 index 9e5c9af..0000000 --- a/fba/federation/misskey.py +++ /dev/null @@ -1,253 +0,0 @@ -# Fedi API Block - An aggregator for fetching blocking data from fediverse nodes -# Copyright (C) 2023 Free Software Foundation -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published -# by the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -import json - -from fba import blacklist -from fba import config -from fba import fba -from fba import instances -from fba import network - -def fetch_peers(domain: str) -> list: - # DEBUG: print(f"DEBUG: domain({len(domain)})={domain} - CALLED!") - if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError("Parameter 'domain' is empty") - - # DEBUG: print(f"DEBUG: domain='{domain}' is misskey, sending API POST request ...") - peers = list() - offset = 0 - step = config.get("misskey_limit") - - # iterating through all "suspended" (follow-only in its terminology) - # instances page-by-page, since that troonware doesn't support - # sending them all at once - while True: - # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...") - if offset == 0: - fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({ - "sort" : "+pubAt", - "host" : None, - "limit": step - }), { - "Origin": domain - }) - else: - fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({ - "sort" : "+pubAt", - "host" : None, - "limit" : step, - "offset": offset - 1 - }), { - "Origin": domain - }) - - # DEBUG: print(f"DEBUG: fetched()={len(fetched)}") - if len(fetched) == 0: - # DEBUG: print(f"DEBUG: Returned zero bytes, exiting loop, domain='{domain}'") - break - elif len(fetched) != config.get("misskey_limit"): - # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'") - offset = offset + (config.get("misskey_limit") - len(fetched)) - else: - # DEBUG: print(f"DEBUG: Raising offset by step={step}") - offset = offset + step - - # Check records - # DEBUG: print(f"DEBUG: fetched({len(fetched)})[]={type(fetched)}") - if isinstance(fetched, dict) and "error" in fetched and "message" in fetched["error"]: - print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}") - instances.update_last_error(domain, fetched["error"]["message"]) - break - - already = 0 - for row in fetched: - # DEBUG: print(f"DEBUG: row()={len(row)}") - if not "host" in row: - print(f"WARNING: row()={len(row)} does not contain key 'host': {row},domain='{domain}'") - continue - elif not isinstance(row["host"], str): - print(f"WARNING: row[host][]={type(row['host'])} is not 'str'") - continue - elif blacklist.is_blacklisted(row["host"]): - # DEBUG: print(f"DEBUG: row[host]='{row['host']}' is blacklisted. domain='{domain}'") - continue - elif row["host"] in peers: - # DEBUG: print(f"DEBUG: Not adding row[host]='{row['host']}', already found.") - already = already + 1 - continue - - # DEBUG: print(f"DEBUG: Adding peer: '{row['host']}'") - peers.append(row["host"]) - - if already == len(fetched): - print(f"WARNING: Host returned same set of '{already}' instances, aborting loop!") - break - - # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'") - instances.set_data("total_peers", domain, len(peers)) - - # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") - instances.update_last_instance_fetch(domain) - - # DEBUG: print(f"DEBUG: Returning peers[]='{type(peers)}'") - return peers - -def fetch_blocks(domain: str) -> dict: - # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") - if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError("Parameter 'domain' is empty") - - # DEBUG: print("DEBUG: Fetching misskey blocks from domain:", domain) - blocklist = { - "suspended": [], - "blocked" : [] - } - - offset = 0 - step = config.get("misskey_limit") - while True: - # iterating through all "suspended" (follow-only in its terminology) - # instances page-by-page, since that troonware doesn't support - # sending them all at once - try: - # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...") - if offset == 0: - # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset) - fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({ - "sort" : "+pubAt", - "host" : None, - "suspended": True, - "limit" : step - }), { - "Origin": domain - }) - else: - # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset) - fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({ - "sort" : "+pubAt", - "host" : None, - "suspended": True, - "limit" : step, - "offset" : offset - 1 - }), { - "Origin": domain - }) - - # DEBUG: print("DEBUG: fetched():", len(fetched)) - if len(fetched) == 0: - # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain) - break - elif len(fetched) != config.get("misskey_limit"): - # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'") - offset = offset + (config.get("misskey_limit") - len(fetched)) - else: - # DEBUG: print("DEBUG: Raising offset by step:", step) - offset = offset + step - - count = 0 - for instance in fetched: - # Is it there? - if instance["isSuspended"] and not fba.has_key(blocklist["suspended"], "domain", instance): - count = count + 1 - blocklist["suspended"].append( - { - "domain": fba.tidyup_domain(instance["host"]), - # no reason field, nothing - "reason": None - } - ) - - # DEBUG: print(f"DEBUG: count={count}") - if count == 0: - # DEBUG: print("DEBUG: API is no more returning new instances, aborting loop!") - break - - except BaseException as exception: - print("WARNING: Caught error, exiting loop:", domain, exception) - instances.update_last_error(domain, exception) - offset = 0 - break - - while True: - # same shit, different asshole ("blocked" aka full suspend) - try: - if offset == 0: - # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset) - fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({ - "sort" : "+pubAt", - "host" : None, - "blocked": True, - "limit" : step - }), { - "Origin": domain - }) - else: - # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset) - fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({ - "sort" : "+pubAt", - "host" : None, - "blocked": True, - "limit" : step, - "offset" : offset - 1 - }), { - "Origin": domain - }) - - # DEBUG: print("DEBUG: fetched():", len(fetched)) - if len(fetched) == 0: - # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain) - break - elif len(fetched) != config.get("misskey_limit"): - # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'") - offset = offset + (config.get("misskey_limit") - len(fetched)) - else: - # DEBUG: print("DEBUG: Raising offset by step:", step) - offset = offset + step - - count = 0 - for instance in fetched: - # Is it there? - if instance["isBlocked"] and not fba.has_key(blocklist["blocked"], "domain", instance): - count = count + 1 - blocklist["blocked"].append({ - "domain": fba.tidyup_domain(instance["host"]), - "reason": None - }) - - # DEBUG: print(f"DEBUG: count={count}") - if count == 0: - # DEBUG: print("DEBUG: API is no more returning new instances, aborting loop!") - break - - except BaseException as exception: - print("ERROR: Exception during POST:", domain, exception) - instances.update_last_error(domain, exception) - offset = 0 - break - - # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") - instances.update_last_instance_fetch(domain) - - # DEBUG: print(f"DEBUG: Returning for domain='{domain}',blocked()={len(blocklist['blocked'])},suspended()={len(blocklist['suspended'])}") - return { - "reject" : blocklist["blocked"], - "followers_only": blocklist["suspended"] - } diff --git a/fba/federation/peertube.py b/fba/federation/peertube.py deleted file mode 100644 index 2d2ebaf..0000000 --- a/fba/federation/peertube.py +++ /dev/null @@ -1,74 +0,0 @@ -# Fedi API Block - An aggregator for fetching blocking data from fediverse nodes -# Copyright (C) 2023 Free Software Foundation -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published -# by the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -from fba import config -from fba import fba -from fba import instances -from fba import network - -def fetch_peers(domain: str) -> list: - # DEBUG: print(f"DEBUG: domain({len(domain)})={domain},software='peertube' - CALLED!") - if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError("Parameter 'domain' is empty") - - # DEBUG: print(f"DEBUG: domain='{domain}' is a PeerTube, fetching JSON ...") - peers = list() - start = 0 - for mode in ["followers", "following"]: - # DEBUG: print(f"DEBUG: domain='{domain}',mode='{mode}'") - while True: - try: - response = network.fetch_response( - domain, - "/api/v1/server/{mode}?start={start}&count=100", - network.api_headers, - (config.get("connection_timeout"), config.get("read_timeout")) - ) - - data = fba.json_from_response(response) - # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code='{response.status_code}',data[]='{type(data)}'") - if response.ok and isinstance(data, dict): - # DEBUG: print("DEBUG: Success, data:", len(data)) - if "data" in data: - # DEBUG: print(f"DEBUG: Found {len(data['data'])} record(s).") - for record in data["data"]: - # DEBUG: print(f"DEBUG: record()={len(record)}") - if mode in record and "host" in record[mode]: - # DEBUG: print(f"DEBUG: Found host={record[mode]['host']}, adding ...") - peers.append(record[mode]["host"]) - else: - print(f"WARNING: record from '{domain}' has no '{mode}' or 'host' record: {record}") - - if len(data["data"]) < 100: - # DEBUG: print("DEBUG: Reached end of JSON response:", domain) - break - - # Continue with next row - start = start + 100 - - except BaseException as exception: - print(f"WARNING: Exception during fetching JSON: domain='{domain}',exception[{type(exception)}]:'{str(exception)}'") - - # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'") - instances.set_data("total_peers", domain, len(peers)) - - # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") - instances.update_last_instance_fetch(domain) - - # DEBUG: print("DEBUG: Returning peers[]:", type(peers)) - return peers diff --git a/fba/federation/pleroma.py b/fba/federation/pleroma.py deleted file mode 100644 index 5e3bbf1..0000000 --- a/fba/federation/pleroma.py +++ /dev/null @@ -1,200 +0,0 @@ -# Fedi API Block - An aggregator for fetching blocking data from fediverse nodes -# Copyright (C) 2023 Free Software Foundation -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published -# by the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -import inspect -import validators - -from fba import blacklist -from fba import blocks -from fba import fba -from fba import instances - -def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): - # DEBUG: print(f"DEBUG: domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}' - CALLED!") - if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError("Parameter 'domain' is empty") - elif not isinstance(origin, str) and origin is not None: - raise ValueError(f"Parameter origin[]={type(origin)} is not 'str'") - elif origin == "": - raise ValueError("Parameter 'origin' is empty") - elif not isinstance(nodeinfo_url, str): - raise ValueError(f"Parameter nodeinfo_url[]={type(nodeinfo_url)} is not 'str'") - elif nodeinfo_url == "": - raise ValueError("Parameter 'nodeinfo_url' is empty") - - try: - # Blocks - blockdict = list() - rows = fba.fetch_nodeinfo(domain, nodeinfo_url) - - if rows is None: - print("WARNING: Could not fetch nodeinfo from domain:", domain) - return - elif not "metadata" in rows: - print(f"WARNING: rows()={len(rows)} does not have key 'metadata', domain='{domain}'") - return - elif not "federation" in rows["metadata"]: - print(f"WARNING: rows()={len(rows['metadata'])} does not have key 'federation', domain='{domain}'") - return - - # DEBUG: print("DEBUG: Updating nodeinfo:", domain) - instances.update_last_nodeinfo(domain) - - federation = rows["metadata"]["federation"] - - if "enabled" in federation: - # DEBUG: print("DEBUG: Instance has no block list to analyze:", domain) - return - - if "mrf_simple" in federation: - for block_level, blocklist in ( - {**federation["mrf_simple"], - **{"quarantined_instances": federation["quarantined_instances"]}} - ).items(): - # DEBUG: print("DEBUG: block_level, blocklist():", block_level, len(blocklist)) - block_level = fba.tidyup_domain(block_level) - # DEBUG: print("DEBUG: BEFORE block_level:", block_level) - - if block_level == "": - print("WARNING: block_level is now empty!") - continue - - # DEBUG: print(f"DEBUG: Checking {len(blocklist)} entries from domain='{domain}',block_level='{block_level}' ...") - for blocked in blocklist: - # DEBUG: print("DEBUG: BEFORE blocked:", blocked) - blocked = fba.tidyup_domain(blocked) - # DEBUG: print("DEBUG: AFTER blocked:", blocked) - - if blocked == "": - print("WARNING: blocked is empty after fba.tidyup_domain():", domain, block_level) - continue - elif blacklist.is_blacklisted(blocked): - # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!") - continue - elif blocked.count("*") > 1: - # -ACK!-oma also started obscuring domains without hash - fba.cursor.execute( - "SELECT domain, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")] - ) - searchres = fba.cursor.fetchone() - # DEBUG: print("DEBUG: searchres[]:", type(searchres)) - - if searchres is None: - print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!") - continue - - blocked = searchres[0] - nodeinfo_url = searchres[1] - # DEBUG: print("DEBUG: Looked up domain:", blocked) - elif not validators.domain(blocked): - print(f"WARNING: blocked='{blocked}',software='pleroma' is not a valid domain name - skipped!") - continue - - # DEBUG: print("DEBUG: Looking up instance by domain:", blocked) - if not validators.domain(blocked): - print(f"WARNING: blocked='{blocked}',software='pleroma' is not a valid domain name - skipped!") - continue - elif not instances.is_registered(blocked): - # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'") - instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url) - - if not blocks.is_instance_blocked(domain, blocked, block_level): - # DEBUG: print("DEBUG: Blocking:", domain, blocked, block_level) - blocks.add_instance(domain, blocked, "unknown", block_level) - - if block_level == "reject": - # DEBUG: print("DEBUG: Adding to blockdict:", blocked) - blockdict.append( - { - "blocked": blocked, - "reason" : None - }) - else: - # DEBUG: print(f"DEBUG: Updating block last seen for domain='{domain}',blocked='{blocked}' ...") - blocks.update_last_seen(domain, blocked, block_level) - - # DEBUG: print("DEBUG: Committing changes ...") - fba.connection.commit() - - # Reasons - if "mrf_simple_info" in federation: - # DEBUG: print("DEBUG: Found mrf_simple_info:", domain) - for block_level, info in ( - {**federation["mrf_simple_info"], - **(federation["quarantined_instances_info"] - if "quarantined_instances_info" in federation - else {})} - ).items(): - # DEBUG: print("DEBUG: block_level, info.items():", block_level, len(info.items())) - block_level = fba.tidyup_domain(block_level) - # DEBUG: print("DEBUG: BEFORE block_level:", block_level) - - if block_level == "": - print("WARNING: block_level is now empty!") - continue - - # DEBUG: print(f"DEBUG: Checking {len(info.items())} entries from domain='{domain}',software='pleroma',block_level='{block_level}' ...") - for blocked, reason in info.items(): - # DEBUG: print(f"DEBUG: blocked='{blocked}',reason='{reason}' - BEFORE!") - blocked = fba.tidyup_domain(blocked) - reason = fba.tidyup_reason(reason) if reason is not None and reason != "" else None - # DEBUG: print(f"DEBUG: blocked='{blocked}',reason='{reason}' - AFTER!") - - if blocked == "": - print("WARNING: blocked is empty after fba.tidyup_domain():", domain, block_level) - continue - elif not validators.domain(blocked): - print(f"WARNING: blocked='{blocked}',software='pleroma' is not a valid domain name - skipped!") - continue - elif blacklist.is_blacklisted(blocked): - # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!") - continue - elif blocked.count("*") > 1: - # same domain guess as above, but for reasons field - fba.cursor.execute( - "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")] - ) - searchres = fba.cursor.fetchone() - - if searchres is None: - print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!") - continue - - blocked = searchres[0] - origin = searchres[1] - nodeinfo_url = searchres[2] - - # DEBUG: print("DEBUG: Looking up instance by domain:", blocked) - if not instances.is_registered(blocked): - # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'") - instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url) - - # DEBUG: print("DEBUG: Updating block reason:", domain, blocked, reason["reason"]) - blocks.update_reason(reason["reason"], domain, blocked, block_level) - - # DEBUG: print(f"DEBUG: blockdict()={len(blockdict)}") - for entry in blockdict: - if entry["blocked"] == blocked: - # DEBUG: print("DEBUG: Updating entry reason:", blocked) - entry["reason"] = reason["reason"] - - fba.connection.commit() - except BaseException as exception: - print(f"ERROR: domain='{domain}',software='pleroma',exception[{type(exception)}]:'{str(exception)}'") - - # DEBUG: print("DEBUG: EXIT!") diff --git a/fba/helpers/__init__.py b/fba/helpers/__init__.py new file mode 100644 index 0000000..9f8b143 --- /dev/null +++ b/fba/helpers/__init__.py @@ -0,0 +1,19 @@ +# Copyright (C) 2023 Free Software Foundation +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +__all__ = [ + 'dicts', + 'tidyup', +] diff --git a/fba/helpers/dicts.py b/fba/helpers/dicts.py new file mode 100644 index 0000000..51843d4 --- /dev/null +++ b/fba/helpers/dicts.py @@ -0,0 +1,38 @@ +# Copyright (C) 2023 Free Software Foundation +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +def has_key(lists: list, key: str, value: any) -> bool: + print(f"DEBUG: lists()={len(lists)},key='{key}',value[]='{type(value)}' - CALLED!") + if not isinstance(lists, list): + raise ValueError(f"Parameter lists[]='{type(lists)}' is not 'list'") + elif not isinstance(key, str): + raise ValueError(f"Parameter key[]='{type(key)}' is not 'str'") + elif key == "": + raise ValueError("Parameter 'key' is empty") + + has = False + print(f"DEBUG: Checking lists()={len(lists)} ...") + for row in lists: + print(f"DEBUG: row['{type(row)}']={row}") + if not isinstance(row, dict): + raise ValueError(f"row[]='{type(row)}' is not 'dict'") + elif not key in row: + raise KeyError(f"Cannot find key='{key}'") + elif row[key] == value: + has = True + break + + print(f"DEBUG: has={has} - EXIT!") + return has diff --git a/fba/helpers/tidyup.py b/fba/helpers/tidyup.py new file mode 100644 index 0000000..c0de2c8 --- /dev/null +++ b/fba/helpers/tidyup.py @@ -0,0 +1,60 @@ +# Copyright (C) 2023 Free Software Foundation +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import re + +def reason(string: str) -> str: + print(f"DEBUG: string='{string}' - CALLED!") + if not isinstance(string, str): + raise ValueError(f"Parameter string[]={type(string)} is not 'str'") + + # Strip string + string = string.strip() + + # Replace â with " + string = re.sub("â", "\"", string) + + print(f"DEBUG: string='{string}' - EXIT!") + return string + +def domain(string: str) -> str: + print(f"DEBUG: string='{string}' - CALLED!") + if not isinstance(string, str): + raise ValueError(f"Parameter string[]={type(string)} is not 'str'") + + # All lower-case and strip spaces out + last dot + string = string.lower().strip().rstrip(".") + + # No port number + string = re.sub("\:\d+$", "", string) + + # No protocol, sometimes without the slashes + string = re.sub("^https?\:(\/*)", "", string) + + # No trailing slash + string = re.sub("\/$", "", string) + + # No @ sign + string = re.sub("^\@", "", string) + + # No individual users in block lists + string = re.sub("(.+)\@", "", string) + if string.find("/profile/"): + string = string.split("/profile/")[0] + elif string.find("/users/"): + string = string.split("/users/")[0] + + print(f"DEBUG: string='{string}' - EXIT!") + return string diff --git a/fba/instances.py b/fba/instances.py index 903fc3c..6ac8bcb 100644 --- a/fba/instances.py +++ b/fba/instances.py @@ -24,6 +24,7 @@ import validators from fba import blacklist from fba import cache from fba import fba +from fba import federation # Found info from node, such as nodeinfo URL, detection mode that needs to be # written to database. Both arrays must be filled at the same time or else @@ -197,7 +198,7 @@ def add(domain: str, origin: str, command: str, path: str = None): raise Exception(f"domain='{domain}' is a single user") # DEBUG: print("DEBUG: domain,origin,command,path:", domain, origin, command, path) - software = fba.determine_software(domain, path) + software = federation.determine_software(domain, path) # DEBUG: print("DEBUG: Determined software:", software) if domain.find("/c/") > 0 and software == "lemmy": domain = domain.split("/c/")[0] diff --git a/fba/locking.py b/fba/locking.py new file mode 100644 index 0000000..132a8ca --- /dev/null +++ b/fba/locking.py @@ -0,0 +1,45 @@ +# Fedi API Block - An aggregator for fetching blocking data from fediverse nodes +# Copyright (C) 2023 Free Software Foundation +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import os +import sys +import tempfile +import zc.lockfile + +# Lock file +lockfile = tempfile.gettempdir() + '/fba.lock' +LOCK = None + +def acquire(): + global LOCK + try: + # DEBUG: print(f"DEBUG: Acquiring lock: '{lockfile}'") + LOCK = zc.lockfile.LockFile(lockfile) + # DEBUG: print("DEBUG: Lock obtained.") + + except zc.lockfile.LockError: + print(f"ERROR: Cannot aquire lock: '{lockfile}'") + sys.exit(100) + +def release(): + # DEBUG: print("DEBUG: CALLED!") + if LOCK is not None: + # DEBUG: print("DEBUG: Releasing lock ...") + LOCK.close() + # DEBUG: print(f"DEBUG: Deleting lockfile='{lockfile}' ...") + os.remove(lockfile) + + # DEBUG: print("DEBUG: EXIT!") diff --git a/fba/network.py b/fba/network.py index 4894c0f..7329979 100644 --- a/fba/network.py +++ b/fba/network.py @@ -14,13 +14,12 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import bs4 +import json import reqto import requests from fba import config from fba import csrf -from fba import fba from fba import instances # HTTP headers for non-API requests @@ -35,7 +34,7 @@ api_headers = { } def post_json_api(domain: str, path: str, parameter: str, extra_headers: dict = {}) -> dict: - # DEBUG: print(f"DEBUG: domain='{domain}',path='{path}',parameter='{parameter}',extra_headers()={len(extra_headers)} - CALLED!") + print(f"DEBUG: domain='{domain}',path='{path}',parameter='{parameter}',extra_headers()={len(extra_headers)} - CALLED!") if not isinstance(domain, str): raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": @@ -47,13 +46,13 @@ def post_json_api(domain: str, path: str, parameter: str, extra_headers: dict = elif not isinstance(parameter, str): raise ValueError(f"parameter[]={type(parameter)} is not 'str'") - # DEBUG: print(f"DEBUG: Determining if CSRF header needs to be sent for domain='{domain}' ...") + print(f"DEBUG: Determining if CSRF header needs to be sent for domain='{domain}' ...") headers = csrf.determine(domain, {**api_headers, **extra_headers}) data = {} try: - # DEBUG: print(f"DEBUG: Sending POST to domain='{domain}',path='{path}',parameter='{parameter}',extra_headers({len(extra_headers)})={extra_headers}") + print(f"DEBUG: Sending POST to domain='{domain}',path='{path}',parameter='{parameter}',extra_headers({len(extra_headers)})={extra_headers}") response = reqto.post( f"https://{domain}{path}", data=parameter, @@ -61,8 +60,8 @@ def post_json_api(domain: str, path: str, parameter: str, extra_headers: dict = timeout=(config.get("connection_timeout"), config.get("read_timeout")) ) - data = fba.json_from_response(response) - # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'") + data = json_from_response(response) + print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'") if not response.ok or response.status_code >= 400: print(f"WARNING: Cannot query JSON API: domain='{domain}',path='{path}',parameter()={len(parameter)},response.status_code='{response.status_code}',data[]='{type(data)}'") instances.update_last_error(domain, response) @@ -70,11 +69,11 @@ def post_json_api(domain: str, path: str, parameter: str, extra_headers: dict = except BaseException as exception: print(f"WARNING: Some error during post(): domain='{domain}',path='{path}',parameter()={len(parameter)},exception[{type(exception)}]:'{str(exception)}'") - # DEBUG: print(f"DEBUG: Returning data({len(data)})=[]:{type(data)}") + print(f"DEBUG: Returning data({len(data)})=[]:{type(data)}") return data def send_bot_post(domain: str, blocklist: dict): - # DEBUG: print(f"DEBUG: domain={domain},blocklist()={len(blocklist)} - CALLED!") + print(f"DEBUG: domain={domain},blocklist()={len(blocklist)} - CALLED!") if not isinstance(domain, str): raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": @@ -89,9 +88,9 @@ def send_bot_post(domain: str, blocklist: dict): truncated = True blocklist = blocklist[0 : 19] - # DEBUG: print(f"DEBUG: blocklist()={len(blocklist)}") + print(f"DEBUG: blocklist()={len(blocklist)}") for block in blocklist: - # DEBUG: print(f"DEBUG: block['{type(block)}']={block}") + print(f"DEBUG: block['{type(block)}']={block}") if block["reason"] is None or block["reason"] == '': message = message + block["blocked"] + " with unspecified reason\n" else: @@ -119,7 +118,7 @@ def send_bot_post(domain: str, blocklist: dict): return True def fetch_response(domain: str, path: str, headers: dict, timeout: list) -> requests.models.Response: - # DEBUG: print(f"DEBUG: domain='{domain}',path='{path}',headers()={len(headers)},timeout={timeout} - CALLED!") + print(f"DEBUG: domain='{domain}',path='{path}',headers()={len(headers)},timeout={timeout} - CALLED!") if not isinstance(domain, str): raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": @@ -129,11 +128,11 @@ def fetch_response(domain: str, path: str, headers: dict, timeout: list) -> requ elif path == "": raise ValueError("Parameter 'path' is empty") - # DEBUG: print(f"DEBUG: Determining if CSRF header needs to be sent for domain='{domain}',headers()='{len(headers)}' ...") + print(f"DEBUG: Determining if CSRF header needs to be sent for domain='{domain}',headers()='{len(headers)}' ...") headers = csrf.determine(domain, headers) try: - # DEBUG: print(f"DEBUG: Sending GET request to '{domain}{path}' ...") + print(f"DEBUG: Sending GET request to '{domain}{path}' ...") response = reqto.get( f"https://{domain}{path}", headers=headers, @@ -141,9 +140,25 @@ def fetch_response(domain: str, path: str, headers: dict, timeout: list) -> requ ) except requests.exceptions.ConnectionError as exception: - # DEBUG: print(f"DEBUG: Fetching '{path}' from '{domain}' failed. exception[{type(exception)}]='{str(exception)}'") + print(f"DEBUG: Fetching '{path}' from '{domain}' failed. exception[{type(exception)}]='{str(exception)}'") instances.update_last_error(domain, exception) raise exception - # DEBUG: print(f"DEBUG: response[]='{type(response)}' - EXXIT!") + print(f"DEBUG: response[]='{type(response)}' - EXXIT!") return response + +def json_from_response(response: requests.models.Response) -> list: + print(f"DEBUG: response[]={type(response)} - CALLED!") + if not isinstance(response, requests.models.Response): + raise ValueError(f"Parameter response[]='{type(response)}' is not type of 'Response'") + + data = list() + if response.text.strip() != "": + print(f"DEBUG: response.text()={len(response.text)} is not empty, invoking response.json() ...") + try: + data = response.json() + except json.decoder.JSONDecodeError: + pass + + print(f"DEBUG: data[]={type(data)} - EXIT!") + return data diff --git a/fba/networks/__init__.py b/fba/networks/__init__.py new file mode 100644 index 0000000..77902b2 --- /dev/null +++ b/fba/networks/__init__.py @@ -0,0 +1,23 @@ +# Copyright (C) 2023 Free Software Foundation +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +__all__ = [ + 'friendica', + 'lemmy', + 'mastodon', + 'misskey', + 'peertube', + 'pleroma', +] diff --git a/fba/networks/friendica.py b/fba/networks/friendica.py new file mode 100644 index 0000000..ceba93d --- /dev/null +++ b/fba/networks/friendica.py @@ -0,0 +1,72 @@ +# Fedi API Block - An aggregator for fetching blocking data from fediverse nodes +# Copyright (C) 2023 Free Software Foundation +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import bs4 + +from fba import config +from fba import network + +from fba.helpers import tidyup + +def fetch_blocks(domain: str) -> dict: + # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") + if not isinstance(domain, str): + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError("Parameter 'domain' is empty") + + # DEBUG: print("DEBUG: Fetching friendica blocks from domain:", domain) + blocked = list() + + doc = bs4.BeautifulSoup( + network.fetch_response( + domain, + "/friendica", + network.web_headers, + (config.get("connection_timeout"), config.get("read_timeout")) + ).text, + "html.parser", + ) + print(f"DEBUG: doc[]='{type(doc)}'") + + blocklist = doc.find(id="about_blocklist") + + # Prevents exceptions: + if blocklist is None: + # DEBUG: print("DEBUG: Instance has no block list:", domain) + return {} + + table = blocklist.find("table") + + # DEBUG: print(f"DEBUG: table[]='{type(table)}'") + if table.find("tbody"): + rows = table.find("tbody").find_all("tr") + else: + rows = table.find_all("tr") + + # DEBUG: print(f"DEBUG: Found rows()={len(rows)}") + for line in rows: + # DEBUG: print(f"DEBUG: line='{line}'") + blocked.append({ + "domain": tidyup.domain(line.find_all("td")[0].text), + "reason": tidyup.reason(line.find_all("td")[1].text) + }) + # DEBUG: print("DEBUG: Next!") + + # DEBUG: print("DEBUG: Returning blocklist() for domain:", domain, len(blocklist)) + return { + "reject": blocked + } diff --git a/fba/networks/lemmy.py b/fba/networks/lemmy.py new file mode 100644 index 0000000..90d750d --- /dev/null +++ b/fba/networks/lemmy.py @@ -0,0 +1,65 @@ +# Fedi API Block - An aggregator for fetching blocking data from fediverse nodes +# Copyright (C) 2023 Free Software Foundation +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from fba import config +from fba import federation +from fba import instances +from fba import network + +def fetch_peers(domain: str) -> list: + # DEBUG: print(f"DEBUG: domain({len(domain)})={domain},software='lemmy' - CALLED!") + if not isinstance(domain, str): + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError("Parameter 'domain' is empty") + + peers = list() + try: + # DEBUG: print(f"DEBUG: domain='{domain}' is Lemmy, fetching JSON ...") + response = network.fetch_response( + domain, + "/api/v3/site", + network.api_headers, + (config.get("connection_timeout"), config.get("read_timeout")) + ) + + data = network.json_from_response(response) + + # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code='{response.status_code}',data[]='{type(data)}'") + if not response.ok or response.status_code >= 400: + print("WARNING: Could not reach any JSON API:", domain) + instances.update_last_error(domain, response) + elif response.ok and isinstance(data, list): + print(f"UNSUPPORTED: domain='{domain}' returned a list: '{data}'") + elif "federated_instances" in data: + # DEBUG: print(f"DEBUG: Found federated_instances for domain='{domain}'") + peers = peers + federation.add_peers(data["federated_instances"]) + # DEBUG: print("DEBUG: Added instance(s) to peers") + else: + print("WARNING: JSON response does not contain 'federated_instances':", domain) + instances.update_last_error(domain, response) + + except BaseException as exception: + print(f"WARNING: Exception during fetching JSON: domain='{domain}',exception[{type(exception)}]:'{str(exception)}'") + + # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'") + instances.set_data("total_peers", domain, len(peers)) + + # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") + instances.update_last_instance_fetch(domain) + + # DEBUG: print("DEBUG: Returning peers[]:", type(peers)) + return peers diff --git a/fba/networks/mastodon.py b/fba/networks/mastodon.py new file mode 100644 index 0000000..2dda82f --- /dev/null +++ b/fba/networks/mastodon.py @@ -0,0 +1,263 @@ +# Fedi API Block - An aggregator for fetching blocking data from fediverse nodes +# Copyright (C) 2023 Free Software Foundation +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import inspect + +import bs4 +import validators + +from fba import blacklist +from fba import blocks +from fba import config +from fba import fba +from fba import instances +from fba import network +from fba.helpers import tidyup + +language_mapping = { + # English -> English + "Silenced instances" : "Silenced servers", + "Suspended instances" : "Suspended servers", + "Limited instances" : "Limited servers", + "Filtered media" : "Filtered media", + # Mappuing German -> English + "Gesperrte Server" : "Suspended servers", + "Gefilterte Medien" : "Filtered media", + "Stummgeschaltete Server" : "Silenced servers", + # Japanese -> English + "停止済みのサーバー" : "Suspended servers", + "制限中のサーバー" : "Limited servers", + "メディアを拒否しているサーバー": "Filtered media", + "サイレンス済みのサーバー" : "Silenced servers", + # ??? -> English + "שרתים מושעים" : "Suspended servers", + "מדיה מסוננת" : "Filtered media", + "שרתים מוגבלים" : "Silenced servers", + # French -> English + "Serveurs suspendus" : "Suspended servers", + "Médias filtrés" : "Filtered media", + "Serveurs limités" : "Limited servers", + "Serveurs modérés" : "Limited servers", +} + +def fetch_blocks_from_about(domain: str) -> dict: + # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") + if not isinstance(domain, str): + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError("Parameter 'domain' is empty") + + # DEBUG: print("DEBUG: Fetching mastodon blocks from domain:", domain) + blocklist = { + "Suspended servers": [], + "Filtered media" : [], + "Limited servers" : [], + "Silenced servers" : [], + } + + try: + doc = bs4.BeautifulSoup( + network.fetch_response( + domain, + "/about/more", + network.web_headers, + (config.get("connection_timeout"), config.get("read_timeout")) + ).text, + "html.parser", + ) + except BaseException as exception: + print("ERROR: Cannot fetch from domain:", domain, exception) + instances.update_last_error(domain, exception) + return {} + + for header in doc.find_all("h3"): + header_text = tidyup.reason(header.text) + + # DEBUG: print(f"DEBUG: header_text='{header_text}'") + if header_text in language_mapping: + # DEBUG: print(f"DEBUG: header_text='{header_text}'") + header_text = language_mapping[header_text] + else: + print(f"WARNING: header_text='{header_text}' not found in language mapping table") + + if header_text in blocklist or header_text.lower() in blocklist: + # replaced find_next_siblings with find_all_next to account for instances that e.g. hide lists in dropdown menu + for line in header.find_all_next("table")[0].find_all("tr")[1:]: + blocklist[header_text].append( + { + "domain": tidyup.domain(line.find("span").text), + "hash" : tidyup.domain(line.find("span")["title"][9:]), + "reason": tidyup.reason(line.find_all("td")[1].text), + } + ) + else: + print(f"WARNING: header_text='{header_text}' not found in blocklist()={len(blocklist)}") + + # DEBUG: print("DEBUG: Returning blocklist for domain:", domain) + return { + "reject" : blocklist["Suspended servers"], + "media_removal" : blocklist["Filtered media"], + "followers_only": blocklist["Limited servers"] + blocklist["Silenced servers"], + } + +def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): + # DEBUG: print(f"DEBUG: domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}' - CALLED!") + if not isinstance(domain, str): + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError("Parameter 'domain' is empty") + elif not isinstance(origin, str) and origin is not None: + raise ValueError(f"Parameter origin[]={type(origin)} is not 'str'") + elif origin == "": + raise ValueError("Parameter 'origin' is empty") + elif not isinstance(nodeinfo_url, str): + raise ValueError(f"Parameter nodeinfo_url[]={type(nodeinfo_url)} is not 'str'") + elif nodeinfo_url == "": + raise ValueError("Parameter 'nodeinfo_url' is empty") + + try: + # json endpoint for newer mastodongs + blockdict = list() + try: + rows = { + "reject" : [], + "media_removal" : [], + "followers_only": [], + "report_removal": [], + } + + # DEBUG: print("DEBUG: Querying API domain_blocks:", domain) + response = network.fetch_response( + domain, + "/api/v1/instance/domain_blocks", + network.api_headers, + (config.get("connection_timeout"), config.get("read_timeout")) + ) + + # DEBUG: print(f"DEBUG: response[]='{type(response)}'") + blocklist = network.json_from_response(response) + + print(f"INFO: Checking {len(blocklist)} entries from domain='{domain}',software='mastodon' ...") + for block in blocklist: + entry = { + 'domain': block['domain'], + 'hash' : block['digest'], + 'reason': block['comment'] + } + + # DEBUG: print("DEBUG: severity,domain,hash,comment:", block['severity'], block['domain'], block['digest'], block['comment']) + if block['severity'] == 'suspend': + # DEBUG: print(f"DEBUG: Adding entry='{entry}' with severity='{block['severity']}' ...") + rows['reject'].append(entry) + elif block['severity'] == 'silence': + # DEBUG: print(f"DEBUG: Adding entry='{entry}' with severity='{block['severity']}' ...") + rows['followers_only'].append(entry) + elif block['severity'] == 'reject_media': + # DEBUG: print(f"DEBUG: Adding entry='{entry}' with severity='{block['severity']}' ...") + rows['media_removal'].append(entry) + elif block['severity'] == 'reject_reports': + # DEBUG: print(f"DEBUG: Adding entry='{entry}' with severity='{block['severity']}' ...") + rows['report_removal'].append(entry) + else: + print("WARNING: Unknown severity:", block['severity'], block['domain']) + + except BaseException as exception: + # DEBUG: print(f"DEBUG: Failed, trying mastodon-specific fetches: domain='{domain}',exception[{type(exception)}]={str(exception)}") + rows = fetch_blocks_from_about(domain) + + print(f"INFO: Checking {len(rows.items())} entries from domain='{domain}',software='mastodon' ...") + for block_level, blocklist in rows.items(): + # DEBUG: print("DEBUG: domain,block_level,blocklist():", domain, block_level, len(blocklist)) + block_level = tidyup.domain(block_level) + + # DEBUG: print("DEBUG: AFTER-block_level:", block_level) + if block_level == "": + print("WARNING: block_level is empty, domain:", domain) + continue + + # DEBUG: print(f"DEBUG: Checking {len(blocklist)} entries from domain='{domain}',software='mastodon',block_level='{block_level}' ...") + for block in blocklist: + # DEBUG: print(f"DEBUG: block[]='{type(block)}'") + blocked, blocked_hash, reason = block.values() + # DEBUG: print(f"DEBUG: blocked='{blocked}',blocked_hash='{blocked_hash}',reason='{reason}':") + blocked = tidyup.domain(blocked) + reason = tidyup.reason(reason) if reason is not None and reason != "" else None + # DEBUG: print(f"DEBUG: blocked='{blocked}',reason='{reason}' - AFTER!") + + if blocked == "": + print("WARNING: blocked is empty:", domain) + continue + elif blacklist.is_blacklisted(blocked): + # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!") + continue + elif blocked.count("*") > 0: + # Doing the hash search for instance names as well to tidy up DB + fba.cursor.execute( + "SELECT domain, origin, nodeinfo_url FROM instances WHERE hash = ? LIMIT 1", [blocked_hash] + ) + searchres = fba.cursor.fetchone() + + if searchres is None: + print(f"WARNING: Cannot deobsfucate blocked='{blocked}',blocked_hash='{blocked_hash}' - SKIPPED!") + continue + + # DEBUG: print("DEBUG: Updating domain: ", searchres[0]) + blocked = searchres[0] + origin = searchres[1] + nodeinfo_url = searchres[2] + + # DEBUG: print("DEBUG: Looking up instance by domain:", blocked) + if not validators.domain(blocked): + print(f"WARNING: blocked='{blocked}',software='mastodon' is not a valid domain name - skipped!") + continue + elif not instances.is_registered(blocked): + # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'") + instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url) + elif not validators.domain(blocked): + print(f"WARNING: blocked='{blocked}',software='mastodon' is not a valid domain name - skipped!") + continue + + # DEBUG: print("DEBUG: Looking up instance by domain:", blocked) + if not validators.domain(blocked): + print(f"WARNING: blocked='{blocked}',software='mastodon' is not a valid domain name - skipped!") + continue + elif not instances.is_registered(blocked): + # DEBUG: print("DEBUG: Hash wasn't found, adding:", blocked, domain) + instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url) + + blocking = blocked if blocked.count("*") <= 1 else blocked_hash + # DEBUG: print(f"DEBUG: blocking='{blocking}',blocked='{blocked}',blocked_hash='{blocked_hash}'") + + if not blocks.is_instance_blocked(domain, blocked, block_level): + # DEBUG: print("DEBUG: Blocking:", domain, blocked, block_level) + blocks.add_instance(domain, blocking, reason, block_level) + + if block_level == "reject": + blockdict.append({ + "blocked": blocked, + "reason" : reason + }) + else: + # DEBUG: print(f"DEBUG: Updating block last seen and reason for domain='{domain}',blocking='{blocking}' ...") + blocks.update_last_seen(domain, blocking, block_level) + blocks.update_reason(reason, domain, blocking, block_level) + + # DEBUG: print("DEBUG: Committing changes ...") + fba.connection.commit() + except BaseException as exception: + print(f"ERROR: domain='{domain}',software='mastodon',exception[{type(exception)}]:'{str(exception)}'") + + # DEBUG: print("DEBUG: EXIT!") diff --git a/fba/networks/misskey.py b/fba/networks/misskey.py new file mode 100644 index 0000000..7b28b8d --- /dev/null +++ b/fba/networks/misskey.py @@ -0,0 +1,255 @@ +# Fedi API Block - An aggregator for fetching blocking data from fediverse nodes +# Copyright (C) 2023 Free Software Foundation +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import json + +from fba import blacklist +from fba import config +from fba import instances +from fba import network + +from fba.helpers import dicts +from fba.helpers import tidyup + +def fetch_peers(domain: str) -> list: + # DEBUG: print(f"DEBUG: domain({len(domain)})={domain} - CALLED!") + if not isinstance(domain, str): + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError("Parameter 'domain' is empty") + + # DEBUG: print(f"DEBUG: domain='{domain}' is misskey, sending API POST request ...") + peers = list() + offset = 0 + step = config.get("misskey_limit") + + # iterating through all "suspended" (follow-only in its terminology) + # instances page-by-page, since that troonware doesn't support + # sending them all at once + while True: + # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...") + if offset == 0: + fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({ + "sort" : "+pubAt", + "host" : None, + "limit": step + }), { + "Origin": domain + }) + else: + fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({ + "sort" : "+pubAt", + "host" : None, + "limit" : step, + "offset": offset - 1 + }), { + "Origin": domain + }) + + # DEBUG: print(f"DEBUG: fetched()={len(fetched)}") + if len(fetched) == 0: + # DEBUG: print(f"DEBUG: Returned zero bytes, exiting loop, domain='{domain}'") + break + elif len(fetched) != config.get("misskey_limit"): + # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'") + offset = offset + (config.get("misskey_limit") - len(fetched)) + else: + # DEBUG: print(f"DEBUG: Raising offset by step={step}") + offset = offset + step + + # Check records + # DEBUG: print(f"DEBUG: fetched({len(fetched)})[]={type(fetched)}") + if isinstance(fetched, dict) and "error" in fetched and "message" in fetched["error"]: + print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}") + instances.update_last_error(domain, fetched["error"]["message"]) + break + + already = 0 + for row in fetched: + # DEBUG: print(f"DEBUG: row()={len(row)}") + if not "host" in row: + print(f"WARNING: row()={len(row)} does not contain key 'host': {row},domain='{domain}'") + continue + elif not isinstance(row["host"], str): + print(f"WARNING: row[host][]={type(row['host'])} is not 'str'") + continue + elif blacklist.is_blacklisted(row["host"]): + # DEBUG: print(f"DEBUG: row[host]='{row['host']}' is blacklisted. domain='{domain}'") + continue + elif row["host"] in peers: + # DEBUG: print(f"DEBUG: Not adding row[host]='{row['host']}', already found.") + already = already + 1 + continue + + # DEBUG: print(f"DEBUG: Adding peer: '{row['host']}'") + peers.append(row["host"]) + + if already == len(fetched): + print(f"WARNING: Host returned same set of '{already}' instances, aborting loop!") + break + + # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'") + instances.set_data("total_peers", domain, len(peers)) + + # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") + instances.update_last_instance_fetch(domain) + + # DEBUG: print(f"DEBUG: Returning peers[]='{type(peers)}'") + return peers + +def fetch_blocks(domain: str) -> dict: + # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") + if not isinstance(domain, str): + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError("Parameter 'domain' is empty") + + # DEBUG: print("DEBUG: Fetching misskey blocks from domain:", domain) + blocklist = { + "suspended": [], + "blocked" : [] + } + + offset = 0 + step = config.get("misskey_limit") + while True: + # iterating through all "suspended" (follow-only in its terminology) + # instances page-by-page, since that troonware doesn't support + # sending them all at once + try: + # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...") + if offset == 0: + # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset) + fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({ + "sort" : "+pubAt", + "host" : None, + "suspended": True, + "limit" : step + }), { + "Origin": domain + }) + else: + # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset) + fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({ + "sort" : "+pubAt", + "host" : None, + "suspended": True, + "limit" : step, + "offset" : offset - 1 + }), { + "Origin": domain + }) + + # DEBUG: print("DEBUG: fetched():", len(fetched)) + if len(fetched) == 0: + # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain) + break + elif len(fetched) != config.get("misskey_limit"): + # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'") + offset = offset + (config.get("misskey_limit") - len(fetched)) + else: + # DEBUG: print("DEBUG: Raising offset by step:", step) + offset = offset + step + + count = 0 + for instance in fetched: + # Is it there? + if instance["isSuspended"] and not dicts.has_key(blocklist["suspended"], "domain", instance): + count = count + 1 + blocklist["suspended"].append( + { + "domain": tidyup.domain(instance["host"]), + # no reason field, nothing + "reason": None + } + ) + + # DEBUG: print(f"DEBUG: count={count}") + if count == 0: + # DEBUG: print("DEBUG: API is no more returning new instances, aborting loop!") + break + + except BaseException as exception: + print("WARNING: Caught error, exiting loop:", domain, exception) + instances.update_last_error(domain, exception) + offset = 0 + break + + while True: + # same shit, different asshole ("blocked" aka full suspend) + try: + if offset == 0: + # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset) + fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({ + "sort" : "+pubAt", + "host" : None, + "blocked": True, + "limit" : step + }), { + "Origin": domain + }) + else: + # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset) + fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({ + "sort" : "+pubAt", + "host" : None, + "blocked": True, + "limit" : step, + "offset" : offset - 1 + }), { + "Origin": domain + }) + + # DEBUG: print("DEBUG: fetched():", len(fetched)) + if len(fetched) == 0: + # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain) + break + elif len(fetched) != config.get("misskey_limit"): + # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'") + offset = offset + (config.get("misskey_limit") - len(fetched)) + else: + # DEBUG: print("DEBUG: Raising offset by step:", step) + offset = offset + step + + count = 0 + for instance in fetched: + # Is it there? + if instance["isBlocked"] and not dicts.has_key(blocklist["blocked"], "domain", instance): + count = count + 1 + blocklist["blocked"].append({ + "domain": tidyup.domain(instance["host"]), + "reason": None + }) + + # DEBUG: print(f"DEBUG: count={count}") + if count == 0: + # DEBUG: print("DEBUG: API is no more returning new instances, aborting loop!") + break + + except BaseException as exception: + print("ERROR: Exception during POST:", domain, exception) + instances.update_last_error(domain, exception) + offset = 0 + break + + # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") + instances.update_last_instance_fetch(domain) + + # DEBUG: print(f"DEBUG: Returning for domain='{domain}',blocked()={len(blocklist['blocked'])},suspended()={len(blocklist['suspended'])}") + return { + "reject" : blocklist["blocked"], + "followers_only": blocklist["suspended"] + } diff --git a/fba/networks/peertube.py b/fba/networks/peertube.py new file mode 100644 index 0000000..da84baf --- /dev/null +++ b/fba/networks/peertube.py @@ -0,0 +1,73 @@ +# Fedi API Block - An aggregator for fetching blocking data from fediverse nodes +# Copyright (C) 2023 Free Software Foundation +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from fba import config +from fba import instances +from fba import network + +def fetch_peers(domain: str) -> list: + # DEBUG: print(f"DEBUG: domain({len(domain)})={domain},software='peertube' - CALLED!") + if not isinstance(domain, str): + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError("Parameter 'domain' is empty") + + # DEBUG: print(f"DEBUG: domain='{domain}' is a PeerTube, fetching JSON ...") + peers = list() + start = 0 + for mode in ["followers", "following"]: + # DEBUG: print(f"DEBUG: domain='{domain}',mode='{mode}'") + while True: + try: + response = network.fetch_response( + domain, + "/api/v1/server/{mode}?start={start}&count=100", + network.api_headers, + (config.get("connection_timeout"), config.get("read_timeout")) + ) + + data = network.json_from_response(response) + # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code='{response.status_code}',data[]='{type(data)}'") + if response.ok and isinstance(data, dict): + # DEBUG: print("DEBUG: Success, data:", len(data)) + if "data" in data: + # DEBUG: print(f"DEBUG: Found {len(data['data'])} record(s).") + for record in data["data"]: + # DEBUG: print(f"DEBUG: record()={len(record)}") + if mode in record and "host" in record[mode]: + # DEBUG: print(f"DEBUG: Found host={record[mode]['host']}, adding ...") + peers.append(record[mode]["host"]) + else: + print(f"WARNING: record from '{domain}' has no '{mode}' or 'host' record: {record}") + + if len(data["data"]) < 100: + # DEBUG: print("DEBUG: Reached end of JSON response:", domain) + break + + # Continue with next row + start = start + 100 + + except BaseException as exception: + print(f"WARNING: Exception during fetching JSON: domain='{domain}',exception[{type(exception)}]:'{str(exception)}'") + + # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'") + instances.set_data("total_peers", domain, len(peers)) + + # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") + instances.update_last_instance_fetch(domain) + + # DEBUG: print("DEBUG: Returning peers[]:", type(peers)) + return peers diff --git a/fba/networks/pleroma.py b/fba/networks/pleroma.py new file mode 100644 index 0000000..b997dc6 --- /dev/null +++ b/fba/networks/pleroma.py @@ -0,0 +1,202 @@ +# Fedi API Block - An aggregator for fetching blocking data from fediverse nodes +# Copyright (C) 2023 Free Software Foundation +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import inspect +import validators + +from fba import blacklist +from fba import blocks +from fba import fba +from fba import federation +from fba import instances +from fba.helpers import tidyup + +def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): + # DEBUG: print(f"DEBUG: domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}' - CALLED!") + if not isinstance(domain, str): + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError("Parameter 'domain' is empty") + elif not isinstance(origin, str) and origin is not None: + raise ValueError(f"Parameter origin[]={type(origin)} is not 'str'") + elif origin == "": + raise ValueError("Parameter 'origin' is empty") + elif not isinstance(nodeinfo_url, str): + raise ValueError(f"Parameter nodeinfo_url[]={type(nodeinfo_url)} is not 'str'") + elif nodeinfo_url == "": + raise ValueError("Parameter 'nodeinfo_url' is empty") + + try: + # Blocks + blockdict = list() + rows = federation.fetch_nodeinfo(domain, nodeinfo_url) + + if rows is None: + print("WARNING: Could not fetch nodeinfo from domain:", domain) + return + elif "metadata" not in rows: + print(f"WARNING: rows()={len(rows)} does not have key 'metadata', domain='{domain}'") + return + elif "federation" not in rows["metadata"]: + print(f"WARNING: rows()={len(rows['metadata'])} does not have key 'federation', domain='{domain}'") + return + + # DEBUG: print("DEBUG: Updating nodeinfo:", domain) + instances.update_last_nodeinfo(domain) + + data = rows["metadata"]["federation"] + + if "enabled" in data: + # DEBUG: print("DEBUG: Instance has no block list to analyze:", domain) + return + + if "mrf_simple" in data: + for block_level, blocklist in ( + {**data["mrf_simple"], + **{"quarantined_instances": data["quarantined_instances"]}} + ).items(): + # DEBUG: print("DEBUG: block_level, blocklist():", block_level, len(blocklist)) + block_level = tidyup.domain(block_level) + # DEBUG: print("DEBUG: BEFORE block_level:", block_level) + + if block_level == "": + print("WARNING: block_level is now empty!") + continue + + # DEBUG: print(f"DEBUG: Checking {len(blocklist)} entries from domain='{domain}',block_level='{block_level}' ...") + for blocked in blocklist: + # DEBUG: print("DEBUG: BEFORE blocked:", blocked) + blocked = tidyup.domain(blocked) + # DEBUG: print("DEBUG: AFTER blocked:", blocked) + + if blocked == "": + print("WARNING: blocked is empty after tidyup.domain():", domain, block_level) + continue + elif blacklist.is_blacklisted(blocked): + # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!") + continue + elif blocked.count("*") > 1: + # -ACK!-oma also started obscuring domains without hash + fba.cursor.execute( + "SELECT domain, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")] + ) + searchres = fba.cursor.fetchone() + # DEBUG: print("DEBUG: searchres[]:", type(searchres)) + + if searchres is None: + print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!") + continue + + blocked = searchres[0] + nodeinfo_url = searchres[1] + # DEBUG: print("DEBUG: Looked up domain:", blocked) + elif not validators.domain(blocked): + print(f"WARNING: blocked='{blocked}',software='pleroma' is not a valid domain name - skipped!") + continue + + # DEBUG: print("DEBUG: Looking up instance by domain:", blocked) + if not validators.domain(blocked): + print(f"WARNING: blocked='{blocked}',software='pleroma' is not a valid domain name - skipped!") + continue + elif not instances.is_registered(blocked): + # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'") + instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url) + + if not blocks.is_instance_blocked(domain, blocked, block_level): + # DEBUG: print("DEBUG: Blocking:", domain, blocked, block_level) + blocks.add_instance(domain, blocked, "unknown", block_level) + + if block_level == "reject": + # DEBUG: print("DEBUG: Adding to blockdict:", blocked) + blockdict.append( + { + "blocked": blocked, + "reason" : None + }) + else: + # DEBUG: print(f"DEBUG: Updating block last seen for domain='{domain}',blocked='{blocked}' ...") + blocks.update_last_seen(domain, blocked, block_level) + + # DEBUG: print("DEBUG: Committing changes ...") + fba.connection.commit() + + # Reasons + if "mrf_simple_info" in data: + # DEBUG: print("DEBUG: Found mrf_simple_info:", domain) + for block_level, info in ( + {**data["mrf_simple_info"], + **(data["quarantined_instances_info"] + if "quarantined_instances_info" in data + else {})} + ).items(): + # DEBUG: print("DEBUG: block_level, info.items():", block_level, len(info.items())) + block_level = tidyup.domain(block_level) + # DEBUG: print("DEBUG: BEFORE block_level:", block_level) + + if block_level == "": + print("WARNING: block_level is now empty!") + continue + + # DEBUG: print(f"DEBUG: Checking {len(info.items())} entries from domain='{domain}',software='pleroma',block_level='{block_level}' ...") + for blocked, reason in info.items(): + # DEBUG: print(f"DEBUG: blocked='{blocked}',reason='{reason}' - BEFORE!") + blocked = tidyup.domain(blocked) + reason = tidyup.reason(reason) if reason is not None and reason != "" else None + # DEBUG: print(f"DEBUG: blocked='{blocked}',reason='{reason}' - AFTER!") + + if blocked == "": + print("WARNING: blocked is empty after tidyup.domain():", domain, block_level) + continue + elif not validators.domain(blocked): + print(f"WARNING: blocked='{blocked}',software='pleroma' is not a valid domain name - skipped!") + continue + elif blacklist.is_blacklisted(blocked): + # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!") + continue + elif blocked.count("*") > 1: + # same domain guess as above, but for reasons field + fba.cursor.execute( + "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")] + ) + searchres = fba.cursor.fetchone() + + if searchres is None: + print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!") + continue + + blocked = searchres[0] + origin = searchres[1] + nodeinfo_url = searchres[2] + + # DEBUG: print("DEBUG: Looking up instance by domain:", blocked) + if not instances.is_registered(blocked): + # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'") + instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url) + + # DEBUG: print("DEBUG: Updating block reason:", domain, blocked, reason["reason"]) + blocks.update_reason(reason["reason"], domain, blocked, block_level) + + # DEBUG: print(f"DEBUG: blockdict()={len(blockdict)}") + for entry in blockdict: + if entry["blocked"] == blocked: + # DEBUG: print("DEBUG: Updating entry reason:", blocked) + entry["reason"] = reason["reason"] + + fba.connection.commit() + except BaseException as exception: + print(f"ERROR: domain='{domain}',software='pleroma',exception[{type(exception)}]:'{str(exception)}'") + + # DEBUG: print("DEBUG: EXIT!")