From f248d55bfebb9cd7885486637e89ad0b1960899a Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Fri, 9 Jun 2023 19:52:11 +0200 Subject: [PATCH] Continued: - renamed fba/federation/ -> fba/networks/ to make room for ... - added fba.federation.py module which has generic functions for federation - introduced fba/locking.py module - renamed aqcuire_lock() -> aqcuire() - added locking.release() - introduced fba/helpers/dict.py and tidyup.py - renamed tidyu_*() to tidyup.*() - this all-in-all was an attempt to fix cyclic imports, still some are left --- fba/__init__.py | 18 + fba/blocks.py | 3 +- fba/boot.py | 27 +- fba/commands.py | 56 +- fba/csrf.py | 19 +- fba/fba.py | 614 ++-------------------- fba/federation.py | 478 +++++++++++++++++ fba/federation/__init__.py | 8 - fba/helpers/__init__.py | 19 + fba/helpers/dicts.py | 38 ++ fba/helpers/tidyup.py | 60 +++ fba/instances.py | 3 +- fba/locking.py | 45 ++ fba/network.py | 47 +- fba/networks/__init__.py | 23 + fba/{federation => networks}/friendica.py | 8 +- fba/{federation => networks}/lemmy.py | 6 +- fba/{federation => networks}/mastodon.py | 17 +- fba/{federation => networks}/misskey.py | 12 +- fba/{federation => networks}/peertube.py | 3 +- fba/{federation => networks}/pleroma.py | 40 +- 21 files changed, 839 insertions(+), 705 deletions(-) create mode 100644 fba/federation.py delete mode 100644 fba/federation/__init__.py create mode 100644 fba/helpers/__init__.py create mode 100644 fba/helpers/dicts.py create mode 100644 fba/helpers/tidyup.py create mode 100644 fba/locking.py create mode 100644 fba/networks/__init__.py rename fba/{federation => networks}/friendica.py (92%) rename fba/{federation => networks}/lemmy.py (94%) rename fba/{federation => networks}/mastodon.py (95%) rename fba/{federation => networks}/misskey.py (96%) rename fba/{federation => networks}/peertube.py (97%) rename fba/{federation => networks}/pleroma.py (88%) diff --git a/fba/__init__.py b/fba/__init__.py index 01c1bf6..40310f9 100644 --- a/fba/__init__.py +++ b/fba/__init__.py @@ -1,3 +1,18 @@ +# Copyright (C) 2023 Free Software Foundation +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + __all__ = [ 'blacklist', 'blocks', @@ -8,6 +23,9 @@ __all__ = [ 'csrf', 'federation', 'fba', + 'helpers', 'instances', + 'locking', 'network', + 'networks', ] diff --git a/fba/blocks.py b/fba/blocks.py index 50ceec8..ed4a32d 100644 --- a/fba/blocks.py +++ b/fba/blocks.py @@ -20,6 +20,7 @@ import validators from fba import blacklist from fba import fba +from fba.helpers import tidyup def update_reason(reason: str, blocker: str, blocked: str, block_level: str): # DEBUG: print(f"DEBUG: reason='{reason}',blocker={blocker},blocked={blocked},block_level={block_level} - CALLED!") @@ -149,7 +150,7 @@ def add_instance(blocker: str, blocked: str, reason: str, block_level: str): if reason is not None: # Maybe needs cleaning - reason = fba.tidyup_reason(reason) + reason = tidyup.reason(reason) print(f"INFO: New block: blocker='{blocker}',blocked='{blocked}', reason='{reason}', block_level='{block_level}'") try: diff --git a/fba/boot.py b/fba/boot.py index 91c2880..0ff07f3 100644 --- a/fba/boot.py +++ b/fba/boot.py @@ -15,30 +15,13 @@ # along with this program. If not, see . import argparse -import os -import sys -import tempfile -import zc.lockfile from fba import commands from fba import fba +from fba import locking -# Lock file -lockfile = tempfile.gettempdir() + '/fba.lock' -LOCK = None _PARSER = None -def acquire_lock(): - global LOCK - try: - # DEBUG: print(f"DEBUG: Acquiring lock: '{lockfile}'") - LOCK = zc.lockfile.LockFile(lockfile) - # DEBUG: print("DEBUG: Lock obtained.") - - except zc.lockfile.LockError: - print(f"ERROR: Cannot aquire lock: '{lockfile}'") - sys.exit(100) - def init_parser(): # DEBUG: # DEBUG: print("DEBUG: init_parser(): CALLED!") global _PARSER @@ -129,11 +112,5 @@ def run_command(): def shutdown(): # DEBUG: print("DEBUG: Closing database connection ...") fba.connection.close() - - if LOCK is not None: - # DEBUG: print("DEBUG: Releasing lock ...") - LOCK.close() - # DEBUG: print(f"DEBUG: Deleting lockfile='{lockfile}' ...") - os.remove(lockfile) - + locking.release() # DEBUG: print("DEBUG: Shutdown completed.") diff --git a/fba/commands.py b/fba/commands.py index d911be6..ab601c9 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -28,13 +28,19 @@ import validators from fba import blacklist from fba import blocks -from fba import boot from fba import config +from fba import federation from fba import fba from fba import instances +from fba import locking from fba import network -from fba.federation import * +from fba.helpers import tidyup + +from fba.networks import friendica +from fba.networks import mastodon +from fba.networks import misskey +from fba.networks import pleroma def check_instance(args: argparse.Namespace) -> int: # DEBUG: print(f"DEBUG: args.domain='{args.domain}' - CALLED!") @@ -65,9 +71,9 @@ def fetch_bkali(args: argparse.Namespace): # DEBUG: print(f"DEBUG: fetched({len(fetched)})[]='{type(fetched)}'") if len(fetched) == 0: raise Exception("WARNING: Returned no records") - elif not "data" in fetched: + elif "data" not in fetched: raise Exception(f"WARNING: fetched()={len(fetched)} does not contain key 'data'") - elif not "nodeinfo" in fetched["data"]: + elif "nodeinfo" not in fetched["data"]: raise Exception(f"WARNING: fetched()={len(fetched['data'])} does not contain key 'nodeinfo'") for entry in fetched["data"]["nodeinfo"]: @@ -94,12 +100,12 @@ def fetch_bkali(args: argparse.Namespace): # DEBUG: print(f"DEBUG: domains()={len(domains)}") if len(domains) > 0: - boot.acquire_lock() + locking.acquire() print(f"INFO: Adding {len(domains)} new instances ...") for domain in domains: print(f"INFO: Fetching instances from domain='{domain}' ...") - fba.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) + federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) # DEBUG: print("DEBUG: EXIT!") @@ -117,7 +123,7 @@ def fetch_blocks(args: argparse.Namespace): print(f"WARNING: domain='{args.domain}' is not registered, please run ./fba.py fetch_instances {args.domain} first.") return - boot.acquire_lock() + locking.acquire() if args.domain is not None and args.domain != "": # Re-check single domain @@ -135,7 +141,7 @@ def fetch_blocks(args: argparse.Namespace): for blocker, software, origin, nodeinfo_url in rows: # DEBUG: print("DEBUG: BEFORE blocker,software,origin,nodeinfo_url:", blocker, software, origin, nodeinfo_url) blockdict = list() - blocker = fba.tidyup_domain(blocker) + blocker = tidyup.domain(blocker) # DEBUG: print("DEBUG: AFTER blocker,software:", blocker, software) if blocker == "": @@ -164,7 +170,7 @@ def fetch_blocks(args: argparse.Namespace): print(f"INFO: Checking {len(rows.items())} entries from blocker='{blocker}',software='{software}' ...") for block_level, blocklist in rows.items(): # DEBUG: print("DEBUG: blocker,block_level,blocklist():", blocker, block_level, len(blocklist)) - block_level = fba.tidyup_domain(block_level) + block_level = tidyup.domain(block_level) # DEBUG: print("DEBUG: AFTER-block_level:", block_level) if block_level == "": print("WARNING: block_level is empty, blocker:", blocker) @@ -174,8 +180,8 @@ def fetch_blocks(args: argparse.Namespace): for block in blocklist: blocked, reason = block.values() # DEBUG: print(f"DEBUG: blocked='{blocked}',reason='{reason}' - BEFORE!") - blocked = fba.tidyup_domain(blocked) - reason = fba.tidyup_reason(reason) if reason is not None and reason != "" else None + blocked = tidyup.domain(blocked) + reason = tidyup.reason(reason) if reason is not None and reason != "" else None # DEBUG: print(f"DEBUG: blocked='{blocked}',reason='{reason}' - AFTER!") if blocked == "": @@ -286,15 +292,15 @@ def fetch_cs(args: argparse.Namespace): # DEBUG: print(f"DEBUG: doc()={len(doc)}[]={type(doc)}") silenced = doc.find("h2", {"id": "silenced-instances"}).findNext("table").find("tbody") # DEBUG: print(f"DEBUG: silenced[]={type(silenced)}") - domains["silenced"] = domains["silenced"] + fba.find_domains(silenced) + domains["silenced"] = domains["silenced"] + federation.find_domains(silenced) blocked = doc.find("h2", {"id": "blocked-instances"}).findNext("table").find("tbody") # DEBUG: print(f"DEBUG: blocked[]={type(blocked)}") - domains["reject"] = domains["reject"] + fba.find_domains(blocked) + domains["reject"] = domains["reject"] + federation.find_domains(blocked) # DEBUG: print(f"DEBUG: domains()={len(domains)}") if len(domains) > 0: - boot.acquire_lock() + locking.acquire() print(f"INFO: Adding {len(domains)} new instances ...") for block_level in domains: @@ -308,7 +314,7 @@ def fetch_cs(args: argparse.Namespace): if not instances.is_registered(row["domain"]): print(f"INFO: Fetching instances from domain='{row['domain']}' ...") - fba.fetch_instances(row["domain"], 'chaos.social', None, inspect.currentframe().f_code.co_name) + federation.fetch_instances(row["domain"], 'chaos.social', None, inspect.currentframe().f_code.co_name) # DEBUG: print("DEBUG: Committing changes ...") fba.connection.commit() @@ -347,12 +353,12 @@ def fetch_fba_rss(args: argparse.Namespace): # DEBUG: print(f"DEBUG: domains()={len(domains)}") if len(domains) > 0: - boot.acquire_lock() + locking.acquire() print(f"INFO: Adding {len(domains)} new instances ...") for domain in domains: print(f"INFO: Fetching instances from domain='{domain}' ...") - fba.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) + federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) # DEBUG: print("DEBUG: EXIT!") @@ -378,7 +384,7 @@ def fetch_fbabot_atom(args: argparse.Namespace): for element in doc.findAll("a"): for href in element["href"].split(","): # DEBUG: print(f"DEBUG: href[{type(href)}]={href}") - domain = fba.tidyup_domain(href) + domain = tidyup.domain(href) # DEBUG: print(f"DEBUG: domain='{domain}'") if blacklist.is_blacklisted(domain): @@ -396,21 +402,21 @@ def fetch_fbabot_atom(args: argparse.Namespace): # DEBUG: print(f"DEBUG: domains({len(domains)})={domains}") if len(domains) > 0: - boot.acquire_lock() + locking.acquire() print(f"INFO: Adding {len(domains)} new instances ...") for domain in domains: print(f"INFO: Fetching instances from domain='{domain}' ...") - fba.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) + federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) # DEBUG: print("DEBUG: EXIT!") def fetch_instances(args: argparse.Namespace): # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!") - boot.acquire_lock() + locking.acquire() # Initial fetch - fba.fetch_instances(args.domain, None, None, inspect.currentframe().f_code.co_name) + federation.fetch_instances(args.domain, None, None, inspect.currentframe().f_code.co_name) if args.single: # DEBUG: print("DEBUG: Not fetching more instances - EXIT!") @@ -430,13 +436,13 @@ def fetch_instances(args: argparse.Namespace): continue print(f"INFO: Fetching instances for instance '{row[0]}' ('{row[2]}') of origin='{row[1]}',nodeinfo_url='{row[3]}'") - fba.fetch_instances(row[0], row[1], row[2], inspect.currentframe().f_code.co_name, row[3]) + federation.fetch_instances(row[0], row[1], row[2], inspect.currentframe().f_code.co_name, row[3]) # DEBUG: print("DEBUG: EXIT!") def fetch_federater(args: argparse.Namespace): # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!") - boot.acquire_lock() + locking.acquire() # Fetch this URL response = fba.fetch_url("https://github.com/federater/blocks_recommended/raw/main/federater.csv", network.web_headers, (config.get("connection_timeout"), config.get("read_timeout"))) @@ -459,6 +465,6 @@ def fetch_federater(args: argparse.Namespace): continue print(f"INFO: Fetching instances for instane='{row['#domain']}' ...") - fba.fetch_instances(row["#domain"], None, None, inspect.currentframe().f_code.co_name) + federation.fetch_instances(row["#domain"], None, None, inspect.currentframe().f_code.co_name) # DEBUG: print("DEBUG: EXIT!") diff --git a/fba/csrf.py b/fba/csrf.py index caaed38..fb0bec3 100644 --- a/fba/csrf.py +++ b/fba/csrf.py @@ -22,7 +22,7 @@ from fba import config from fba import network def determine(domain: str, headers: dict) -> dict: - # DEBUG: print(f"DEBUG: domain='{domain}',headers()={len(headers)} - CALLED!") + print(f"DEBUG: domain='{domain}',headers()={len(headers)} - CALLED!") if not isinstance(domain, str): raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": @@ -35,32 +35,31 @@ def determine(domain: str, headers: dict) -> dict: try: # Fetch / to check for meta tag indicating csrf - # DEBUG: print(f"DEBUG: Fetching / from domain='{domain}' for CSRF check ...") + print(f"DEBUG: Fetching / from domain='{domain}' for CSRF check ...") response = reqto.get( f"https://{domain}/", headers=network.web_headers, timeout=(config.get("connection_timeout"), config.get("read_timeout")) ) - # DEBUG: print(f"DEBUG: response.ok='{response.ok}',response.status_code={response.status_code},response.text()={len(response.text)}") + print(f"DEBUG: response.ok='{response.ok}',response.status_code={response.status_code},response.text()={len(response.text)}") if response.ok and len(response.text) > 0: meta = bs4.BeautifulSoup( response.text, "html.parser" ) - # DEBUG: print(f"DEBUG: meta[]='{type(meta)}'") + print(f"DEBUG: meta[]='{type(meta)}'") tag = meta.find("meta", attrs={"name": "csrf-token"}) - # DEBUG: print(f"DEBUG: tag={tag}") + print(f"DEBUG: tag={tag}") csrf = tag["content"] - # DEBUG: print(f"DEBUG: Adding CSRF token='{csrf}' for domain='{domain}'") - - reqheaders = {**headers, **{"X-CSRF-Token": csrf}} + print(f"DEBUG: Adding CSRF token='{csrf}' for domain='{domain}'") + reqheaders["X-CSRF-Token"] = csrf except BaseException as exception: - # DEBUG: print(f"DEBUG: No CSRF token found, using normal headers: domain='{domain}',exception[{type(exception)}]={exception}") + print(f"DEBUG: No CSRF token found, using normal headers: domain='{domain}',exception[{type(exception)}]={exception}") pass - # DEBUG: print(f"DEBUG: reqheaders()={len(reqheaders)} - EXIT!") + print(f"DEBUG: reqheaders()={len(reqheaders)} - EXIT!") return reqheaders diff --git a/fba/fba.py b/fba/fba.py index 2552466..a89d20f 100644 --- a/fba/fba.py +++ b/fba/fba.py @@ -22,35 +22,15 @@ import time from urllib.parse import urlparse -import bs4 import requests -import validators -from fba import blacklist from fba import config -from fba import instances from fba import network -from fba.federation import lemmy -from fba.federation import misskey -from fba.federation import peertube - # Array with pending errors needed to be written to database pending_errors = { } -# "rel" identifiers (no real URLs) -nodeinfo_identifier = [ - "https://nodeinfo.diaspora.software/ns/schema/2.1", - "https://nodeinfo.diaspora.software/ns/schema/2.0", - "https://nodeinfo.diaspora.software/ns/schema/1.1", - "https://nodeinfo.diaspora.software/ns/schema/1.0", - "http://nodeinfo.diaspora.software/ns/schema/2.1", - "http://nodeinfo.diaspora.software/ns/schema/2.0", - "http://nodeinfo.diaspora.software/ns/schema/1.1", - "http://nodeinfo.diaspora.software/ns/schema/1.0", -] - # Connect to database connection = sqlite3.connect("blocks.db") cursor = connection.cursor() @@ -70,97 +50,11 @@ patterns = [ ##### Other functions ##### def is_primitive(var: any) -> bool: - # DEBUG: print(f"DEBUG: var[]='{type(var)}' - CALLED!") + print(f"DEBUG: var[]='{type(var)}' - CALLED!") return type(var) in {int, str, float, bool} or var is None -def fetch_instances(domain: str, origin: str, software: str, script: str, path: str = None): - # DEBUG: print(f"DEBUG: domain='{domain}',origin='{origin}',software='{software}',path='{path}' - CALLED!") - if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError("Parameter 'domain' is empty") - elif not isinstance(origin, str) and origin is not None: - raise ValueError(f"Parameter origin[]={type(origin)} is not 'str'") - elif software is None: - # DEBUG: print(f"DEBUG: software for domain='{domain}' is not set, determining ...") - software = determine_software(domain, path) - # DEBUG: print(f"DEBUG: Determined software='{software}' for domain='{domain}'") - elif not isinstance(software, str): - raise ValueError(f"Parameter software[]={type(software)} is not 'str'") - elif not isinstance(script, str): - raise ValueError(f"Parameter script[]={type(script)} is not 'str'") - elif domain == "": - raise ValueError("Parameter 'domain' is empty") - - if not instances.is_registered(domain): - # DEBUG: print("DEBUG: Adding new domain:", domain, origin) - instances.add(domain, origin, script, path) - - # DEBUG: print("DEBUG: Fetching instances for domain:", domain, software) - peerlist = fetch_peers(domain, software) - - if peerlist is None: - print("ERROR: Cannot fetch peers:", domain) - return - elif instances.has_pending_instance_data(domain): - # DEBUG: print(f"DEBUG: domain='{domain}' has pending nodeinfo data, flushing ...") - instances.update_data(domain) - - print(f"INFO: Checking {len(peerlist)} instances from {domain} ...") - for instance in peerlist: - if instance is None: - # Skip "None" types as tidup() cannot parse them - continue - - # DEBUG: print(f"DEBUG: instance='{instance}' - BEFORE") - instance = tidyup_domain(instance) - # DEBUG: print(f"DEBUG: instance='{instance}' - AFTER") - - if instance == "": - print("WARNING: Empty instance after tidyup_domain(), domain:", domain) - continue - elif not validators.domain(instance.split("/")[0]): - print(f"WARNING: Bad instance='{instance}' from domain='{domain}',origin='{origin}',software='{software}'") - continue - elif blacklist.is_blacklisted(instance): - # DEBUG: print("DEBUG: instance is blacklisted:", instance) - continue - - # DEBUG: print("DEBUG: Handling instance:", instance) - try: - if not instances.is_registered(instance): - # DEBUG: print("DEBUG: Adding new instance:", instance, domain) - instances.add(instance, domain, script) - except BaseException as exception: - print(f"ERROR: instance='{instance}',exception[{type(exception)}]:'{str(exception)}'") - continue - - # DEBUG: print("DEBUG: EXIT!") - -def add_peers(rows: dict) -> list: - # DEBUG: print(f"DEBUG: rows()={len(rows)} - CALLED!") - peers = list() - for key in ["linked", "allowed", "blocked"]: - # DEBUG: print(f"DEBUG: Checking key='{key}'") - if key in rows and rows[key] is not None: - # DEBUG: print(f"DEBUG: Adding {len(rows[key])} peer(s) to peers list ...") - for peer in rows[key]: - # DEBUG: print(f"DEBUG: peer='{peer}' - BEFORE!") - peer = tidyup_domain(peer) - - # DEBUG: print(f"DEBUG: peer='{peer}' - AFTER!") - if blacklist.is_blacklisted(peer): - # DEBUG: print(f"DEBUG: peer='{peer}' is blacklisted, skipped!") - continue - - # DEBUG: print(f"DEBUG: Adding peer='{peer}' ...") - peers.append(peer) - - # DEBUG: print(f"DEBUG: peers()={len(peers)} - EXIT!") - return peers - def remove_version(software: str) -> str: - # DEBUG: print(f"DEBUG: software='{software}' - CALLED!") + print(f"DEBUG: software='{software}' - CALLED!") if not "." in software and " " not in software: print(f"WARNING: software='{software}' does not contain a version number.") return software @@ -173,7 +67,7 @@ def remove_version(software: str) -> str: elif " - " in software: temp = software.split(" - ")[0] - # DEBUG: print(f"DEBUG: software='{software}'") + print(f"DEBUG: software='{software}'") version = None if " " in software: version = temp.split(" ")[-1] @@ -182,81 +76,81 @@ def remove_version(software: str) -> str: elif "-" in software: version = temp.split("-")[-1] else: - # DEBUG: print(f"DEBUG: Was not able to find common seperator, returning untouched software='{software}'") + print(f"DEBUG: Was not able to find common seperator, returning untouched software='{software}'") return software match = None - # DEBUG: print(f"DEBUG: Checking {len(patterns)} patterns ...") + print(f"DEBUG: Checking {len(patterns)} patterns ...") for pattern in patterns: # Run match() match = pattern.match(version) - # DEBUG: print(f"DEBUG: match[]={type(match)}") + print(f"DEBUG: match[]={type(match)}") if isinstance(match, re.Match): - # DEBUG: print(f"DEBUG: version='{version}' is matching pattern='{pattern}'") + print(f"DEBUG: version='{version}' is matching pattern='{pattern}'") break - # DEBUG: print(f"DEBUG: version[{type(version)}]='{version}',match='{match}'") + print(f"DEBUG: version[{type(version)}]='{version}',match='{match}'") if not isinstance(match, re.Match): print(f"WARNING: version='{version}' does not match regex, leaving software='{software}' untouched.") return software - # DEBUG: print(f"DEBUG: Found valid version number: '{version}', removing it ...") + print(f"DEBUG: Found valid version number: '{version}', removing it ...") end = len(temp) - len(version) - 1 - # DEBUG: print(f"DEBUG: end[{type(end)}]={end}") + print(f"DEBUG: end[{type(end)}]={end}") software = temp[0:end].strip() if " version" in software: - # DEBUG: print(f"DEBUG: software='{software}' contains word ' version'") + print(f"DEBUG: software='{software}' contains word ' version'") software = strip_until(software, " version") - # DEBUG: print(f"DEBUG: software='{software}' - EXIT!") + print(f"DEBUG: software='{software}' - EXIT!") return software def strip_powered_by(software: str) -> str: - # DEBUG: print(f"DEBUG: software='{software}' - CALLED!") + print(f"DEBUG: software='{software}' - CALLED!") if not isinstance(software, str): raise ValueError(f"Parameter software[]='{type(software)}' is not 'str'") elif software == "": raise ValueError("Parameter 'software' is empty") - elif not "powered by" in software: + elif "powered by" not in software: print(f"WARNING: Cannot find 'powered by' in software='{software}'!") return software start = software.find("powered by ") - # DEBUG: print(f"DEBUG: start[{type(start)}]='{start}'") + print(f"DEBUG: start[{type(start)}]='{start}'") software = software[start + 11:].strip() - # DEBUG: print(f"DEBUG: software='{software}'") + print(f"DEBUG: software='{software}'") software = strip_until(software, " - ") - # DEBUG: print(f"DEBUG: software='{software}' - EXIT!") + print(f"DEBUG: software='{software}' - EXIT!") return software def strip_hosted_on(software: str) -> str: - # DEBUG: print(f"DEBUG: software='{software}' - CALLED!") + print(f"DEBUG: software='{software}' - CALLED!") if not isinstance(software, str): raise ValueError(f"Parameter software[]='{type(software)}' is not 'str'") elif software == "": raise ValueError("Parameter 'software' is empty") - elif not "hosted on" in software: + elif "hosted on" not in software: print(f"WARNING: Cannot find 'hosted on' in '{software}'!") return software end = software.find("hosted on ") - # DEBUG: print(f"DEBUG: end[{type(end)}]='{end}'") + print(f"DEBUG: end[{type(end)}]='{end}'") software = software[0, end].strip() - # DEBUG: print(f"DEBUG: software='{software}'") + print(f"DEBUG: software='{software}'") software = strip_until(software, " - ") - # DEBUG: print(f"DEBUG: software='{software}' - EXIT!") + print(f"DEBUG: software='{software}' - EXIT!") return software def strip_until(software: str, until: str) -> str: - # DEBUG: print(f"DEBUG: software='{software}',until='{until}' - CALLED!") + print(f"DEBUG: software='{software}',until='{until}' - CALLED!") if not isinstance(software, str): raise ValueError(f"Parameter software[]='{type(software)}' is not 'str'") elif software == "": @@ -272,11 +166,11 @@ def strip_until(software: str, until: str) -> str: # Next, strip until part end = software.find(until) - # DEBUG: print(f"DEBUG: end[{type(end)}]='{end}'") + print(f"DEBUG: end[{type(end)}]='{end}'") if end > 0: software = software[0:end].strip() - # DEBUG: print(f"DEBUG: software='{software}' - EXIT!") + print(f"DEBUG: software='{software}' - EXIT!") return software def remove_pending_error(domain: str): @@ -292,7 +186,7 @@ def remove_pending_error(domain: str): except: pass - # DEBUG: print("DEBUG: EXIT!") + print("DEBUG: EXIT!") def get_hash(domain: str) -> str: if not isinstance(domain, str): @@ -303,21 +197,21 @@ def get_hash(domain: str) -> str: return hashlib.sha256(domain.encode("utf-8")).hexdigest() def log_error(domain: str, response: requests.models.Response): - # DEBUG: print("DEBUG: domain,response[]:", domain, type(response)) + print("DEBUG: domain,response[]:", domain, type(response)) if not isinstance(domain, str): raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") elif config.get("write_error_log").lower() != "true": - # DEBUG: print(f"DEBUG: Writing to error_log is disabled in configuruation file - EXIT!") + print(f"DEBUG: Writing to error_log is disabled in configuruation file - EXIT!") return try: - # DEBUG: print("DEBUG: BEFORE response[]:", type(response)) + print("DEBUG: BEFORE response[]:", type(response)) if isinstance(response, BaseException) or isinstance(response, json.decoder.JSONDecodeError): response = f"response[{type(response)}]='{str(response)}'" - # DEBUG: print("DEBUG: AFTER response[]:", type(response)) + print("DEBUG: AFTER response[]:", type(response)) if isinstance(response, str): cursor.execute("INSERT INTO error_log (domain, error_code, error_message, created) VALUES (?, 999, ?, ?)",[ domain, @@ -333,452 +227,16 @@ def log_error(domain: str, response: requests.models.Response): ]) # Cleanup old entries - # DEBUG: print(f"DEBUG: Purging old records (distance: {config.get('error_log_cleanup')})") + print(f"DEBUG: Purging old records (distance: {config.get('error_log_cleanup')})") cursor.execute("DELETE FROM error_log WHERE created < ?", [time.time() - config.get("error_log_cleanup")]) except BaseException as exception: print(f"ERROR: failed SQL query: domain='{domain}',exception[{type(exception)}]:'{str(exception)}'") sys.exit(255) - # DEBUG: print("DEBUG: EXIT!") - -def fetch_peers(domain: str, software: str) -> list: - # DEBUG: print(f"DEBUG: domain({len(domain)})={domain},software={software} - CALLED!") - if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError("Parameter 'domain' is empty") - elif not isinstance(software, str) and software is not None: - raise ValueError(f"software[]={type(software)} is not 'str'") - - if software == "misskey": - # DEBUG: print(f"DEBUG: Invoking misskey.fetch_peers({domain}) ...") - return misskey.fetch_peers(domain) - elif software == "lemmy": - # DEBUG: print(f"DEBUG: Invoking lemmy.fetch_peers({domain}) ...") - return lemmy.fetch_peers(domain) - elif software == "peertube": - # DEBUG: print(f"DEBUG: Invoking peertube.fetch_peers({domain}) ...") - return peertube.fetch_peers(domain) - - # DEBUG: print(f"DEBUG: Fetching peers from '{domain}',software='{software}' ...") - peers = list() - response = network.fetch_response(domain, "/api/v1/instance/peers", network.api_headers, (config.get("connection_timeout"), config.get("read_timeout"))) - # DEBUG: print(f"DEBUG: response[]='{type(response)}'") - - data = json_from_response(response) - # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'") - - if not response.ok or response.status_code >= 400: - # DEBUG: print("DEBUG: Was not able to fetch peers, trying alternative ...") - response = network.fetch_response(domain, "/api/v3/site", network.api_headers, (config.get("connection_timeout"), config.get("read_timeout"))) - - data = json_from_response(response) - # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'") - if not response.ok or response.status_code >= 400: - print("WARNING: Could not reach any JSON API:", domain) - instances.update_last_error(domain, response) - elif response.ok and isinstance(data, list): - # DEBUG: print(f"DEBUG: domain='{domain}' returned a list: '{data}'") - sys.exit(255) - elif "federated_instances" in data: - # DEBUG: print(f"DEBUG: Found federated_instances for domain='{domain}'") - peers = peers + add_peers(data["federated_instances"]) - # DEBUG: print("DEBUG: Added instance(s) to peers") - else: - print("WARNING: JSON response does not contain 'federated_instances':", domain) - instances.update_last_error(domain, response) - else: - # DEBUG: print("DEBUG: Querying API was successful:", domain, len(data)) - peers = data - - # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'") - instances.set_data("total_peers", domain, len(peers)) - - # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") - instances.update_last_instance_fetch(domain) - - # DEBUG: print("DEBUG: Returning peers[]:", type(peers)) - return peers - -def fetch_nodeinfo(domain: str, path: str = None) -> list: - # DEBUG: print(f"DEBUG: domain='{domain}',path={path} - CALLED!") - if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError("Parameter 'domain' is empty") - elif not isinstance(path, str) and path is not None: - raise ValueError(f"Parameter path[]={type(path)} is not 'str'") - - # DEBUG: print(f"DEBUG: Fetching nodeinfo from domain='{domain}' ...") - nodeinfo = fetch_wellknown_nodeinfo(domain) - - # DEBUG: print(f"DEBUG: nodeinfo({len(nodeinfo)})={nodeinfo}") - if len(nodeinfo) > 0: - # DEBUG: print("DEBUG: nodeinfo()={len(nodeinfo))} - EXIT!") - return nodeinfo - - request_paths = [ - "/nodeinfo/2.1.json", - "/nodeinfo/2.1", - "/nodeinfo/2.0.json", - "/nodeinfo/2.0", - "/nodeinfo/1.0", - "/api/v1/instance" - ] - - data = {} - for request in request_paths: - if path is not None and path != "" and path != f"https://{domain}{path}": - # DEBUG: print(f"DEBUG: path='{path}' does not match request='{request}' - SKIPPED!") - continue - - # DEBUG: print(f"DEBUG: Fetching request='{request}' from domain='{domain}' ...") - response = network.fetch_response(domain, request, network.api_headers, (config.get("nodeinfo_connection_timeout"), config.get("nodeinfo_read_timeout"))) - - data = json_from_response(response) - # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'") - if response.ok and isinstance(data, dict): - # DEBUG: print("DEBUG: Success:", request) - instances.set_data("detection_mode", domain, "STATIC_CHECK") - instances.set_data("nodeinfo_url" , domain, request) - break - elif response.ok and isinstance(data, list): - print(f"UNSUPPORTED: domain='{domain}' returned a list: '{data}'") - sys.exit(255) - elif not response.ok or response.status_code >= 400: - print("WARNING: Failed fetching nodeinfo from domain:", domain) - instances.update_last_error(domain, response) - continue - - # DEBUG: print(f"DEBUG: data()={len(data)} - EXIT!") - return data - -def fetch_wellknown_nodeinfo(domain: str) -> list: - # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") - if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError("Parameter 'domain' is empty") - - # DEBUG: print("DEBUG: Fetching .well-known info for domain:", domain) - data = {} - - response = network.fetch_response(domain, "/.well-known/nodeinfo", network.api_headers, (config.get("nodeinfo_connection_timeout"), config.get("nodeinfo_read_timeout"))) - - data = json_from_response(response) - # DEBUG: print("DEBUG: domain,response.ok,data[]:", domain, response.ok, type(data)) - if response.ok and isinstance(data, dict): - nodeinfo = data - # DEBUG: print("DEBUG: Found entries:", len(nodeinfo), domain) - if "links" in nodeinfo: - # DEBUG: print("DEBUG: Found links in nodeinfo():", len(nodeinfo["links"])) - for link in nodeinfo["links"]: - # DEBUG: print("DEBUG: rel,href:", link["rel"], link["href"]) - if link["rel"] in nodeinfo_identifier: - # DEBUG: print("DEBUG: Fetching nodeinfo from:", link["href"]) - response = fetch_url(link["href"], network.api_headers, (config.get("connection_timeout"), config.get("read_timeout"))) - - data = json_from_response(response) - # DEBUG: print("DEBUG: href,response.ok,response.status_code:", link["href"], response.ok, response.status_code) - if response.ok and isinstance(data, dict): - # DEBUG: print("DEBUG: Found JSON nodeinfo():", len(data)) - instances.set_data("detection_mode", domain, "AUTO_DISCOVERY") - instances.set_data("nodeinfo_url" , domain, link["href"]) - break - else: - print("WARNING: Unknown 'rel' value:", domain, link["rel"]) - else: - print("WARNING: nodeinfo does not contain 'links':", domain) - - # DEBUG: print("DEBUG: Returning data[]:", type(data)) - return data - -def fetch_generator_from_path(domain: str, path: str = "/") -> str: - # DEBUG: print(f"DEBUG: domain({len(domain)})={domain},path={path} - CALLED!") - if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError("Parameter 'domain' is empty") - elif not isinstance(path, str): - raise ValueError(f"path[]={type(path)} is not 'str'") - elif path == "": - raise ValueError("Parameter 'path' is empty") - - # DEBUG: print(f"DEBUG: domain='{domain}',path='{path}' - CALLED!") - software = None - - # DEBUG: print(f"DEBUG: Fetching path='{path}' from '{domain}' ...") - response = network.fetch_response(domain, path, network.web_headers, (config.get("connection_timeout"), config.get("read_timeout"))) - - # DEBUG: print("DEBUG: domain,response.ok,response.status_code,response.text[]:", domain, response.ok, response.status_code, type(response.text)) - if response.ok and response.status_code < 300 and len(response.text) > 0: - # DEBUG: print("DEBUG: Search for :", domain) - doc = bs4.BeautifulSoup(response.text, "html.parser") - - # DEBUG: print("DEBUG: doc[]:", type(doc)) - generator = doc.find("meta", {"name": "generator"}) - site_name = doc.find("meta", {"property": "og:site_name"}) - - # DEBUG: print(f"DEBUG: generator='{generator}',site_name='{site_name}'") - if isinstance(generator, bs4.element.Tag): - # DEBUG: print("DEBUG: Found generator meta tag:", domain) - software = tidyup_domain(generator.get("content")) - print(f"INFO: domain='{domain}' is generated by '{software}'") - instances.set_data("detection_mode", domain, "GENERATOR") - remove_pending_error(domain) - elif isinstance(site_name, bs4.element.Tag): - # DEBUG: print("DEBUG: Found property=og:site_name:", domain) - sofware = tidyup_domain(site_name.get("content")) - print(f"INFO: domain='{domain}' has og:site_name='{software}'") - instances.set_data("detection_mode", domain, "SITE_NAME") - remove_pending_error(domain) - - # DEBUG: print(f"DEBUG: software[]={type(software)}") - if isinstance(software, str) and software == "": - # DEBUG: print(f"DEBUG: Corrected empty string to None for software of domain='{domain}'") - software = None - elif isinstance(software, str) and ("." in software or " " in software): - # DEBUG: print(f"DEBUG: software='{software}' may contain a version number, domain='{domain}', removing it ...") - software = remove_version(software) - - # DEBUG: print(f"DEBUG: software[]={type(software)}") - if isinstance(software, str) and " powered by " in software: - # DEBUG: print(f"DEBUG: software='{software}' has 'powered by' in it") - software = remove_version(strip_powered_by(software)) - elif isinstance(software, str) and " hosted on " in software: - # DEBUG: print(f"DEBUG: software='{software}' has 'hosted on' in it") - software = remove_version(strip_hosted_on(software)) - elif isinstance(software, str) and " by " in software: - # DEBUG: print(f"DEBUG: software='{software}' has ' by ' in it") - software = strip_until(software, " by ") - elif isinstance(software, str) and " see " in software: - # DEBUG: print(f"DEBUG: software='{software}' has ' see ' in it") - software = strip_until(software, " see ") - - # DEBUG: print(f"DEBUG: software='{software}' - EXIT!") - return software - -def determine_software(domain: str, path: str = None) -> str: - # DEBUG: print(f"DEBUG: domain({len(domain)})={domain},path={path} - CALLED!") - if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError("Parameter 'domain' is empty") - elif not isinstance(path, str) and path is not None: - raise ValueError(f"Parameter path[]={type(path)} is not 'str'") - - # DEBUG: print("DEBUG: Determining software for domain,path:", domain, path) - software = None - - # DEBUG: print(f"DEBUG: Fetching nodeinfo from '{domain}' ...") - data = fetch_nodeinfo(domain, path) - - # DEBUG: print("DEBUG: data[]:", type(data)) - if not isinstance(data, dict) or len(data) == 0: - # DEBUG: print("DEBUG: Could not determine software type:", domain) - return fetch_generator_from_path(domain) - - # DEBUG: print("DEBUG: data():", len(data), data) - if "status" in data and data["status"] == "error" and "message" in data: - print("WARNING: JSON response is an error:", data["message"]) - instances.update_last_error(domain, data["message"]) - return fetch_generator_from_path(domain) - elif "message" in data: - print("WARNING: JSON response contains only a message:", data["message"]) - instances.update_last_error(domain, data["message"]) - return fetch_generator_from_path(domain) - elif "software" not in data or "name" not in data["software"]: - # DEBUG: print(f"DEBUG: JSON response from domain='{domain}' does not include [software][name], fetching / ...") - software = fetch_generator_from_path(domain) - - # DEBUG: print(f"DEBUG: Generator for domain='{domain}' is: {software}, EXIT!") - return software - - software = tidyup_domain(data["software"]["name"]) - - # DEBUG: print("DEBUG: sofware after tidyup_domain():", software) - if software in ["akkoma", "rebased"]: - # DEBUG: print("DEBUG: Setting pleroma:", domain, software) - software = "pleroma" - elif software in ["hometown", "ecko"]: - # DEBUG: print("DEBUG: Setting mastodon:", domain, software) - software = "mastodon" - elif software in ["calckey", "groundpolis", "foundkey", "cherrypick", "meisskey"]: - # DEBUG: print("DEBUG: Setting misskey:", domain, software) - software = "misskey" - elif software.find("/") > 0: - print("WARNING: Spliting of slash:", software) - software = tidyup_domain(software.split("/")[-1]) - elif software.find("|") > 0: - print("WARNING: Spliting of pipe:", software) - software = tidyup_domain(software.split("|")[0]) - elif "powered by" in software: - # DEBUG: print(f"DEBUG: software='{software}' has 'powered by' in it") - software = strip_powered_by(software) - elif isinstance(software, str) and " by " in software: - # DEBUG: print(f"DEBUG: software='{software}' has ' by ' in it") - software = strip_until(software, " by ") - elif isinstance(software, str) and " see " in software: - # DEBUG: print(f"DEBUG: software='{software}' has ' see ' in it") - software = strip_until(software, " see ") - - # DEBUG: print(f"DEBUG: software[]={type(software)}") - if software == "": - print("WARNING: tidyup_domain() left no software name behind:", domain) - software = None - - # DEBUG: print(f"DEBUG: software[]={type(software)}") - if str(software) == "": - # DEBUG: print(f"DEBUG: software for '{domain}' was not detected, trying generator ...") - software = fetch_generator_from_path(domain) - elif len(str(software)) > 0 and ("." in software or " " in software): - # DEBUG: print(f"DEBUG: software='{software}' may contain a version number, domain='{domain}', removing it ...") - software = remove_version(software) - - # DEBUG: print(f"DEBUG: software[]={type(software)}") - if isinstance(software, str) and "powered by" in software: - # DEBUG: print(f"DEBUG: software='{software}' has 'powered by' in it") - software = remove_version(strip_powered_by(software)) - - # DEBUG: print("DEBUG: Returning domain,software:", domain, software) - return software - -def tidyup_reason(reason: str) -> str: - # DEBUG: print(f"DEBUG: reason='{reason}' - CALLED!") - if not isinstance(reason, str): - raise ValueError(f"Parameter reason[]={type(reason)} is not 'str'") - - # Strip string - reason = reason.strip() - - # Replace â with " - reason = re.sub("â", "\"", reason) - - # DEBUG: print(f"DEBUG: reason='{reason}' - EXIT!") - return reason - -def tidyup_domain(domain: str) -> str: - # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") - if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - - # All lower-case and strip spaces out + last dot - domain = domain.lower().strip().rstrip(".") - - # No port number - domain = re.sub("\:\d+$", "", domain) - - # No protocol, sometimes without the slashes - domain = re.sub("^https?\:(\/*)", "", domain) - - # No trailing slash - domain = re.sub("\/$", "", domain) - - # No @ sign - domain = re.sub("^\@", "", domain) - - # No individual users in block lists - domain = re.sub("(.+)\@", "", domain) - if domain.find("/profile/"): - domain = domain.split("/profile/")[0] - elif domain.find("/users/"): - domain = domain.split("/users/")[0] - - # DEBUG: print(f"DEBUG: domain='{domain}' - EXIT!") - return domain - -def json_from_response(response: requests.models.Response) -> list: - # DEBUG: print(f"DEBUG: response[]={type(response)} - CALLED!") - if not isinstance(response, requests.models.Response): - raise ValueError(f"Parameter response[]='{type(response)}' is not type of 'Response'") - - data = list() - if response.text.strip() != "": - # DEBUG: print(f"DEBUG: response.text()={len(response.text)} is not empty, invoking response.json() ...") - try: - data = response.json() - except json.decoder.JSONDecodeError: - pass - - # DEBUG: print(f"DEBUG: data[]={type(data)} - EXIT!") - return data - -def has_key(lists: list, key: str, value: any) -> bool: - # DEBUG: print(f"DEBUG: lists()={len(lists)},key='{key}',value[]='{type(value)}' - CALLED!") - if not isinstance(lists, list): - raise ValueError(f"Parameter lists[]='{type(lists)}' is not 'list'") - elif not isinstance(key, str): - raise ValueError(f"Parameter key[]='{type(key)}' is not 'str'") - elif key == "": - raise ValueError("Parameter 'key' is empty") - - has = False - # DEBUG: print(f"DEBUG: Checking lists()={len(lists)} ...") - for row in lists: - # DEBUG: print(f"DEBUG: row['{type(row)}']={row}") - if not isinstance(row, dict): - raise ValueError(f"row[]='{type(row)}' is not 'dict'") - elif not key in row: - raise KeyError(f"Cannot find key='{key}'") - elif row[key] == value: - has = True - break - - # DEBUG: print(f"DEBUG: has={has} - EXIT!") - return has - -def find_domains(tag: bs4.element.Tag) -> list: - # DEBUG: print(f"DEBUG: tag[]={type(tag)} - CALLED!") - if not isinstance(tag, bs4.element.Tag): - raise ValueError(f"Parameter tag[]={type(tag)} is not type of bs4.element.Tag") - elif len(tag.select("tr")) == 0: - raise KeyError("No table rows found in table!") - - domains = list() - for element in tag.select("tr"): - # DEBUG: print(f"DEBUG: element[]={type(element)}") - if not element.find("td"): - # DEBUG: print("DEBUG: Skipping element, no found") - continue - - domain = tidyup_domain(element.find("td").text) - reason = tidyup_reason(element.findAll("td")[1].text) - - # DEBUG: print(f"DEBUG: domain='{domain}',reason='{reason}'") - - if blacklist.is_blacklisted(domain): - print(f"WARNING: domain='{domain}' is blacklisted - skipped!") - continue - elif domain == "gab.com/.ai, develop.gab.com": - # DEBUG: print("DEBUG: Multiple domains detected in one row") - domains.append({ - "domain": "gab.com", - "reason": reason, - }) - domains.append({ - "domain": "gab.ai", - "reason": reason, - }) - domains.append({ - "domain": "develop.gab.com", - "reason": reason, - }) - continue - elif not validators.domain(domain): - print(f"WARNING: domain='{domain}' is not a valid domain - skipped!") - continue - - # DEBUG: print(f"DEBUG: Adding domain='{domain}' ...") - domains.append({ - "domain": domain, - "reason": reason, - }) - - # DEBUG: print(f"DEBUG: domains()={len(domains)} - EXIT!") - return domains + print("DEBUG: EXIT!") def fetch_url(url: str, headers: dict, timeout: tuple) -> requests.models.Response: - # DEBUG: print(f"DEBUG: url='{url}',headers()={len(headers)},timeout={timeout} - CALLED!") + print(f"DEBUG: url='{url}',headers()={len(headers)},timeout={timeout} - CALLED!") if not isinstance(url, str): raise ValueError(f"Parameter url[]='{type(url)}' is not 'str'") elif url == "": @@ -788,15 +246,15 @@ def fetch_url(url: str, headers: dict, timeout: tuple) -> requests.models.Respon elif not isinstance(timeout, tuple): raise ValueError(f"Parameter timeout[]='{type(timeout)}' is not 'tuple'") - # DEBUG: print(f"DEBUG: Parsing url='{url}'") + print(f"DEBUG: Parsing url='{url}'") components = urlparse(url) # Invoke other function, avoid trailing ? - # DEBUG: print(f"DEBUG: components[{type(components)}]={components}") + print(f"DEBUG: components[{type(components)}]={components}") if components.query != "": response = network.fetch_response(components.hostname, f"{components.path}?{components.query}", headers, timeout) else: response = network.fetch_response(components.hostname, f"{components.path}", headers, timeout) - # DEBUG: print(f"DEBUG: response[]='{type(response)}' - EXXIT!") + print(f"DEBUG: response[]='{type(response)}' - EXXIT!") return response diff --git a/fba/federation.py b/fba/federation.py new file mode 100644 index 0000000..b7e86a2 --- /dev/null +++ b/fba/federation.py @@ -0,0 +1,478 @@ +# Copyright (C) 2023 Free Software Foundation +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import sys + +import bs4 +import validators + +from fba import blacklist +from fba import config +from fba import fba +from fba import instances +from fba import network + +from fba.helpers import tidyup + +from fba.networks import lemmy +from fba.networks import misskey +from fba.networks import peertube + +# "rel" identifiers (no real URLs) +nodeinfo_identifier = [ + "https://nodeinfo.diaspora.software/ns/schema/2.1", + "https://nodeinfo.diaspora.software/ns/schema/2.0", + "https://nodeinfo.diaspora.software/ns/schema/1.1", + "https://nodeinfo.diaspora.software/ns/schema/1.0", + "http://nodeinfo.diaspora.software/ns/schema/2.1", + "http://nodeinfo.diaspora.software/ns/schema/2.0", + "http://nodeinfo.diaspora.software/ns/schema/1.1", + "http://nodeinfo.diaspora.software/ns/schema/1.0", +] + +def fetch_instances(domain: str, origin: str, software: str, script: str, path: str = None): + print(f"DEBUG: domain='{domain}',origin='{origin}',software='{software}',path='{path}' - CALLED!") + if not isinstance(domain, str): + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError("Parameter 'domain' is empty") + elif not isinstance(origin, str) and origin is not None: + raise ValueError(f"Parameter origin[]={type(origin)} is not 'str'") + elif software is None: + print(f"DEBUG: software for domain='{domain}' is not set, determining ...") + software = determine_software(domain, path) + print(f"DEBUG: Determined software='{software}' for domain='{domain}'") + elif not isinstance(software, str): + raise ValueError(f"Parameter software[]={type(software)} is not 'str'") + elif not isinstance(script, str): + raise ValueError(f"Parameter script[]={type(script)} is not 'str'") + elif domain == "": + raise ValueError("Parameter 'domain' is empty") + + if not instances.is_registered(domain): + print("DEBUG: Adding new domain:", domain, origin) + instances.add(domain, origin, script, path) + + print("DEBUG: Fetching instances for domain:", domain, software) + peerlist = fetch_peers(domain, software) + + if peerlist is None: + print("ERROR: Cannot fetch peers:", domain) + return + elif instances.has_pending_instance_data(domain): + print(f"DEBUG: domain='{domain}' has pending nodeinfo data, flushing ...") + instances.update_data(domain) + + print(f"INFO: Checking {len(peerlist)} instances from {domain} ...") + for instance in peerlist: + if instance is None: + # Skip "None" types as tidup() cannot parse them + continue + + print(f"DEBUG: instance='{instance}' - BEFORE") + instance = tidyup.domain(instance) + print(f"DEBUG: instance='{instance}' - AFTER") + + if instance == "": + print("WARNING: Empty instance after tidyup.domain(), domain:", domain) + continue + elif not validators.domain(instance.split("/")[0]): + print(f"WARNING: Bad instance='{instance}' from domain='{domain}',origin='{origin}',software='{software}'") + continue + elif blacklist.is_blacklisted(instance): + print("DEBUG: instance is blacklisted:", instance) + continue + + print("DEBUG: Handling instance:", instance) + try: + if not instances.is_registered(instance): + print("DEBUG: Adding new instance:", instance, domain) + instances.add(instance, domain, script) + except BaseException as exception: + print(f"ERROR: instance='{instance}',exception[{type(exception)}]:'{str(exception)}'") + continue + + print("DEBUG: EXIT!") + +def fetch_peers(domain: str, software: str) -> list: + print(f"DEBUG: domain({len(domain)})={domain},software={software} - CALLED!") + if not isinstance(domain, str): + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError("Parameter 'domain' is empty") + elif not isinstance(software, str) and software is not None: + raise ValueError(f"software[]={type(software)} is not 'str'") + + if software == "misskey": + print(f"DEBUG: Invoking misskey.fetch_peers({domain}) ...") + return misskey.fetch_peers(domain) + elif software == "lemmy": + print(f"DEBUG: Invoking lemmy.fetch_peers({domain}) ...") + return lemmy.fetch_peers(domain) + elif software == "peertube": + print(f"DEBUG: Invoking peertube.fetch_peers({domain}) ...") + return peertube.fetch_peers(domain) + + print(f"DEBUG: Fetching peers from '{domain}',software='{software}' ...") + peers = list() + response = network.fetch_response(domain, "/api/v1/instance/peers", network.api_headers, (config.get("connection_timeout"), config.get("read_timeout"))) + print(f"DEBUG: response[]='{type(response)}'") + + data = network.json_from_response(response) + print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'") + + if not response.ok or response.status_code >= 400: + print("DEBUG: Was not able to fetch peers, trying alternative ...") + response = network.fetch_response(domain, "/api/v3/site", network.api_headers, (config.get("connection_timeout"), config.get("read_timeout"))) + + data = network.json_from_response(response) + print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'") + if not response.ok or response.status_code >= 400: + print("WARNING: Could not reach any JSON API:", domain) + instances.update_last_error(domain, response) + elif response.ok and isinstance(data, list): + print(f"DEBUG: domain='{domain}' returned a list: '{data}'") + sys.exit(255) + elif "federated_instances" in data: + print(f"DEBUG: Found federated_instances for domain='{domain}'") + peers = peers + add_peers(data["federated_instances"]) + print("DEBUG: Added instance(s) to peers") + else: + print("WARNING: JSON response does not contain 'federated_instances':", domain) + instances.update_last_error(domain, response) + else: + print("DEBUG: Querying API was successful:", domain, len(data)) + peers = data + + print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'") + instances.set_data("total_peers", domain, len(peers)) + + print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") + instances.update_last_instance_fetch(domain) + + print("DEBUG: Returning peers[]:", type(peers)) + return peers + +def fetch_nodeinfo(domain: str, path: str = None) -> list: + print(f"DEBUG: domain='{domain}',path={path} - CALLED!") + if not isinstance(domain, str): + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError("Parameter 'domain' is empty") + elif not isinstance(path, str) and path is not None: + raise ValueError(f"Parameter path[]={type(path)} is not 'str'") + + print(f"DEBUG: Fetching nodeinfo from domain='{domain}' ...") + nodeinfo = fetch_wellknown_nodeinfo(domain) + + print(f"DEBUG: nodeinfo({len(nodeinfo)})={nodeinfo}") + if len(nodeinfo) > 0: + print("DEBUG: nodeinfo()={len(nodeinfo))} - EXIT!") + return nodeinfo + + request_paths = [ + "/nodeinfo/2.1.json", + "/nodeinfo/2.1", + "/nodeinfo/2.0.json", + "/nodeinfo/2.0", + "/nodeinfo/1.0", + "/api/v1/instance" + ] + + for request in request_paths: + if path is not None and path != "" and path != request: + print(f"DEBUG: path='{path}' does not match request='{request}' - SKIPPED!") + continue + + print(f"DEBUG: Fetching request='{request}' from domain='{domain}' ...") + response = network.fetch_response(domain, request, network.api_headers, (config.get("nodeinfo_connection_timeout"), config.get("nodeinfo_read_timeout"))) + + data = network.json_from_response(response) + print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'") + if response.ok and isinstance(data, dict): + print("DEBUG: Success:", request) + instances.set_data("detection_mode", domain, "STATIC_CHECK") + instances.set_data("nodeinfo_url" , domain, request) + break + elif response.ok and isinstance(data, list): + print(f"UNSUPPORTED: domain='{domain}' returned a list: '{data}'") + sys.exit(255) + elif not response.ok or response.status_code >= 400: + print("WARNING: Failed fetching nodeinfo from domain:", domain) + instances.update_last_error(domain, response) + continue + + print(f"DEBUG: data()={len(data)} - EXIT!") + return data + +def fetch_wellknown_nodeinfo(domain: str) -> list: + print(f"DEBUG: domain='{domain}' - CALLED!") + if not isinstance(domain, str): + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError("Parameter 'domain' is empty") + + print("DEBUG: Fetching .well-known info for domain:", domain) + response = network.fetch_response(domain, "/.well-known/nodeinfo", network.api_headers, (config.get("nodeinfo_connection_timeout"), config.get("nodeinfo_read_timeout"))) + + data = network.json_from_response(response) + print("DEBUG: domain,response.ok,data[]:", domain, response.ok, type(data)) + if response.ok and isinstance(data, dict): + nodeinfo = data + print("DEBUG: Found entries:", len(nodeinfo), domain) + if "links" in nodeinfo: + print("DEBUG: Found links in nodeinfo():", len(nodeinfo["links"])) + for link in nodeinfo["links"]: + print("DEBUG: rel,href:", link["rel"], link["href"]) + if link["rel"] in nodeinfo_identifier: + print("DEBUG: Fetching nodeinfo from:", link["href"]) + response = fba.fetch_url(link["href"], network.api_headers, (config.get("connection_timeout"), config.get("read_timeout"))) + + data = network.json_from_response(response) + print("DEBUG: href,response.ok,response.status_code:", link["href"], response.ok, response.status_code) + if response.ok and isinstance(data, dict): + print("DEBUG: Found JSON nodeinfo():", len(data)) + instances.set_data("detection_mode", domain, "AUTO_DISCOVERY") + instances.set_data("nodeinfo_url" , domain, link["href"]) + break + else: + print("WARNING: Unknown 'rel' value:", domain, link["rel"]) + else: + print("WARNING: nodeinfo does not contain 'links':", domain) + + print("DEBUG: Returning data[]:", type(data)) + return data + +def fetch_generator_from_path(domain: str, path: str = "/") -> str: + print(f"DEBUG: domain({len(domain)})={domain},path={path} - CALLED!") + if not isinstance(domain, str): + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError("Parameter 'domain' is empty") + elif not isinstance(path, str): + raise ValueError(f"path[]={type(path)} is not 'str'") + elif path == "": + raise ValueError("Parameter 'path' is empty") + + print(f"DEBUG: domain='{domain}',path='{path}' - CALLED!") + software = None + + print(f"DEBUG: Fetching path='{path}' from '{domain}' ...") + response = network.fetch_response(domain, path, network.web_headers, (config.get("connection_timeout"), config.get("read_timeout"))) + + print("DEBUG: domain,response.ok,response.status_code,response.text[]:", domain, response.ok, response.status_code, type(response.text)) + if response.ok and response.status_code < 300 and len(response.text) > 0: + print("DEBUG: Search for :", domain) + doc = bs4.BeautifulSoup(response.text, "html.parser") + + print("DEBUG: doc[]:", type(doc)) + generator = doc.find("meta", {"name": "generator"}) + site_name = doc.find("meta", {"property": "og:site_name"}) + + print(f"DEBUG: generator='{generator}',site_name='{site_name}'") + if isinstance(generator, bs4.element.Tag): + print("DEBUG: Found generator meta tag:", domain) + software = tidyup.domain(generator.get("content")) + print(f"INFO: domain='{domain}' is generated by '{software}'") + instances.set_data("detection_mode", domain, "GENERATOR") + fba.remove_pending_error(domain) + elif isinstance(site_name, bs4.element.Tag): + print("DEBUG: Found property=og:site_name:", domain) + sofware = tidyup.domain(site_name.get("content")) + print(f"INFO: domain='{domain}' has og:site_name='{software}'") + instances.set_data("detection_mode", domain, "SITE_NAME") + fba.remove_pending_error(domain) + + print(f"DEBUG: software[]={type(software)}") + if isinstance(software, str) and software == "": + print(f"DEBUG: Corrected empty string to None for software of domain='{domain}'") + software = None + elif isinstance(software, str) and ("." in software or " " in software): + print(f"DEBUG: software='{software}' may contain a version number, domain='{domain}', removing it ...") + software = fba.remove_version(software) + + print(f"DEBUG: software[]={type(software)}") + if isinstance(software, str) and " powered by " in software: + print(f"DEBUG: software='{software}' has 'powered by' in it") + software = fba.remove_version(fba.strip_powered_by(software)) + elif isinstance(software, str) and " hosted on " in software: + print(f"DEBUG: software='{software}' has 'hosted on' in it") + software = fba.remove_version(fba.strip_hosted_on(software)) + elif isinstance(software, str) and " by " in software: + print(f"DEBUG: software='{software}' has ' by ' in it") + software = fba.strip_until(software, " by ") + elif isinstance(software, str) and " see " in software: + print(f"DEBUG: software='{software}' has ' see ' in it") + software = fba.strip_until(software, " see ") + + print(f"DEBUG: software='{software}' - EXIT!") + return software + +def determine_software(domain: str, path: str = None) -> str: + print(f"DEBUG: domain({len(domain)})={domain},path={path} - CALLED!") + if not isinstance(domain, str): + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError("Parameter 'domain' is empty") + elif not isinstance(path, str) and path is not None: + raise ValueError(f"Parameter path[]={type(path)} is not 'str'") + + print("DEBUG: Determining software for domain,path:", domain, path) + software = None + + print(f"DEBUG: Fetching nodeinfo from '{domain}' ...") + data = fetch_nodeinfo(domain, path) + + print("DEBUG: data[]:", type(data)) + if not isinstance(data, dict) or len(data) == 0: + print("DEBUG: Could not determine software type:", domain) + return fetch_generator_from_path(domain) + + print("DEBUG: data():", len(data), data) + if "status" in data and data["status"] == "error" and "message" in data: + print("WARNING: JSON response is an error:", data["message"]) + instances.update_last_error(domain, data["message"]) + return fetch_generator_from_path(domain) + elif "message" in data: + print("WARNING: JSON response contains only a message:", data["message"]) + instances.update_last_error(domain, data["message"]) + return fetch_generator_from_path(domain) + elif "software" not in data or "name" not in data["software"]: + print(f"DEBUG: JSON response from domain='{domain}' does not include [software][name], fetching / ...") + software = fetch_generator_from_path(domain) + + print(f"DEBUG: Generator for domain='{domain}' is: {software}, EXIT!") + return software + + software = tidyup.domain(data["software"]["name"]) + + print("DEBUG: sofware after tidyup.domain():", software) + if software in ["akkoma", "rebased"]: + print("DEBUG: Setting pleroma:", domain, software) + software = "pleroma" + elif software in ["hometown", "ecko"]: + print("DEBUG: Setting mastodon:", domain, software) + software = "mastodon" + elif software in ["calckey", "groundpolis", "foundkey", "cherrypick", "meisskey"]: + print("DEBUG: Setting misskey:", domain, software) + software = "misskey" + elif software.find("/") > 0: + print("WARNING: Spliting of slash:", software) + software = tidyup.domain(software.split("/")[-1]) + elif software.find("|") > 0: + print("WARNING: Spliting of pipe:", software) + software = tidyup.domain(software.split("|")[0]) + elif "powered by" in software: + print(f"DEBUG: software='{software}' has 'powered by' in it") + software = fba.strip_powered_by(software) + elif isinstance(software, str) and " by " in software: + print(f"DEBUG: software='{software}' has ' by ' in it") + software = fba.strip_until(software, " by ") + elif isinstance(software, str) and " see " in software: + print(f"DEBUG: software='{software}' has ' see ' in it") + software = fba.strip_until(software, " see ") + + print(f"DEBUG: software[]={type(software)}") + if software == "": + print("WARNING: tidyup.domain() left no software name behind:", domain) + software = None + + print(f"DEBUG: software[]={type(software)}") + if str(software) == "": + print(f"DEBUG: software for '{domain}' was not detected, trying generator ...") + software = fetch_generator_from_path(domain) + elif len(str(software)) > 0 and ("." in software or " " in software): + print(f"DEBUG: software='{software}' may contain a version number, domain='{domain}', removing it ...") + software = fba.remove_version(software) + + print(f"DEBUG: software[]={type(software)}") + if isinstance(software, str) and "powered by" in software: + print(f"DEBUG: software='{software}' has 'powered by' in it") + software = fba.remove_version(fba.strip_powered_by(software)) + + print("DEBUG: Returning domain,software:", domain, software) + return software + +def find_domains(tag: bs4.element.Tag) -> list: + print(f"DEBUG: tag[]={type(tag)} - CALLED!") + if not isinstance(tag, bs4.element.Tag): + raise ValueError(f"Parameter tag[]={type(tag)} is not type of bs4.element.Tag") + elif len(tag.select("tr")) == 0: + raise KeyError("No table rows found in table!") + + domains = list() + for element in tag.select("tr"): + print(f"DEBUG: element[]={type(element)}") + if not element.find("td"): + print("DEBUG: Skipping element, no found") + continue + + domain = tidyup.domain(element.find("td").text) + reason = tidyup.reason(element.findAll("td")[1].text) + + print(f"DEBUG: domain='{domain}',reason='{reason}'") + + if blacklist.is_blacklisted(domain): + print(f"WARNING: domain='{domain}' is blacklisted - skipped!") + continue + elif domain == "gab.com/.ai, develop.gab.com": + print("DEBUG: Multiple domains detected in one row") + domains.append({ + "domain": "gab.com", + "reason": reason, + }) + domains.append({ + "domain": "gab.ai", + "reason": reason, + }) + domains.append({ + "domain": "develop.gab.com", + "reason": reason, + }) + continue + elif not validators.domain(domain): + print(f"WARNING: domain='{domain}' is not a valid domain - skipped!") + continue + + print(f"DEBUG: Adding domain='{domain}' ...") + domains.append({ + "domain": domain, + "reason": reason, + }) + + print(f"DEBUG: domains()={len(domains)} - EXIT!") + return domains + +def add_peers(rows: dict) -> list: + # DEBUG: print(f"DEBUG: rows()={len(rows)} - CALLED!") + peers = list() + for key in ["linked", "allowed", "blocked"]: + # DEBUG: print(f"DEBUG: Checking key='{key}'") + if key in rows and rows[key] is not None: + # DEBUG: print(f"DEBUG: Adding {len(rows[key])} peer(s) to peers list ...") + for peer in rows[key]: + # DEBUG: print(f"DEBUG: peer='{peer}' - BEFORE!") + peer = tidyup.domain(peer) + + # DEBUG: print(f"DEBUG: peer='{peer}' - AFTER!") + if blacklist.is_blacklisted(peer): + # DEBUG: print(f"DEBUG: peer='{peer}' is blacklisted, skipped!") + continue + + # DEBUG: print(f"DEBUG: Adding peer='{peer}' ...") + peers.append(peer) + + # DEBUG: print(f"DEBUG: peers()={len(peers)} - EXIT!") + return peers diff --git a/fba/federation/__init__.py b/fba/federation/__init__.py deleted file mode 100644 index af23ef3..0000000 --- a/fba/federation/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -__all__ = [ - 'friendica', - 'lemmy', - 'mastodon', - 'misskey', - 'peertube', - 'pleroma', -] diff --git a/fba/helpers/__init__.py b/fba/helpers/__init__.py new file mode 100644 index 0000000..9f8b143 --- /dev/null +++ b/fba/helpers/__init__.py @@ -0,0 +1,19 @@ +# Copyright (C) 2023 Free Software Foundation +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +__all__ = [ + 'dicts', + 'tidyup', +] diff --git a/fba/helpers/dicts.py b/fba/helpers/dicts.py new file mode 100644 index 0000000..51843d4 --- /dev/null +++ b/fba/helpers/dicts.py @@ -0,0 +1,38 @@ +# Copyright (C) 2023 Free Software Foundation +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +def has_key(lists: list, key: str, value: any) -> bool: + print(f"DEBUG: lists()={len(lists)},key='{key}',value[]='{type(value)}' - CALLED!") + if not isinstance(lists, list): + raise ValueError(f"Parameter lists[]='{type(lists)}' is not 'list'") + elif not isinstance(key, str): + raise ValueError(f"Parameter key[]='{type(key)}' is not 'str'") + elif key == "": + raise ValueError("Parameter 'key' is empty") + + has = False + print(f"DEBUG: Checking lists()={len(lists)} ...") + for row in lists: + print(f"DEBUG: row['{type(row)}']={row}") + if not isinstance(row, dict): + raise ValueError(f"row[]='{type(row)}' is not 'dict'") + elif not key in row: + raise KeyError(f"Cannot find key='{key}'") + elif row[key] == value: + has = True + break + + print(f"DEBUG: has={has} - EXIT!") + return has diff --git a/fba/helpers/tidyup.py b/fba/helpers/tidyup.py new file mode 100644 index 0000000..c0de2c8 --- /dev/null +++ b/fba/helpers/tidyup.py @@ -0,0 +1,60 @@ +# Copyright (C) 2023 Free Software Foundation +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import re + +def reason(string: str) -> str: + print(f"DEBUG: string='{string}' - CALLED!") + if not isinstance(string, str): + raise ValueError(f"Parameter string[]={type(string)} is not 'str'") + + # Strip string + string = string.strip() + + # Replace â with " + string = re.sub("â", "\"", string) + + print(f"DEBUG: string='{string}' - EXIT!") + return string + +def domain(string: str) -> str: + print(f"DEBUG: string='{string}' - CALLED!") + if not isinstance(string, str): + raise ValueError(f"Parameter string[]={type(string)} is not 'str'") + + # All lower-case and strip spaces out + last dot + string = string.lower().strip().rstrip(".") + + # No port number + string = re.sub("\:\d+$", "", string) + + # No protocol, sometimes without the slashes + string = re.sub("^https?\:(\/*)", "", string) + + # No trailing slash + string = re.sub("\/$", "", string) + + # No @ sign + string = re.sub("^\@", "", string) + + # No individual users in block lists + string = re.sub("(.+)\@", "", string) + if string.find("/profile/"): + string = string.split("/profile/")[0] + elif string.find("/users/"): + string = string.split("/users/")[0] + + print(f"DEBUG: string='{string}' - EXIT!") + return string diff --git a/fba/instances.py b/fba/instances.py index 903fc3c..6ac8bcb 100644 --- a/fba/instances.py +++ b/fba/instances.py @@ -24,6 +24,7 @@ import validators from fba import blacklist from fba import cache from fba import fba +from fba import federation # Found info from node, such as nodeinfo URL, detection mode that needs to be # written to database. Both arrays must be filled at the same time or else @@ -197,7 +198,7 @@ def add(domain: str, origin: str, command: str, path: str = None): raise Exception(f"domain='{domain}' is a single user") # DEBUG: print("DEBUG: domain,origin,command,path:", domain, origin, command, path) - software = fba.determine_software(domain, path) + software = federation.determine_software(domain, path) # DEBUG: print("DEBUG: Determined software:", software) if domain.find("/c/") > 0 and software == "lemmy": domain = domain.split("/c/")[0] diff --git a/fba/locking.py b/fba/locking.py new file mode 100644 index 0000000..132a8ca --- /dev/null +++ b/fba/locking.py @@ -0,0 +1,45 @@ +# Fedi API Block - An aggregator for fetching blocking data from fediverse nodes +# Copyright (C) 2023 Free Software Foundation +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import os +import sys +import tempfile +import zc.lockfile + +# Lock file +lockfile = tempfile.gettempdir() + '/fba.lock' +LOCK = None + +def acquire(): + global LOCK + try: + # DEBUG: print(f"DEBUG: Acquiring lock: '{lockfile}'") + LOCK = zc.lockfile.LockFile(lockfile) + # DEBUG: print("DEBUG: Lock obtained.") + + except zc.lockfile.LockError: + print(f"ERROR: Cannot aquire lock: '{lockfile}'") + sys.exit(100) + +def release(): + # DEBUG: print("DEBUG: CALLED!") + if LOCK is not None: + # DEBUG: print("DEBUG: Releasing lock ...") + LOCK.close() + # DEBUG: print(f"DEBUG: Deleting lockfile='{lockfile}' ...") + os.remove(lockfile) + + # DEBUG: print("DEBUG: EXIT!") diff --git a/fba/network.py b/fba/network.py index 4894c0f..7329979 100644 --- a/fba/network.py +++ b/fba/network.py @@ -14,13 +14,12 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import bs4 +import json import reqto import requests from fba import config from fba import csrf -from fba import fba from fba import instances # HTTP headers for non-API requests @@ -35,7 +34,7 @@ api_headers = { } def post_json_api(domain: str, path: str, parameter: str, extra_headers: dict = {}) -> dict: - # DEBUG: print(f"DEBUG: domain='{domain}',path='{path}',parameter='{parameter}',extra_headers()={len(extra_headers)} - CALLED!") + print(f"DEBUG: domain='{domain}',path='{path}',parameter='{parameter}',extra_headers()={len(extra_headers)} - CALLED!") if not isinstance(domain, str): raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": @@ -47,13 +46,13 @@ def post_json_api(domain: str, path: str, parameter: str, extra_headers: dict = elif not isinstance(parameter, str): raise ValueError(f"parameter[]={type(parameter)} is not 'str'") - # DEBUG: print(f"DEBUG: Determining if CSRF header needs to be sent for domain='{domain}' ...") + print(f"DEBUG: Determining if CSRF header needs to be sent for domain='{domain}' ...") headers = csrf.determine(domain, {**api_headers, **extra_headers}) data = {} try: - # DEBUG: print(f"DEBUG: Sending POST to domain='{domain}',path='{path}',parameter='{parameter}',extra_headers({len(extra_headers)})={extra_headers}") + print(f"DEBUG: Sending POST to domain='{domain}',path='{path}',parameter='{parameter}',extra_headers({len(extra_headers)})={extra_headers}") response = reqto.post( f"https://{domain}{path}", data=parameter, @@ -61,8 +60,8 @@ def post_json_api(domain: str, path: str, parameter: str, extra_headers: dict = timeout=(config.get("connection_timeout"), config.get("read_timeout")) ) - data = fba.json_from_response(response) - # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'") + data = json_from_response(response) + print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'") if not response.ok or response.status_code >= 400: print(f"WARNING: Cannot query JSON API: domain='{domain}',path='{path}',parameter()={len(parameter)},response.status_code='{response.status_code}',data[]='{type(data)}'") instances.update_last_error(domain, response) @@ -70,11 +69,11 @@ def post_json_api(domain: str, path: str, parameter: str, extra_headers: dict = except BaseException as exception: print(f"WARNING: Some error during post(): domain='{domain}',path='{path}',parameter()={len(parameter)},exception[{type(exception)}]:'{str(exception)}'") - # DEBUG: print(f"DEBUG: Returning data({len(data)})=[]:{type(data)}") + print(f"DEBUG: Returning data({len(data)})=[]:{type(data)}") return data def send_bot_post(domain: str, blocklist: dict): - # DEBUG: print(f"DEBUG: domain={domain},blocklist()={len(blocklist)} - CALLED!") + print(f"DEBUG: domain={domain},blocklist()={len(blocklist)} - CALLED!") if not isinstance(domain, str): raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": @@ -89,9 +88,9 @@ def send_bot_post(domain: str, blocklist: dict): truncated = True blocklist = blocklist[0 : 19] - # DEBUG: print(f"DEBUG: blocklist()={len(blocklist)}") + print(f"DEBUG: blocklist()={len(blocklist)}") for block in blocklist: - # DEBUG: print(f"DEBUG: block['{type(block)}']={block}") + print(f"DEBUG: block['{type(block)}']={block}") if block["reason"] is None or block["reason"] == '': message = message + block["blocked"] + " with unspecified reason\n" else: @@ -119,7 +118,7 @@ def send_bot_post(domain: str, blocklist: dict): return True def fetch_response(domain: str, path: str, headers: dict, timeout: list) -> requests.models.Response: - # DEBUG: print(f"DEBUG: domain='{domain}',path='{path}',headers()={len(headers)},timeout={timeout} - CALLED!") + print(f"DEBUG: domain='{domain}',path='{path}',headers()={len(headers)},timeout={timeout} - CALLED!") if not isinstance(domain, str): raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": @@ -129,11 +128,11 @@ def fetch_response(domain: str, path: str, headers: dict, timeout: list) -> requ elif path == "": raise ValueError("Parameter 'path' is empty") - # DEBUG: print(f"DEBUG: Determining if CSRF header needs to be sent for domain='{domain}',headers()='{len(headers)}' ...") + print(f"DEBUG: Determining if CSRF header needs to be sent for domain='{domain}',headers()='{len(headers)}' ...") headers = csrf.determine(domain, headers) try: - # DEBUG: print(f"DEBUG: Sending GET request to '{domain}{path}' ...") + print(f"DEBUG: Sending GET request to '{domain}{path}' ...") response = reqto.get( f"https://{domain}{path}", headers=headers, @@ -141,9 +140,25 @@ def fetch_response(domain: str, path: str, headers: dict, timeout: list) -> requ ) except requests.exceptions.ConnectionError as exception: - # DEBUG: print(f"DEBUG: Fetching '{path}' from '{domain}' failed. exception[{type(exception)}]='{str(exception)}'") + print(f"DEBUG: Fetching '{path}' from '{domain}' failed. exception[{type(exception)}]='{str(exception)}'") instances.update_last_error(domain, exception) raise exception - # DEBUG: print(f"DEBUG: response[]='{type(response)}' - EXXIT!") + print(f"DEBUG: response[]='{type(response)}' - EXXIT!") return response + +def json_from_response(response: requests.models.Response) -> list: + print(f"DEBUG: response[]={type(response)} - CALLED!") + if not isinstance(response, requests.models.Response): + raise ValueError(f"Parameter response[]='{type(response)}' is not type of 'Response'") + + data = list() + if response.text.strip() != "": + print(f"DEBUG: response.text()={len(response.text)} is not empty, invoking response.json() ...") + try: + data = response.json() + except json.decoder.JSONDecodeError: + pass + + print(f"DEBUG: data[]={type(data)} - EXIT!") + return data diff --git a/fba/networks/__init__.py b/fba/networks/__init__.py new file mode 100644 index 0000000..77902b2 --- /dev/null +++ b/fba/networks/__init__.py @@ -0,0 +1,23 @@ +# Copyright (C) 2023 Free Software Foundation +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +__all__ = [ + 'friendica', + 'lemmy', + 'mastodon', + 'misskey', + 'peertube', + 'pleroma', +] diff --git a/fba/federation/friendica.py b/fba/networks/friendica.py similarity index 92% rename from fba/federation/friendica.py rename to fba/networks/friendica.py index 4dd72fb..ceba93d 100644 --- a/fba/federation/friendica.py +++ b/fba/networks/friendica.py @@ -17,10 +17,10 @@ import bs4 from fba import config -from fba import fba -from fba import instances from fba import network +from fba.helpers import tidyup + def fetch_blocks(domain: str) -> dict: # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") if not isinstance(domain, str): @@ -61,8 +61,8 @@ def fetch_blocks(domain: str) -> dict: for line in rows: # DEBUG: print(f"DEBUG: line='{line}'") blocked.append({ - "domain": fba.tidyup_domain(line.find_all("td")[0].text), - "reason": fba.tidyup_reason(line.find_all("td")[1].text) + "domain": tidyup.domain(line.find_all("td")[0].text), + "reason": tidyup.reason(line.find_all("td")[1].text) }) # DEBUG: print("DEBUG: Next!") diff --git a/fba/federation/lemmy.py b/fba/networks/lemmy.py similarity index 94% rename from fba/federation/lemmy.py rename to fba/networks/lemmy.py index 8e07a83..90d750d 100644 --- a/fba/federation/lemmy.py +++ b/fba/networks/lemmy.py @@ -15,7 +15,7 @@ # along with this program. If not, see . from fba import config -from fba import fba +from fba import federation from fba import instances from fba import network @@ -36,7 +36,7 @@ def fetch_peers(domain: str) -> list: (config.get("connection_timeout"), config.get("read_timeout")) ) - data = fba.json_from_response(response) + data = network.json_from_response(response) # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code='{response.status_code}',data[]='{type(data)}'") if not response.ok or response.status_code >= 400: @@ -46,7 +46,7 @@ def fetch_peers(domain: str) -> list: print(f"UNSUPPORTED: domain='{domain}' returned a list: '{data}'") elif "federated_instances" in data: # DEBUG: print(f"DEBUG: Found federated_instances for domain='{domain}'") - peers = peers + fba.add_peers(data["federated_instances"]) + peers = peers + federation.add_peers(data["federated_instances"]) # DEBUG: print("DEBUG: Added instance(s) to peers") else: print("WARNING: JSON response does not contain 'federated_instances':", domain) diff --git a/fba/federation/mastodon.py b/fba/networks/mastodon.py similarity index 95% rename from fba/federation/mastodon.py rename to fba/networks/mastodon.py index f55a4cd..2dda82f 100644 --- a/fba/federation/mastodon.py +++ b/fba/networks/mastodon.py @@ -25,6 +25,7 @@ from fba import config from fba import fba from fba import instances from fba import network +from fba.helpers import tidyup language_mapping = { # English -> English @@ -83,7 +84,7 @@ def fetch_blocks_from_about(domain: str) -> dict: return {} for header in doc.find_all("h3"): - header_text = fba.tidyup_reason(header.text) + header_text = tidyup.reason(header.text) # DEBUG: print(f"DEBUG: header_text='{header_text}'") if header_text in language_mapping: @@ -97,9 +98,9 @@ def fetch_blocks_from_about(domain: str) -> dict: for line in header.find_all_next("table")[0].find_all("tr")[1:]: blocklist[header_text].append( { - "domain": fba.tidyup_domain(line.find("span").text), - "hash" : fba.tidyup_domain(line.find("span")["title"][9:]), - "reason": fba.tidyup_reason(line.find_all("td")[1].text), + "domain": tidyup.domain(line.find("span").text), + "hash" : tidyup.domain(line.find("span")["title"][9:]), + "reason": tidyup.reason(line.find_all("td")[1].text), } ) else: @@ -147,7 +148,7 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): ) # DEBUG: print(f"DEBUG: response[]='{type(response)}'") - blocklist = fba.json_from_response(response) + blocklist = network.json_from_response(response) print(f"INFO: Checking {len(blocklist)} entries from domain='{domain}',software='mastodon' ...") for block in blocklist: @@ -180,7 +181,7 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): print(f"INFO: Checking {len(rows.items())} entries from domain='{domain}',software='mastodon' ...") for block_level, blocklist in rows.items(): # DEBUG: print("DEBUG: domain,block_level,blocklist():", domain, block_level, len(blocklist)) - block_level = fba.tidyup_domain(block_level) + block_level = tidyup.domain(block_level) # DEBUG: print("DEBUG: AFTER-block_level:", block_level) if block_level == "": @@ -192,8 +193,8 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): # DEBUG: print(f"DEBUG: block[]='{type(block)}'") blocked, blocked_hash, reason = block.values() # DEBUG: print(f"DEBUG: blocked='{blocked}',blocked_hash='{blocked_hash}',reason='{reason}':") - blocked = fba.tidyup_domain(blocked) - reason = fba.tidyup_reason(reason) if reason is not None and reason != "" else None + blocked = tidyup.domain(blocked) + reason = tidyup.reason(reason) if reason is not None and reason != "" else None # DEBUG: print(f"DEBUG: blocked='{blocked}',reason='{reason}' - AFTER!") if blocked == "": diff --git a/fba/federation/misskey.py b/fba/networks/misskey.py similarity index 96% rename from fba/federation/misskey.py rename to fba/networks/misskey.py index 9e5c9af..7b28b8d 100644 --- a/fba/federation/misskey.py +++ b/fba/networks/misskey.py @@ -18,10 +18,12 @@ import json from fba import blacklist from fba import config -from fba import fba from fba import instances from fba import network +from fba.helpers import dicts +from fba.helpers import tidyup + def fetch_peers(domain: str) -> list: # DEBUG: print(f"DEBUG: domain({len(domain)})={domain} - CALLED!") if not isinstance(domain, str): @@ -165,11 +167,11 @@ def fetch_blocks(domain: str) -> dict: count = 0 for instance in fetched: # Is it there? - if instance["isSuspended"] and not fba.has_key(blocklist["suspended"], "domain", instance): + if instance["isSuspended"] and not dicts.has_key(blocklist["suspended"], "domain", instance): count = count + 1 blocklist["suspended"].append( { - "domain": fba.tidyup_domain(instance["host"]), + "domain": tidyup.domain(instance["host"]), # no reason field, nothing "reason": None } @@ -225,10 +227,10 @@ def fetch_blocks(domain: str) -> dict: count = 0 for instance in fetched: # Is it there? - if instance["isBlocked"] and not fba.has_key(blocklist["blocked"], "domain", instance): + if instance["isBlocked"] and not dicts.has_key(blocklist["blocked"], "domain", instance): count = count + 1 blocklist["blocked"].append({ - "domain": fba.tidyup_domain(instance["host"]), + "domain": tidyup.domain(instance["host"]), "reason": None }) diff --git a/fba/federation/peertube.py b/fba/networks/peertube.py similarity index 97% rename from fba/federation/peertube.py rename to fba/networks/peertube.py index 2d2ebaf..da84baf 100644 --- a/fba/federation/peertube.py +++ b/fba/networks/peertube.py @@ -15,7 +15,6 @@ # along with this program. If not, see . from fba import config -from fba import fba from fba import instances from fba import network @@ -40,7 +39,7 @@ def fetch_peers(domain: str) -> list: (config.get("connection_timeout"), config.get("read_timeout")) ) - data = fba.json_from_response(response) + data = network.json_from_response(response) # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code='{response.status_code}',data[]='{type(data)}'") if response.ok and isinstance(data, dict): # DEBUG: print("DEBUG: Success, data:", len(data)) diff --git a/fba/federation/pleroma.py b/fba/networks/pleroma.py similarity index 88% rename from fba/federation/pleroma.py rename to fba/networks/pleroma.py index 5e3bbf1..b997dc6 100644 --- a/fba/federation/pleroma.py +++ b/fba/networks/pleroma.py @@ -20,7 +20,9 @@ import validators from fba import blacklist from fba import blocks from fba import fba +from fba import federation from fba import instances +from fba.helpers import tidyup def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): # DEBUG: print(f"DEBUG: domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}' - CALLED!") @@ -40,34 +42,34 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): try: # Blocks blockdict = list() - rows = fba.fetch_nodeinfo(domain, nodeinfo_url) + rows = federation.fetch_nodeinfo(domain, nodeinfo_url) if rows is None: print("WARNING: Could not fetch nodeinfo from domain:", domain) return - elif not "metadata" in rows: + elif "metadata" not in rows: print(f"WARNING: rows()={len(rows)} does not have key 'metadata', domain='{domain}'") return - elif not "federation" in rows["metadata"]: + elif "federation" not in rows["metadata"]: print(f"WARNING: rows()={len(rows['metadata'])} does not have key 'federation', domain='{domain}'") return # DEBUG: print("DEBUG: Updating nodeinfo:", domain) instances.update_last_nodeinfo(domain) - federation = rows["metadata"]["federation"] + data = rows["metadata"]["federation"] - if "enabled" in federation: + if "enabled" in data: # DEBUG: print("DEBUG: Instance has no block list to analyze:", domain) return - if "mrf_simple" in federation: + if "mrf_simple" in data: for block_level, blocklist in ( - {**federation["mrf_simple"], - **{"quarantined_instances": federation["quarantined_instances"]}} + {**data["mrf_simple"], + **{"quarantined_instances": data["quarantined_instances"]}} ).items(): # DEBUG: print("DEBUG: block_level, blocklist():", block_level, len(blocklist)) - block_level = fba.tidyup_domain(block_level) + block_level = tidyup.domain(block_level) # DEBUG: print("DEBUG: BEFORE block_level:", block_level) if block_level == "": @@ -77,11 +79,11 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): # DEBUG: print(f"DEBUG: Checking {len(blocklist)} entries from domain='{domain}',block_level='{block_level}' ...") for blocked in blocklist: # DEBUG: print("DEBUG: BEFORE blocked:", blocked) - blocked = fba.tidyup_domain(blocked) + blocked = tidyup.domain(blocked) # DEBUG: print("DEBUG: AFTER blocked:", blocked) if blocked == "": - print("WARNING: blocked is empty after fba.tidyup_domain():", domain, block_level) + print("WARNING: blocked is empty after tidyup.domain():", domain, block_level) continue elif blacklist.is_blacklisted(blocked): # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!") @@ -132,16 +134,16 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): fba.connection.commit() # Reasons - if "mrf_simple_info" in federation: + if "mrf_simple_info" in data: # DEBUG: print("DEBUG: Found mrf_simple_info:", domain) for block_level, info in ( - {**federation["mrf_simple_info"], - **(federation["quarantined_instances_info"] - if "quarantined_instances_info" in federation + {**data["mrf_simple_info"], + **(data["quarantined_instances_info"] + if "quarantined_instances_info" in data else {})} ).items(): # DEBUG: print("DEBUG: block_level, info.items():", block_level, len(info.items())) - block_level = fba.tidyup_domain(block_level) + block_level = tidyup.domain(block_level) # DEBUG: print("DEBUG: BEFORE block_level:", block_level) if block_level == "": @@ -151,12 +153,12 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): # DEBUG: print(f"DEBUG: Checking {len(info.items())} entries from domain='{domain}',software='pleroma',block_level='{block_level}' ...") for blocked, reason in info.items(): # DEBUG: print(f"DEBUG: blocked='{blocked}',reason='{reason}' - BEFORE!") - blocked = fba.tidyup_domain(blocked) - reason = fba.tidyup_reason(reason) if reason is not None and reason != "" else None + blocked = tidyup.domain(blocked) + reason = tidyup.reason(reason) if reason is not None and reason != "" else None # DEBUG: print(f"DEBUG: blocked='{blocked}',reason='{reason}' - AFTER!") if blocked == "": - print("WARNING: blocked is empty after fba.tidyup_domain():", domain, block_level) + print("WARNING: blocked is empty after tidyup.domain():", domain, block_level) continue elif not validators.domain(blocked): print(f"WARNING: blocked='{blocked}',software='pleroma' is not a valid domain name - skipped!") -- 2.39.5