From: Roland Häder Date: Sat, 3 Jun 2023 06:15:19 +0000 (+0200) Subject: Continued: X-Git-Url: https://git.mxchange.org/?a=commitdiff_plain;h=ad0e8ee96f0dd23c04e491b01a23000b63cab854;p=fba.git Continued: - moved 'fba' to own folder - splitted up file a bit: boot, cache --- diff --git a/api.py b/api.py index b37e555..871398e 100644 --- a/api.py +++ b/api.py @@ -25,7 +25,7 @@ import fastapi import uvicorn import requests import re -import fba +from fba import * router = fastapi.FastAPI(docs_url=fba.config["base_url"] + "/docs", redoc_url=fba.config["base_url"] + "/redoc") templates = Jinja2Templates(directory="templates") diff --git a/fba.py b/fba.py deleted file mode 100644 index c8a088c..0000000 --- a/fba.py +++ /dev/null @@ -1,1573 +0,0 @@ -# Fedi API Block - An aggregator for fetching blocking data from fediverse nodes -# Copyright (C) 2023 Free Software Foundation -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published -# by the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -import bs4 -import hashlib -import re -import reqto -import json -import os -import sqlite3 -import sys -import tempfile -import time -import validators -import zc.lockfile - -with open("config.json") as f: - config = json.loads(f.read()) - -# Don't check these, known trolls/flooders/testing/developing -blacklist = [ - # Floods network with fake nodes as "research" project - "activitypub-troll.cf", - # Similar troll - "gab.best", - # Similar troll - "4chan.icu", - # Flooder (?) - "social.shrimpcam.pw", - # Flooder (?) - "mastotroll.netz.org", - # Testing/developing installations - "ngrok.io", - "ngrok-free.app", - "misskeytest.chn.moe", -] - -# Array with pending errors needed to be written to database -pending_errors = { -} - -# "rel" identifiers (no real URLs) -nodeinfo_identifier = [ - "https://nodeinfo.diaspora.software/ns/schema/2.1", - "https://nodeinfo.diaspora.software/ns/schema/2.0", - "https://nodeinfo.diaspora.software/ns/schema/1.1", - "https://nodeinfo.diaspora.software/ns/schema/1.0", - "http://nodeinfo.diaspora.software/ns/schema/2.1", - "http://nodeinfo.diaspora.software/ns/schema/2.0", - "http://nodeinfo.diaspora.software/ns/schema/1.1", - "http://nodeinfo.diaspora.software/ns/schema/1.0", -] - -# HTTP headers for non-API requests -headers = { - "User-Agent": config["useragent"], -} -# HTTP headers for API requests -api_headers = { - "User-Agent": config["useragent"], - "Content-Type": "application/json", -} - -# Found info from node, such as nodeinfo URL, detection mode that needs to be -# written to database. Both arrays must be filled at the same time or else -# update_instance_data() will fail -instance_data = { - # Detection mode: 'AUTO_DISCOVERY', 'STATIC_CHECKS' or 'GENERATOR' - # NULL means all detection methods have failed (maybe still reachable instance) - "detection_mode" : {}, - # Found nodeinfo URL - "nodeinfo_url" : {}, - # Found total peers - "total_peers" : {}, - # Last fetched instances - "last_instance_fetch": {}, - # Last updated - "last_updated" : {}, - # Last blocked - "last_blocked" : {}, - # Last nodeinfo (fetched) - "last_nodeinfo" : {}, - # Last status code - "last_status_code" : {}, - # Last error details - "last_error_details" : {}, -} - -language_mapping = { - # English -> English - "Silenced instances" : "Silenced servers", - "Suspended instances" : "Suspended servers", - "Limited instances" : "Limited servers", - # Mappuing German -> English - "Gesperrte Server" : "Suspended servers", - "Gefilterte Medien" : "Filtered media", - "Stummgeschaltete Server" : "Silenced servers", - # Japanese -> English - "停止済みのサーバー" : "Suspended servers", - "制限中のサーバー" : "Limited servers", - "メディアを拒否しているサーバー": "Filtered media", - "サイレンス済みのサーバー" : "Silenced servers", - # ??? -> English - "שרתים מושעים" : "Suspended servers", - "מדיה מסוננת" : "Filtered media", - "שרתים מוגבלים" : "Silenced servers", - # French -> English - "Serveurs suspendus" : "Suspended servers", - "Médias filtrés" : "Filtered media", - "Serveurs limités" : "Limited servers", - "Serveurs modérés" : "Limited servers", -} - -# URL for fetching peers -get_peers_url = "/api/v1/instance/peers" - -# Cache for redundant SQL queries -cache = {} - -# Connect to database -connection = sqlite3.connect("blocks.db") -cursor = connection.cursor() - -# Pattern instance for version numbers -patterns = [ - # semantic version number (with v|V) prefix) - re.compile("^(?Pv|V{0,1})(\.{0,1})(?P0|[1-9]\d*)\.(?P0+|[1-9]\d*)(\.(?P0+|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)?$"), - # non-sematic, e.g. 1.2.3.4 - re.compile("^(?Pv|V{0,1})(\.{0,1})(?P0|[1-9]\d*)\.(?P0+|[1-9]\d*)(\.(?P0+|[1-9]\d*)(\.(?P0|[1-9]\d*))?)$"), - # non-sematic, e.g. 2023-05[-dev] - re.compile("^(?P[1-9]{1}[0-9]{3})\.(?P[0-9]{2})(-dev){0,1}$"), - # non-semantic, e.g. abcdef0 - re.compile("^[a-f0-9]{7}$"), -] - -# Lock file -lockfile = tempfile.gettempdir() + '/.' + __name__ + '.lock' -LOCK = None - -##### Cache ##### - -def is_cache_initialized(key: str) -> bool: - return key in cache - -def set_all_cache_key(key: str, rows: list, value: any): - # NOISY-DEBUG: print(f"DEBUG: key='{key}',rows()={len(rows)},value[]={type(value)} - CALLED!") - if type(key) != str: - raise ValueError("Parameter key[]='{type(key)}' is not 'str'") - elif not is_cache_initialized(key): - # NOISY-DEBUG: print(f"DEBUG: Cache for key='{key}' not initialized.") - cache[key] = {} - - for sub in rows: - # NOISY-DEBUG: print(f"DEBUG: Setting key='{key}',sub[{type(sub)}]='{sub}'") - - if isinstance(sub, tuple): - cache[key][sub[0]] = value - else: - print(f"WARNING: Unsupported type row[]='{type(row)}'") - - # NOISY-DEBUG: print("DEBUG: EXIT!") - -def set_cache_key(key: str, sub: str, value: any): - if type(key) != str: - raise ValueError("Parameter key[]='{type(key)}' is not 'str'") - elif type(sub) != str: - raise ValueError("Parameter sub[]='{type(sub)}' is not 'str'") - elif not is_cache_initialized(key): - print(f"WARNING: Bad method call, key='{key}' is not initialized yet.") - raise Exception(f"Cache for key='{key}' is not initialized, but function called") - - cache[key][sub] = value - -def is_cache_key_set(key: str, sub: str) -> bool: - if type(key) != str: - raise ValueError("Parameter key[]='{type(key)}' is not 'str'") - elif type(sub) != str: - raise ValueError("Parameter sub[]='{type(sub)}' is not 'str'") - elif not is_cache_initialized(key): - print(f"WARNING: Bad method call, key='{key}' is not initialized yet.") - raise Exception(f"Cache for key='{key}' is not initialized, but function called") - - return sub in cache[key] - -##### Other functions ##### - -def is_primitive(var: any) -> bool: - # NOISY-DEBUG: print(f"DEBUG: var[]='{type(var)}' - CALLED!") - return type(var) in {int, str, float, bool} or var == None - -def fetch_instances(domain: str, origin: str, software: str, script: str, path: str = None): - if type(domain) != str: - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") - elif type(origin) != str and origin != None: - raise ValueError(f"Parameter origin[]={type(origin)} is not 'str'") - elif type(script) != str: - raise ValueError(f"Parameter script[]={type(script)} is not 'str'") - elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") - - # DEBUG: print("DEBUG: domain,origin,software,path:", domain, origin, software, path) - if not is_instance_registered(domain): - # DEBUG: print("DEBUG: Adding new domain:", domain, origin) - add_instance(domain, origin, script, path) - - # DEBUG: print("DEBUG: Fetching instances for domain:", domain, software) - peerlist = get_peers(domain, software) - - if (peerlist is None): - print("ERROR: Cannot fetch peers:", domain) - return - elif has_pending_instance_data(domain): - # DEBUG: print(f"DEBUG: domain='{domain}' has pending nodeinfo data, flushing ...") - update_instance_data(domain) - - print(f"INFO: Checking {len(peerlist)} instances from {domain} ...") - for instance in peerlist: - if instance == None: - # Skip "None" types as tidup() cannot parse them - continue - - # DEBUG: print(f"DEBUG: instance[{type(instance}]={instance} - BEFORE") - instance = tidyup(instance) - # DEBUG: print(f"DEBUG: instance[{type(instance}]={instance} - AFTER") - - if instance == "": - print("WARNING: Empty instance after tidyup(), domain:", domain) - continue - elif not validators.domain(instance.split("/")[0]): - print(f"WARNING: Bad instance='{instance}' from domain='{domain}',origin='{origin}',software='{software}'") - continue - elif is_blacklisted(instance): - # DEBUG: print("DEBUG: instance is blacklisted:", instance) - continue - - # DEBUG: print("DEBUG: Handling instance:", instance) - try: - if not is_instance_registered(instance): - # DEBUG: print("DEBUG: Adding new instance:", instance, domain) - add_instance(instance, domain, sys.argv[0]) - except BaseException as e: - print(f"ERROR: instance='{instance}',exception[{type(e)}]:'{str(e)}'") - continue - - # DEBUG: print("DEBUG: EXIT!") - -def set_instance_data(key: str, domain: str, value: any): - # NOISY-DEBUG: print(f"DEBUG: key='{key}',domain='{domain}',value[]='{type(value)}' - CALLED!") - if type(key) != str: - raise ValueError("Parameter key[]='{type(key)}' is not 'str'") - elif key == "": - raise ValueError(f"Parameter 'key' cannot be empty") - elif type(domain) != str: - raise ValueError("Parameter domain[]='{type(domain)}' is not 'str'") - elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") - elif not key in instance_data: - raise ValueError(f"key='{key}' not found in instance_data") - elif not is_primitive(value): - raise ValueError(f"value[]='{type(value)}' is not a primitive type") - - # Set it - instance_data[key][domain] = value - - # DEBUG: print("DEBUG: EXIT!") - -def add_peers(rows: dict) -> list: - # DEBUG: print(f"DEBUG: rows()={len(rows)} - CALLED!") - peers = list() - for element in ["linked", "allowed", "blocked"]: - # DEBUG: print(f"DEBUG: Checking element='{element}'") - if element in rows and rows[element] != None: - # DEBUG: print(f"DEBUG: Adding {len(rows[element])} peer(s) to peers list ...") - for peer in rows[element]: - # DEBUG: print(f"DEBUG: peer='{peer}' - BEFORE!") - peer = tidyup(peer) - - # DEBUG: print(f"DEBUG: peer='{peer}' - AFTER!") - if is_blacklisted(peer): - # DEBUG: print(f"DEBUG: peer='{peer}' is blacklisted, skipped!") - continue - - # DEBUG: print(f"DEBUG: Adding peer='{peer}' ...") - peers.append(peer) - - # DEBUG: print(f"DEBUG: peers()={len(peers)} - EXIT!") - return peers - -def remove_version(software: str) -> str: - # DEBUG: print(f"DEBUG: software='{software}' - CALLED!") - if not "." in software and " " not in software: - print(f"WARNING: software='{software}' does not contain a version number.") - return software - - temp = software - if ";" in software: - temp = software.split(";")[0] - elif "," in software: - temp = software.split(",")[0] - elif " - " in software: - temp = software.split(" - ")[0] - - # DEBUG: print(f"DEBUG: software='{software}'") - version = None - if " " in software: - version = temp.split(" ")[-1] - elif "/" in software: - version = temp.split("/")[-1] - elif "-" in software: - version = temp.split("-")[-1] - else: - # DEBUG: print(f"DEBUG: Was not able to find common seperator, returning untouched software='{software}'") - return software - - matches = None - match = None - # DEBUG: print(f"DEBUG: Checking {len(patterns)} patterns ...") - for pattern in patterns: - # Run match() - match = pattern.match(version) - - # DEBUG: print(f"DEBUG: match[]={type(match)}") - if type(match) is re.Match: - break - - # DEBUG: print(f"DEBUG: version[{type(version)}]='{version}',match='{match}'") - if type(match) is not re.Match: - print(f"WARNING: version='{version}' does not match regex, leaving software='{software}' untouched.") - return software - - # DEBUG: print(f"DEBUG: Found valid version number: '{version}', removing it ...") - end = len(temp) - len(version) - 1 - - # DEBUG: print(f"DEBUG: end[{type(end)}]={end}") - software = temp[0:end].strip() - if " version" in software: - # DEBUG: print(f"DEBUG: software='{software}' contains word ' version'") - software = strip_until(software, " version") - - # DEBUG: print(f"DEBUG: software='{software}' - EXIT!") - return software - -def strip_powered_by(software: str) -> str: - # DEBUG: print(f"DEBUG: software='{software}' - CALLED!") - if software == "": - print(f"ERROR: Bad method call, 'software' is empty") - raise Exception("Parameter 'software' is empty") - elif not "powered by" in software: - print(f"WARNING: Cannot find 'powered by' in '{software}'!") - return software - - start = software.find("powered by ") - # DEBUG: print(f"DEBUG: start[{type(start)}]='{start}'") - - software = software[start + 11:].strip() - # DEBUG: print(f"DEBUG: software='{software}'") - - software = strip_until(software, " - ") - - # DEBUG: print(f"DEBUG: software='{software}' - EXIT!") - return software - -def strip_hosted_on(software: str) -> str: - # DEBUG: print(f"DEBUG: software='{software}' - CALLED!") - if software == "": - print(f"ERROR: Bad method call, 'software' is empty") - raise Exception("Parameter 'software' is empty") - elif not "hosted on" in software: - print(f"WARNING: Cannot find 'hosted on' in '{software}'!") - return software - - end = software.find("hosted on ") - # DEBUG: print(f"DEBUG: end[{type(end)}]='{end}'") - - software = software[0, start].strip() - # DEBUG: print(f"DEBUG: software='{software}'") - - software = strip_until(software, " - ") - - # DEBUG: print(f"DEBUG: software='{software}' - EXIT!") - return software - -def strip_until(software: str, until: str) -> str: - # DEBUG: print(f"DEBUG: software='{software}',until='{until}' - CALLED!") - if software == "": - print(f"ERROR: Bad method call, 'software' is empty") - raise Exception("Parameter 'software' is empty") - elif until == "": - print(f"ERROR: Bad method call, 'until' is empty") - raise Exception("Parameter 'until' is empty") - elif not until in software: - print(f"WARNING: Cannot find '{until}' in '{software}'!") - return software - - # Next, strip until part - end = software.find(until) - - # DEBUG: print(f"DEBUG: end[{type(end)}]='{end}'") - if end > 0: - software = software[0:end].strip() - - # DEBUG: print(f"DEBUG: software='{software}' - EXIT!") - return software - -def is_blacklisted(domain: str) -> bool: - if type(domain) != str: - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") - - blacklisted = False - for peer in blacklist: - if peer in domain: - blacklisted = True - - return blacklisted - -def remove_pending_error(domain: str): - if type(domain) != str: - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") - - try: - # Prevent updating any pending errors, nodeinfo was found - del pending_errors[domain] - - except: - pass - - # DEBUG: print("DEBUG: EXIT!") - -def get_hash(domain: str) -> str: - if type(domain) != str: - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") - - return hashlib.sha256(domain.encode("utf-8")).hexdigest() - -def update_last_blocked(domain: str): - if type(domain) != str: - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") - - # DEBUG: print("DEBUG: Updating last_blocked for domain", domain) - set_instance_data("last_blocked", domain, time.time()) - - # Running pending updated - # DEBUG: print(f"DEBUG: Invoking update_instance_data({domain}) ...") - update_instance_data(domain) - - # DEBUG: print("DEBUG: EXIT!") - -def has_pending_instance_data(domain: str) -> bool: - # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") - if type(domain) != str: - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") - - has_pending = False - for key in instance_data: - # DEBUG: print(f"DEBUG: key='{key}',domain='{domain}',instance_data[key]()='{len(instance_data[key])}'") - if domain in instance_data[key]: - has_pending = True - break - - # DEBUG: print(f"DEBUG: has_pending='{has_pending}' - EXIT!") - return has_pending - -def update_instance_data(domain: str): - if type(domain) != str: - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") - - # DEBUG: print(f"DEBUG: Updating nodeinfo for domain='{domain}' ...") - sql_string = '' - fields = list() - for key in instance_data: - # DEBUG: print("DEBUG: key:", key) - if domain in instance_data[key]: - # DEBUG: print(f"DEBUG: Adding '{instance_data[key][domain]}' for key='{key}' ...") - fields.append(instance_data[key][domain]) - sql_string += f" {key} = ?," - - fields.append(domain) - - if sql_string == '': - raise ValueError(f"No fields have been set, but method invoked, domain='{domain}'") - - # DEBUG: print(f"DEBUG: sql_string='{sql_string}',fields()={len(fields)}") - sql_string = "UPDATE instances SET" + sql_string + " last_updated = TIME() WHERE domain = ? LIMIT 1" - # DEBUG: print("DEBUG: sql_string:", sql_string) - - try: - # DEBUG: print("DEBUG: Executing SQL:", sql_string) - cursor.execute(sql_string, fields) - - # DEBUG: print(f"DEBUG: Success! (rowcount={cursor.rowcount })") - if cursor.rowcount == 0: - print(f"WARNING: Did not update any rows: domain='{domain}',fields()={len(fields)} - EXIT!") - return - - connection.commit() - - # DEBUG: print("DEBUG: Deleting instance_data for domain:", domain) - for key in instance_data: - try: - # DEBUG: print("DEBUG: Deleting key:", key) - del instance_data[key][domain] - except: - pass - - except BaseException as e: - print(f"ERROR: failed SQL query: domain='{domain}',sql_string='{sql_string}',exception[{type(e)}]:'{str(e)}'") - sys.exit(255) - - # DEBUG: print("DEBUG: EXIT!") - -def log_error(domain: str, res: any): - # DEBUG: print("DEBUG: domain,res[]:", domain, type(res)) - if type(domain) != str: - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") - - try: - # DEBUG: print("DEBUG: BEFORE res[]:", type(res)) - if isinstance(res, BaseException) or isinstance(res, json.JSONDecodeError): - res = str(res) - - # DEBUG: print("DEBUG: AFTER res[]:", type(res)) - if type(res) is str: - cursor.execute("INSERT INTO error_log (domain, error_code, error_message, created) VALUES (?, 999, ?, ?)",[ - domain, - res, - time.time() - ]) - else: - cursor.execute("INSERT INTO error_log (domain, error_code, error_message, created) VALUES (?, ?, ?, ?)",[ - domain, - res.status_code, - res.reason, - time.time() - ]) - - # Cleanup old entries - # DEBUG: print(f"DEBUG: Purging old records (distance: {config['error_log_cleanup']})") - cursor.execute("DELETE FROM error_log WHERE created < ?", [time.time() - config["error_log_cleanup"]]) - except BaseException as e: - print(f"ERROR: failed SQL query: domain='{domain}',exception[{type(e)}]:'{str(e)}'") - sys.exit(255) - - # DEBUG: print("DEBUG: EXIT!") - -def update_last_error(domain: str, res: any): - # DEBUG: print("DEBUG: domain,res[]:", domain, type(res)) - if type(domain) != str: - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") - - # DEBUG: print("DEBUG: BEFORE res[]:", type(res)) - if isinstance(res, BaseException) or isinstance(res, json.JSONDecodeError): - res = str(res) - - # DEBUG: print("DEBUG: AFTER res[]:", type(res)) - if type(res) is str: - # DEBUG: print(f"DEBUG: Setting last_error_details='{res}'"); - set_instance_data("last_status_code" , domain, 999) - set_instance_data("last_error_details", domain, res) - else: - # DEBUG: print(f"DEBUG: Setting last_error_details='{res.reason}'"); - set_instance_data("last_status_code" , domain, res.status_code) - set_instance_data("last_error_details", domain, res.reason) - - # Running pending updated - # DEBUG: print(f"DEBUG: Invoking update_instance_data({domain}) ...") - update_instance_data(domain) - - log_error(domain, res) - - # DEBUG: print("DEBUG: EXIT!") - -def update_last_instance_fetch(domain: str): - if type(domain) != str: - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") - - # DEBUG: print("DEBUG: Updating last_instance_fetch for domain:", domain) - set_instance_data("last_instance_fetch", domain, time.time()) - - # Running pending updated - # DEBUG: print(f"DEBUG: Invoking update_instance_data({domain}) ...") - update_instance_data(domain) - - # DEBUG: print("DEBUG: EXIT!") - -def update_last_nodeinfo(domain: str): - if type(domain) != str: - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") - - # DEBUG: print("DEBUG: Updating last_nodeinfo for domain:", domain) - set_instance_data("last_nodeinfo", domain, time.time()) - set_instance_data("last_updated" , domain, time.time()) - - # Running pending updated - # DEBUG: print(f"DEBUG: Invoking update_instance_data({domain}) ...") - update_instance_data(domain) - - # DEBUG: print("DEBUG: EXIT!") - -def get_peers(domain: str, software: str) -> list: - if type(domain) != str: - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") - elif type(software) != str and software != None: - raise ValueError(f"software[]={type(software)} is not 'str'") - - # DEBUG: print(f"DEBUG: domain='{domain}',software='{software}' - CALLED!") - peers = list() - - if software == "misskey": - # DEBUG: print(f"DEBUG: domain='{domain}' is misskey, sending API POST request ...") - offset = 0 - step = config["misskey_offset"] - - # iterating through all "suspended" (follow-only in its terminology) - # instances page-by-page, since that troonware doesn't support - # sending them all at once - while True: - # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...") - if offset == 0: - fetched = post_json_api(domain, "/api/federation/instances", json.dumps({ - "sort" : "+pubAt", - "host" : None, - "limit": step - }), {"Origin": domain}) - else: - fetched = post_json_api(domain, "/api/federation/instances", json.dumps({ - "sort" : "+pubAt", - "host" : None, - "limit" : step, - "offset": offset - 1 - }), {"Origin": domain}) - - # DEBUG: print("DEBUG: fetched():", len(fetched)) - if len(fetched) == 0: - # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain) - break - elif len(fetched) != config["misskey_offset"]: - # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config['misskey_offset']}'") - offset = offset + (config["misskey_offset"] - len(fetched)) - else: - # DEBUG: print("DEBUG: Raising offset by step:", step) - offset = offset + step - - # Check records - # DEBUG: print(f"DEBUG: fetched({len(fetched)})[]={type(fetched)}") - if isinstance(fetched, dict) and "error" in fetched and "message" in fetched["error"]: - print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}") - update_last_error(domain, fetched["error"]["message"]) - break - - for row in fetched: - # DEBUG: print(f"DEBUG: row():{len(row)}") - if not "host" in row: - print(f"WARNING: row()={len(row)} does not contain element 'host': {row},domain='{domain}'") - continue - elif type(row["host"]) != str: - print(f"WARNING: row[host][]={type(row['host'])} is not 'str'") - continue - elif is_blacklisted(row["host"]): - # DEBUG: print(f"DEBUG: row[host]='{row['host']}' is blacklisted. domain='{domain}'") - continue - - # DEBUG: print(f"DEBUG: Adding peer: '{row['host']}'") - peers.append(row["host"]) - - # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'") - set_instance_data("total_peers", domain, len(peers)) - - # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") - update_last_instance_fetch(domain) - - # DEBUG: print("DEBUG: Returning peers[]:", type(peers)) - return peers - elif software == "lemmy": - # DEBUG: print(f"DEBUG: domain='{domain}' is Lemmy, fetching JSON ...") - try: - res = reqto.get(f"https://{domain}/api/v3/site", headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"])) - - data = res.json() - # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code='{res.status_code}',data[]='{type(data)}'") - if not res.ok or res.status_code >= 400: - print("WARNING: Could not reach any JSON API:", domain) - update_last_error(domain, res) - elif res.ok and isinstance(data, list): - # DEBUG: print(f"DEBUG: domain='{domain}' returned a list: '{data}'") - sys.exit(255) - elif "federated_instances" in data: - # DEBUG: print(f"DEBUG: Found federated_instances for domain='{domain}'") - peers = peers + add_peers(data["federated_instances"]) - # DEBUG: print("DEBUG: Added instance(s) to peers") - else: - print("WARNING: JSON response does not contain 'federated_instances':", domain) - update_last_error(domain, res) - - except BaseException as e: - print(f"WARNING: Exception during fetching JSON: domain='{domain}',exception[{type(e)}]:'{str(e)}'") - - # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'") - set_instance_data("total_peers", domain, len(peers)) - - # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") - update_last_instance_fetch(domain) - - # DEBUG: print("DEBUG: Returning peers[]:", type(peers)) - return peers - elif software == "peertube": - # DEBUG: print(f"DEBUG: domain='{domain}' is a PeerTube, fetching JSON ...") - - start = 0 - for mode in ["followers", "following"]: - # DEBUG: print(f"DEBUG: domain='{domain}',mode='{mode}'") - while True: - try: - res = reqto.get(f"https://{domain}/api/v1/server/{mode}?start={start}&count=100", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])) - - data = res.json() - # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code='{res.status_code}',data[]='{type(data)}'") - if res.ok and isinstance(data, dict): - # DEBUG: print("DEBUG: Success, data:", len(data)) - if "data" in data: - # DEBUG: print(f"DEBUG: Found {len(data['data'])} record(s).") - for record in data["data"]: - # DEBUG: print(f"DEBUG: record()={len(record)}") - if mode in record and "host" in record[mode]: - # DEBUG: print(f"DEBUG: Found host={record[mode]['host']}, adding ...") - peers.append(record[mode]["host"]) - else: - print(f"WARNING: record from '{domain}' has no '{mode}' or 'host' record: {record}") - - if len(data["data"]) < 100: - # DEBUG: print("DEBUG: Reached end of JSON response:", domain) - break - - # Continue with next row - start = start + 100 - - except BaseException as e: - print(f"WARNING: Exception during fetching JSON: domain='{domain}',exception[{type(e)}]:'{str(e)}'") - - # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'") - set_instance_data("total_peers", domain, len(peers)) - - # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") - update_last_instance_fetch(domain) - - # DEBUG: print("DEBUG: Returning peers[]:", type(peers)) - return peers - - # DEBUG: print(f"DEBUG: Fetching get_peers_url='{get_peers_url}' from '{domain}' ...") - try: - res = reqto.get(f"https://{domain}{get_peers_url}", headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"])) - - data = res.json() - # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'") - if not res.ok or res.status_code >= 400: - # DEBUG: print(f"DEBUG: Was not able to fetch '{get_peers_url}', trying alternative ...") - res = reqto.get(f"https://{domain}/api/v3/site", headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"])) - - data = res.json() - # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'") - if not res.ok or res.status_code >= 400: - print("WARNING: Could not reach any JSON API:", domain) - update_last_error(domain, res) - elif res.ok and isinstance(data, list): - # DEBUG: print(f"DEBUG: domain='{domain}' returned a list: '{data}'") - sys.exit(255) - elif "federated_instances" in data: - # DEBUG: print(f"DEBUG: Found federated_instances for domain='{domain}'") - peers = peers + add_peers(data["federated_instances"]) - # DEBUG: print("DEBUG: Added instance(s) to peers") - else: - print("WARNING: JSON response does not contain 'federated_instances':", domain) - update_last_error(domain, res) - else: - # DEBUG: print("DEBUG: Querying API was successful:", domain, len(data)) - peers = data - - except BaseException as e: - print("WARNING: Some error during get():", domain, e) - update_last_error(domain, e) - - # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'") - set_instance_data("total_peers", domain, len(peers)) - - # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") - update_last_instance_fetch(domain) - - # DEBUG: print("DEBUG: Returning peers[]:", type(peers)) - return peers - -def post_json_api(domain: str, path: str, parameter: str, extra_headers: dict = {}) -> dict: - if type(domain) != str: - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") - elif type(path) != str: - raise ValueError(f"path[]={type(path)} is not 'str'") - elif path == "": - raise ValueError(f"path cannot be empty") - elif type(parameter) != str: - raise ValueError(f"parameter[]={type(parameter)} is not 'str'") - - # DEBUG: print("DEBUG: Sending POST to domain,path,parameter:", domain, path, parameter, extra_headers) - data = {} - try: - res = reqto.post(f"https://{domain}{path}", data=parameter, headers={**api_headers, **extra_headers}, timeout=(config["connection_timeout"], config["read_timeout"])) - - data = res.json() - # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'") - if not res.ok or res.status_code >= 400: - print(f"WARNING: Cannot query JSON API: domain='{domain}',path='{path}',parameter()={len(parameter)},res.status_code='{res.status_code}',data[]='{type(data)}'") - update_last_error(domain, res) - - except BaseException as e: - print(f"WARNING: Some error during post(): domain='{domain}',path='{path}',parameter()={len(parameter)},exception[{type(e)}]:'{str(e)}'") - - # DEBUG: print(f"DEBUG: Returning data({len(data)})=[]:{type(data)}") - return data - -def fetch_nodeinfo(domain: str, path: str = None) -> list: - if type(domain) != str: - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") - - # DEBUG: print("DEBUG: Fetching nodeinfo from domain,path:", domain, path) - - nodeinfo = fetch_wellknown_nodeinfo(domain) - # DEBUG: print("DEBUG: nodeinfo:", nodeinfo) - - if len(nodeinfo) > 0: - # DEBUG: print("DEBUG: Returning auto-discovered nodeinfo:", len(nodeinfo)) - return nodeinfo - - requests = [ - f"https://{domain}/nodeinfo/2.1.json", - f"https://{domain}/nodeinfo/2.1", - f"https://{domain}/nodeinfo/2.0.json", - f"https://{domain}/nodeinfo/2.0", - f"https://{domain}/nodeinfo/1.0", - f"https://{domain}/api/v1/instance" - ] - - data = {} - for request in requests: - if path != None and path != "" and request != path: - # DEBUG: print(f"DEBUG: path='{path}' does not match request='{request}' - SKIPPED!") - continue - - try: - # DEBUG: print("DEBUG: Fetching request:", request) - res = reqto.get(request, headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"])) - - data = res.json() - # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'") - if res.ok and isinstance(data, dict): - # DEBUG: print("DEBUG: Success:", request) - set_instance_data("detection_mode", domain, "STATIC_CHECK") - set_instance_data("nodeinfo_url" , domain, request) - break - elif res.ok and isinstance(data, list): - # DEBUG: print(f"DEBUG: domain='{domain}' returned a list: '{data}'") - sys.exit(255) - elif not res.ok or res.status_code >= 400: - print("WARNING: Failed fetching nodeinfo from domain:", domain) - update_last_error(domain, res) - continue - - except BaseException as e: - # DEBUG: print("DEBUG: Cannot fetch API request:", request) - update_last_error(domain, e) - pass - - # DEBUG: print("DEBUG: Returning data[]:", type(data)) - return data - -def fetch_wellknown_nodeinfo(domain: str) -> list: - if type(domain) != str: - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") - - # DEBUG: print("DEBUG: Fetching .well-known info for domain:", domain) - data = {} - - try: - res = reqto.get(f"https://{domain}/.well-known/nodeinfo", headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"])) - - data = res.json() - # DEBUG: print("DEBUG: domain,res.ok,data[]:", domain, res.ok, type(data)) - if res.ok and isinstance(data, dict): - nodeinfo = data - # DEBUG: print("DEBUG: Found entries:", len(nodeinfo), domain) - if "links" in nodeinfo: - # DEBUG: print("DEBUG: Found links in nodeinfo():", len(nodeinfo["links"])) - for link in nodeinfo["links"]: - # DEBUG: print("DEBUG: rel,href:", link["rel"], link["href"]) - if link["rel"] in nodeinfo_identifier: - # DEBUG: print("DEBUG: Fetching nodeinfo from:", link["href"]) - res = reqto.get(link["href"]) - - data = res.json() - # DEBUG: print("DEBUG: href,res.ok,res.status_code:", link["href"], res.ok, res.status_code) - if res.ok and isinstance(data, dict): - # DEBUG: print("DEBUG: Found JSON nodeinfo():", len(data)) - set_instance_data("detection_mode", domain, "AUTO_DISCOVERY") - set_instance_data("nodeinfo_url" , domain, link["href"]) - break - else: - print("WARNING: Unknown 'rel' value:", domain, link["rel"]) - else: - print("WARNING: nodeinfo does not contain 'links':", domain) - - except BaseException as e: - print("WARNING: Failed fetching .well-known info:", domain) - update_last_error(domain, e) - pass - - # DEBUG: print("DEBUG: Returning data[]:", type(data)) - return data - -def fetch_generator_from_path(domain: str, path: str = "/") -> str: - if type(domain) != str: - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") - elif type(path) != str: - raise ValueError(f"path[]={type(path)} is not 'str'") - elif path == "": - raise ValueError(f"Parameter 'domain' cannot be empty") - - # DEBUG: print(f"DEBUG: domain='{domain}',path='{path}' - CALLED!") - software = None - - try: - # DEBUG: print(f"DEBUG: Fetching path='{path}' from '{domain}' ...") - res = reqto.get(f"https://{domain}{path}", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])) - - # DEBUG: print("DEBUG: domain,res.ok,res.status_code,res.text[]:", domain, res.ok, res.status_code, type(res.text)) - if res.ok and res.status_code < 300 and len(res.text) > 0: - # DEBUG: print("DEBUG: Search for :", domain) - doc = bs4.BeautifulSoup(res.text, "html.parser") - - # DEBUG: print("DEBUG: doc[]:", type(doc)) - generator = doc.find("meta", {"name": "generator"}) - site_name = doc.find("meta", {"property": "og:site_name"}) - - # DEBUG: print(f"DEBUG: generator='{generator}',site_name='{site_name}'") - if isinstance(generator, bs4.element.Tag): - # DEBUG: print("DEBUG: Found generator meta tag:", domain) - software = tidyup(generator.get("content")) - print(f"INFO: domain='{domain}' is generated by '{software}'") - set_instance_data("detection_mode", domain, "GENERATOR") - remove_pending_error(domain) - elif isinstance(site_name, bs4.element.Tag): - # DEBUG: print("DEBUG: Found property=og:site_name:", domain) - sofware = tidyup(site_name.get("content")) - print(f"INFO: domain='{domain}' has og:site_name='{software}'") - set_instance_data("detection_mode", domain, "SITE_NAME") - remove_pending_error(domain) - - except BaseException as e: - # DEBUG: print(f"DEBUG: Cannot fetch / from '{domain}':", e) - update_last_error(domain, e) - pass - - # DEBUG: print(f"DEBUG: software[]={type(software)}") - if type(software) is str and software == "": - # DEBUG: print(f"DEBUG: Corrected empty string to None for software of domain='{domain}'") - software = None - elif type(software) is str and ("." in software or " " in software): - # DEBUG: print(f"DEBUG: software='{software}' may contain a version number, domain='{domain}', removing it ...") - software = remove_version(software) - - # DEBUG: print(f"DEBUG: software[]={type(software)}") - if type(software) is str and " powered by " in software: - # DEBUG: print(f"DEBUG: software='{software}' has 'powered by' in it") - software = remove_version(strip_powered_by(software)) - elif type(software) is str and " hosted on " in software: - # DEBUG: print(f"DEBUG: software='{software}' has 'hosted on' in it") - software = remove_version(strip_hosted_on(software)) - elif type(software) is str and " by " in software: - # DEBUG: print(f"DEBUG: software='{software}' has ' by ' in it") - software = strip_until(software, " by ") - elif type(software) is str and " see " in software: - # DEBUG: print(f"DEBUG: software='{software}' has ' see ' in it") - software = strip_until(software, " see ") - - # DEBUG: print(f"DEBUG: software='{software}' - EXIT!") - return software - -def determine_software(domain: str, path: str = None) -> str: - if type(domain) != str: - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") - - # DEBUG: print("DEBUG: Determining software for domain,path:", domain, path) - software = None - - # DEBUG: print(f"DEBUG: Fetching nodeinfo from '{domain}' ...") - data = fetch_nodeinfo(domain, path) - - # DEBUG: print("DEBUG: data[]:", type(data)) - if not isinstance(data, dict) or len(data) == 0: - # DEBUG: print("DEBUG: Could not determine software type:", domain) - return fetch_generator_from_path(domain) - - # DEBUG: print("DEBUG: data():", len(data), data) - if "status" in data and data["status"] == "error" and "message" in data: - print("WARNING: JSON response is an error:", data["message"]) - update_last_error(domain, data["message"]) - return fetch_generator_from_path(domain) - elif "message" in data: - print("WARNING: JSON response contains only a message:", data["message"]) - update_last_error(domain, data["message"]) - return fetch_generator_from_path(domain) - elif "software" not in data or "name" not in data["software"]: - # DEBUG: print(f"DEBUG: JSON response from domain='{domain}' does not include [software][name], fetching / ...") - software = fetch_generator_from_path(domain) - - # DEBUG: print(f"DEBUG: Generator for domain='{domain}' is: {software}, EXIT!") - return software - - software = tidyup(data["software"]["name"]) - - # DEBUG: print("DEBUG: sofware after tidyup():", software) - if software in ["akkoma", "rebased"]: - # DEBUG: print("DEBUG: Setting pleroma:", domain, software) - software = "pleroma" - elif software in ["hometown", "ecko"]: - # DEBUG: print("DEBUG: Setting mastodon:", domain, software) - software = "mastodon" - elif software in ["calckey", "groundpolis", "foundkey", "cherrypick", "meisskey"]: - # DEBUG: print("DEBUG: Setting misskey:", domain, software) - software = "misskey" - elif software.find("/") > 0: - print("WARNING: Spliting of slash:", software) - software = software.split("/")[-1]; - elif software.find("|") > 0: - print("WARNING: Spliting of pipe:", software) - software = tidyup(software.split("|")[0]); - elif "powered by" in software: - # DEBUG: print(f"DEBUG: software='{software}' has 'powered by' in it") - software = strip_powered_by(software) - elif type(software) is str and " by " in software: - # DEBUG: print(f"DEBUG: software='{software}' has ' by ' in it") - software = strip_until(software, " by ") - elif type(software) is str and " see " in software: - # DEBUG: print(f"DEBUG: software='{software}' has ' see ' in it") - software = strip_until(software, " see ") - - # DEBUG: print(f"DEBUG: software[]={type(software)}") - if software == "": - print("WARNING: tidyup() left no software name behind:", domain) - software = None - - # DEBUG: print(f"DEBUG: software[]={type(software)}") - if str(software) == "": - # DEBUG: print(f"DEBUG: software for '{domain}' was not detected, trying generator ...") - software = fetch_generator_from_path(domain) - elif len(str(software)) > 0 and ("." in software or " " in software): - # DEBUG: print(f"DEBUG: software='{software}' may contain a version number, domain='{domain}', removing it ...") - software = remove_version(software) - - # DEBUG: print(f"DEBUG: software[]={type(software)}") - if type(software) is str and "powered by" in software: - # DEBUG: print(f"DEBUG: software='{software}' has 'powered by' in it") - software = remove_version(strip_powered_by(software)) - - # DEBUG: print("DEBUG: Returning domain,software:", domain, software) - return software - -def update_block_reason(reason: str, blocker: str, blocked: str, block_level: str): - if type(reason) != str and reason != None: - raise ValueError(f"Parameter reason[]='{type(reason)}' is not 'str'") - elif type(blocker) != str: - raise ValueError(f"Parameter blocker[]='{type(blocker)}' is not 'str'") - elif type(blocked) != str: - raise ValueError(f"Parameter blocked[]='{type(blocked)}' is not 'str'") - elif type(block_level) != str: - raise ValueError(f"Parameter block_level[]='{type(block_level)}' is not 'str'") - - # DEBUG: print("DEBUG: Updating block reason:", reason, blocker, blocked, block_level) - try: - cursor.execute( - "UPDATE blocks SET reason = ?, last_seen = ? WHERE blocker = ? AND blocked = ? AND block_level = ? AND reason IN ('','unknown') LIMIT 1", - ( - reason, - time.time(), - blocker, - blocked, - block_level - ), - ) - - # DEBUG: print(f"DEBUG: cursor.rowcount={cursor.rowcount}") - if cursor.rowcount == 0: - # DEBUG: print(f"DEBUG: Did not update any rows: blocker='{blocker}',blocked='{blocked}',block_level='{block_level}',reason='{reason}' - EXIT!") - return - - except BaseException as e: - print(f"ERROR: failed SQL query: reason='{reason}',blocker='{blocker}',blocked='{blocked}',block_level='{block_level}',exception[{type(e)}]:'{str(e)}'") - sys.exit(255) - - # DEBUG: print("DEBUG: EXIT!") - -def update_last_seen(blocker: str, blocked: str, block_level: str): - # DEBUG: print("DEBUG: Updating last_seen for:", blocker, blocked, block_level) - try: - cursor.execute( - "UPDATE blocks SET last_seen = ? WHERE blocker = ? AND blocked = ? AND block_level = ? LIMIT 1", - ( - time.time(), - blocker, - blocked, - block_level - ) - ) - - # DEBUG: print(f"DEBUG: cursor.rowcount={cursor.rowcount}") - if cursor.rowcount == 0: - # DEBUG: print(f"DEBUG: Did not update any rows: blocker='{blocker}',blocked='{blocked}',block_level='{block_level}' - EXIT!") - return - - except BaseException as e: - print(f"ERROR: failed SQL query: blocker='{blocker}',blocked='{blocked}',block_level='{block_level}',exception[{type(e)}]:'{str(e)}'") - sys.exit(255) - - # DEBUG: print("DEBUG: EXIT!") - -def block_instance(blocker: str, blocked: str, reason: str, block_level: str): - # DEBUG: print("DEBUG: blocker,blocked,reason,block_level:", blocker, blocked, reason, block_level) - if type(blocker) != str: - raise ValueError(f"Parameter blocker[]={type(blocker)} is not 'str'") - elif blocker == "": - raise ValueError(f"Parameter 'blocker' cannot be empty") - elif not validators.domain(blocker.split("/")[0]): - raise ValueError(f"Bad blocker='{blocker}'") - elif type(blocked) != str: - raise ValueError(f"Parameter blocked[]={type(blocked)} is not 'str'") - elif blocked == "": - raise ValueError(f"Parameter 'blocked' cannot be empty") - elif not validators.domain(blocked.split("/")[0]): - raise ValueError(f"Bad blocked='{blocked}'") - elif is_blacklisted(blocker): - raise Exception(f"blocker='{blocker}' is blacklisted but function invoked") - elif is_blacklisted(blocked): - raise Exception(f"blocked='{blocked}' is blacklisted but function invoked") - - print(f"INFO: New block: blocker='{blocker}',blocked='{blocked}', reason='{reason}', block_level='{block_level}'") - try: - cursor.execute( - "INSERT INTO blocks (blocker, blocked, reason, block_level, first_seen, last_seen) VALUES(?, ?, ?, ?, ?, ?)", - ( - blocker, - blocked, - reason, - block_level, - time.time(), - time.time() - ), - ) - except BaseException as e: - print(f"ERROR: failed SQL query: blocker='{blocker}',blocked='{blocked}',reason='{reason}',block_level='{block_level}',exception[{type(e)}]:'{str(e)}'") - sys.exit(255) - - # DEBUG: print("DEBUG: EXIT!") - -def is_instance_registered(domain: str) -> bool: - if type(domain) != str: - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") - - # NOISY-DEBUG: # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") - if not is_cache_initialized("is_registered"): - # NOISY-DEBUG: # DEBUG: print(f"DEBUG: Cache for 'is_registered' not initialized, fetching all rows ...") - try: - cursor.execute("SELECT domain FROM instances") - - # Check Set all - set_all_cache_key("is_registered", cursor.fetchall(), True) - except BaseException as e: - print(f"ERROR: failed SQL query: domain='{domain}',exception[{type(e)}]:'{str(e)}'") - sys.exit(255) - - # Is cache found? - registered = is_cache_key_set("is_registered", domain) - - # NOISY-DEBUG: # DEBUG: print(f"DEBUG: registered='{registered}' - EXIT!") - return registered - -def add_instance(domain: str, origin: str, originator: str, path: str = None): - if type(domain) != str: - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") - elif type(origin) != str and origin != None: - raise ValueError(f"origin[]={type(origin)} is not 'str'") - elif type(originator) != str: - raise ValueError(f"originator[]={type(originator)} is not 'str'") - elif originator == "": - raise ValueError(f"originator cannot be empty") - elif not validators.domain(domain.split("/")[0]): - raise ValueError(f"Bad domain name='{domain}'") - elif origin is not None and not validators.domain(origin.split("/")[0]): - raise ValueError(f"Bad origin name='{origin}'") - elif is_blacklisted(domain): - raise Exception(f"domain='{domain}' is blacklisted, but method invoked") - - # DEBUG: print("DEBUG: domain,origin,originator,path:", domain, origin, originator, path) - software = determine_software(domain, path) - # DEBUG: print("DEBUG: Determined software:", software) - - print(f"INFO: Adding instance domain='{domain}' (origin='{origin}',software='{software}')") - try: - cursor.execute( - "INSERT INTO instances (domain, origin, originator, hash, software, first_seen) VALUES (?, ?, ?, ?, ?, ?)", - ( - domain, - origin, - originator, - get_hash(domain), - software, - time.time() - ), - ) - - set_cache_key("is_registered", domain, True) - - if has_pending_instance_data(domain): - # DEBUG: print(f"DEBUG: domain='{domain}' has pending nodeinfo being updated ...") - set_instance_data("last_status_code" , domain, None) - set_instance_data("last_error_details", domain, None) - update_instance_data(domain) - remove_pending_error(domain) - - if domain in pending_errors: - # DEBUG: print("DEBUG: domain has pending error being updated:", domain) - update_last_error(domain, pending_errors[domain]) - remove_pending_error(domain) - - except BaseException as e: - print(f"ERROR: failed SQL query: domain='{domain}',exception[{type(e)}]:'{str(e)}'") - sys.exit(255) - else: - # DEBUG: print("DEBUG: Updating nodeinfo for domain:", domain) - update_last_nodeinfo(domain) - - # DEBUG: print("DEBUG: EXIT!") - -def send_bot_post(instance: str, blocks: dict): - message = instance + " has blocked the following instances:\n\n" - truncated = False - - if len(blocks) > 20: - truncated = True - blocks = blocks[0 : 19] - - for block in blocks: - if block["reason"] == None or block["reason"] == '': - message = message + block["blocked"] + " with unspecified reason\n" - else: - if len(block["reason"]) > 420: - block["reason"] = block["reason"][0:419] + "[…]" - - message = message + block["blocked"] + ' for "' + block["reason"].replace("@", "@\u200b") + '"\n' - - if truncated: - message = message + "(the list has been truncated to the first 20 entries)" - - botheaders = {**api_headers, **{"Authorization": "Bearer " + config["bot_token"]}} - - req = reqto.post( - f"{config['bot_instance']}/api/v1/statuses", - data={ - "status" : message, - "visibility" : config['bot_visibility'], - "content_type": "text/plain" - }, - headers=botheaders, - timeout=10 - ).json() - - return True - -def get_mastodon_blocks(domain: str) -> dict: - if type(domain) != str: - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") - - # DEBUG: print("DEBUG: Fetching mastodon blocks from domain:", domain) - blocks = { - "Suspended servers": [], - "Filtered media" : [], - "Limited servers" : [], - "Silenced servers" : [], - } - - try: - doc = bs4.BeautifulSoup( - reqto.get(f"https://{domain}/about", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])).text, - "html.parser", - ) - except BaseException as e: - print("ERROR: Cannot fetch from domain:", domain, e) - update_last_error(domain, e) - return {} - - for header in doc.find_all("h3"): - header_text = tidyup(header.text) - - if header_text in language_mapping: - # DEBUG: print(f"DEBUG: header_text='{header_text}'") - header_text = language_mapping[header_text] - - if header_text in blocks or header_text.lower() in blocks: - # replaced find_next_siblings with find_all_next to account for instances that e.g. hide lists in dropdown menu - for line in header.find_all_next("table")[0].find_all("tr")[1:]: - blocks[header_text].append( - { - "domain": tidyup(line.find("span").text), - "hash" : tidyup(line.find("span")["title"][9:]), - "reason": tidyup(line.find_all("td")[1].text), - } - ) - - # DEBUG: print("DEBUG: Returning blocks for domain:", domain) - return { - "reject" : blocks["Suspended servers"], - "media_removal" : blocks["Filtered media"], - "followers_only": blocks["Limited servers"] + blocks["Silenced servers"], - } - -def get_friendica_blocks(domain: str) -> dict: - if type(domain) != str: - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") - - # DEBUG: print("DEBUG: Fetching friendica blocks from domain:", domain) - blocks = [] - - try: - doc = bs4.BeautifulSoup( - reqto.get(f"https://{domain}/friendica", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])).text, - "html.parser", - ) - except BaseException as e: - print("WARNING: Failed to fetch /friendica from domain:", domain, e) - update_last_error(domain, e) - return {} - - blocklist = doc.find(id="about_blocklist") - - # Prevents exceptions: - if blocklist is None: - # DEBUG: print("DEBUG: Instance has no block list:", domain) - return {} - - for line in blocklist.find("table").find_all("tr")[1:]: - # DEBUG: print(f"DEBUG: line='{line}'") - blocks.append({ - "domain": tidyup(line.find_all("td")[0].text), - "reason": tidyup(line.find_all("td")[1].text) - }) - - # DEBUG: print("DEBUG: Returning blocks() for domain:", domain, len(blocks)) - return { - "reject": blocks - } - -def get_misskey_blocks(domain: str) -> dict: - if type(domain) != str: - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError(f"Parameter 'domain' cannot be empty") - - # DEBUG: print("DEBUG: Fetching misskey blocks from domain:", domain) - blocks = { - "suspended": [], - "blocked" : [] - } - - offset = 0 - step = config["misskey_offset"] - while True: - # iterating through all "suspended" (follow-only in its terminology) - # instances page-by-page, since that troonware doesn't support - # sending them all at once - try: - # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...") - if offset == 0: - # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset) - fetched = post_json_api(domain, "/api/federation/instances", json.dumps({ - "sort" : "+pubAt", - "host" : None, - "suspended": True, - "limit" : step - }), {"Origin": domain}) - else: - # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset) - fetched = post_json_api(domain, "/api/federation/instances", json.dumps({ - "sort" : "+pubAt", - "host" : None, - "suspended": True, - "limit" : step, - "offset" : offset - 1 - }), {"Origin": domain}) - - # DEBUG: print("DEBUG: fetched():", len(fetched)) - if len(fetched) == 0: - # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain) - break - elif len(fetched) != config["misskey_offset"]: - # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config['misskey_offset']}'") - offset = offset + (config["misskey_offset"] - len(fetched)) - else: - # DEBUG: print("DEBUG: Raising offset by step:", step) - offset = offset + step - - for instance in fetched: - # just in case - if instance["isSuspended"]: - blocks["suspended"].append( - { - "domain": tidyup(instance["host"]), - # no reason field, nothing - "reason": None - } - ) - - except BaseException as e: - print("WARNING: Caught error, exiting loop:", domain, e) - update_last_error(domain, e) - offset = 0 - break - - while True: - # same shit, different asshole ("blocked" aka full suspend) - try: - if offset == 0: - # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset) - fetched = post_json_api(domain,"/api/federation/instances", json.dumps({ - "sort" : "+pubAt", - "host" : None, - "blocked": True, - "limit" : step - }), {"Origin": domain}) - else: - # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset) - fetched = post_json_api(domain,"/api/federation/instances", json.dumps({ - "sort" : "+pubAt", - "host" : None, - "blocked": True, - "limit" : step, - "offset" : offset-1 - }), {"Origin": domain}) - - # DEBUG: print("DEBUG: fetched():", len(fetched)) - if len(fetched) == 0: - # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain) - break - elif len(fetched) != config["misskey_offset"]: - # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config['misskey_offset']}'") - offset = offset + (config["misskey_offset"] - len(fetched)) - else: - # DEBUG: print("DEBUG: Raising offset by step:", step) - offset = offset + step - - for instance in fetched: - if instance["isBlocked"]: - blocks["blocked"].append({ - "domain": tidyup(instance["host"]), - "reason": None - }) - - except BaseException as e: - print("ERROR: Exception during POST:", domain, e) - update_last_error(domain, e) - offset = 0 - break - - # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") - update_last_instance_fetch(domain) - - # DEBUG: print("DEBUG: Returning for domain,blocked(),suspended():", domain, len(blocks["blocked"]), len(blocks["suspended"])) - return { - "reject" : blocks["blocked"], - "followers_only": blocks["suspended"] - } - -def tidyup(string: str) -> str: - if type(string) != str: - raise ValueError(f"Parameter string[]={type(string)} is not expected") - - # some retards put their blocks in variable case - string = string.lower().strip() - - # other retards put the port - string = re.sub("\:\d+$", "", string) - - # bigger retards put the schema in their blocklist, sometimes even without slashes - string = re.sub("^https?\:(\/*)", "", string) - - # and trailing slash - string = re.sub("\/$", "", string) - - # and the @ - string = re.sub("^\@", "", string) - - # the biggest retards of them all try to block individual users - string = re.sub("(.+)\@", "", string) - - return string - -def lock_process(): - global LOCK - try: - print(f"DEBUG: Acquiring lock: '{lockfile}'") - LOCK = zc.lockfile.LockFile(lockfile) - print("DEBUG: Lock obtained.") - - except zc.lockfile.LockError: - print(f"ERROR: Cannot aquire lock: '{lockfile}'") - sys.exit(100) - -def shutdown(): - print("DEBUG: Closing database connection ...") - connection.close() - print("DEBUG: Releasing lock ...") - LOCK.close() - print(f"DEBUG: Deleting lockfile='{lockfile}' ...") - os.remove(lockfile) - print("DEBUG: Shutdown completed.") diff --git a/fba/__init__.py b/fba/__init__.py new file mode 100644 index 0000000..803e5ea --- /dev/null +++ b/fba/__init__.py @@ -0,0 +1 @@ +__all__ = ['boot', 'cache', 'fba'] diff --git a/fba/boot.py b/fba/boot.py new file mode 100644 index 0000000..8f962fc --- /dev/null +++ b/fba/boot.py @@ -0,0 +1,45 @@ +# Fedi API Block - An aggregator for fetching blocking data from fediverse nodes +# Copyright (C) 2023 Free Software Foundation +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import os +import sys +import tempfile +import zc.lockfile +from fba import fba + +# Lock file +lockfile = tempfile.gettempdir() + '/fba.lock' +LOCK = None + +def lock_process(): + global LOCK + try: + print(f"DEBUG: Acquiring lock: '{lockfile}'") + LOCK = zc.lockfile.LockFile(lockfile) + print("DEBUG: Lock obtained.") + + except zc.lockfile.LockError: + print(f"ERROR: Cannot aquire lock: '{lockfile}'") + sys.exit(100) + +def shutdown(): + print("DEBUG: Closing database connection ...") + fba.connection.close() + print("DEBUG: Releasing lock ...") + LOCK.close() + print(f"DEBUG: Deleting lockfile='{lockfile}' ...") + os.remove(lockfile) + print("DEBUG: Shutdown completed.") diff --git a/fba/cache.py b/fba/cache.py new file mode 100644 index 0000000..068fd35 --- /dev/null +++ b/fba/cache.py @@ -0,0 +1,64 @@ +# Fedi API Block - An aggregator for fetching blocking data from fediverse nodes +# Copyright (C) 2023 Free Software Foundation +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + + +# Cache for redundant SQL queries +cache = {} + +##### Cache ##### + +def is_cache_initialized(key: str) -> bool: + return key in cache + +def set_all_cache_key(key: str, rows: list, value: any): + # NOISY-DEBUG: print(f"DEBUG: key='{key}',rows()={len(rows)},value[]={type(value)} - CALLED!") + if type(key) != str: + raise ValueError("Parameter key[]='{type(key)}' is not 'str'") + elif not is_cache_initialized(key): + # NOISY-DEBUG: print(f"DEBUG: Cache for key='{key}' not initialized.") + cache[key] = {} + + for sub in rows: + # NOISY-DEBUG: print(f"DEBUG: Setting key='{key}',sub[{type(sub)}]='{sub}'") + + if isinstance(sub, tuple): + cache[key][sub[0]] = value + else: + print(f"WARNING: Unsupported type row[]='{type(row)}'") + + # NOISY-DEBUG: print("DEBUG: EXIT!") + +def set_cache_key(key: str, sub: str, value: any): + if type(key) != str: + raise ValueError("Parameter key[]='{type(key)}' is not 'str'") + elif type(sub) != str: + raise ValueError("Parameter sub[]='{type(sub)}' is not 'str'") + elif not is_cache_initialized(key): + print(f"WARNING: Bad method call, key='{key}' is not initialized yet.") + raise Exception(f"Cache for key='{key}' is not initialized, but function called") + + cache[key][sub] = value + +def is_cache_key_set(key: str, sub: str) -> bool: + if type(key) != str: + raise ValueError("Parameter key[]='{type(key)}' is not 'str'") + elif type(sub) != str: + raise ValueError("Parameter sub[]='{type(sub)}' is not 'str'") + elif not is_cache_initialized(key): + print(f"WARNING: Bad method call, key='{key}' is not initialized yet.") + raise Exception(f"Cache for key='{key}' is not initialized, but function called") + + return sub in cache[key] diff --git a/fba/fba.py b/fba/fba.py new file mode 100644 index 0000000..4e97f23 --- /dev/null +++ b/fba/fba.py @@ -0,0 +1,1499 @@ +# Fedi API Block - An aggregator for fetching blocking data from fediverse nodes +# Copyright (C) 2023 Free Software Foundation +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import bs4 +from fba import cache +import hashlib +import re +import reqto +import json +import sqlite3 +import sys +import time +import validators + +with open("config.json") as f: + config = json.loads(f.read()) + +# Don't check these, known trolls/flooders/testing/developing +blacklist = [ + # Floods network with fake nodes as "research" project + "activitypub-troll.cf", + # Similar troll + "gab.best", + # Similar troll + "4chan.icu", + # Flooder (?) + "social.shrimpcam.pw", + # Flooder (?) + "mastotroll.netz.org", + # Testing/developing installations + "ngrok.io", + "ngrok-free.app", + "misskeytest.chn.moe", +] + +# Array with pending errors needed to be written to database +pending_errors = { +} + +# "rel" identifiers (no real URLs) +nodeinfo_identifier = [ + "https://nodeinfo.diaspora.software/ns/schema/2.1", + "https://nodeinfo.diaspora.software/ns/schema/2.0", + "https://nodeinfo.diaspora.software/ns/schema/1.1", + "https://nodeinfo.diaspora.software/ns/schema/1.0", + "http://nodeinfo.diaspora.software/ns/schema/2.1", + "http://nodeinfo.diaspora.software/ns/schema/2.0", + "http://nodeinfo.diaspora.software/ns/schema/1.1", + "http://nodeinfo.diaspora.software/ns/schema/1.0", +] + +# HTTP headers for non-API requests +headers = { + "User-Agent": config["useragent"], +} +# HTTP headers for API requests +api_headers = { + "User-Agent": config["useragent"], + "Content-Type": "application/json", +} + +# Found info from node, such as nodeinfo URL, detection mode that needs to be +# written to database. Both arrays must be filled at the same time or else +# update_instance_data() will fail +instance_data = { + # Detection mode: 'AUTO_DISCOVERY', 'STATIC_CHECKS' or 'GENERATOR' + # NULL means all detection methods have failed (maybe still reachable instance) + "detection_mode" : {}, + # Found nodeinfo URL + "nodeinfo_url" : {}, + # Found total peers + "total_peers" : {}, + # Last fetched instances + "last_instance_fetch": {}, + # Last updated + "last_updated" : {}, + # Last blocked + "last_blocked" : {}, + # Last nodeinfo (fetched) + "last_nodeinfo" : {}, + # Last status code + "last_status_code" : {}, + # Last error details + "last_error_details" : {}, +} + +language_mapping = { + # English -> English + "Silenced instances" : "Silenced servers", + "Suspended instances" : "Suspended servers", + "Limited instances" : "Limited servers", + # Mappuing German -> English + "Gesperrte Server" : "Suspended servers", + "Gefilterte Medien" : "Filtered media", + "Stummgeschaltete Server" : "Silenced servers", + # Japanese -> English + "停止済みのサーバー" : "Suspended servers", + "制限中のサーバー" : "Limited servers", + "メディアを拒否しているサーバー": "Filtered media", + "サイレンス済みのサーバー" : "Silenced servers", + # ??? -> English + "שרתים מושעים" : "Suspended servers", + "מדיה מסוננת" : "Filtered media", + "שרתים מוגבלים" : "Silenced servers", + # French -> English + "Serveurs suspendus" : "Suspended servers", + "Médias filtrés" : "Filtered media", + "Serveurs limités" : "Limited servers", + "Serveurs modérés" : "Limited servers", +} + +# URL for fetching peers +get_peers_url = "/api/v1/instance/peers" + +# Connect to database +connection = sqlite3.connect("blocks.db") +cursor = connection.cursor() + +# Pattern instance for version numbers +patterns = [ + # semantic version number (with v|V) prefix) + re.compile("^(?Pv|V{0,1})(\.{0,1})(?P0|[1-9]\d*)\.(?P0+|[1-9]\d*)(\.(?P0+|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)?$"), + # non-sematic, e.g. 1.2.3.4 + re.compile("^(?Pv|V{0,1})(\.{0,1})(?P0|[1-9]\d*)\.(?P0+|[1-9]\d*)(\.(?P0+|[1-9]\d*)(\.(?P0|[1-9]\d*))?)$"), + # non-sematic, e.g. 2023-05[-dev] + re.compile("^(?P[1-9]{1}[0-9]{3})\.(?P[0-9]{2})(-dev){0,1}$"), + # non-semantic, e.g. abcdef0 + re.compile("^[a-f0-9]{7}$"), +] + +##### Other functions ##### + +def is_primitive(var: any) -> bool: + # NOISY-DEBUG: print(f"DEBUG: var[]='{type(var)}' - CALLED!") + return type(var) in {int, str, float, bool} or var == None + +def fetch_instances(domain: str, origin: str, software: str, script: str, path: str = None): + if type(domain) != str: + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError(f"Parameter 'domain' cannot be empty") + elif type(origin) != str and origin != None: + raise ValueError(f"Parameter origin[]={type(origin)} is not 'str'") + elif type(script) != str: + raise ValueError(f"Parameter script[]={type(script)} is not 'str'") + elif domain == "": + raise ValueError(f"Parameter 'domain' cannot be empty") + + # DEBUG: print("DEBUG: domain,origin,software,path:", domain, origin, software, path) + if not is_instance_registered(domain): + # DEBUG: print("DEBUG: Adding new domain:", domain, origin) + add_instance(domain, origin, script, path) + + # DEBUG: print("DEBUG: Fetching instances for domain:", domain, software) + peerlist = get_peers(domain, software) + + if (peerlist is None): + print("ERROR: Cannot fetch peers:", domain) + return + elif has_pending_instance_data(domain): + # DEBUG: print(f"DEBUG: domain='{domain}' has pending nodeinfo data, flushing ...") + update_instance_data(domain) + + print(f"INFO: Checking {len(peerlist)} instances from {domain} ...") + for instance in peerlist: + if instance == None: + # Skip "None" types as tidup() cannot parse them + continue + + # DEBUG: print(f"DEBUG: instance[{type(instance}]={instance} - BEFORE") + instance = tidyup(instance) + # DEBUG: print(f"DEBUG: instance[{type(instance}]={instance} - AFTER") + + if instance == "": + print("WARNING: Empty instance after tidyup(), domain:", domain) + continue + elif not validators.domain(instance.split("/")[0]): + print(f"WARNING: Bad instance='{instance}' from domain='{domain}',origin='{origin}',software='{software}'") + continue + elif is_blacklisted(instance): + # DEBUG: print("DEBUG: instance is blacklisted:", instance) + continue + + # DEBUG: print("DEBUG: Handling instance:", instance) + try: + if not is_instance_registered(instance): + # DEBUG: print("DEBUG: Adding new instance:", instance, domain) + add_instance(instance, domain, sys.argv[0]) + except BaseException as e: + print(f"ERROR: instance='{instance}',exception[{type(e)}]:'{str(e)}'") + continue + + # DEBUG: print("DEBUG: EXIT!") + +def set_instance_data(key: str, domain: str, value: any): + # NOISY-DEBUG: print(f"DEBUG: key='{key}',domain='{domain}',value[]='{type(value)}' - CALLED!") + if type(key) != str: + raise ValueError("Parameter key[]='{type(key)}' is not 'str'") + elif key == "": + raise ValueError(f"Parameter 'key' cannot be empty") + elif type(domain) != str: + raise ValueError("Parameter domain[]='{type(domain)}' is not 'str'") + elif domain == "": + raise ValueError(f"Parameter 'domain' cannot be empty") + elif not key in instance_data: + raise ValueError(f"key='{key}' not found in instance_data") + elif not is_primitive(value): + raise ValueError(f"value[]='{type(value)}' is not a primitive type") + + # Set it + instance_data[key][domain] = value + + # DEBUG: print("DEBUG: EXIT!") + +def add_peers(rows: dict) -> list: + # DEBUG: print(f"DEBUG: rows()={len(rows)} - CALLED!") + peers = list() + for element in ["linked", "allowed", "blocked"]: + # DEBUG: print(f"DEBUG: Checking element='{element}'") + if element in rows and rows[element] != None: + # DEBUG: print(f"DEBUG: Adding {len(rows[element])} peer(s) to peers list ...") + for peer in rows[element]: + # DEBUG: print(f"DEBUG: peer='{peer}' - BEFORE!") + peer = tidyup(peer) + + # DEBUG: print(f"DEBUG: peer='{peer}' - AFTER!") + if is_blacklisted(peer): + # DEBUG: print(f"DEBUG: peer='{peer}' is blacklisted, skipped!") + continue + + # DEBUG: print(f"DEBUG: Adding peer='{peer}' ...") + peers.append(peer) + + # DEBUG: print(f"DEBUG: peers()={len(peers)} - EXIT!") + return peers + +def remove_version(software: str) -> str: + # DEBUG: print(f"DEBUG: software='{software}' - CALLED!") + if not "." in software and " " not in software: + print(f"WARNING: software='{software}' does not contain a version number.") + return software + + temp = software + if ";" in software: + temp = software.split(";")[0] + elif "," in software: + temp = software.split(",")[0] + elif " - " in software: + temp = software.split(" - ")[0] + + # DEBUG: print(f"DEBUG: software='{software}'") + version = None + if " " in software: + version = temp.split(" ")[-1] + elif "/" in software: + version = temp.split("/")[-1] + elif "-" in software: + version = temp.split("-")[-1] + else: + # DEBUG: print(f"DEBUG: Was not able to find common seperator, returning untouched software='{software}'") + return software + + matches = None + match = None + # DEBUG: print(f"DEBUG: Checking {len(patterns)} patterns ...") + for pattern in patterns: + # Run match() + match = pattern.match(version) + + # DEBUG: print(f"DEBUG: match[]={type(match)}") + if type(match) is re.Match: + break + + # DEBUG: print(f"DEBUG: version[{type(version)}]='{version}',match='{match}'") + if type(match) is not re.Match: + print(f"WARNING: version='{version}' does not match regex, leaving software='{software}' untouched.") + return software + + # DEBUG: print(f"DEBUG: Found valid version number: '{version}', removing it ...") + end = len(temp) - len(version) - 1 + + # DEBUG: print(f"DEBUG: end[{type(end)}]={end}") + software = temp[0:end].strip() + if " version" in software: + # DEBUG: print(f"DEBUG: software='{software}' contains word ' version'") + software = strip_until(software, " version") + + # DEBUG: print(f"DEBUG: software='{software}' - EXIT!") + return software + +def strip_powered_by(software: str) -> str: + # DEBUG: print(f"DEBUG: software='{software}' - CALLED!") + if software == "": + print(f"ERROR: Bad method call, 'software' is empty") + raise Exception("Parameter 'software' is empty") + elif not "powered by" in software: + print(f"WARNING: Cannot find 'powered by' in '{software}'!") + return software + + start = software.find("powered by ") + # DEBUG: print(f"DEBUG: start[{type(start)}]='{start}'") + + software = software[start + 11:].strip() + # DEBUG: print(f"DEBUG: software='{software}'") + + software = strip_until(software, " - ") + + # DEBUG: print(f"DEBUG: software='{software}' - EXIT!") + return software + +def strip_hosted_on(software: str) -> str: + # DEBUG: print(f"DEBUG: software='{software}' - CALLED!") + if software == "": + print(f"ERROR: Bad method call, 'software' is empty") + raise Exception("Parameter 'software' is empty") + elif not "hosted on" in software: + print(f"WARNING: Cannot find 'hosted on' in '{software}'!") + return software + + end = software.find("hosted on ") + # DEBUG: print(f"DEBUG: end[{type(end)}]='{end}'") + + software = software[0, start].strip() + # DEBUG: print(f"DEBUG: software='{software}'") + + software = strip_until(software, " - ") + + # DEBUG: print(f"DEBUG: software='{software}' - EXIT!") + return software + +def strip_until(software: str, until: str) -> str: + # DEBUG: print(f"DEBUG: software='{software}',until='{until}' - CALLED!") + if software == "": + print(f"ERROR: Bad method call, 'software' is empty") + raise Exception("Parameter 'software' is empty") + elif until == "": + print(f"ERROR: Bad method call, 'until' is empty") + raise Exception("Parameter 'until' is empty") + elif not until in software: + print(f"WARNING: Cannot find '{until}' in '{software}'!") + return software + + # Next, strip until part + end = software.find(until) + + # DEBUG: print(f"DEBUG: end[{type(end)}]='{end}'") + if end > 0: + software = software[0:end].strip() + + # DEBUG: print(f"DEBUG: software='{software}' - EXIT!") + return software + +def is_blacklisted(domain: str) -> bool: + if type(domain) != str: + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError(f"Parameter 'domain' cannot be empty") + + blacklisted = False + for peer in blacklist: + if peer in domain: + blacklisted = True + + return blacklisted + +def remove_pending_error(domain: str): + if type(domain) != str: + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError(f"Parameter 'domain' cannot be empty") + + try: + # Prevent updating any pending errors, nodeinfo was found + del pending_errors[domain] + + except: + pass + + # DEBUG: print("DEBUG: EXIT!") + +def get_hash(domain: str) -> str: + if type(domain) != str: + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError(f"Parameter 'domain' cannot be empty") + + return hashlib.sha256(domain.encode("utf-8")).hexdigest() + +def update_last_blocked(domain: str): + if type(domain) != str: + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError(f"Parameter 'domain' cannot be empty") + + # DEBUG: print("DEBUG: Updating last_blocked for domain", domain) + set_instance_data("last_blocked", domain, time.time()) + + # Running pending updated + # DEBUG: print(f"DEBUG: Invoking update_instance_data({domain}) ...") + update_instance_data(domain) + + # DEBUG: print("DEBUG: EXIT!") + +def has_pending_instance_data(domain: str) -> bool: + # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") + if type(domain) != str: + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError(f"Parameter 'domain' cannot be empty") + + has_pending = False + for key in instance_data: + # DEBUG: print(f"DEBUG: key='{key}',domain='{domain}',instance_data[key]()='{len(instance_data[key])}'") + if domain in instance_data[key]: + has_pending = True + break + + # DEBUG: print(f"DEBUG: has_pending='{has_pending}' - EXIT!") + return has_pending + +def update_instance_data(domain: str): + if type(domain) != str: + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError(f"Parameter 'domain' cannot be empty") + + # DEBUG: print(f"DEBUG: Updating nodeinfo for domain='{domain}' ...") + sql_string = '' + fields = list() + for key in instance_data: + # DEBUG: print("DEBUG: key:", key) + if domain in instance_data[key]: + # DEBUG: print(f"DEBUG: Adding '{instance_data[key][domain]}' for key='{key}' ...") + fields.append(instance_data[key][domain]) + sql_string += f" {key} = ?," + + fields.append(domain) + + if sql_string == '': + raise ValueError(f"No fields have been set, but method invoked, domain='{domain}'") + + # DEBUG: print(f"DEBUG: sql_string='{sql_string}',fields()={len(fields)}") + sql_string = "UPDATE instances SET" + sql_string + " last_updated = TIME() WHERE domain = ? LIMIT 1" + # DEBUG: print("DEBUG: sql_string:", sql_string) + + try: + # DEBUG: print("DEBUG: Executing SQL:", sql_string) + cursor.execute(sql_string, fields) + + # DEBUG: print(f"DEBUG: Success! (rowcount={cursor.rowcount })") + if cursor.rowcount == 0: + print(f"WARNING: Did not update any rows: domain='{domain}',fields()={len(fields)} - EXIT!") + return + + connection.commit() + + # DEBUG: print("DEBUG: Deleting instance_data for domain:", domain) + for key in instance_data: + try: + # DEBUG: print("DEBUG: Deleting key:", key) + del instance_data[key][domain] + except: + pass + + except BaseException as e: + print(f"ERROR: failed SQL query: domain='{domain}',sql_string='{sql_string}',exception[{type(e)}]:'{str(e)}'") + sys.exit(255) + + # DEBUG: print("DEBUG: EXIT!") + +def log_error(domain: str, res: any): + # DEBUG: print("DEBUG: domain,res[]:", domain, type(res)) + if type(domain) != str: + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError(f"Parameter 'domain' cannot be empty") + + try: + # DEBUG: print("DEBUG: BEFORE res[]:", type(res)) + if isinstance(res, BaseException) or isinstance(res, json.JSONDecodeError): + res = str(res) + + # DEBUG: print("DEBUG: AFTER res[]:", type(res)) + if type(res) is str: + cursor.execute("INSERT INTO error_log (domain, error_code, error_message, created) VALUES (?, 999, ?, ?)",[ + domain, + res, + time.time() + ]) + else: + cursor.execute("INSERT INTO error_log (domain, error_code, error_message, created) VALUES (?, ?, ?, ?)",[ + domain, + res.status_code, + res.reason, + time.time() + ]) + + # Cleanup old entries + # DEBUG: print(f"DEBUG: Purging old records (distance: {config['error_log_cleanup']})") + cursor.execute("DELETE FROM error_log WHERE created < ?", [time.time() - config["error_log_cleanup"]]) + except BaseException as e: + print(f"ERROR: failed SQL query: domain='{domain}',exception[{type(e)}]:'{str(e)}'") + sys.exit(255) + + # DEBUG: print("DEBUG: EXIT!") + +def update_last_error(domain: str, res: any): + # DEBUG: print("DEBUG: domain,res[]:", domain, type(res)) + if type(domain) != str: + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError(f"Parameter 'domain' cannot be empty") + + # DEBUG: print("DEBUG: BEFORE res[]:", type(res)) + if isinstance(res, BaseException) or isinstance(res, json.JSONDecodeError): + res = str(res) + + # DEBUG: print("DEBUG: AFTER res[]:", type(res)) + if type(res) is str: + # DEBUG: print(f"DEBUG: Setting last_error_details='{res}'"); + set_instance_data("last_status_code" , domain, 999) + set_instance_data("last_error_details", domain, res) + else: + # DEBUG: print(f"DEBUG: Setting last_error_details='{res.reason}'"); + set_instance_data("last_status_code" , domain, res.status_code) + set_instance_data("last_error_details", domain, res.reason) + + # Running pending updated + # DEBUG: print(f"DEBUG: Invoking update_instance_data({domain}) ...") + update_instance_data(domain) + + log_error(domain, res) + + # DEBUG: print("DEBUG: EXIT!") + +def update_last_instance_fetch(domain: str): + if type(domain) != str: + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError(f"Parameter 'domain' cannot be empty") + + # DEBUG: print("DEBUG: Updating last_instance_fetch for domain:", domain) + set_instance_data("last_instance_fetch", domain, time.time()) + + # Running pending updated + # DEBUG: print(f"DEBUG: Invoking update_instance_data({domain}) ...") + update_instance_data(domain) + + # DEBUG: print("DEBUG: EXIT!") + +def update_last_nodeinfo(domain: str): + if type(domain) != str: + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError(f"Parameter 'domain' cannot be empty") + + # DEBUG: print("DEBUG: Updating last_nodeinfo for domain:", domain) + set_instance_data("last_nodeinfo", domain, time.time()) + set_instance_data("last_updated" , domain, time.time()) + + # Running pending updated + # DEBUG: print(f"DEBUG: Invoking update_instance_data({domain}) ...") + update_instance_data(domain) + + # DEBUG: print("DEBUG: EXIT!") + +def get_peers(domain: str, software: str) -> list: + if type(domain) != str: + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError(f"Parameter 'domain' cannot be empty") + elif type(software) != str and software != None: + raise ValueError(f"software[]={type(software)} is not 'str'") + + # DEBUG: print(f"DEBUG: domain='{domain}',software='{software}' - CALLED!") + peers = list() + + if software == "misskey": + # DEBUG: print(f"DEBUG: domain='{domain}' is misskey, sending API POST request ...") + offset = 0 + step = config["misskey_offset"] + + # iterating through all "suspended" (follow-only in its terminology) + # instances page-by-page, since that troonware doesn't support + # sending them all at once + while True: + # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...") + if offset == 0: + fetched = post_json_api(domain, "/api/federation/instances", json.dumps({ + "sort" : "+pubAt", + "host" : None, + "limit": step + }), {"Origin": domain}) + else: + fetched = post_json_api(domain, "/api/federation/instances", json.dumps({ + "sort" : "+pubAt", + "host" : None, + "limit" : step, + "offset": offset - 1 + }), {"Origin": domain}) + + # DEBUG: print("DEBUG: fetched():", len(fetched)) + if len(fetched) == 0: + # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain) + break + elif len(fetched) != config["misskey_offset"]: + # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config['misskey_offset']}'") + offset = offset + (config["misskey_offset"] - len(fetched)) + else: + # DEBUG: print("DEBUG: Raising offset by step:", step) + offset = offset + step + + # Check records + # DEBUG: print(f"DEBUG: fetched({len(fetched)})[]={type(fetched)}") + if isinstance(fetched, dict) and "error" in fetched and "message" in fetched["error"]: + print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}") + update_last_error(domain, fetched["error"]["message"]) + break + + for row in fetched: + # DEBUG: print(f"DEBUG: row():{len(row)}") + if not "host" in row: + print(f"WARNING: row()={len(row)} does not contain element 'host': {row},domain='{domain}'") + continue + elif type(row["host"]) != str: + print(f"WARNING: row[host][]={type(row['host'])} is not 'str'") + continue + elif is_blacklisted(row["host"]): + # DEBUG: print(f"DEBUG: row[host]='{row['host']}' is blacklisted. domain='{domain}'") + continue + + # DEBUG: print(f"DEBUG: Adding peer: '{row['host']}'") + peers.append(row["host"]) + + # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'") + set_instance_data("total_peers", domain, len(peers)) + + # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") + update_last_instance_fetch(domain) + + # DEBUG: print("DEBUG: Returning peers[]:", type(peers)) + return peers + elif software == "lemmy": + # DEBUG: print(f"DEBUG: domain='{domain}' is Lemmy, fetching JSON ...") + try: + res = reqto.get(f"https://{domain}/api/v3/site", headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"])) + + data = res.json() + # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code='{res.status_code}',data[]='{type(data)}'") + if not res.ok or res.status_code >= 400: + print("WARNING: Could not reach any JSON API:", domain) + update_last_error(domain, res) + elif res.ok and isinstance(data, list): + # DEBUG: print(f"DEBUG: domain='{domain}' returned a list: '{data}'") + sys.exit(255) + elif "federated_instances" in data: + # DEBUG: print(f"DEBUG: Found federated_instances for domain='{domain}'") + peers = peers + add_peers(data["federated_instances"]) + # DEBUG: print("DEBUG: Added instance(s) to peers") + else: + print("WARNING: JSON response does not contain 'federated_instances':", domain) + update_last_error(domain, res) + + except BaseException as e: + print(f"WARNING: Exception during fetching JSON: domain='{domain}',exception[{type(e)}]:'{str(e)}'") + + # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'") + set_instance_data("total_peers", domain, len(peers)) + + # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") + update_last_instance_fetch(domain) + + # DEBUG: print("DEBUG: Returning peers[]:", type(peers)) + return peers + elif software == "peertube": + # DEBUG: print(f"DEBUG: domain='{domain}' is a PeerTube, fetching JSON ...") + + start = 0 + for mode in ["followers", "following"]: + # DEBUG: print(f"DEBUG: domain='{domain}',mode='{mode}'") + while True: + try: + res = reqto.get(f"https://{domain}/api/v1/server/{mode}?start={start}&count=100", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])) + + data = res.json() + # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code='{res.status_code}',data[]='{type(data)}'") + if res.ok and isinstance(data, dict): + # DEBUG: print("DEBUG: Success, data:", len(data)) + if "data" in data: + # DEBUG: print(f"DEBUG: Found {len(data['data'])} record(s).") + for record in data["data"]: + # DEBUG: print(f"DEBUG: record()={len(record)}") + if mode in record and "host" in record[mode]: + # DEBUG: print(f"DEBUG: Found host={record[mode]['host']}, adding ...") + peers.append(record[mode]["host"]) + else: + print(f"WARNING: record from '{domain}' has no '{mode}' or 'host' record: {record}") + + if len(data["data"]) < 100: + # DEBUG: print("DEBUG: Reached end of JSON response:", domain) + break + + # Continue with next row + start = start + 100 + + except BaseException as e: + print(f"WARNING: Exception during fetching JSON: domain='{domain}',exception[{type(e)}]:'{str(e)}'") + + # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'") + set_instance_data("total_peers", domain, len(peers)) + + # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") + update_last_instance_fetch(domain) + + # DEBUG: print("DEBUG: Returning peers[]:", type(peers)) + return peers + + # DEBUG: print(f"DEBUG: Fetching get_peers_url='{get_peers_url}' from '{domain}' ...") + try: + res = reqto.get(f"https://{domain}{get_peers_url}", headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"])) + + data = res.json() + # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'") + if not res.ok or res.status_code >= 400: + # DEBUG: print(f"DEBUG: Was not able to fetch '{get_peers_url}', trying alternative ...") + res = reqto.get(f"https://{domain}/api/v3/site", headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"])) + + data = res.json() + # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'") + if not res.ok or res.status_code >= 400: + print("WARNING: Could not reach any JSON API:", domain) + update_last_error(domain, res) + elif res.ok and isinstance(data, list): + # DEBUG: print(f"DEBUG: domain='{domain}' returned a list: '{data}'") + sys.exit(255) + elif "federated_instances" in data: + # DEBUG: print(f"DEBUG: Found federated_instances for domain='{domain}'") + peers = peers + add_peers(data["federated_instances"]) + # DEBUG: print("DEBUG: Added instance(s) to peers") + else: + print("WARNING: JSON response does not contain 'federated_instances':", domain) + update_last_error(domain, res) + else: + # DEBUG: print("DEBUG: Querying API was successful:", domain, len(data)) + peers = data + + except BaseException as e: + print("WARNING: Some error during get():", domain, e) + update_last_error(domain, e) + + # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'") + set_instance_data("total_peers", domain, len(peers)) + + # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") + update_last_instance_fetch(domain) + + # DEBUG: print("DEBUG: Returning peers[]:", type(peers)) + return peers + +def post_json_api(domain: str, path: str, parameter: str, extra_headers: dict = {}) -> dict: + if type(domain) != str: + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError(f"Parameter 'domain' cannot be empty") + elif type(path) != str: + raise ValueError(f"path[]={type(path)} is not 'str'") + elif path == "": + raise ValueError(f"path cannot be empty") + elif type(parameter) != str: + raise ValueError(f"parameter[]={type(parameter)} is not 'str'") + + # DEBUG: print("DEBUG: Sending POST to domain,path,parameter:", domain, path, parameter, extra_headers) + data = {} + try: + res = reqto.post(f"https://{domain}{path}", data=parameter, headers={**api_headers, **extra_headers}, timeout=(config["connection_timeout"], config["read_timeout"])) + + data = res.json() + # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'") + if not res.ok or res.status_code >= 400: + print(f"WARNING: Cannot query JSON API: domain='{domain}',path='{path}',parameter()={len(parameter)},res.status_code='{res.status_code}',data[]='{type(data)}'") + update_last_error(domain, res) + + except BaseException as e: + print(f"WARNING: Some error during post(): domain='{domain}',path='{path}',parameter()={len(parameter)},exception[{type(e)}]:'{str(e)}'") + + # DEBUG: print(f"DEBUG: Returning data({len(data)})=[]:{type(data)}") + return data + +def fetch_nodeinfo(domain: str, path: str = None) -> list: + if type(domain) != str: + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError(f"Parameter 'domain' cannot be empty") + + # DEBUG: print("DEBUG: Fetching nodeinfo from domain,path:", domain, path) + + nodeinfo = fetch_wellknown_nodeinfo(domain) + # DEBUG: print("DEBUG: nodeinfo:", nodeinfo) + + if len(nodeinfo) > 0: + # DEBUG: print("DEBUG: Returning auto-discovered nodeinfo:", len(nodeinfo)) + return nodeinfo + + requests = [ + f"https://{domain}/nodeinfo/2.1.json", + f"https://{domain}/nodeinfo/2.1", + f"https://{domain}/nodeinfo/2.0.json", + f"https://{domain}/nodeinfo/2.0", + f"https://{domain}/nodeinfo/1.0", + f"https://{domain}/api/v1/instance" + ] + + data = {} + for request in requests: + if path != None and path != "" and request != path: + # DEBUG: print(f"DEBUG: path='{path}' does not match request='{request}' - SKIPPED!") + continue + + try: + # DEBUG: print("DEBUG: Fetching request:", request) + res = reqto.get(request, headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"])) + + data = res.json() + # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'") + if res.ok and isinstance(data, dict): + # DEBUG: print("DEBUG: Success:", request) + set_instance_data("detection_mode", domain, "STATIC_CHECK") + set_instance_data("nodeinfo_url" , domain, request) + break + elif res.ok and isinstance(data, list): + # DEBUG: print(f"DEBUG: domain='{domain}' returned a list: '{data}'") + sys.exit(255) + elif not res.ok or res.status_code >= 400: + print("WARNING: Failed fetching nodeinfo from domain:", domain) + update_last_error(domain, res) + continue + + except BaseException as e: + # DEBUG: print("DEBUG: Cannot fetch API request:", request) + update_last_error(domain, e) + pass + + # DEBUG: print("DEBUG: Returning data[]:", type(data)) + return data + +def fetch_wellknown_nodeinfo(domain: str) -> list: + if type(domain) != str: + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError(f"Parameter 'domain' cannot be empty") + + # DEBUG: print("DEBUG: Fetching .well-known info for domain:", domain) + data = {} + + try: + res = reqto.get(f"https://{domain}/.well-known/nodeinfo", headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"])) + + data = res.json() + # DEBUG: print("DEBUG: domain,res.ok,data[]:", domain, res.ok, type(data)) + if res.ok and isinstance(data, dict): + nodeinfo = data + # DEBUG: print("DEBUG: Found entries:", len(nodeinfo), domain) + if "links" in nodeinfo: + # DEBUG: print("DEBUG: Found links in nodeinfo():", len(nodeinfo["links"])) + for link in nodeinfo["links"]: + # DEBUG: print("DEBUG: rel,href:", link["rel"], link["href"]) + if link["rel"] in nodeinfo_identifier: + # DEBUG: print("DEBUG: Fetching nodeinfo from:", link["href"]) + res = reqto.get(link["href"]) + + data = res.json() + # DEBUG: print("DEBUG: href,res.ok,res.status_code:", link["href"], res.ok, res.status_code) + if res.ok and isinstance(data, dict): + # DEBUG: print("DEBUG: Found JSON nodeinfo():", len(data)) + set_instance_data("detection_mode", domain, "AUTO_DISCOVERY") + set_instance_data("nodeinfo_url" , domain, link["href"]) + break + else: + print("WARNING: Unknown 'rel' value:", domain, link["rel"]) + else: + print("WARNING: nodeinfo does not contain 'links':", domain) + + except BaseException as e: + print("WARNING: Failed fetching .well-known info:", domain) + update_last_error(domain, e) + pass + + # DEBUG: print("DEBUG: Returning data[]:", type(data)) + return data + +def fetch_generator_from_path(domain: str, path: str = "/") -> str: + if type(domain) != str: + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError(f"Parameter 'domain' cannot be empty") + elif type(path) != str: + raise ValueError(f"path[]={type(path)} is not 'str'") + elif path == "": + raise ValueError(f"Parameter 'domain' cannot be empty") + + # DEBUG: print(f"DEBUG: domain='{domain}',path='{path}' - CALLED!") + software = None + + try: + # DEBUG: print(f"DEBUG: Fetching path='{path}' from '{domain}' ...") + res = reqto.get(f"https://{domain}{path}", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])) + + # DEBUG: print("DEBUG: domain,res.ok,res.status_code,res.text[]:", domain, res.ok, res.status_code, type(res.text)) + if res.ok and res.status_code < 300 and len(res.text) > 0: + # DEBUG: print("DEBUG: Search for :", domain) + doc = bs4.BeautifulSoup(res.text, "html.parser") + + # DEBUG: print("DEBUG: doc[]:", type(doc)) + generator = doc.find("meta", {"name": "generator"}) + site_name = doc.find("meta", {"property": "og:site_name"}) + + # DEBUG: print(f"DEBUG: generator='{generator}',site_name='{site_name}'") + if isinstance(generator, bs4.element.Tag): + # DEBUG: print("DEBUG: Found generator meta tag:", domain) + software = tidyup(generator.get("content")) + print(f"INFO: domain='{domain}' is generated by '{software}'") + set_instance_data("detection_mode", domain, "GENERATOR") + remove_pending_error(domain) + elif isinstance(site_name, bs4.element.Tag): + # DEBUG: print("DEBUG: Found property=og:site_name:", domain) + sofware = tidyup(site_name.get("content")) + print(f"INFO: domain='{domain}' has og:site_name='{software}'") + set_instance_data("detection_mode", domain, "SITE_NAME") + remove_pending_error(domain) + + except BaseException as e: + # DEBUG: print(f"DEBUG: Cannot fetch / from '{domain}':", e) + update_last_error(domain, e) + pass + + # DEBUG: print(f"DEBUG: software[]={type(software)}") + if type(software) is str and software == "": + # DEBUG: print(f"DEBUG: Corrected empty string to None for software of domain='{domain}'") + software = None + elif type(software) is str and ("." in software or " " in software): + # DEBUG: print(f"DEBUG: software='{software}' may contain a version number, domain='{domain}', removing it ...") + software = remove_version(software) + + # DEBUG: print(f"DEBUG: software[]={type(software)}") + if type(software) is str and " powered by " in software: + # DEBUG: print(f"DEBUG: software='{software}' has 'powered by' in it") + software = remove_version(strip_powered_by(software)) + elif type(software) is str and " hosted on " in software: + # DEBUG: print(f"DEBUG: software='{software}' has 'hosted on' in it") + software = remove_version(strip_hosted_on(software)) + elif type(software) is str and " by " in software: + # DEBUG: print(f"DEBUG: software='{software}' has ' by ' in it") + software = strip_until(software, " by ") + elif type(software) is str and " see " in software: + # DEBUG: print(f"DEBUG: software='{software}' has ' see ' in it") + software = strip_until(software, " see ") + + # DEBUG: print(f"DEBUG: software='{software}' - EXIT!") + return software + +def determine_software(domain: str, path: str = None) -> str: + if type(domain) != str: + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError(f"Parameter 'domain' cannot be empty") + + # DEBUG: print("DEBUG: Determining software for domain,path:", domain, path) + software = None + + # DEBUG: print(f"DEBUG: Fetching nodeinfo from '{domain}' ...") + data = fetch_nodeinfo(domain, path) + + # DEBUG: print("DEBUG: data[]:", type(data)) + if not isinstance(data, dict) or len(data) == 0: + # DEBUG: print("DEBUG: Could not determine software type:", domain) + return fetch_generator_from_path(domain) + + # DEBUG: print("DEBUG: data():", len(data), data) + if "status" in data and data["status"] == "error" and "message" in data: + print("WARNING: JSON response is an error:", data["message"]) + update_last_error(domain, data["message"]) + return fetch_generator_from_path(domain) + elif "message" in data: + print("WARNING: JSON response contains only a message:", data["message"]) + update_last_error(domain, data["message"]) + return fetch_generator_from_path(domain) + elif "software" not in data or "name" not in data["software"]: + # DEBUG: print(f"DEBUG: JSON response from domain='{domain}' does not include [software][name], fetching / ...") + software = fetch_generator_from_path(domain) + + # DEBUG: print(f"DEBUG: Generator for domain='{domain}' is: {software}, EXIT!") + return software + + software = tidyup(data["software"]["name"]) + + # DEBUG: print("DEBUG: sofware after tidyup():", software) + if software in ["akkoma", "rebased"]: + # DEBUG: print("DEBUG: Setting pleroma:", domain, software) + software = "pleroma" + elif software in ["hometown", "ecko"]: + # DEBUG: print("DEBUG: Setting mastodon:", domain, software) + software = "mastodon" + elif software in ["calckey", "groundpolis", "foundkey", "cherrypick", "meisskey"]: + # DEBUG: print("DEBUG: Setting misskey:", domain, software) + software = "misskey" + elif software.find("/") > 0: + print("WARNING: Spliting of slash:", software) + software = software.split("/")[-1]; + elif software.find("|") > 0: + print("WARNING: Spliting of pipe:", software) + software = tidyup(software.split("|")[0]); + elif "powered by" in software: + # DEBUG: print(f"DEBUG: software='{software}' has 'powered by' in it") + software = strip_powered_by(software) + elif type(software) is str and " by " in software: + # DEBUG: print(f"DEBUG: software='{software}' has ' by ' in it") + software = strip_until(software, " by ") + elif type(software) is str and " see " in software: + # DEBUG: print(f"DEBUG: software='{software}' has ' see ' in it") + software = strip_until(software, " see ") + + # DEBUG: print(f"DEBUG: software[]={type(software)}") + if software == "": + print("WARNING: tidyup() left no software name behind:", domain) + software = None + + # DEBUG: print(f"DEBUG: software[]={type(software)}") + if str(software) == "": + # DEBUG: print(f"DEBUG: software for '{domain}' was not detected, trying generator ...") + software = fetch_generator_from_path(domain) + elif len(str(software)) > 0 and ("." in software or " " in software): + # DEBUG: print(f"DEBUG: software='{software}' may contain a version number, domain='{domain}', removing it ...") + software = remove_version(software) + + # DEBUG: print(f"DEBUG: software[]={type(software)}") + if type(software) is str and "powered by" in software: + # DEBUG: print(f"DEBUG: software='{software}' has 'powered by' in it") + software = remove_version(strip_powered_by(software)) + + # DEBUG: print("DEBUG: Returning domain,software:", domain, software) + return software + +def update_block_reason(reason: str, blocker: str, blocked: str, block_level: str): + if type(reason) != str and reason != None: + raise ValueError(f"Parameter reason[]='{type(reason)}' is not 'str'") + elif type(blocker) != str: + raise ValueError(f"Parameter blocker[]='{type(blocker)}' is not 'str'") + elif type(blocked) != str: + raise ValueError(f"Parameter blocked[]='{type(blocked)}' is not 'str'") + elif type(block_level) != str: + raise ValueError(f"Parameter block_level[]='{type(block_level)}' is not 'str'") + + # DEBUG: print("DEBUG: Updating block reason:", reason, blocker, blocked, block_level) + try: + cursor.execute( + "UPDATE blocks SET reason = ?, last_seen = ? WHERE blocker = ? AND blocked = ? AND block_level = ? AND reason IN ('','unknown') LIMIT 1", + ( + reason, + time.time(), + blocker, + blocked, + block_level + ), + ) + + # DEBUG: print(f"DEBUG: cursor.rowcount={cursor.rowcount}") + if cursor.rowcount == 0: + # DEBUG: print(f"DEBUG: Did not update any rows: blocker='{blocker}',blocked='{blocked}',block_level='{block_level}',reason='{reason}' - EXIT!") + return + + except BaseException as e: + print(f"ERROR: failed SQL query: reason='{reason}',blocker='{blocker}',blocked='{blocked}',block_level='{block_level}',exception[{type(e)}]:'{str(e)}'") + sys.exit(255) + + # DEBUG: print("DEBUG: EXIT!") + +def update_last_seen(blocker: str, blocked: str, block_level: str): + # DEBUG: print("DEBUG: Updating last_seen for:", blocker, blocked, block_level) + try: + cursor.execute( + "UPDATE blocks SET last_seen = ? WHERE blocker = ? AND blocked = ? AND block_level = ? LIMIT 1", + ( + time.time(), + blocker, + blocked, + block_level + ) + ) + + # DEBUG: print(f"DEBUG: cursor.rowcount={cursor.rowcount}") + if cursor.rowcount == 0: + # DEBUG: print(f"DEBUG: Did not update any rows: blocker='{blocker}',blocked='{blocked}',block_level='{block_level}' - EXIT!") + return + + except BaseException as e: + print(f"ERROR: failed SQL query: blocker='{blocker}',blocked='{blocked}',block_level='{block_level}',exception[{type(e)}]:'{str(e)}'") + sys.exit(255) + + # DEBUG: print("DEBUG: EXIT!") + +def block_instance(blocker: str, blocked: str, reason: str, block_level: str): + # DEBUG: print("DEBUG: blocker,blocked,reason,block_level:", blocker, blocked, reason, block_level) + if type(blocker) != str: + raise ValueError(f"Parameter blocker[]={type(blocker)} is not 'str'") + elif blocker == "": + raise ValueError(f"Parameter 'blocker' cannot be empty") + elif not validators.domain(blocker.split("/")[0]): + raise ValueError(f"Bad blocker='{blocker}'") + elif type(blocked) != str: + raise ValueError(f"Parameter blocked[]={type(blocked)} is not 'str'") + elif blocked == "": + raise ValueError(f"Parameter 'blocked' cannot be empty") + elif not validators.domain(blocked.split("/")[0]): + raise ValueError(f"Bad blocked='{blocked}'") + elif is_blacklisted(blocker): + raise Exception(f"blocker='{blocker}' is blacklisted but function invoked") + elif is_blacklisted(blocked): + raise Exception(f"blocked='{blocked}' is blacklisted but function invoked") + + print(f"INFO: New block: blocker='{blocker}',blocked='{blocked}', reason='{reason}', block_level='{block_level}'") + try: + cursor.execute( + "INSERT INTO blocks (blocker, blocked, reason, block_level, first_seen, last_seen) VALUES(?, ?, ?, ?, ?, ?)", + ( + blocker, + blocked, + reason, + block_level, + time.time(), + time.time() + ), + ) + except BaseException as e: + print(f"ERROR: failed SQL query: blocker='{blocker}',blocked='{blocked}',reason='{reason}',block_level='{block_level}',exception[{type(e)}]:'{str(e)}'") + sys.exit(255) + + # DEBUG: print("DEBUG: EXIT!") + +def is_instance_registered(domain: str) -> bool: + if type(domain) != str: + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError(f"Parameter 'domain' cannot be empty") + + # NOISY-DEBUG: # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") + if not cache.is_cache_initialized("is_registered"): + # NOISY-DEBUG: # DEBUG: print(f"DEBUG: Cache for 'is_registered' not initialized, fetching all rows ...") + try: + cursor.execute("SELECT domain FROM instances") + + # Check Set all + cache.set_all_cache_key("is_registered", cursor.fetchall(), True) + except BaseException as e: + print(f"ERROR: failed SQL query: domain='{domain}',exception[{type(e)}]:'{str(e)}'") + sys.exit(255) + + # Is cache found? + registered = cache.is_cache_key_set("is_registered", domain) + + # NOISY-DEBUG: # DEBUG: print(f"DEBUG: registered='{registered}' - EXIT!") + return registered + +def add_instance(domain: str, origin: str, originator: str, path: str = None): + if type(domain) != str: + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError(f"Parameter 'domain' cannot be empty") + elif type(origin) != str and origin != None: + raise ValueError(f"origin[]={type(origin)} is not 'str'") + elif type(originator) != str: + raise ValueError(f"originator[]={type(originator)} is not 'str'") + elif originator == "": + raise ValueError(f"originator cannot be empty") + elif not validators.domain(domain.split("/")[0]): + raise ValueError(f"Bad domain name='{domain}'") + elif origin is not None and not validators.domain(origin.split("/")[0]): + raise ValueError(f"Bad origin name='{origin}'") + elif is_blacklisted(domain): + raise Exception(f"domain='{domain}' is blacklisted, but method invoked") + + # DEBUG: print("DEBUG: domain,origin,originator,path:", domain, origin, originator, path) + software = determine_software(domain, path) + # DEBUG: print("DEBUG: Determined software:", software) + + print(f"INFO: Adding instance domain='{domain}' (origin='{origin}',software='{software}')") + try: + cursor.execute( + "INSERT INTO instances (domain, origin, originator, hash, software, first_seen) VALUES (?, ?, ?, ?, ?, ?)", + ( + domain, + origin, + originator, + get_hash(domain), + software, + time.time() + ), + ) + + set_cache_key("is_registered", domain, True) + + if has_pending_instance_data(domain): + # DEBUG: print(f"DEBUG: domain='{domain}' has pending nodeinfo being updated ...") + set_instance_data("last_status_code" , domain, None) + set_instance_data("last_error_details", domain, None) + update_instance_data(domain) + remove_pending_error(domain) + + if domain in pending_errors: + # DEBUG: print("DEBUG: domain has pending error being updated:", domain) + update_last_error(domain, pending_errors[domain]) + remove_pending_error(domain) + + except BaseException as e: + print(f"ERROR: failed SQL query: domain='{domain}',exception[{type(e)}]:'{str(e)}'") + sys.exit(255) + else: + # DEBUG: print("DEBUG: Updating nodeinfo for domain:", domain) + update_last_nodeinfo(domain) + + # DEBUG: print("DEBUG: EXIT!") + +def send_bot_post(instance: str, blocks: dict): + message = instance + " has blocked the following instances:\n\n" + truncated = False + + if len(blocks) > 20: + truncated = True + blocks = blocks[0 : 19] + + for block in blocks: + if block["reason"] == None or block["reason"] == '': + message = message + block["blocked"] + " with unspecified reason\n" + else: + if len(block["reason"]) > 420: + block["reason"] = block["reason"][0:419] + "[…]" + + message = message + block["blocked"] + ' for "' + block["reason"].replace("@", "@\u200b") + '"\n' + + if truncated: + message = message + "(the list has been truncated to the first 20 entries)" + + botheaders = {**api_headers, **{"Authorization": "Bearer " + config["bot_token"]}} + + req = reqto.post( + f"{config['bot_instance']}/api/v1/statuses", + data={ + "status" : message, + "visibility" : config['bot_visibility'], + "content_type": "text/plain" + }, + headers=botheaders, + timeout=10 + ).json() + + return True + +def get_mastodon_blocks(domain: str) -> dict: + if type(domain) != str: + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError(f"Parameter 'domain' cannot be empty") + + # DEBUG: print("DEBUG: Fetching mastodon blocks from domain:", domain) + blocks = { + "Suspended servers": [], + "Filtered media" : [], + "Limited servers" : [], + "Silenced servers" : [], + } + + try: + doc = bs4.BeautifulSoup( + reqto.get(f"https://{domain}/about", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])).text, + "html.parser", + ) + except BaseException as e: + print("ERROR: Cannot fetch from domain:", domain, e) + update_last_error(domain, e) + return {} + + for header in doc.find_all("h3"): + header_text = tidyup(header.text) + + if header_text in language_mapping: + # DEBUG: print(f"DEBUG: header_text='{header_text}'") + header_text = language_mapping[header_text] + + if header_text in blocks or header_text.lower() in blocks: + # replaced find_next_siblings with find_all_next to account for instances that e.g. hide lists in dropdown menu + for line in header.find_all_next("table")[0].find_all("tr")[1:]: + blocks[header_text].append( + { + "domain": tidyup(line.find("span").text), + "hash" : tidyup(line.find("span")["title"][9:]), + "reason": tidyup(line.find_all("td")[1].text), + } + ) + + # DEBUG: print("DEBUG: Returning blocks for domain:", domain) + return { + "reject" : blocks["Suspended servers"], + "media_removal" : blocks["Filtered media"], + "followers_only": blocks["Limited servers"] + blocks["Silenced servers"], + } + +def get_friendica_blocks(domain: str) -> dict: + if type(domain) != str: + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError(f"Parameter 'domain' cannot be empty") + + # DEBUG: print("DEBUG: Fetching friendica blocks from domain:", domain) + blocks = [] + + try: + doc = bs4.BeautifulSoup( + reqto.get(f"https://{domain}/friendica", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])).text, + "html.parser", + ) + except BaseException as e: + print("WARNING: Failed to fetch /friendica from domain:", domain, e) + update_last_error(domain, e) + return {} + + blocklist = doc.find(id="about_blocklist") + + # Prevents exceptions: + if blocklist is None: + # DEBUG: print("DEBUG: Instance has no block list:", domain) + return {} + + for line in blocklist.find("table").find_all("tr")[1:]: + # DEBUG: print(f"DEBUG: line='{line}'") + blocks.append({ + "domain": tidyup(line.find_all("td")[0].text), + "reason": tidyup(line.find_all("td")[1].text) + }) + + # DEBUG: print("DEBUG: Returning blocks() for domain:", domain, len(blocks)) + return { + "reject": blocks + } + +def get_misskey_blocks(domain: str) -> dict: + if type(domain) != str: + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError(f"Parameter 'domain' cannot be empty") + + # DEBUG: print("DEBUG: Fetching misskey blocks from domain:", domain) + blocks = { + "suspended": [], + "blocked" : [] + } + + offset = 0 + step = config["misskey_offset"] + while True: + # iterating through all "suspended" (follow-only in its terminology) + # instances page-by-page, since that troonware doesn't support + # sending them all at once + try: + # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...") + if offset == 0: + # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset) + fetched = post_json_api(domain, "/api/federation/instances", json.dumps({ + "sort" : "+pubAt", + "host" : None, + "suspended": True, + "limit" : step + }), {"Origin": domain}) + else: + # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset) + fetched = post_json_api(domain, "/api/federation/instances", json.dumps({ + "sort" : "+pubAt", + "host" : None, + "suspended": True, + "limit" : step, + "offset" : offset - 1 + }), {"Origin": domain}) + + # DEBUG: print("DEBUG: fetched():", len(fetched)) + if len(fetched) == 0: + # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain) + break + elif len(fetched) != config["misskey_offset"]: + # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config['misskey_offset']}'") + offset = offset + (config["misskey_offset"] - len(fetched)) + else: + # DEBUG: print("DEBUG: Raising offset by step:", step) + offset = offset + step + + for instance in fetched: + # just in case + if instance["isSuspended"]: + blocks["suspended"].append( + { + "domain": tidyup(instance["host"]), + # no reason field, nothing + "reason": None + } + ) + + except BaseException as e: + print("WARNING: Caught error, exiting loop:", domain, e) + update_last_error(domain, e) + offset = 0 + break + + while True: + # same shit, different asshole ("blocked" aka full suspend) + try: + if offset == 0: + # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset) + fetched = post_json_api(domain,"/api/federation/instances", json.dumps({ + "sort" : "+pubAt", + "host" : None, + "blocked": True, + "limit" : step + }), {"Origin": domain}) + else: + # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset) + fetched = post_json_api(domain,"/api/federation/instances", json.dumps({ + "sort" : "+pubAt", + "host" : None, + "blocked": True, + "limit" : step, + "offset" : offset-1 + }), {"Origin": domain}) + + # DEBUG: print("DEBUG: fetched():", len(fetched)) + if len(fetched) == 0: + # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain) + break + elif len(fetched) != config["misskey_offset"]: + # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config['misskey_offset']}'") + offset = offset + (config["misskey_offset"] - len(fetched)) + else: + # DEBUG: print("DEBUG: Raising offset by step:", step) + offset = offset + step + + for instance in fetched: + if instance["isBlocked"]: + blocks["blocked"].append({ + "domain": tidyup(instance["host"]), + "reason": None + }) + + except BaseException as e: + print("ERROR: Exception during POST:", domain, e) + update_last_error(domain, e) + offset = 0 + break + + # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") + update_last_instance_fetch(domain) + + # DEBUG: print("DEBUG: Returning for domain,blocked(),suspended():", domain, len(blocks["blocked"]), len(blocks["suspended"])) + return { + "reject" : blocks["blocked"], + "followers_only": blocks["suspended"] + } + +def tidyup(string: str) -> str: + if type(string) != str: + raise ValueError(f"Parameter string[]={type(string)} is not expected") + + # some retards put their blocks in variable case + string = string.lower().strip() + + # other retards put the port + string = re.sub("\:\d+$", "", string) + + # bigger retards put the schema in their blocklist, sometimes even without slashes + string = re.sub("^https?\:(\/*)", "", string) + + # and trailing slash + string = re.sub("\/$", "", string) + + # and the @ + string = re.sub("^\@", "", string) + + # the biggest retards of them all try to block individual users + string = re.sub("(.+)\@", "", string) + + return string diff --git a/fetch_bkali.py b/fetch_bkali.py index 01aea5a..f6b6fdb 100755 --- a/fetch_bkali.py +++ b/fetch_bkali.py @@ -20,9 +20,9 @@ import json import sys import validators -import fba +from fba import * -fba.lock_process() +boot.lock_process() domains = list() try: @@ -68,4 +68,4 @@ if len(domains) > 0: print(f"INFO: Fetching instances from domain='{domain}' ...") fba.fetch_instances(domain, None, None, sys.argv[0]) -fba.shutdown() +boot.shutdown() diff --git a/fetch_blocks.py b/fetch_blocks.py index b5a49c4..c7f25f5 100755 --- a/fetch_blocks.py +++ b/fetch_blocks.py @@ -23,9 +23,9 @@ import bs4 import itertools import re import validators -import fba +from fba import * -fba.lock_process() +boot.lock_process() fba.cursor.execute( "SELECT domain, software, origin, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial', 'bookwyrm', 'takahe') AND (last_blocked IS NULL OR last_blocked < ?) ORDER BY rowid DESC", [time.time() - fba.config["recheck_block"]] @@ -548,4 +548,4 @@ for blocker, software, origin, nodeinfo_url in rows: blockdict = [] -fba.shutdown() +boot.shutdown() diff --git a/fetch_fba_rss.py b/fetch_fba_rss.py index 7e888e2..022f46e 100755 --- a/fetch_fba_rss.py +++ b/fetch_fba_rss.py @@ -20,9 +20,9 @@ import reqto import rss_parser import sys -import fba +from fba import * -fba.lock_process() +boot.lock_process() feed = sys.argv[1] @@ -63,4 +63,4 @@ if len(domains) > 0: print(f"INFO: Fetching instances from domain='{domain}' ...") fba.fetch_instances(domain, None, None, sys.argv[0]) -fba.shutdown() +boot.shutdown() diff --git a/fetch_instances.py b/fetch_instances.py index e20bb75..926ccf6 100755 --- a/fetch_instances.py +++ b/fetch_instances.py @@ -22,9 +22,9 @@ import sys import json import time import validators -import fba +from fba import * -fba.lock_process() +boot.lock_process() instance = sys.argv[1] @@ -47,4 +47,4 @@ for row in rows: print(f"INFO: Fetching instances for instance '{row[0]}' ('{row[2]}') of origin='{row[1]}',nodeinfo_url='{row[3]}'") fba.fetch_instances(row[0], row[1], row[2], sys.argv[0], row[3]) -fba.shutdown() +boot.shutdown()