X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;f=fba%2Ffba.py;h=59a3e8309b28b8627f65512065f0cc2574ce3e82;hb=88c30cbcf5f275c39bee5426443dd5b30c51281a;hp=1cc62a456b036b478d335edc5b943bb14a4c82f7;hpb=f534a8cbbbdbd4cdcf6a9bcf749c8cc35e03738d;p=fba.git diff --git a/fba/fba.py b/fba/fba.py index 1cc62a4..59a3e83 100644 --- a/fba/fba.py +++ b/fba/fba.py @@ -1,4 +1,3 @@ -# Fedi API Block - An aggregator for fetching blocking data from fediverse nodes # Copyright (C) 2023 Free Software Foundation # # This program is free software: you can redistribute it and/or modify @@ -14,43 +13,27 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import bs4 import hashlib import re -import reqto -import requests import json import sqlite3 import sys import time + +import bs4 +import requests import validators from urllib.parse import urlparse -from fba import cache +from fba import blacklist from fba import config from fba import instances +from fba import network -from fba.network import lemmy -from fba.network import misskey - -# Don't check these, known trolls/flooders/testing/developing -blacklist = [ - # Floods network with fake nodes as "research" project - "activitypub-troll.cf", - # Similar troll - "gab.best", - # Similar troll - "4chan.icu", - # Flooder (?) - "social.shrimpcam.pw", - # Flooder (?) - "mastotroll.netz.org", - # Testing/developing installations - "ngrok.io", - "ngrok-free.app", - "misskeytest.chn.moe", -] +from fba.federation import lemmy +from fba.federation import misskey +from fba.federation import peertube # Array with pending errors needed to be written to database pending_errors = { @@ -68,20 +51,6 @@ nodeinfo_identifier = [ "http://nodeinfo.diaspora.software/ns/schema/1.0", ] -# HTTP headers for non-API requests -headers = { - "User-Agent": config.get("useragent"), -} - -# HTTP headers for API requests -api_headers = { - "User-Agent": config.get("useragent"), - "Content-Type": "application/json", -} - -# URL for fetching peers -get_peers_url = "/api/v1/instance/peers" - # Connect to database connection = sqlite3.connect("blocks.db") cursor = connection.cursor() @@ -102,38 +71,44 @@ patterns = [ def is_primitive(var: any) -> bool: # DEBUG: print(f"DEBUG: var[]='{type(var)}' - CALLED!") - return type(var) in {int, str, float, bool} or var == None + return type(var) in {int, str, float, bool} or var is None def fetch_instances(domain: str, origin: str, software: str, script: str, path: str = None): # DEBUG: print(f"DEBUG: domain='{domain}',origin='{origin}',software='{software}',path='{path}' - CALLED!") - if type(domain) != str: + if not isinstance(domain, str): raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": - raise ValueError(f"Parameter 'domain' is empty") - elif type(origin) != str and origin != None: + raise ValueError("Parameter 'domain' is empty") + elif not isinstance(origin, str) and origin is not None: raise ValueError(f"Parameter origin[]={type(origin)} is not 'str'") - elif type(script) != str: + elif software is None: + print(f"DEBUG: software for domain='{domain}' is not set, determining ...") + software = determine_software(domain, path) + print(f"DEBUG: Determined software='{software}' for domain='{domain}'") + elif not isinstance(software, str): + raise ValueError(f"Parameter software[]={type(software)} is not 'str'") + elif not isinstance(script, str): raise ValueError(f"Parameter script[]={type(script)} is not 'str'") elif domain == "": - raise ValueError(f"Parameter 'domain' is empty") + raise ValueError("Parameter 'domain' is empty") - if not is_instance_registered(domain): + if not instances.is_registered(domain): # DEBUG: print("DEBUG: Adding new domain:", domain, origin) - add_instance(domain, origin, script, path) + instances.add(domain, origin, script, path) # DEBUG: print("DEBUG: Fetching instances for domain:", domain, software) - peerlist = get_peers(domain, software) + peerlist = fetch_peers(domain, software) if (peerlist is None): print("ERROR: Cannot fetch peers:", domain) return elif instances.has_pending_instance_data(domain): # DEBUG: print(f"DEBUG: domain='{domain}' has pending nodeinfo data, flushing ...") - instances.update_instance_data(domain) + instances.update_data(domain) print(f"INFO: Checking {len(peerlist)} instances from {domain} ...") for instance in peerlist: - if instance == None: + if instance is None: # Skip "None" types as tidup() cannot parse them continue @@ -147,17 +122,17 @@ def fetch_instances(domain: str, origin: str, software: str, script: str, path: elif not validators.domain(instance.split("/")[0]): print(f"WARNING: Bad instance='{instance}' from domain='{domain}',origin='{origin}',software='{software}'") continue - elif is_blacklisted(instance): + elif blacklist.is_blacklisted(instance): # DEBUG: print("DEBUG: instance is blacklisted:", instance) continue # DEBUG: print("DEBUG: Handling instance:", instance) try: - if not is_instance_registered(instance): + if not instances.is_registered(instance): # DEBUG: print("DEBUG: Adding new instance:", instance, domain) - add_instance(instance, domain, script) - except BaseException as e: - print(f"ERROR: instance='{instance}',exception[{type(e)}]:'{str(e)}'") + instances.add(instance, domain, script) + except BaseException as exception: + print(f"ERROR: instance='{instance}',exception[{type(exception)}]:'{str(exception)}'") continue # DEBUG: print("DEBUG: EXIT!") @@ -167,14 +142,14 @@ def add_peers(rows: dict) -> list: peers = list() for key in ["linked", "allowed", "blocked"]: # DEBUG: print(f"DEBUG: Checking key='{key}'") - if key in rows and rows[key] != None: + if key in rows and rows[key] is not None: # DEBUG: print(f"DEBUG: Adding {len(rows[key])} peer(s) to peers list ...") for peer in rows[key]: # DEBUG: print(f"DEBUG: peer='{peer}' - BEFORE!") peer = tidyup_domain(peer) # DEBUG: print(f"DEBUG: peer='{peer}' - AFTER!") - if is_blacklisted(peer): + if blacklist.is_blacklisted(peer): # DEBUG: print(f"DEBUG: peer='{peer}' is blacklisted, skipped!") continue @@ -210,7 +185,6 @@ def remove_version(software: str) -> str: # DEBUG: print(f"DEBUG: Was not able to find common seperator, returning untouched software='{software}'") return software - matches = None match = None # DEBUG: print(f"DEBUG: Checking {len(patterns)} patterns ...") for pattern in patterns: @@ -218,11 +192,12 @@ def remove_version(software: str) -> str: match = pattern.match(version) # DEBUG: print(f"DEBUG: match[]={type(match)}") - if type(match) is re.Match: + if isinstance(match, re.Match): + # DEBUG: print(f"DEBUG: version='{version}' is matching pattern='{pattern}'") break # DEBUG: print(f"DEBUG: version[{type(version)}]='{version}',match='{match}'") - if type(match) is not re.Match: + if not isinstance(match, re.Match): print(f"WARNING: version='{version}' does not match regex, leaving software='{software}' untouched.") return software @@ -240,11 +215,12 @@ def remove_version(software: str) -> str: def strip_powered_by(software: str) -> str: # DEBUG: print(f"DEBUG: software='{software}' - CALLED!") - if software == "": - print(f"ERROR: Bad method call, 'software' is empty") - raise Exception("Parameter 'software' is empty") + if not isinstance(software, str): + raise ValueError(f"Parameter software[]='{type(software)}' is not 'str'") + elif software == "": + raise ValueError("Parameter 'software' is empty") elif not "powered by" in software: - print(f"WARNING: Cannot find 'powered by' in '{software}'!") + print(f"WARNING: Cannot find 'powered by' in software='{software}'!") return software start = software.find("powered by ") @@ -260,9 +236,10 @@ def strip_powered_by(software: str) -> str: def strip_hosted_on(software: str) -> str: # DEBUG: print(f"DEBUG: software='{software}' - CALLED!") - if software == "": - print(f"ERROR: Bad method call, 'software' is empty") - raise Exception("Parameter 'software' is empty") + if not isinstance(software, str): + raise ValueError(f"Parameter software[]='{type(software)}' is not 'str'") + elif software == "": + raise ValueError("Parameter 'software' is empty") elif not "hosted on" in software: print(f"WARNING: Cannot find 'hosted on' in '{software}'!") return software @@ -270,7 +247,7 @@ def strip_hosted_on(software: str) -> str: end = software.find("hosted on ") # DEBUG: print(f"DEBUG: end[{type(end)}]='{end}'") - software = software[0, start].strip() + software = software[0, end].strip() # DEBUG: print(f"DEBUG: software='{software}'") software = strip_until(software, " - ") @@ -280,12 +257,14 @@ def strip_hosted_on(software: str) -> str: def strip_until(software: str, until: str) -> str: # DEBUG: print(f"DEBUG: software='{software}',until='{until}' - CALLED!") - if software == "": - print(f"ERROR: Bad method call, 'software' is empty") - raise Exception("Parameter 'software' is empty") + if not isinstance(software, str): + raise ValueError(f"Parameter software[]='{type(software)}' is not 'str'") + elif software == "": + raise ValueError("Parameter 'software' is empty") + elif not isinstance(until, str): + raise ValueError(f"Parameter until[]='{type(until)}' is not 'str'") elif until == "": - print(f"ERROR: Bad method call, 'until' is empty") - raise Exception("Parameter 'until' is empty") + raise ValueError("Parameter 'until' is empty") elif not until in software: print(f"WARNING: Cannot find '{until}' in '{software}'!") return software @@ -300,24 +279,11 @@ def strip_until(software: str, until: str) -> str: # DEBUG: print(f"DEBUG: software='{software}' - EXIT!") return software -def is_blacklisted(domain: str) -> bool: - if type(domain) != str: - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError(f"Parameter 'domain' is empty") - - blacklisted = False - for peer in blacklist: - if peer in domain: - blacklisted = True - - return blacklisted - def remove_pending_error(domain: str): - if type(domain) != str: + if not isinstance(domain, str): raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": - raise ValueError(f"Parameter 'domain' is empty") + raise ValueError("Parameter 'domain' is empty") try: # Prevent updating any pending errors, nodeinfo was found @@ -329,34 +295,19 @@ def remove_pending_error(domain: str): # DEBUG: print("DEBUG: EXIT!") def get_hash(domain: str) -> str: - if type(domain) != str: + if not isinstance(domain, str): raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": - raise ValueError(f"Parameter 'domain' is empty") + raise ValueError("Parameter 'domain' is empty") return hashlib.sha256(domain.encode("utf-8")).hexdigest() -def update_last_blocked(domain: str): - if type(domain) != str: - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError(f"Parameter 'domain' is empty") - - # DEBUG: print("DEBUG: Updating last_blocked for domain", domain) - instances.set("last_blocked", domain, time.time()) - - # Running pending updated - # DEBUG: print(f"DEBUG: Invoking instances.update_instance_data({domain}) ...") - instances.update_instance_data(domain) - - # DEBUG: print("DEBUG: EXIT!") - def log_error(domain: str, response: requests.models.Response): # DEBUG: print("DEBUG: domain,response[]:", domain, type(response)) - if type(domain) != str: + if not isinstance(domain, str): raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": - raise ValueError(f"Parameter 'domain' is empty") + raise ValueError("Parameter 'domain' is empty") try: # DEBUG: print("DEBUG: BEFORE response[]:", type(response)) @@ -364,7 +315,7 @@ def log_error(domain: str, response: requests.models.Response): response = str(response) # DEBUG: print("DEBUG: AFTER response[]:", type(response)) - if type(response) is str: + if isinstance(response, str): cursor.execute("INSERT INTO error_log (domain, error_code, error_message, created) VALUES (?, 999, ?, ?)",[ domain, response, @@ -381,110 +332,48 @@ def log_error(domain: str, response: requests.models.Response): # Cleanup old entries # DEBUG: print(f"DEBUG: Purging old records (distance: {config.get('error_log_cleanup')})") cursor.execute("DELETE FROM error_log WHERE created < ?", [time.time() - config.get("error_log_cleanup")]) - except BaseException as e: - print(f"ERROR: failed SQL query: domain='{domain}',exception[{type(e)}]:'{str(e)}'") + except BaseException as exception: + print(f"ERROR: failed SQL query: domain='{domain}',exception[{type(exception)}]:'{str(exception)}'") sys.exit(255) # DEBUG: print("DEBUG: EXIT!") -def update_last_error(domain: str, response: requests.models.Response): - # DEBUG: print("DEBUG: domain,response[]:", domain, type(response)) - if type(domain) != str: - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError(f"Parameter 'domain' is empty") - - # DEBUG: print("DEBUG: BEFORE response[]:", type(response)) - if isinstance(response, BaseException) or isinstance(response, json.decoder.JSONDecodeError): - response = f"{type}:str(response)" - - # DEBUG: print("DEBUG: AFTER response[]:", type(response)) - if type(response) is str: - # DEBUG: print(f"DEBUG: Setting last_error_details='{response}'"); - instances.set("last_status_code" , domain, 999) - instances.set("last_error_details", domain, response) - else: - # DEBUG: print(f"DEBUG: Setting last_error_details='{response.reason}'"); - instances.set("last_status_code" , domain, response.status_code) - instances.set("last_error_details", domain, response.reason) - - # Running pending updated - # DEBUG: print(f"DEBUG: Invoking instances.update_instance_data({domain}) ...") - instances.update_instance_data(domain) - - log_error(domain, response) - - # DEBUG: print("DEBUG: EXIT!") - -def update_last_instance_fetch(domain: str): - # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") - if type(domain) != str: - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError(f"Parameter 'domain' is empty") - - # DEBUG: print("DEBUG: Updating last_instance_fetch for domain:", domain) - instances.set("last_instance_fetch", domain, time.time()) - - # Running pending updated - # DEBUG: print(f"DEBUG: Invoking instances.update_instance_data({domain}) ...") - instances.update_instance_data(domain) - - # DEBUG: print("DEBUG: EXIT!") - -def update_last_nodeinfo(domain: str): - # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") - if type(domain) != str: - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError(f"Parameter 'domain' is empty") - - # DEBUG: print("DEBUG: Updating last_nodeinfo for domain:", domain) - instances.set("last_nodeinfo", domain, time.time()) - instances.set("last_updated" , domain, time.time()) - - # Running pending updated - # DEBUG: print(f"DEBUG: Invoking instances.update_instance_data({domain}) ...") - instances.update_instance_data(domain) - - # DEBUG: print("DEBUG: EXIT!") - -def get_peers(domain: str, software: str) -> list: +def fetch_peers(domain: str, software: str) -> list: # DEBUG: print(f"DEBUG: domain({len(domain)})={domain},software={software} - CALLED!") - if type(domain) != str: + if not isinstance(domain, str): raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": - raise ValueError(f"Parameter 'domain' is empty") - elif type(software) != str and software != None: + raise ValueError("Parameter 'domain' is empty") + elif not isinstance(software, str) and software is not None: raise ValueError(f"software[]={type(software)} is not 'str'") if software == "misskey": - print(f"DEBUG: Invoking misskey.get_peers({domain}) ...") - return misskey.get_peers(domain) + # DEBUG: print(f"DEBUG: Invoking misskey.fetch_peers({domain}) ...") + return misskey.fetch_peers(domain) elif software == "lemmy": - print(f"DEBUG: Invoking lemmy.get_peers({domain}) ...") - return lemmy.get_peers(domain) + # DEBUG: print(f"DEBUG: Invoking lemmy.fetch_peers({domain}) ...") + return lemmy.fetch_peers(domain) elif software == "peertube": - print(f"DEBUG: Invoking peertube.get_peers({domain}) ...") - return peertube.get_peers(domain) + # DEBUG: print(f"DEBUG: Invoking peertube.fetch_peers({domain}) ...") + return peertube.fetch_peers(domain) - # DEBUG: print(f"DEBUG: Fetching get_peers_url='{get_peers_url}' from '{domain}',software='{software}' ...") + # DEBUG: print(f"DEBUG: Fetching peers from '{domain}',software='{software}' ...") peers = list() try: - response = get_response(domain, get_peers_url, api_headers, (config.get("connection_timeout"), config.get("read_timeout"))) + response = network.fetch_response(domain, "/api/v1/instance/peers", network.api_headers, (config.get("connection_timeout"), config.get("read_timeout"))) data = json_from_response(response) # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'") if not response.ok or response.status_code >= 400: - # DEBUG: print(f"DEBUG: Was not able to fetch '{get_peers_url}', trying alternative ...") - response = get_response(domain, "/api/v3/site", api_headers, (config.get("connection_timeout"), config.get("read_timeout"))) + # DEBUG: print(f"DEBUG: Was not able to fetch peers, trying alternative ...") + response = network.fetch_response(domain, "/api/v3/site", network.api_headers, (config.get("connection_timeout"), config.get("read_timeout"))) data = json_from_response(response) # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'") if not response.ok or response.status_code >= 400: print("WARNING: Could not reach any JSON API:", domain) - update_last_error(domain, response) + instances.update_last_error(domain, response) elif response.ok and isinstance(data, list): # DEBUG: print(f"DEBUG: domain='{domain}' returned a list: '{data}'") sys.exit(255) @@ -494,65 +383,31 @@ def get_peers(domain: str, software: str) -> list: # DEBUG: print("DEBUG: Added instance(s) to peers") else: print("WARNING: JSON response does not contain 'federated_instances':", domain) - update_last_error(domain, response) + instances.update_last_error(domain, response) else: # DEBUG: print("DEBUG: Querying API was successful:", domain, len(data)) peers = data - except BaseException as e: - print("WARNING: Some error during get():", domain, e) - update_last_error(domain, e) + except BaseException as exception: + print("WARNING: Some error during get():", domain, exception) + instances.update_last_error(domain, exception) # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'") instances.set("total_peers", domain, len(peers)) # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") - update_last_instance_fetch(domain) + instances.update_last_instance_fetch(domain) # DEBUG: print("DEBUG: Returning peers[]:", type(peers)) return peers -def post_json_api(domain: str, path: str, parameter: str, extra_headers: dict = {}) -> dict: - if type(domain) != str: - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError(f"Parameter 'domain' is empty") - elif type(path) != str: - raise ValueError(f"path[]={type(path)} is not 'str'") - elif path == "": - raise ValueError("Parameter 'path' cannot be empty") - elif type(parameter) != str: - raise ValueError(f"parameter[]={type(parameter)} is not 'str'") - - # DEBUG: print("DEBUG: Sending POST to domain,path,parameter:", domain, path, parameter, extra_headers) - data = {} - try: - response = reqto.post( - f"https://{domain}{path}", - data=parameter, - headers={**api_headers, **extra_headers}, - timeout=(config.get("connection_timeout"), config.get("read_timeout")) - ) - - data = json_from_response(response) - # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'") - if not response.ok or response.status_code >= 400: - print(f"WARNING: Cannot query JSON API: domain='{domain}',path='{path}',parameter()={len(parameter)},response.status_code='{response.status_code}',data[]='{type(data)}'") - update_last_error(domain, response) - - except BaseException as e: - print(f"WARNING: Some error during post(): domain='{domain}',path='{path}',parameter()={len(parameter)},exception[{type(e)}]:'{str(e)}'") - - # DEBUG: print(f"DEBUG: Returning data({len(data)})=[]:{type(data)}") - return data - def fetch_nodeinfo(domain: str, path: str = None) -> list: # DEBUG: print(f"DEBUG: domain='{domain}',path={path} - CALLED!") - if type(domain) != str: + if not isinstance(domain, str): raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": - raise ValueError(f"Parameter 'domain' is empty") - elif type(path) != str and path != None: + raise ValueError("Parameter 'domain' is empty") + elif not isinstance(path, str) and path is not None: raise ValueError(f"Parameter path[]={type(path)} is not 'str'") # DEBUG: print(f"DEBUG: Fetching nodeinfo from domain='{domain}' ...") @@ -574,13 +429,13 @@ def fetch_nodeinfo(domain: str, path: str = None) -> list: data = {} for request in request_paths: - if path != None and path != "" and path != f"https://{domain}{path}": + if path is not None and path != "" and path != f"https://{domain}{path}": # DEBUG: print(f"DEBUG: path='{path}' does not match request='{request}' - SKIPPED!") continue try: # DEBUG: print(f"DEBUG: Fetching request='{request}' from domain='{domain}' ...") - response = get_response(domain, request, api_headers, (config.get("nodeinfo_connection_timeout"), config.get("nodeinfo_read_timeout"))) + response = network.fetch_response(domain, request, network.api_headers, (config.get("nodeinfo_connection_timeout"), config.get("nodeinfo_read_timeout"))) data = json_from_response(response) # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'") @@ -594,12 +449,12 @@ def fetch_nodeinfo(domain: str, path: str = None) -> list: sys.exit(255) elif not response.ok or response.status_code >= 400: print("WARNING: Failed fetching nodeinfo from domain:", domain) - update_last_error(domain, response) + instances.update_last_error(domain, response) continue - except BaseException as e: + except BaseException as exception: # DEBUG: print("DEBUG: Cannot fetch API request:", request) - update_last_error(domain, e) + instances.update_last_error(domain, exception) pass # DEBUG: print(f"DEBUG: data()={len(data)} - EXIT!") @@ -607,16 +462,16 @@ def fetch_nodeinfo(domain: str, path: str = None) -> list: def fetch_wellknown_nodeinfo(domain: str) -> list: # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") - if type(domain) != str: + if not isinstance(domain, str): raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": - raise ValueError(f"Parameter 'domain' is empty") + raise ValueError("Parameter 'domain' is empty") # DEBUG: print("DEBUG: Fetching .well-known info for domain:", domain) data = {} try: - response = get_response(domain, "/.well-known/nodeinfo", api_headers, (config.get("nodeinfo_connection_timeout"), config.get("nodeinfo_read_timeout"))) + response = network.fetch_response(domain, "/.well-known/nodeinfo", network.api_headers, (config.get("nodeinfo_connection_timeout"), config.get("nodeinfo_read_timeout"))) data = json_from_response(response) # DEBUG: print("DEBUG: domain,response.ok,data[]:", domain, response.ok, type(data)) @@ -629,7 +484,7 @@ def fetch_wellknown_nodeinfo(domain: str) -> list: # DEBUG: print("DEBUG: rel,href:", link["rel"], link["href"]) if link["rel"] in nodeinfo_identifier: # DEBUG: print("DEBUG: Fetching nodeinfo from:", link["href"]) - response = get_url(link["href"], api_headers, (config.get("connection_timeout"), config.get("read_timeout"))) + response = fetch_url(link["href"], network.api_headers, (config.get("connection_timeout"), config.get("read_timeout"))) data = json_from_response(response) # DEBUG: print("DEBUG: href,response.ok,response.status_code:", link["href"], response.ok, response.status_code) @@ -643,9 +498,9 @@ def fetch_wellknown_nodeinfo(domain: str) -> list: else: print("WARNING: nodeinfo does not contain 'links':", domain) - except BaseException as e: + except BaseException as exception: print("WARNING: Failed fetching .well-known info:", domain) - update_last_error(domain, e) + instances.update_last_error(domain, exception) pass # DEBUG: print("DEBUG: Returning data[]:", type(data)) @@ -653,21 +508,21 @@ def fetch_wellknown_nodeinfo(domain: str) -> list: def fetch_generator_from_path(domain: str, path: str = "/") -> str: # DEBUG: print(f"DEBUG: domain({len(domain)})={domain},path={path} - CALLED!") - if type(domain) != str: + if not isinstance(domain, str): raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": - raise ValueError(f"Parameter 'domain' is empty") - elif type(path) != str: + raise ValueError("Parameter 'domain' is empty") + elif not isinstance(path, str): raise ValueError(f"path[]={type(path)} is not 'str'") elif path == "": - raise ValueError(f"Parameter 'domain' is empty") + raise ValueError("Parameter 'path' is empty") # DEBUG: print(f"DEBUG: domain='{domain}',path='{path}' - CALLED!") software = None try: # DEBUG: print(f"DEBUG: Fetching path='{path}' from '{domain}' ...") - response = get_response(domain, path, headers, (config.get("connection_timeout"), config.get("read_timeout"))) + response = network.fetch_response(domain, path, network.web_headers, (config.get("connection_timeout"), config.get("read_timeout"))) # DEBUG: print("DEBUG: domain,response.ok,response.status_code,response.text[]:", domain, response.ok, response.status_code, type(response.text)) if response.ok and response.status_code < 300 and len(response.text) > 0: @@ -692,30 +547,30 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str: instances.set("detection_mode", domain, "SITE_NAME") remove_pending_error(domain) - except BaseException as e: - # DEBUG: print(f"DEBUG: Cannot fetch / from '{domain}':", e) - update_last_error(domain, e) + except BaseException as exception: + # DEBUG: print(f"DEBUG: Cannot fetch / from '{domain}':", exception) + instances.update_last_error(domain, exception) pass # DEBUG: print(f"DEBUG: software[]={type(software)}") - if type(software) is str and software == "": + if isinstance(software, str) and software == "": # DEBUG: print(f"DEBUG: Corrected empty string to None for software of domain='{domain}'") software = None - elif type(software) is str and ("." in software or " " in software): + elif isinstance(software, str) and ("." in software or " " in software): # DEBUG: print(f"DEBUG: software='{software}' may contain a version number, domain='{domain}', removing it ...") software = remove_version(software) # DEBUG: print(f"DEBUG: software[]={type(software)}") - if type(software) is str and " powered by " in software: + if isinstance(software, str) and " powered by " in software: # DEBUG: print(f"DEBUG: software='{software}' has 'powered by' in it") software = remove_version(strip_powered_by(software)) - elif type(software) is str and " hosted on " in software: + elif isinstance(software, str) and " hosted on " in software: # DEBUG: print(f"DEBUG: software='{software}' has 'hosted on' in it") software = remove_version(strip_hosted_on(software)) - elif type(software) is str and " by " in software: + elif isinstance(software, str) and " by " in software: # DEBUG: print(f"DEBUG: software='{software}' has ' by ' in it") software = strip_until(software, " by ") - elif type(software) is str and " see " in software: + elif isinstance(software, str) and " see " in software: # DEBUG: print(f"DEBUG: software='{software}' has ' see ' in it") software = strip_until(software, " see ") @@ -724,11 +579,11 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str: def determine_software(domain: str, path: str = None) -> str: # DEBUG: print(f"DEBUG: domain({len(domain)})={domain},path={path} - CALLED!") - if type(domain) != str: + if not isinstance(domain, str): raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": - raise ValueError(f"Parameter 'domain' is empty") - elif type(path) != str and path != None: + raise ValueError("Parameter 'domain' is empty") + elif not isinstance(path, str) and path is not None: raise ValueError(f"Parameter path[]={type(path)} is not 'str'") # DEBUG: print("DEBUG: Determining software for domain,path:", domain, path) @@ -745,11 +600,11 @@ def determine_software(domain: str, path: str = None) -> str: # DEBUG: print("DEBUG: data():", len(data), data) if "status" in data and data["status"] == "error" and "message" in data: print("WARNING: JSON response is an error:", data["message"]) - update_last_error(domain, data["message"]) + instances.update_last_error(domain, data["message"]) return fetch_generator_from_path(domain) elif "message" in data: print("WARNING: JSON response contains only a message:", data["message"]) - update_last_error(domain, data["message"]) + instances.update_last_error(domain, data["message"]) return fetch_generator_from_path(domain) elif "software" not in data or "name" not in data["software"]: # DEBUG: print(f"DEBUG: JSON response from domain='{domain}' does not include [software][name], fetching / ...") @@ -772,17 +627,17 @@ def determine_software(domain: str, path: str = None) -> str: software = "misskey" elif software.find("/") > 0: print("WARNING: Spliting of slash:", software) - software = software.split("/")[-1]; + software = tidup_domain(software.split("/")[-1]); elif software.find("|") > 0: print("WARNING: Spliting of pipe:", software) software = tidyup_domain(software.split("|")[0]); elif "powered by" in software: # DEBUG: print(f"DEBUG: software='{software}' has 'powered by' in it") software = strip_powered_by(software) - elif type(software) is str and " by " in software: + elif isinstance(software, str) and " by " in software: # DEBUG: print(f"DEBUG: software='{software}' has ' by ' in it") software = strip_until(software, " by ") - elif type(software) is str and " see " in software: + elif isinstance(software, str) and " see " in software: # DEBUG: print(f"DEBUG: software='{software}' has ' see ' in it") software = strip_until(software, " see ") @@ -800,330 +655,17 @@ def determine_software(domain: str, path: str = None) -> str: software = remove_version(software) # DEBUG: print(f"DEBUG: software[]={type(software)}") - if type(software) is str and "powered by" in software: + if isinstance(software, str) and "powered by" in software: # DEBUG: print(f"DEBUG: software='{software}' has 'powered by' in it") software = remove_version(strip_powered_by(software)) # DEBUG: print("DEBUG: Returning domain,software:", domain, software) return software -def is_instance_registered(domain: str) -> bool: - # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") - if type(domain) != str: - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError(f"Parameter 'domain' is empty") - - # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") - if not cache.key_exists("is_registered"): - # DEBUG: print(f"DEBUG: Cache for 'is_registered' not initialized, fetching all rows ...") - try: - cursor.execute("SELECT domain FROM instances") - - # Check Set all - cache.set_all("is_registered", cursor.fetchall(), True) - except BaseException as e: - print(f"ERROR: failed SQL query: domain='{domain}',exception[{type(e)}]:'{str(e)}'") - sys.exit(255) - - # Is cache found? - registered = cache.sub_key_exists("is_registered", domain) - - # DEBUG: print(f"DEBUG: registered='{registered}' - EXIT!") - return registered - -def add_instance(domain: str, origin: str, originator: str, path: str = None): - # DEBUG: print(f"DEBUG: domain='{domain}',origin='{origin}',originator='{originator}',path='{path}' - CALLED!") - if type(domain) != str: - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError(f"Parameter 'domain' is empty") - elif type(origin) != str and origin != None: - raise ValueError(f"origin[]={type(origin)} is not 'str'") - elif type(originator) != str: - raise ValueError(f"originator[]={type(originator)} is not 'str'") - elif originator == "": - raise ValueError(f"originator cannot be empty") - elif not validators.domain(domain.split("/")[0]): - raise ValueError(f"Bad domain name='{domain}'") - elif origin is not None and not validators.domain(origin.split("/")[0]): - raise ValueError(f"Bad origin name='{origin}'") - elif is_blacklisted(domain): - raise Exception(f"domain='{domain}' is blacklisted, but method invoked") - - # DEBUG: print("DEBUG: domain,origin,originator,path:", domain, origin, originator, path) - software = determine_software(domain, path) - # DEBUG: print("DEBUG: Determined software:", software) - - print(f"INFO: Adding instance domain='{domain}' (origin='{origin}',software='{software}')") - try: - cursor.execute( - "INSERT INTO instances (domain, origin, originator, hash, software, first_seen) VALUES (?, ?, ?, ?, ?, ?)", - ( - domain, - origin, - originator, - get_hash(domain), - software, - time.time() - ), - ) - - cache.set_sub_key("is_registered", domain, True) - - if instances.has_pending_instance_data(domain): - # DEBUG: print(f"DEBUG: domain='{domain}' has pending nodeinfo being updated ...") - instances.set("last_status_code" , domain, None) - instances.set("last_error_details", domain, None) - instances.update_instance_data(domain) - remove_pending_error(domain) - - if domain in pending_errors: - # DEBUG: print("DEBUG: domain has pending error being updated:", domain) - update_last_error(domain, pending_errors[domain]) - remove_pending_error(domain) - - except BaseException as e: - print(f"ERROR: failed SQL query: domain='{domain}',exception[{type(e)}]:'{str(e)}'") - sys.exit(255) - else: - # DEBUG: print("DEBUG: Updating nodeinfo for domain:", domain) - update_last_nodeinfo(domain) - - # DEBUG: print("DEBUG: EXIT!") - -def send_bot_post(instance: str, blocks: dict): - # DEBUG: print(f"DEBUG: instance={instance},blocks()={len(blocks)} - CALLED!") - if type(domain) != str: - raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") - elif domain == "": - raise ValueError("Parameter 'domain' is empty") - elif type(blocks) != dict: - raise ValueError(f"Parameter blocks[]='{type(blocks)}' is not 'dict'") - - message = instance + " has blocked the following instances:\n\n" - truncated = False - - if len(blocks) > 20: - truncated = True - blocks = blocks[0 : 19] - - # DEBUG: print(f"DEBUG: blocks()={len(blocks)}") - for block in blocks: - # DEBUG: print(f"DEBUG: block['{type(block)}']={block}") - if block["reason"] == None or block["reason"] == '': - message = message + block["blocked"] + " with unspecified reason\n" - else: - if len(block["reason"]) > 420: - block["reason"] = block["reason"][0:419] + "[…]" - - message = message + block["blocked"] + ' for "' + block["reason"].replace("@", "@\u200b") + '"\n' - - if truncated: - message = message + "(the list has been truncated to the first 20 entries)" - - botheaders = {**api_headers, **{"Authorization": "Bearer " + config.get("bot_token")}} - - req = reqto.post( - f"{config.get('bot_instance')}/api/v1/statuses", - data={ - "status" : message, - "visibility" : config.get('bot_visibility'), - "content_type": "text/plain" - }, - headers=botheaders, - timeout=10 - ).json() - - return True - -def fetch_friendica_blocks(domain: str) -> dict: - # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") - if type(domain) != str: - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError(f"Parameter 'domain' is empty") - - # DEBUG: print("DEBUG: Fetching friendica blocks from domain:", domain) - blocks = [] - - try: - doc = bs4.BeautifulSoup( - get_response(domain, "/friendica", headers, (config.get("connection_timeout"), config.get("read_timeout"))).text, - "html.parser", - ) - except BaseException as e: - print("WARNING: Failed to fetch /friendica from domain:", domain, e) - update_last_error(domain, e) - return {} - - blocklist = doc.find(id="about_blocklist") - - # Prevents exceptions: - if blocklist is None: - # DEBUG: print("DEBUG: Instance has no block list:", domain) - return {} - - for line in blocklist.find("table").find_all("tr")[1:]: - # DEBUG: print(f"DEBUG: line='{line}'") - blocks.append({ - "domain": tidyup_domain(line.find_all("td")[0].text), - "reason": tidyup_domain(line.find_all("td")[1].text) - }) - - # DEBUG: print("DEBUG: Returning blocks() for domain:", domain, len(blocks)) - return { - "reject": blocks - } - -def fetch_misskey_blocks(domain: str) -> dict: - # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") - if type(domain) != str: - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError(f"Parameter 'domain' is empty") - - # DEBUG: print("DEBUG: Fetching misskey blocks from domain:", domain) - blocks = { - "suspended": [], - "blocked" : [] - } - - offset = 0 - step = config.get("misskey_limit") - while True: - # iterating through all "suspended" (follow-only in its terminology) - # instances page-by-page, since that troonware doesn't support - # sending them all at once - try: - # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...") - if offset == 0: - # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset) - fetched = post_json_api(domain, "/api/federation/instances", json.dumps({ - "sort" : "+pubAt", - "host" : None, - "suspended": True, - "limit" : step - }), { - "Origin": domain - }) - else: - # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset) - fetched = post_json_api(domain, "/api/federation/instances", json.dumps({ - "sort" : "+pubAt", - "host" : None, - "suspended": True, - "limit" : step, - "offset" : offset - 1 - }), { - "Origin": domain - }) - - # DEBUG: print("DEBUG: fetched():", len(fetched)) - if len(fetched) == 0: - # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain) - break - elif len(fetched) != config.get("misskey_limit"): - # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'") - offset = offset + (config.get("misskey_limit") - len(fetched)) - else: - # DEBUG: print("DEBUG: Raising offset by step:", step) - offset = offset + step - - count = 0 - for instance in fetched: - # Is it there? - if instance["isSuspended"] and not has_key(blocks["suspended"], "domain", instance): - count = count + 1 - blocks["suspended"].append( - { - "domain": tidyup_domain(instance["host"]), - # no reason field, nothing - "reason": None - } - ) - - # DEBUG: print(f"DEBUG: count={count}") - if count == 0: - # DEBUG: print(f"DEBUG: API is no more returning new instances, aborting loop!") - break - - except BaseException as e: - print("WARNING: Caught error, exiting loop:", domain, e) - update_last_error(domain, e) - offset = 0 - break - - while True: - # same shit, different asshole ("blocked" aka full suspend) - try: - if offset == 0: - # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset) - fetched = post_json_api(domain,"/api/federation/instances", json.dumps({ - "sort" : "+pubAt", - "host" : None, - "blocked": True, - "limit" : step - }), { - "Origin": domain - }) - else: - # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset) - fetched = post_json_api(domain,"/api/federation/instances", json.dumps({ - "sort" : "+pubAt", - "host" : None, - "blocked": True, - "limit" : step, - "offset" : offset - 1 - }), { - "Origin": domain - }) - - # DEBUG: print("DEBUG: fetched():", len(fetched)) - if len(fetched) == 0: - # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain) - break - elif len(fetched) != config.get("misskey_limit"): - # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'") - offset = offset + (config.get("misskey_limit") - len(fetched)) - else: - # DEBUG: print("DEBUG: Raising offset by step:", step) - offset = offset + step - - count = 0 - for instance in fetched: - # Is it there? - if instance["isBlocked"] and not has_key(blocks["blocked"], "domain", instance): - count = count + 1 - blocks["blocked"].append({ - "domain": tidyup_domain(instance["host"]), - "reason": None - }) - - # DEBUG: print(f"DEBUG: count={count}") - if count == 0: - # DEBUG: print(f"DEBUG: API is no more returning new instances, aborting loop!") - break - - except BaseException as e: - print("ERROR: Exception during POST:", domain, e) - update_last_error(domain, e) - offset = 0 - break - - # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") - update_last_instance_fetch(domain) - - # DEBUG: print("DEBUG: Returning for domain,blocked(),suspended():", domain, len(blocks["blocked"]), len(blocks["suspended"])) - return { - "reject" : blocks["blocked"], - "followers_only": blocks["suspended"] - } - def tidyup_reason(reason: str) -> str: # DEBUG: print(f"DEBUG: reason='{reason}' - CALLED!") - if type(reason) != str: - raise ValueError(f"Parameter reason[]={type(reason)} is not expected") + if not isinstance(reason, str): + raise ValueError(f"Parameter reason[]={type(reason)} is not 'str'") # Strip string reason = reason.strip() @@ -1131,13 +673,13 @@ def tidyup_reason(reason: str) -> str: # Replace â with " reason = re.sub("â", "\"", reason) - ## DEBUG: print(f"DEBUG: reason='{reason}' - EXIT!") + # DEBUG: print(f"DEBUG: reason='{reason}' - EXIT!") return reason def tidyup_domain(domain: str) -> str: # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") - if type(domain) != str: - raise ValueError(f"Parameter domain[]={type(domain)} is not expected") + if not isinstance(domain, str): + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") # All lower-case and strip spaces out + last dot domain = domain.lower().strip().rstrip(".") @@ -1156,6 +698,10 @@ def tidyup_domain(domain: str) -> str: # No individual users in block lists domain = re.sub("(.+)\@", "", domain) + if domain.find("/profile/"): + domain = domain.split("/profile/")[0] + elif domain.find("/users/"): + domain = domain.split("/users/")[0] # DEBUG: print(f"DEBUG: domain='{domain}' - EXIT!") return domain @@ -1176,50 +722,24 @@ def json_from_response(response: requests.models.Response) -> list: # DEBUG: print(f"DEBUG: data[]={type(data)} - EXIT!") return data -def get_response(domain: str, path: str, headers: dict, timeout: list) -> requests.models.Response: - # DEBUG: print(f"DEBUG: domain='{domain}',path='{path}',headers()={len(headers)},timeout={timeout} - CALLED!") - if type(domain) != str: - raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") - elif domain == "": - raise ValueError("Parameter 'domain' is empty") - elif type(path) != str: - raise ValueError(f"Parameter path[]='{type(path)}' is not 'str'") - elif path == "": - raise ValueError("Parameter 'path' is empty") - - try: - # DEBUG: print(f"DEBUG: Sending request to '{domain}{path}' ...") - response = reqto.get( - f"https://{domain}{path}", - headers=headers, - timeout=timeout - ); - except requests.exceptions.ConnectionError as e: - # DEBUG: print(f"DEBUG: Fetching '{path}' from '{domain}' failed. exception[{type(e)}]='{str(e)}'") - update_last_error(domain, e) - raise e - - # DEBUG: print(f"DEBUG: response[]='{type(response)}' - EXXIT!") - return response - -def has_key(keys: list, search: str, value: any) -> bool: - # DEBUG: print(f"DEBUG: keys()={len(keys)},search='{search}',value[]='{type(value)}' - CALLED!") - if type(keys) != list: - raise ValueError(f"Parameter keys[]='{type(keys)}' is not 'list'") - elif type(search) != str: - raise ValueError(f"Parameter search[]='{type(search)}' is not 'str'") - elif search == "": - raise ValueError("Parameter 'search' is empty") +def has_key(lists: list, key: str, value: any) -> bool: + # DEBUG: print(f"DEBUG: lists()={len(lists)},key='{key}',value[]='{type(value)}' - CALLED!") + if not isinstance(lists, list): + raise ValueError(f"Parameter lists[]='{type(lists)}' is not 'list'") + elif not isinstance(key, str): + raise ValueError(f"Parameter key[]='{type(key)}' is not 'str'") + elif key == "": + raise ValueError("Parameter 'key' is empty") has = False - # DEBUG: print(f"DEBUG: Checking keys()={len(keys)} ...") - for key in keys: - # DEBUG: print(f"DEBUG: key['{type(key)}']={key}") - if type(key) != dict: - raise ValueError(f"key[]='{type(key)}' is not 'dict'") - elif not search in key: - raise KeyError(f"Cannot find search='{search}'") - elif key[search] == value: + # DEBUG: print(f"DEBUG: Checking lists()={len(lists)} ...") + for row in lists: + # DEBUG: print(f"DEBUG: row['{type(row)}']={row}") + if not isinstance(row, dict): + raise ValueError(f"row[]='{type(row)}' is not 'dict'") + elif not key in row: + raise KeyError(f"Cannot find key='{key}'") + elif row[key] == value: has = True break @@ -1230,8 +750,6 @@ def find_domains(tag: bs4.element.Tag) -> list: # DEBUG: print(f"DEBUG: tag[]={type(tag)} - CALLED!") if not isinstance(tag, bs4.element.Tag): raise ValueError(f"Parameter tag[]={type(tag)} is not type of bs4.element.Tag") - elif not isinstance(tag, bs4.element.Tag): - raise KeyError("Cannot find table with instances!") elif len(tag.select("tr")) == 0: raise KeyError("No table rows found in table!") @@ -1247,7 +765,7 @@ def find_domains(tag: bs4.element.Tag) -> list: # DEBUG: print(f"DEBUG: domain='{domain}',reason='{reason}'") - if is_blacklisted(domain): + if blacklist.is_blacklisted(domain): print(f"WARNING: domain='{domain}' is blacklisted - skipped!") continue elif domain == "gab.com/.ai, develop.gab.com": @@ -1278,12 +796,16 @@ def find_domains(tag: bs4.element.Tag) -> list: # DEBUG: print(f"DEBUG: domains()={len(domains)} - EXIT!") return domains -def get_url(url: str, headers: dict, timeout: list) -> requests.models.Response: +def fetch_url(url: str, headers: dict, timeout: tuple) -> requests.models.Response: # DEBUG: print(f"DEBUG: url='{url}',headers()={len(headers)},timeout={timeout} - CALLED!") - if type(url) != str: + if not isinstance(url, str): raise ValueError(f"Parameter url[]='{type(url)}' is not 'str'") elif url == "": raise ValueError("Parameter 'url' is empty") + elif not isinstance(headers, dict): + raise ValueError(f"Parameter headers[]='{type(headers)}' is not 'dict'") + elif not isinstance(timeout, tuple): + raise ValueError(f"Parameter timeout[]='{type(timeout)}' is not 'tuple'") # DEBUG: print(f"DEBUG: Parsing url='{url}'") components = urlparse(url) @@ -1291,9 +813,9 @@ def get_url(url: str, headers: dict, timeout: list) -> requests.models.Response: # Invoke other function, avoid trailing ? # DEBUG: print(f"DEBUG: components[{type(components)}]={components}") if components.query != "": - response = get_response(components.hostname, f"{components.path}?{components.query}", headers, timeout) + response = network.fetch_response(components.hostname, f"{components.path}?{components.query}", headers, timeout) else: - response = get_response(components.hostname, f"{components.path}", headers, timeout) + response = network.fetch_response(components.hostname, f"{components.path}", headers, timeout) # DEBUG: print(f"DEBUG: response[]='{type(response)}' - EXXIT!") return response