X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;f=fba.py;h=08b29c18acf6172bc3e07785b5ecf1253842df47;hb=05230958548d4e7538072ca7fab22af4450c8537;hp=9427b147b3ed645c13f20ea88020a431f8ef62e1;hpb=3b7e8ffd45989dd63ffa8e9ff4ec5b6d18464f3a;p=fba.git diff --git a/fba.py b/fba.py old mode 100644 new mode 100755 index 9427b14..08b29c1 --- a/fba.py +++ b/fba.py @@ -1,831 +1,33 @@ -import bs4 -import hashlib -import re -import reqto -import json -import sqlite3 -import sys -import time -import validators - -with open("config.json") as f: - config = json.loads(f.read()) - -# Don't check these, known trolls/flooders/testing/developing -blacklist = [ - # Floods network with fake nodes as "research" project - "activitypub-troll.cf", - # Similar troll - "gab.best", - # Similar troll - "4chan.icu", - # Flooder (?) - "social.shrimpcam.pw", - # Flooder (?) - "mastotroll.netz.org", - # Testing/developing installations - "ngrok.io", -] - -# Array with pending errors needed to be written to database -pending_errors = { -} - -# "rel" identifiers (no real URLs) -nodeinfo_identifier = [ - "http://nodeinfo.diaspora.software/ns/schema/2.1", - "http://nodeinfo.diaspora.software/ns/schema/2.0", - "http://nodeinfo.diaspora.software/ns/schema/1.1", - "http://nodeinfo.diaspora.software/ns/schema/1.0", -] - -# HTTP headers for requests -headers = { - "user-agent": config["useragent"], -} - -# Found info from node, such as nodeinfo URL, detection mode that needs to be -# written to database. Both arrays must be filled at the same time or else -# update_nodeinfos() will fail -nodeinfos = { - # Detection mode: AUTO_DISCOVERY or STATIC_CHECKS - "detection_mode": {}, - # Found nodeinfo URL - "nodeinfo_url": {}, - # Where to fetch peers (other instances) - "get_peers_url": {}, -} - -language_mapping = { - # English -> English - "Silenced instances" : "Silenced servers", - "Suspended instances" : "Suspended servers", - "Limited instances" : "Limited servers", - # Mappuing German -> English - "Gesperrte Server" : "Suspended servers", - "Gefilterte Medien" : "Filtered media", - "Stummgeschaltete Server" : "Silenced servers", - # Japanese -> English - "停止済みのサーバー" : "Suspended servers", - "制限中のサーバー" : "Limited servers", - "メディアを拒否しているサーバー": "Filtered media", - "サイレンス済みのサーバー" : "Silenced servers", - # ??? -> English - "שרתים מושעים" : "Suspended servers", - "מדיה מסוננת" : "Filtered media", - "שרתים מוגבלים" : "Silenced servers", - # French -> English - "Serveurs suspendus" : "Suspended servers", - "Médias filtrés" : "Filtered media", - "Serveurs limités" : "Limited servers", - "Serveurs modérés" : "Limited servers", -} - -# URL for fetching peers -get_peers_url = "/api/v1/instance/peers" - -# Connect to database -connection = sqlite3.connect("blocks.db") -cursor = connection.cursor() - -def is_blacklisted(domain: str) -> bool: - blacklisted = False - for peer in blacklist: - if peer in domain: - blacklisted = True - - return blacklisted - -def get_hash(domain: str) -> str: - return hashlib.sha256(domain.encode("utf-8")).hexdigest() - -def update_last_blocked(domain: str): - # NOISY-DEBUG: print("DEBUG: Updating last_blocked for domain", domain) - try: - cursor.execute("UPDATE instances SET last_blocked = ?, last_updated = ? WHERE domain = ? LIMIT 1", [ - time.time(), - time.time(), - domain - ]) - - if cursor.rowcount == 0: - print("WARNING: Did not update any rows:", domain) - - except BaseException as e: - print("ERROR: failed SQL query:", domain, e) - sys.exit(255) - - # NOISY-DEBUG: print("DEBUG: EXIT!") - -def update_nodeinfos(domain: str): - # NOISY-DEBUG: print("DEBUG: Updating nodeinfo for domain:", domain) - sql_string = '' - fields = list() - for key in nodeinfos: - # NOISY-DEBUG: print("DEBUG: key:", key) - if domain in nodeinfos[key]: - # NOISY-DEBUG: print(f"DEBUG: Adding '{nodeinfos[key][domain]}' for key='{key}' ...") - fields.append(nodeinfos[key][domain]) - sql_string += f" {key} = ?," - - fields.append(domain) - # NOISY-DEBUG: print(f"DEBUG: sql_string='{sql_string}',fields()={len(fields)}") - - sql = "UPDATE instances SET" + sql_string + " last_status_code = NULL, last_error_details = NULL WHERE domain = ? LIMIT 1" - # NOISY-DEBUG: print("DEBUG: sql:", sql) - - try: - # NOISY-DEBUG: print("DEBUG: Executing SQL:", sql) - cursor.execute(sql, fields) - # NOISY-DEBUG: print(f"DEBUG: Success! (rowcount={cursor.rowcount })") - - if cursor.rowcount == 0: - print("WARNING: Did not update any rows:", domain) - - except BaseException as e: - print(f"ERROR: failed SQL query: domain='{domain}',sql='{sql}',exception='{e}'") - sys.exit(255) - - # NOISY-DEBUG: print("DEBUG: Deleting nodeinfos for domain:", domain) - for key in nodeinfos: - try: - # NOISY-DEBUG: print("DEBUG: Deleting key:", key) - del nodeinfos[key][domain] - except: - pass - - # NOISY-DEBUG: print("DEBUG: EXIT!") - -def update_last_error(domain: str, res: any): - # NOISY-DEBUG: print("DEBUG: domain,res[]:", domain, type(res)) - try: - # NOISY-DEBUG: print("DEBUG: BEFORE res[]:", type(res)) - if isinstance(res, BaseException) or isinstance(res, json.JSONDecodeError): - res = str(res) - - # NOISY-DEBUG: print("DEBUG: AFTER res[]:", type(res)) - if type(res) is str: - # NOISY-DEBUG: print(f"DEBUG: Setting last_error_details='{res}'"); - cursor.execute("UPDATE instances SET last_status_code = 999, last_error_details = ?, last_updated = ? WHERE domain = ? LIMIT 1", [ - res, - time.time(), - domain - ]) - else: - # NOISY-DEBUG: print(f"DEBUG: Setting last_error_details='{res.reason}'"); - cursor.execute("UPDATE instances SET last_status_code = ?, last_error_details = ?, last_updated = ? WHERE domain = ? LIMIT 1", [ - res.status_code, - res.reason, - time.time(), - domain - ]) - - if cursor.rowcount == 0: - # NOISY-DEBUG: print("DEBUG: Did not update any rows:", domain) - pending_errors[domain] = res - - except BaseException as e: - print("ERROR: failed SQL query:", domain, e) - sys.exit(255) - - # NOISY-DEBUG: print("DEBUG: EXIT!") - -def update_last_nodeinfo(domain: str): - # NOISY-DEBUG: print("DEBUG: Updating last_nodeinfo for domain:", domain) - try: - cursor.execute("UPDATE instances SET last_nodeinfo = ?, last_updated = ? WHERE domain = ? LIMIT 1", [ - time.time(), - time.time(), - domain - ]) - - if cursor.rowcount == 0: - print("WARNING: Did not update any rows:", domain) - - except BaseException as e: - print("ERROR: failed SQL query:", domain, e) - sys.exit(255) - - connection.commit() - # NOISY-DEBUG: print("DEBUG: EXIT!") - -def get_peers(domain: str, software: str) -> list: - # NOISY-DEBUG: print("DEBUG: Getting peers for domain:", domain, software) - peers = list() - - if software == "lemmy": - # NOISY-DEBUG: print(f"DEBUG: domain='{domain}' is Lemmy, fetching JSON ...") - try: - res = reqto.get(f"https://{domain}/api/v3/site", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])) - - # NOISY-DEBUG: print(f"DEBUG: res.ok={res.ok},res.json[]={type(res.json())}") - if res.ok and isinstance(res.json(), dict): - # NOISY-DEBUG: print("DEBUG: Success, res.json():", len(res.json())) - json = res.json() - - if "federated_instances" in json and "linked" in json["federated_instances"]: - # NOISY-DEBUG: print("DEBUG: Found federated_instances", domain) - peers = json["federated_instances"]["linked"] + json["federated_instances"]["allowed"] + json["federated_instances"]["blocked"] - - except BaseException as e: - print("WARNING: Exception during fetching JSON:", domain, e) - - update_last_nodeinfo(domain) - - # NOISY-DEBUG: print("DEBUG: Returning peers[]:", type(peers)) - return peers - elif software == "peertube": - # NOISY-DEBUG: print(f"DEBUG: domain='{domain}' is a PeerTube, fetching JSON ...") - - start = 0 - for mode in ["followers", "following"]: - # NOISY-DEBUG: print(f"DEBUG: domain='{domain}',mode='{mode}'") - while True: - try: - res = reqto.get(f"https://{domain}/api/v1/server/{mode}?start={start}&count=100", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])) - - # NOISY-DEBUG: print(f"DEBUG: res.ok={res.ok},res.json[]={type(res.json())}") - if res.ok and isinstance(res.json(), dict): - # NOISY-DEBUG: print("DEBUG: Success, res.json():", len(res.json())) - json = res.json() - - if "data" in json: - # NOISY-DEBUG: print(f"DEBUG: Found {len(json['data'])} record(s).") - for record in json["data"]: - # NOISY-DEBUG: print(f"DEBUG: record()={len(record)}") - if mode in record and "host" in record[mode]: - # NOISY-DEBUG: print(f"DEBUG: Found host={record[mode]['host']}, adding ...") - peers.append(record[mode]["host"]) - else: - print(f"WARNING: record from '{domain}' has no '{mode}' or 'host' record: {record}") - - if len(json["data"]) < 100: - # NOISY-DEBUG: print("DEBUG: Reached end of JSON response:", domain) - break - - # Continue with next row - start = start + 100 - - except BaseException as e: - print("WARNING: Exception during fetching JSON:", domain, e) - - update_last_nodeinfo(domain) - - # NOISY-DEBUG: print("DEBUG: Returning peers[]:", type(peers)) - return peers - - # NOISY-DEBUG: print(f"DEBUG: Fetching '{get_peers_url}' from '{domain}' ...") - try: - res = reqto.get(f"https://{domain}{get_peers_url}", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])) - - if not res.ok or res.status_code >= 400: - res = reqto.get(f"https://{domain}/api/v3/site", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])) - - if "federated_instances" in json and "linked" in json["federated_instances"]: - # NOISY-DEBUG: print("DEBUG: Found federated_instances", domain) - peers = json["federated_instances"]["linked"] + json["federated_instances"]["allowed"] + json["federated_instances"]["blocked"] - else: - print("WARNING: Could not reach any JSON API:", domain) - update_last_error(domain, res) - else: - # NOISY-DEBUG: print("DEBUG: Querying API was successful:", domain, len(res.json())) - peers = res.json() - nodeinfos["get_peers_url"][domain] = get_peers_url - - except BaseException as e: - print("WARNING: Some error during get():", domain, e) - update_last_error(domain, e) - - update_last_nodeinfo(domain) - - # NOISY-DEBUG: print("DEBUG: Returning peers[]:", type(peers)) - return peers - -def post_json_api(domain: str, path: str, data: str) -> list: - # NOISY-DEBUG: print("DEBUG: Sending POST to domain,path,data:", domain, path, data) - json = {} - try: - res = reqto.post(f"https://{domain}{path}", data=data, headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])) - - if not res.ok or res.status_code >= 400: - print("WARNING: Cannot query JSON API:", domain, path, data, res.status_code) - update_last_error(domain, res) - raise - - update_last_nodeinfo(domain) - json = res.json() - except BaseException as e: - print("WARNING: Some error during post():", domain, path, data, e) - - # NOISY-DEBUG: print("DEBUG: Returning json():", len(json)) - return json - -def fetch_nodeinfo(domain: str) -> list: - # NOISY-DEBUG: print("DEBUG: Fetching nodeinfo from domain:", domain) - - nodeinfo = fetch_wellknown_nodeinfo(domain) - # NOISY-DEBUG: print("DEBUG: nodeinfo:", len(nodeinfo)) - - if len(nodeinfo) > 0: - # NOISY-DEBUG: print("DEBUG: Returning auto-discovered nodeinfo:", len(nodeinfo)) - return nodeinfo - - requests = [ - f"https://{domain}/nodeinfo/2.1.json", - f"https://{domain}/nodeinfo/2.1", - f"https://{domain}/nodeinfo/2.0.json", - f"https://{domain}/nodeinfo/2.0", - f"https://{domain}/nodeinfo/1.0", - f"https://{domain}/api/v1/instance" - ] - - json = {} - for request in requests: - try: - # NOISY-DEBUG: print("DEBUG: Fetching request:", request) - res = reqto.get(request, headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])) - - # NOISY-DEBUG: print("DEBUG: res.ok,res.json[]:", res.ok, type(res.json())) - if res.ok and isinstance(res.json(), dict): - # NOISY-DEBUG: print("DEBUG: Success:", request) - json = res.json() - nodeinfos["detection_mode"][domain] = "STATIC_CHECK" - nodeinfos["nodeinfo_url"][domain] = request - break - elif not res.ok or res.status_code >= 400: - print("WARNING: Failed fetching nodeinfo from domain:", domain) - update_last_error(domain, res) - continue - - except BaseException as e: - # NOISY-DEBUG: print("DEBUG: Cannot fetch API request:", request) - update_last_error(domain, e) - pass - - # NOISY-DEBUG: print("DEBUG: json[]:", type(json)) - if not isinstance(json, dict) or len(json) == 0: - print("WARNING: Failed fetching nodeinfo from domain:", domain) - - # NOISY-DEBUG: print("DEBUG: Returning json[]:", type(json)) - return json - -def fetch_wellknown_nodeinfo(domain: str) -> list: - # NOISY-DEBUG: print("DEBUG: Fetching .well-known info for domain:", domain) - json = {} +#!/usr/bin/python3 +# -*- coding: utf-8 -*- + +# Fedi API Block - An aggregator for fetching blocking data from fediverse nodes +# Copyright (C) 2023 Free Software Foundation +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . - try: - res = reqto.get(f"https://{domain}/.well-known/nodeinfo", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])) - # NOISY-DEBUG: print("DEBUG: domain,res.ok,res.json[]:", domain, res.ok, type(res.json())) - if res.ok and isinstance(res.json(), dict): - nodeinfo = res.json() - # NOISY-DEBUG: print("DEBUG: Found entries:", len(nodeinfo), domain) - if "links" in nodeinfo: - # NOISY-DEBUG: print("DEBUG: Found links in nodeinfo():", len(nodeinfo["links"])) - for link in nodeinfo["links"]: - # NOISY-DEBUG: print("DEBUG: rel,href:", link["rel"], link["href"]) - if link["rel"] in nodeinfo_identifier: - # NOISY-DEBUG: print("DEBUG: Fetching nodeinfo from:", link["href"]) - res = reqto.get(link["href"]) - - # NOISY-DEBUG: print("DEBUG: href,res.ok,res.status_code:", link["href"], res.ok, res.status_code) - if res.ok and isinstance(res.json(), dict): - # NOISY-DEBUG: print("DEBUG: Found JSON nodeinfo():", len(res.json())) - json = res.json() - nodeinfos["detection_mode"][domain] = "AUTO_DISCOVERY" - nodeinfos["nodeinfo_url"][domain] = link["href"] - break - else: - print("WARNING: Unknown 'rel' value:", domain, link["rel"]) - else: - print("WARNING: nodeinfo does not contain 'links':", domain) - - except BaseException as e: - print("WARNING: Failed fetching .well-known info:", domain) - update_last_error(domain, e) - pass - - # NOISY-DEBUG: print("DEBUG: Returning json[]:", type(json)) - return json - -def determine_software(domain: str) -> str: - # NOISY-DEBUG: print("DEBUG: Determining software for domain:", domain) - software = None - - json = fetch_nodeinfo(domain) - # NOISY-DEBUG: print("DEBUG: json[]:", type(json)) - - if not isinstance(json, dict) or len(json) == 0: - # NOISY-DEBUG: print("DEBUG: Could not determine software type:", domain) - return None - - # NOISY-DEBUG: print("DEBUG: json():", len(json), json) - if "status" in json and json["status"] == "error" and "message" in json: - print("WARNING: JSON response is an error:", json["message"]) - update_last_error(domain, json["message"]) - return None - elif "software" not in json or "name" not in json["software"]: - print("WARNING: JSON response does not include [software][name], guessing ...") - found = 0 - for element in {"uri", "title", "short_description", "description", "email", "version", "urls", "stats", "thumbnail", "languages", "contact_account", "registrations", "approval_required"}: - # NOISY-DEBUG: print("DEBUG: element:", element) - if element in json: - found = found + 1 - - # NOISY-DEBUG: print("DEBUG: Found elements:", found) - if found == len(json): - # NOISY-DEBUG: print("DEBUG: Maybe is Mastodon:", domain) - return "mastodon" - - print(f"WARNING: Cannot guess software type: domain='{domain}',found={found},json()={len(json)}") - return None - - software = tidyup(json["software"]["name"]) - - # NOISY-DEBUG: print("DEBUG: tidyup software:", software) - if software in ["akkoma", "rebased"]: - # NOISY-DEBUG: print("DEBUG: Setting pleroma:", domain, software) - software = "pleroma" - elif software in ["hometown", "ecko"]: - # NOISY-DEBUG: print("DEBUG: Setting mastodon:", domain, software) - software = "mastodon" - elif software in ["calckey", "groundpolis", "foundkey", "cherrypick", "meisskey"]: - # NOISY-DEBUG: print("DEBUG: Setting misskey:", domain, software) - software = "misskey" - elif software.find("/") > 0: - print("WARNING: Spliting of path:", software) - software = software.split("/")[-1]; - elif software.find("|") > 0: - print("WARNING: Spliting of path:", software) - software = software.split("|")[0].strip(); - - if software == "": - print("WARNING: tidyup() left no software name behind:", domain) - software = None - - # NOISY-DEBUG: print("DEBUG: Returning domain,software:", domain, software) - return software - -def update_block_reason(reason: str, blocker: str, blocked: str, block_level: str): - # NOISY-DEBUG: print("DEBUG: Updating block reason:", reason, blocker, blocked, block_level) - try: - cursor.execute( - "UPDATE blocks SET reason = ?, last_seen = ? WHERE blocker = ? AND blocked = ? AND block_level = ? AND reason = ''", - ( - reason, - time.time(), - blocker, - blocked, - block_level - ), - ) - - if cursor.rowcount == 0: - print("WARNING: Did not update any rows:", domain) - - except BaseException as e: - print("ERROR: failed SQL query:", reason, blocker, blocked, block_level, e) - sys.exit(255) - -def update_last_seen(blocker: str, blocked: str, block_level: str): - # NOISY-DEBUG: print("DEBUG: Updating last_seen for:", blocker, blocked, block_level) - try: - cursor.execute( - "UPDATE blocks SET last_seen = ? WHERE blocker = ? AND blocked = ? AND block_level = ?", - ( - time.time(), - blocker, - blocked, - block_level - ) - ) - - if cursor.rowcount == 0: - print("WARNING: Did not update any rows:", domain) - - except BaseException as e: - print("ERROR: failed SQL query:", last_seen, blocker, blocked, block_level, e) - sys.exit(255) - - # NOISY-DEBUG: print("DEBUG: EXIT!") - -def block_instance(blocker: str, blocked: str, reason: str, block_level: str): - # NOISY-DEBUG: print("DEBUG: blocker,blocked,reason,block_level:", blocker, blocked, reason, block_level) - if not validators.domain(blocker): - print("WARNING: Bad blocker:", blocker) - raise - elif not validators.domain(blocked): - print("WARNING: Bad blocked:", blocked) - raise - - print("INFO: New block:", blocker, blocked, reason, block_level, first_added, last_seen) - try: - cursor.execute( - "INSERT INTO blocks (blocker, blocked, reason, block_level, first_added, last_seen) VALUES(?, ?, ?, ?, ?, ?)", - ( - blocker, - blocked, - reason, - block_level, - time.time(), - time.time() - ), - ) - - except BaseException as e: - print("ERROR: failed SQL query:", blocker, blocked, reason, block_level, first_added, last_seen, e) - sys.exit(255) - - # NOISY-DEBUG: print("DEBUG: EXIT!") - -def add_instance(domain: str, origin: str, originator: str): - # NOISY-DEBUG: print("DEBUG: domain,origin:", domain, origin, originator) - if not validators.domain(domain): - print("WARNING: Bad domain name:", domain) - raise - elif origin is not None and not validators.domain(origin): - print("WARNING: Bad origin name:", origin) - raise - - software = determine_software(domain) - # NOISY-DEBUG: print("DEBUG: Determined software:", software) - - print(f"INFO: Adding new instance {domain} (origin: {origin})") - try: - cursor.execute( - "INSERT INTO instances (domain, origin, originator, hash, software, first_seen) VALUES (?, ?, ?, ?, ?, ?)", - ( - domain, - origin, - originator, - get_hash(domain), - software, - time.time() - ), - ) - - if domain in nodeinfos["nodeinfo_url"]: - # NOISY-DEBUG # NOISY-DEBUG: print("DEBUG: domain has pending nodeinfo being updated:", domain) - update_nodeinfos(domain) - try: - # Prevent updating any pending errors, nodeinfo was found - del pending_errors[domain] - except: - pass - elif domain in pending_errors: - # NOISY-DEBUG: print("DEBUG: domain has pending error being updated:", domain) - update_last_error(domain, pending_errors[domain]) - del pending_errors[domain] - - except BaseException as e: - print("ERROR: failed SQL query:", domain, e) - sys.exit(255) - else: - # NOISY-DEBUG: print("DEBUG: Updating nodeinfo for domain:", domain) - update_last_nodeinfo(domain) - - # NOISY-DEBUG: print("DEBUG: EXIT!") - -def send_bot_post(instance: str, blocks: dict): - message = instance + " has blocked the following instances:\n\n" - truncated = False - - if len(blocks) > 20: - truncated = True - blocks = blocks[0 : 19] - - for block in blocks: - if block["reason"] == None or block["reason"] == '': - message = message + block["blocked"] + " with unspecified reason\n" - else: - if len(block["reason"]) > 420: - block["reason"] = block["reason"][0:419] + "[…]" - - message = message + block["blocked"] + ' for "' + block["reason"].replace("@", "@\u200b") + '"\n' - - if truncated: - message = message + "(the list has been truncated to the first 20 entries)" - - botheaders = {**headers, **{"Authorization": "Bearer " + config["bot_token"]}} - - req = reqto.post( - f"{config['bot_instance']}/api/v1/statuses", - data={ - "status" : message, - "visibility" : config['bot_visibility'], - "content_type": "text/plain" - }, - headers=botheaders, - timeout=10 - ).json() - - return True - -def get_mastodon_blocks(domain: str) -> dict: - # NOISY-DEBUG: print("DEBUG: Fetching mastodon blocks from domain:", domain) - blocks = { - "Suspended servers": [], - "Filtered media" : [], - "Limited servers" : [], - "Silenced servers" : [], - } - - try: - doc = bs4.BeautifulSoup( - reqto.get(f"https://{domain}/about/more", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])).text, - "html.parser", - ) - except BaseException as e: - print("ERROR: Cannot fetch from domain:", domain, e) - update_last_error(domain, e) - return {} - - for header in doc.find_all("h3"): - header_text = tidyup(header.text) - - if header_text in language_mapping: - # NOISY-DEBUG: print(f"DEBUG: header_text='{header_text}'") - header_text = language_mapping[header_text] - - if header_text in blocks or header_text.lower() in blocks: - # replaced find_next_siblings with find_all_next to account for instances that e.g. hide lists in dropdown menu - for line in header.find_all_next("table")[0].find_all("tr")[1:]: - blocks[header_text].append( - { - "domain": tidyup(line.find("span").text), - "hash" : tidyup(line.find("span")["title"][9:]), - "reason": tidyup(line.find_all("td")[1].text), - } - ) - - # NOISY-DEBUG: print("DEBUG: Returning blocks for domain:", domain) - return { - "reject" : blocks["Suspended servers"], - "media_removal" : blocks["Filtered media"], - "followers_only": blocks["Limited servers"] + blocks["Silenced servers"], - } - -def get_friendica_blocks(domain: str) -> dict: - # NOISY-DEBUG: print("DEBUG: Fetching friendica blocks from domain:", domain) - blocks = [] - - try: - doc = bs4.BeautifulSoup( - reqto.get(f"https://{domain}/friendica", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])).text, - "html.parser", - ) - except BaseException as e: - print("WARNING: Failed to fetch /friendica from domain:", domain, e) - update_last_error(domain, e) - return {} - - blocklist = doc.find(id="about_blocklist") - - # Prevents exceptions: - if blocklist is None: - # NOISY-DEBUG: print("DEBUG: Instance has no block list:", domain) - return {} - - for line in blocklist.find("table").find_all("tr")[1:]: - blocks.append({ - "domain": tidyup(line.find_all("td")[0].text), - "reason": tidyup(line.find_all("td")[1].text) - }) - - # NOISY-DEBUG: print("DEBUG: Returning blocks() for domain:", domain, len(blocks)) - return { - "reject": blocks - } - -def get_misskey_blocks(domain: str) -> dict: - # NOISY-DEBUG: print("DEBUG: Fetching misskey blocks from domain:", domain) - blocks = { - "suspended": [], - "blocked" : [] - } - - counter = 0 - step = 99 - while True: - # iterating through all "suspended" (follow-only in its terminology) - # instances page-by-page, since that troonware doesn't support - # sending them all at once - try: - if counter == 0: - # NOISY-DEBUG: print("DEBUG: Sending JSON API request to domain,step,counter:", domain, step, counter) - doc = post_json_api(domain, "/api/federation/instances/", json.dumps({ - "sort" : "+caughtAt", - "host" : None, - "suspended": True, - "limit" : step - })) - else: - # NOISY-DEBUG: print("DEBUG: Sending JSON API request to domain,step,counter:", domain, step, counter) - doc = post_json_api(domain, "/api/federation/instances/", json.dumps({ - "sort" : "+caughtAt", - "host" : None, - "suspended": True, - "limit" : step, - "offset" : counter-1 - })) - - # NOISY-DEBUG: print("DEBUG: doc():", len(doc)) - if len(doc) == 0: - # NOISY-DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain) - break - - for instance in doc: - # just in case - if instance["isSuspended"]: - blocks["suspended"].append( - { - "domain": tidyup(instance["host"]), - # no reason field, nothing - "reason": "" - } - ) - - if len(doc) < step: - # NOISY-DEBUG: print("DEBUG: End of request:", len(doc), step) - break - - # NOISY-DEBUG: print("DEBUG: Raising counter by step:", step) - counter = counter + step - - except BaseException as e: - print("WARNING: Caught error, exiting loop:", domain, e) - update_last_error(domain, e) - counter = 0 - break - - while True: - # same shit, different asshole ("blocked" aka full suspend) - try: - if counter == 0: - # NOISY-DEBUG: print("DEBUG: Sending JSON API request to domain,step,counter:", domain, step, counter) - doc = post_json_api(domain,"/api/federation/instances", json.dumps({ - "sort" : "+caughtAt", - "host" : None, - "blocked": True, - "limit" : step - })) - else: - # NOISY-DEBUG: print("DEBUG: Sending JSON API request to domain,step,counter:", domain, step, counter) - doc = post_json_api(domain,"/api/federation/instances", json.dumps({ - "sort" : "+caughtAt", - "host" : None, - "blocked": True, - "limit" : step, - "offset" : counter-1 - })) - - # NOISY-DEBUG: print("DEBUG: doc():", len(doc)) - if len(doc) == 0: - # NOISY-DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain) - break - - for instance in doc: - if instance["isBlocked"]: - blocks["blocked"].append({ - "domain": tidyup(instance["host"]), - "reason": "" - }) - - if len(doc) < step: - # NOISY-DEBUG: print("DEBUG: End of request:", len(doc), step) - break - - # NOISY-DEBUG: print("DEBUG: Raising counter by step:", step) - counter = counter + step - - except BaseException as e: - print("ERROR: Exception during POST:", domain, e) - update_last_error(domain, e) - counter = 0 - break - - # NOISY-DEBUG: print("DEBUG: Returning for domain,blocked(),suspended():", domain, len(blocks["blocked"]), len(blocks["suspended"])) - return { - "reject" : blocks["blocked"], - "followers_only": blocks["suspended"] - } - -def tidyup(string: str) -> str: - # some retards put their blocks in variable case - string = string.lower().strip() - - # other retards put the port - string = re.sub("\:\d+$", "", string) - - # bigger retards put the schema in their blocklist, sometimes even without slashes - string = re.sub("^https?\:(\/*)", "", string) +import sys +from fba import boot - # and trailing slash - string = re.sub("\/$", "", string) +# Init parser +boot.init_parser() - # and the @ - string = re.sub("^\@", "", string) +# Run command +status = boot.run_command() - # the biggest retards of them all try to block individual users - string = re.sub("(.+)\@", "", string) +# Shutdown again +boot.shutdown() - return string +# Exit with status code +sys.exit(status)