From: Roland Häder Date: Fri, 9 Jun 2023 00:35:07 +0000 (+0200) Subject: Continued: X-Git-Url: https://git.mxchange.org/?a=commitdiff_plain;h=bf40c5220d0def178c31df888ddeb5113481c52c;p=fba.git Continued: - moved wrapper functions around reqto.get|post() to module fba/network.py - renamed get_response() -> fetch_response() --- diff --git a/fba/__init__.py b/fba/__init__.py index 48c3116..d6d5a6e 100644 --- a/fba/__init__.py +++ b/fba/__init__.py @@ -8,4 +8,5 @@ __all__ = [ 'federation', 'fba', 'instances', + 'network', ] diff --git a/fba/commands.py b/fba/commands.py index 56bc9ff..dedaf0a 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -33,6 +33,7 @@ from fba import boot from fba import config from fba import fba from fba import instances +from fba import network from fba.federation import * @@ -58,7 +59,7 @@ def fetch_bkali(args: argparse.Namespace): # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!") domains = list() try: - fetched = fba.post_json_api("gql.api.bka.li", "/v1/graphql", json.dumps({ + fetched = network.post_json_api("gql.api.bka.li", "/v1/graphql", json.dumps({ "query": "query domainlist {nodeinfo(order_by: {domain: asc}) {domain}}" })) @@ -158,7 +159,7 @@ def fetch_blocks(args: argparse.Namespace): print(f"INFO: blocker='{blocker}',software='{software}'") try: if software == "friendica": - json = fba.fetch_friendica_blocks(blocker) + json = friendica.fetch_blocks(blocker) elif software == "misskey": json = misskey.fetch_blocks(blocker) @@ -248,7 +249,7 @@ def fetch_blocks(args: argparse.Namespace): print("WARNING: Unknown software:", blocker, software) if config.get("bot_enabled") and len(blockdict) > 0: - send_bot_post(blocker, blockdict) + network.send_bot_post(blocker, blockdict) blockdict = [] @@ -263,7 +264,7 @@ def fetch_cs(args: argparse.Namespace): try: doc = bs4.BeautifulSoup( - fba.get_response("meta.chaos.social", "/federation", fba.headers, (config.get("connection_timeout"), config.get("read_timeout"))).text, + network.fetch_response("meta.chaos.social", "/federation", fba.headers, (config.get("connection_timeout"), config.get("read_timeout"))).text, "html.parser", ) # DEBUG: print(f"DEBUG: doc()={len(doc)}[]={type(doc)}") diff --git a/fba/fba.py b/fba/fba.py index 7868dd6..122b2f6 100644 --- a/fba/fba.py +++ b/fba/fba.py @@ -1,4 +1,3 @@ -# Fedi API Block - An aggregator for fetching blocking data from fediverse nodes # Copyright (C) 2023 Free Software Foundation # # This program is free software: you can redistribute it and/or modify @@ -17,7 +16,6 @@ import bs4 import hashlib import re -import reqto import requests import json import sqlite3 @@ -31,6 +29,7 @@ from fba import blacklist from fba import cache from fba import config from fba import instances +from fba import network from fba.federation import lemmy from fba.federation import misskey @@ -368,14 +367,14 @@ def fetch_peers(domain: str, software: str) -> list: # DEBUG: print(f"DEBUG: Fetching peers from '{domain}',software='{software}' ...") peers = list() try: - response = get_response(domain, "/api/v1/instance/peers", api_headers, (config.get("connection_timeout"), config.get("read_timeout"))) + response = network.fetch_response(domain, "/api/v1/instance/peers", api_headers, (config.get("connection_timeout"), config.get("read_timeout"))) data = json_from_response(response) # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'") if not response.ok or response.status_code >= 400: # DEBUG: print(f"DEBUG: Was not able to fetch peers, trying alternative ...") - response = get_response(domain, "/api/v3/site", api_headers, (config.get("connection_timeout"), config.get("read_timeout"))) + response = network.fetch_response(domain, "/api/v3/site", api_headers, (config.get("connection_timeout"), config.get("read_timeout"))) data = json_from_response(response) # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'") @@ -409,41 +408,6 @@ def fetch_peers(domain: str, software: str) -> list: # DEBUG: print("DEBUG: Returning peers[]:", type(peers)) return peers -def post_json_api(domain: str, path: str, parameter: str, extra_headers: dict = {}) -> dict: - # DEBUG: print(f"DEBUG: domain='{domain}',path='{path}',parameter='{parameter}',extra_headers()={len(extra_headers)} - CALLED!") - if type(domain) != str: - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError(f"Parameter 'domain' is empty") - elif type(path) != str: - raise ValueError(f"path[]={type(path)} is not 'str'") - elif path == "": - raise ValueError("Parameter 'path' cannot be empty") - elif type(parameter) != str: - raise ValueError(f"parameter[]={type(parameter)} is not 'str'") - - # DEBUG: print("DEBUG: Sending POST to domain,path,parameter:", domain, path, parameter, extra_headers) - data = {} - try: - response = reqto.post( - f"https://{domain}{path}", - data=parameter, - headers={**api_headers, **extra_headers}, - timeout=(config.get("connection_timeout"), config.get("read_timeout")) - ) - - data = json_from_response(response) - # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'") - if not response.ok or response.status_code >= 400: - print(f"WARNING: Cannot query JSON API: domain='{domain}',path='{path}',parameter()={len(parameter)},response.status_code='{response.status_code}',data[]='{type(data)}'") - instances.update_last_error(domain, response) - - except BaseException as e: - print(f"WARNING: Some error during post(): domain='{domain}',path='{path}',parameter()={len(parameter)},exception[{type(e)}]:'{str(e)}'") - - # DEBUG: print(f"DEBUG: Returning data({len(data)})=[]:{type(data)}") - return data - def fetch_nodeinfo(domain: str, path: str = None) -> list: # DEBUG: print(f"DEBUG: domain='{domain}',path={path} - CALLED!") if type(domain) != str: @@ -478,7 +442,7 @@ def fetch_nodeinfo(domain: str, path: str = None) -> list: try: # DEBUG: print(f"DEBUG: Fetching request='{request}' from domain='{domain}' ...") - response = get_response(domain, request, api_headers, (config.get("nodeinfo_connection_timeout"), config.get("nodeinfo_read_timeout"))) + response = network.fetch_response(domain, request, api_headers, (config.get("nodeinfo_connection_timeout"), config.get("nodeinfo_read_timeout"))) data = json_from_response(response) # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'") @@ -514,7 +478,7 @@ def fetch_wellknown_nodeinfo(domain: str) -> list: data = {} try: - response = get_response(domain, "/.well-known/nodeinfo", api_headers, (config.get("nodeinfo_connection_timeout"), config.get("nodeinfo_read_timeout"))) + response = network.fetch_response(domain, "/.well-known/nodeinfo", api_headers, (config.get("nodeinfo_connection_timeout"), config.get("nodeinfo_read_timeout"))) data = json_from_response(response) # DEBUG: print("DEBUG: domain,response.ok,data[]:", domain, response.ok, type(data)) @@ -565,7 +529,7 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str: try: # DEBUG: print(f"DEBUG: Fetching path='{path}' from '{domain}' ...") - response = get_response(domain, path, headers, (config.get("connection_timeout"), config.get("read_timeout"))) + response = network.fetch_response(domain, path, headers, (config.get("connection_timeout"), config.get("read_timeout"))) # DEBUG: print("DEBUG: domain,response.ok,response.status_code,response.text[]:", domain, response.ok, response.status_code, type(response.text)) if response.ok and response.status_code < 300 and len(response.text) > 0: @@ -705,100 +669,6 @@ def determine_software(domain: str, path: str = None) -> str: # DEBUG: print("DEBUG: Returning domain,software:", domain, software) return software -def send_bot_post(instance: str, blocklist: dict): - # DEBUG: print(f"DEBUG: instance={instance},blocklist()={len(blocklist)} - CALLED!") - if type(domain) != str: - raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") - elif domain == "": - raise ValueError("Parameter 'domain' is empty") - elif type(blocklist) != dict: - raise ValueError(f"Parameter blocklist[]='{type(blocklist)}' is not 'dict'") - - message = instance + " has blocked the following instances:\n\n" - truncated = False - - if len(blocklist) > 20: - truncated = True - blocklist = blocklist[0 : 19] - - # DEBUG: print(f"DEBUG: blocklist()={len(blocklist)}") - for block in blocklist: - # DEBUG: print(f"DEBUG: block['{type(block)}']={block}") - if block["reason"] == None or block["reason"] == '': - message = message + block["blocked"] + " with unspecified reason\n" - else: - if len(block["reason"]) > 420: - block["reason"] = block["reason"][0:419] + "[…]" - - message = message + block["blocked"] + ' for "' + block["reason"].replace("@", "@\u200b") + '"\n' - - if truncated: - message = message + "(the list has been truncated to the first 20 entries)" - - botheaders = {**api_headers, **{"Authorization": "Bearer " + config.get("bot_token")}} - - req = reqto.post( - f"{config.get('bot_instance')}/api/v1/statuses", - data={ - "status" : message, - "visibility" : config.get('bot_visibility'), - "content_type": "text/plain" - }, - headers=botheaders, - timeout=10 - ).json() - - return True - -def fetch_friendica_blocks(domain: str) -> dict: - # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") - if type(domain) != str: - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError(f"Parameter 'domain' is empty") - - # DEBUG: print("DEBUG: Fetching friendica blocks from domain:", domain) - blocked = list() - - try: - doc = bs4.BeautifulSoup( - get_response(domain, "/friendica", headers, (config.get("connection_timeout"), config.get("read_timeout"))).text, - "html.parser", - ) - except BaseException as e: - print("WARNING: Failed to fetch /friendica from domain:", domain, e) - instances.update_last_error(domain, e) - return {} - - blocklist = doc.find(id="about_blocklist") - - # Prevents exceptions: - if blocklist is None: - # DEBUG: print("DEBUG: Instance has no block list:", domain) - return {} - - table = blocklist.find("table") - - # DEBUG: print(f"DEBUG: table[]='{type(table)}'") - if table.find("tbody"): - rows = table.find("tbody").find_all("tr") - else: - rows = table.find_all("tr") - - # DEBUG: print(f"DEBUG: Found rows()={len(rows)}") - for line in rows: - # DEBUG: print(f"DEBUG: line='{line}'") - blocked.append({ - "domain": tidyup_domain(line.find_all("td")[0].text), - "reason": tidyup_reason(line.find_all("td")[1].text) - }) - # DEBUG: print("DEBUG: Next!") - - # DEBUG: print("DEBUG: Returning blocklist() for domain:", domain, len(blocklist)) - return { - "reject": blocked - } - def tidyup_reason(reason: str) -> str: # DEBUG: print(f"DEBUG: reason='{reason}' - CALLED!") if type(reason) != str: @@ -859,32 +729,6 @@ def json_from_response(response: requests.models.Response) -> list: # DEBUG: print(f"DEBUG: data[]={type(data)} - EXIT!") return data -def get_response(domain: str, path: str, headers: dict, timeout: list) -> requests.models.Response: - # DEBUG: print(f"DEBUG: domain='{domain}',path='{path}',headers()={len(headers)},timeout={timeout} - CALLED!") - if type(domain) != str: - raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") - elif domain == "": - raise ValueError("Parameter 'domain' is empty") - elif type(path) != str: - raise ValueError(f"Parameter path[]='{type(path)}' is not 'str'") - elif path == "": - raise ValueError("Parameter 'path' is empty") - - try: - # DEBUG: print(f"DEBUG: Sending request to '{domain}{path}' ...") - response = reqto.get( - f"https://{domain}{path}", - headers=headers, - timeout=timeout - ); - except requests.exceptions.ConnectionError as e: - # DEBUG: print(f"DEBUG: Fetching '{path}' from '{domain}' failed. exception[{type(e)}]='{str(e)}'") - instances.update_last_error(domain, e) - raise e - - # DEBUG: print(f"DEBUG: response[]='{type(response)}' - EXXIT!") - return response - def has_key(keys: list, search: str, value: any) -> bool: # DEBUG: print(f"DEBUG: keys()={len(keys)},search='{search}',value[]='{type(value)}' - CALLED!") if type(keys) != list: @@ -974,9 +818,9 @@ def fetch_url(url: str, headers: dict, timeout: list) -> requests.models.Respons # Invoke other function, avoid trailing ? # DEBUG: print(f"DEBUG: components[{type(components)}]={components}") if components.query != "": - response = get_response(components.hostname, f"{components.path}?{components.query}", headers, timeout) + response = network.fetch_response(components.hostname, f"{components.path}?{components.query}", headers, timeout) else: - response = get_response(components.hostname, f"{components.path}", headers, timeout) + response = network.fetch_response(components.hostname, f"{components.path}", headers, timeout) # DEBUG: print(f"DEBUG: response[]='{type(response)}' - EXXIT!") return response diff --git a/fba/federation/lemmy.py b/fba/federation/lemmy.py index 8f7ce56..cc6d132 100644 --- a/fba/federation/lemmy.py +++ b/fba/federation/lemmy.py @@ -17,6 +17,7 @@ from fba import config from fba import fba from fba import instances +from fba import network def fetch_peers(domain: str) -> list: # DEBUG: print(f"DEBUG: domain({len(domain)})={domain},software='lemmy' - CALLED!") @@ -28,7 +29,7 @@ def fetch_peers(domain: str) -> list: peers = list() try: # DEBUG: print(f"DEBUG: domain='{domain}' is Lemmy, fetching JSON ...") - response = fba.get_response(domain, "/api/v3/site", fba.api_headers, (config.get("connection_timeout"), config.get("read_timeout"))) + response = network.fetch_response(domain, "/api/v3/site", fba.api_headers, (config.get("connection_timeout"), config.get("read_timeout"))) data = fba.json_from_response(response) diff --git a/fba/federation/mastodon.py b/fba/federation/mastodon.py index 51f1473..d00fa7c 100644 --- a/fba/federation/mastodon.py +++ b/fba/federation/mastodon.py @@ -23,6 +23,7 @@ from fba import blocks from fba import config from fba import fba from fba import instances +from fba import network language_mapping = { # English -> English @@ -67,7 +68,7 @@ def fetch_blocks_from_about(domain: str) -> dict: try: doc = bs4.BeautifulSoup( - fba.get_response(domain, "/about/more", fba.headers, (config.get("connection_timeout"), config.get("read_timeout"))).text, + network.fetch_response(domain, "/about/more", fba.headers, (config.get("connection_timeout"), config.get("read_timeout"))).text, "html.parser", ) except BaseException as e: @@ -134,7 +135,7 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): # handling CSRF, I've saw at least one server requiring it to access the endpoint # DEBUG: print("DEBUG: Fetching meta:", domain) meta = bs4.BeautifulSoup( - fba.get_response(domain, "/", fba.headers, (config.get("connection_timeout"), config.get("read_timeout"))).text, + network.fetch_response(domain, "/", fba.headers, (config.get("connection_timeout"), config.get("read_timeout"))).text, "html.parser", ) try: @@ -146,7 +147,7 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): reqheaders = fba.api_headers # DEBUG: print("DEBUG: Querying API domain_blocks:", domain) - blocklist = fba.get_response(domain, "/api/v1/instance/domain_blocks", reqheaders, (config.get("connection_timeout"), config.get("read_timeout"))).json() + blocklist = network.fetch_response(domain, "/api/v1/instance/domain_blocks", reqheaders, (config.get("connection_timeout"), config.get("read_timeout"))).json() print(f"INFO: Checking {len(blocklist)} entries from domain='{domain}',software='mastodon' ...") for block in blocklist: diff --git a/fba/federation/misskey.py b/fba/federation/misskey.py index 325c528..cb75a96 100644 --- a/fba/federation/misskey.py +++ b/fba/federation/misskey.py @@ -20,6 +20,7 @@ from fba import blacklist from fba import config from fba import fba from fba import instances +from fba import network def fetch_peers(domain: str) -> list: # DEBUG: print(f"DEBUG: domain({len(domain)})={domain} - CALLED!") @@ -39,7 +40,7 @@ def fetch_peers(domain: str) -> list: while True: # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...") if offset == 0: - fetched = fba.post_json_api(domain, "/api/federation/instances", json.dumps({ + fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({ "sort" : "+pubAt", "host" : None, "limit": step @@ -47,7 +48,7 @@ def fetch_peers(domain: str) -> list: "Origin": domain }) else: - fetched = fba.post_json_api(domain, "/api/federation/instances", json.dumps({ + fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({ "sort" : "+pubAt", "host" : None, "limit" : step, @@ -130,7 +131,7 @@ def fetch_blocks(domain: str) -> dict: # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...") if offset == 0: # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset) - fetched = fba.post_json_api(domain, "/api/federation/instances", json.dumps({ + fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({ "sort" : "+pubAt", "host" : None, "suspended": True, @@ -140,7 +141,7 @@ def fetch_blocks(domain: str) -> dict: }) else: # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset) - fetched = fba.post_json_api(domain, "/api/federation/instances", json.dumps({ + fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({ "sort" : "+pubAt", "host" : None, "suspended": True, @@ -190,7 +191,7 @@ def fetch_blocks(domain: str) -> dict: try: if offset == 0: # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset) - fetched = fba.post_json_api(domain, "/api/federation/instances", json.dumps({ + fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({ "sort" : "+pubAt", "host" : None, "blocked": True, @@ -200,7 +201,7 @@ def fetch_blocks(domain: str) -> dict: }) else: # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset) - fetched = fba.post_json_api(domain, "/api/federation/instances", json.dumps({ + fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({ "sort" : "+pubAt", "host" : None, "blocked": True, diff --git a/fba/federation/peertube.py b/fba/federation/peertube.py index da50cf3..13adea4 100644 --- a/fba/federation/peertube.py +++ b/fba/federation/peertube.py @@ -17,6 +17,7 @@ from fba import config from fba import fba from fba import instances +from fba import network def fetch_peers(domain: str) -> list: # DEBUG: print(f"DEBUG: domain({len(domain)})={domain},software='peertube' - CALLED!") @@ -32,7 +33,7 @@ def fetch_peers(domain: str) -> list: # DEBUG: print(f"DEBUG: domain='{domain}',mode='{mode}'") while True: try: - response = fba.get_response(domain, "/api/v1/server/{mode}?start={start}&count=100", headers, (config.get("connection_timeout"), config.get("read_timeout"))) + response = network.fetch_response(domain, "/api/v1/server/{mode}?start={start}&count=100", headers, (config.get("connection_timeout"), config.get("read_timeout"))) data = fba.json_from_response(response) # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code='{response.status_code}',data[]='{type(data)}'") diff --git a/fba/network.py b/fba/network.py new file mode 100644 index 0000000..a60e0e8 --- /dev/null +++ b/fba/network.py @@ -0,0 +1,177 @@ +# Fedi API Block - An aggregator for fetching blocking data from fediverse nodes +# Copyright (C) 2023 Free Software Foundation +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import bs4 +import reqto +import requests + +from fba import config +from fba import instances + +def post_json_api(domain: str, path: str, parameter: str, extra_headers: dict = {}) -> dict: + # DEBUG: print(f"DEBUG: domain='{domain}',path='{path}',parameter='{parameter}',extra_headers()={len(extra_headers)} - CALLED!") + if type(domain) != str: + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError(f"Parameter 'domain' is empty") + elif type(path) != str: + raise ValueError(f"path[]={type(path)} is not 'str'") + elif path == "": + raise ValueError("Parameter 'path' cannot be empty") + elif type(parameter) != str: + raise ValueError(f"parameter[]={type(parameter)} is not 'str'") + + # DEBUG: print("DEBUG: Sending POST to domain,path,parameter:", domain, path, parameter, extra_headers) + data = {} + try: + response = reqto.post( + f"https://{domain}{path}", + data=parameter, + headers={**api_headers, **extra_headers}, + timeout=(config.get("connection_timeout"), config.get("read_timeout")) + ) + + data = json_from_response(response) + # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'") + if not response.ok or response.status_code >= 400: + print(f"WARNING: Cannot query JSON API: domain='{domain}',path='{path}',parameter()={len(parameter)},response.status_code='{response.status_code}',data[]='{type(data)}'") + instances.update_last_error(domain, response) + + except BaseException as e: + print(f"WARNING: Some error during post(): domain='{domain}',path='{path}',parameter()={len(parameter)},exception[{type(e)}]:'{str(e)}'") + + # DEBUG: print(f"DEBUG: Returning data({len(data)})=[]:{type(data)}") + return data + +def send_bot_post(instance: str, blocklist: dict): + # DEBUG: print(f"DEBUG: instance={instance},blocklist()={len(blocklist)} - CALLED!") + if type(domain) != str: + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") + elif domain == "": + raise ValueError("Parameter 'domain' is empty") + elif type(blocklist) != dict: + raise ValueError(f"Parameter blocklist[]='{type(blocklist)}' is not 'dict'") + + message = instance + " has blocked the following instances:\n\n" + truncated = False + + if len(blocklist) > 20: + truncated = True + blocklist = blocklist[0 : 19] + + # DEBUG: print(f"DEBUG: blocklist()={len(blocklist)}") + for block in blocklist: + # DEBUG: print(f"DEBUG: block['{type(block)}']={block}") + if block["reason"] == None or block["reason"] == '': + message = message + block["blocked"] + " with unspecified reason\n" + else: + if len(block["reason"]) > 420: + block["reason"] = block["reason"][0:419] + "[…]" + + message = message + block["blocked"] + ' for "' + block["reason"].replace("@", "@\u200b") + '"\n' + + if truncated: + message = message + "(the list has been truncated to the first 20 entries)" + + botheaders = {**api_headers, **{"Authorization": "Bearer " + config.get("bot_token")}} + + req = reqto.post( + f"{config.get('bot_instance')}/api/v1/statuses", + data={ + "status" : message, + "visibility" : config.get('bot_visibility'), + "content_type": "text/plain" + }, + headers=botheaders, + timeout=10 + ).json() + + return True + +def fetch_friendica_blocks(domain: str) -> dict: + # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") + if type(domain) != str: + raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + elif domain == "": + raise ValueError(f"Parameter 'domain' is empty") + + # DEBUG: print("DEBUG: Fetching friendica blocks from domain:", domain) + blocked = list() + + try: + doc = bs4.BeautifulSoup( + fetch_response(domain, "/friendica", headers, (config.get("connection_timeout"), config.get("read_timeout"))).text, + "html.parser", + ) + except BaseException as e: + print("WARNING: Failed to fetch /friendica from domain:", domain, e) + instances.update_last_error(domain, e) + return {} + + blocklist = doc.find(id="about_blocklist") + + # Prevents exceptions: + if blocklist is None: + # DEBUG: print("DEBUG: Instance has no block list:", domain) + return {} + + table = blocklist.find("table") + + # DEBUG: print(f"DEBUG: table[]='{type(table)}'") + if table.find("tbody"): + rows = table.find("tbody").find_all("tr") + else: + rows = table.find_all("tr") + + # DEBUG: print(f"DEBUG: Found rows()={len(rows)}") + for line in rows: + # DEBUG: print(f"DEBUG: line='{line}'") + blocked.append({ + "domain": tidyup_domain(line.find_all("td")[0].text), + "reason": tidyup_reason(line.find_all("td")[1].text) + }) + # DEBUG: print("DEBUG: Next!") + + # DEBUG: print("DEBUG: Returning blocklist() for domain:", domain, len(blocklist)) + return { + "reject": blocked + } + +def fetch_response(domain: str, path: str, headers: dict, timeout: list) -> requests.models.Response: + # DEBUG: print(f"DEBUG: domain='{domain}',path='{path}',headers()={len(headers)},timeout={timeout} - CALLED!") + if type(domain) != str: + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") + elif domain == "": + raise ValueError("Parameter 'domain' is empty") + elif type(path) != str: + raise ValueError(f"Parameter path[]='{type(path)}' is not 'str'") + elif path == "": + raise ValueError("Parameter 'path' is empty") + + try: + # DEBUG: print(f"DEBUG: Sending request to '{domain}{path}' ...") + response = reqto.get( + f"https://{domain}{path}", + headers=headers, + timeout=timeout + ); + except requests.exceptions.ConnectionError as e: + # DEBUG: print(f"DEBUG: Fetching '{path}' from '{domain}' failed. exception[{type(e)}]='{str(e)}'") + instances.update_last_error(domain, e) + raise e + + # DEBUG: print(f"DEBUG: response[]='{type(response)}' - EXXIT!") + return response