X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;ds=sidebyside;f=fba%2Fnetworks%2Fmastodon.py;h=be22e94f9eba52f0d413ad2503733fc3e3623310;hb=7ce34f60c519ff20cc76b2030a91a0cefde3062b;hp=7a1f3eb6b07b9c16e3867ffa6d87e2cd95a0a7c8;hpb=f71083f2182908ee7bd6e8c24bdeeb3778cb9b41;p=fba.git diff --git a/fba/networks/mastodon.py b/fba/networks/mastodon.py index 7a1f3eb..be22e94 100644 --- a/fba/networks/mastodon.py +++ b/fba/networks/mastodon.py @@ -14,20 +14,25 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import inspect +import logging +import validators import bs4 -import requests -import validators -from fba import blacklist -from fba import blocks -from fba import config -from fba import fba -from fba import instances -from fba import network +from fba.helpers import blacklist +from fba.helpers import config +from fba.helpers import domain as domain_helper from fba.helpers import tidyup +from fba.http import network + +from fba.models import blocks +from fba.models import instances + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# Language mapping X -> English language_mapping = { # English -> English "Silenced instances" : "Silenced servers", @@ -55,13 +60,38 @@ language_mapping = { } def fetch_blocks_from_about(domain: str) -> dict: - # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") - if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError("Parameter 'domain' is empty") + logger.debug("domain='%s' - CALLED!", domain) + domain_helper.raise_on(domain) + + if blacklist.is_blacklisted(domain): + raise Exception(f"domain='{domain}' is blacklisted but function is invoked.") + elif not instances.is_registered(domain): + raise Exception(f"domain='{domain}' is not registered but function is invoked.") + + logger.info("Fetching mastodon blocks from domain='%s'", domain) + doc = None + for path in ["/about/more", "/about"]: + try: + logger.debug("Fetching path='%s' from domain='%s' ...", path, domain) + doc = bs4.BeautifulSoup( + network.fetch_response( + domain, + path, + network.web_headers, + (config.get("connection_timeout"), config.get("read_timeout")) + ).text, + "html.parser", + ) + + if len(doc.find_all("h3")) > 0: + logger.debug("path='%s' had some headlines - BREAK!", path) + break + + except network.exceptions as exception: + logger.warning("Cannot fetch from domain='%s',exception='%s'", domain, type(exception)) + instances.set_last_error(domain, exception) + break - # DEBUG: print("DEBUG: Fetching mastodon blocks from domain:", domain) blocklist = { "Suspended servers": [], "Filtered media" : [], @@ -69,199 +99,105 @@ def fetch_blocks_from_about(domain: str) -> dict: "Silenced servers" : [], } - try: - doc = bs4.BeautifulSoup( - network.fetch_response( - domain, - "/about/more", - network.web_headers, - (config.get("connection_timeout"), config.get("read_timeout")) - ).text, - "html.parser", - ) - except BaseException as exception: - print("ERROR: Cannot fetch from domain:", domain, exception) - instances.update_last_error(domain, exception) - return {} + logger.debug("doc[]='%s'", type(doc)) + if doc is None: + logger.warning("Cannot fetch any /about pages for domain='%s' - EXIT!", domain) + return list() for header in doc.find_all("h3"): header_text = tidyup.reason(header.text) - # DEBUG: print(f"DEBUG: header_text='{header_text}'") + logger.debug("header_text='%s'", header_text) if header_text in language_mapping: - # DEBUG: print(f"DEBUG: header_text='{header_text}'") + logger.debug("Translating header_text='%s' ...", header_text) header_text = language_mapping[header_text] else: - print(f"WARNING: header_text='{header_text}' not found in language mapping table") + logger.warning("header_text='%s' not found in language mapping table", header_text) if header_text in blocklist or header_text.lower() in blocklist: # replaced find_next_siblings with find_all_next to account for instances that e.g. hide lists in dropdown menu for line in header.find_all_next("table")[0].find_all("tr")[1:]: - blocklist[header_text].append( - { - "domain": tidyup.domain(line.find("span").text), - "hash" : tidyup.domain(line.find("span")["title"][9:]), - "reason": tidyup.reason(line.find_all("td")[1].text), - } - ) + domain = line.find("span").text + digest = line.find("span")["title"][9:] + reason = line.find_all("td")[1].text + + logger.debug("domain='%s',reason='%s' - BEFORE!", domain, reason) + domain = tidyup.domain(domain) if domain != "" else None + reason = tidyup.reason(reason) if reason != "" else None + + logger.debug("domain='%s',reason='%s' - AFTER!", domain, reason) + if domain is None or domain == "": + logger.warning("domain='%s' is empty,line='%s' - SKIPPED!", domain, line) + continue + + logger.debug("Appending domain='%s',digest='%s',reason='%s' to blocklist header_text='%s' ...", domain, digest, reason, blocklist) + blocklist[header_text].append({ + "domain": domain, + "digest": digest, + "reason": reason, + }) else: - print(f"WARNING: header_text='{header_text}' not found in blocklist()={len(blocklist)}") + logger.warning("header_text='%s' not found in blocklist()=%d", header_text, len(blocklist)) - # DEBUG: print("DEBUG: Returning blocklist for domain:", domain) + logger.debug("Returning blocklist for domain='%s' - EXIT!", domain) return { "reject" : blocklist["Suspended servers"], "media_removal" : blocklist["Filtered media"], "followers_only": blocklist["Limited servers"] + blocklist["Silenced servers"], } -def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): - # DEBUG: print(f"DEBUG: domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}' - CALLED!") - if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError("Parameter 'domain' is empty") - elif not isinstance(origin, str) and origin is not None: - raise ValueError(f"Parameter origin[]={type(origin)} is not 'str'") - elif origin == "": - raise ValueError("Parameter 'origin' is empty") - elif not isinstance(nodeinfo_url, str): - raise ValueError(f"Parameter nodeinfo_url[]={type(nodeinfo_url)} is not 'str'") - elif nodeinfo_url == "": - raise ValueError("Parameter 'nodeinfo_url' is empty") +def fetch_blocks(domain: str) -> list: + logger.debug("domain='%s' - CALLED!", domain) + domain_helper.raise_on(domain) - try: - # json endpoint for newer mastodongs - blockdict = list() - rows = { - "reject" : [], - "media_removal" : [], - "followers_only": [], - "report_removal": [], - } + if blacklist.is_blacklisted(domain): + raise Exception(f"domain='{domain}' is blacklisted but function is invoked.") + elif not instances.is_registered(domain): + raise Exception(f"domain='{domain}' is not registered but function is invoked.") - # DEBUG: print("DEBUG: Querying API domain_blocks:", domain) - data = network.get_json_api( - domain, - "/api/v1/instance/domain_blocks", - (config.get("connection_timeout"), config.get("read_timeout")) - ) + blocklist = list() - if "error_message" in data: - print(f"WARNING: Was not able to fetch domain_blocks from domain='{domain}': status_code='{data['status_code']}',error_message='{data['error_message']}'") - instances.update_last_error(domain, data) - else: - # Getting blocklist - blocklist = data["json"] - - if len(blocklist) > 0: - print(f"INFO: Checking {len(blocklist)} entries from domain='{domain}',software='mastodon' ...") - for block in blocklist: - # Map block -> entry - entry = { - "domain": block["domain"], - "hash" : block["digest"], - "reason": block["comment"] - } - - # DEBUG: print("DEBUG: severity,domain,hash,comment:", block['severity'], block['domain'], block['digest'], block['comment']) - if block['severity'] == 'suspend': - # DEBUG: print(f"DEBUG: Adding entry='{entry}' with severity='{block['severity']}' ...") - rows['reject'].append(entry) - elif block['severity'] == 'silence': - # DEBUG: print(f"DEBUG: Adding entry='{entry}' with severity='{block['severity']}' ...") - rows['followers_only'].append(entry) - elif block['severity'] == 'reject_media': - # DEBUG: print(f"DEBUG: Adding entry='{entry}' with severity='{block['severity']}' ...") - rows['media_removal'].append(entry) - elif block['severity'] == 'reject_reports': - # DEBUG: print(f"DEBUG: Adding entry='{entry}' with severity='{block['severity']}' ...") - rows['report_removal'].append(entry) - else: - print("WARNING: Unknown severity:", block['severity'], block['domain']) - else: - # DEBUG: print(f"DEBUG: domain='{domain}' has returned zero rows, trying /about/more page ...") - rows = fetch_blocks_from_about(domain) + logger.debug("Invoking fetch_blocks_from_about(%s) ...", domain) + rows = fetch_blocks_from_about(domain) - print(f"INFO: Checking {len(rows.items())} entries from domain='{domain}',software='mastodon' ...") - for block_level, blocklist in rows.items(): - # DEBUG: print("DEBUG: domain,block_level,blocklist():", domain, block_level, len(blocklist)) - block_level = tidyup.domain(block_level) - - # DEBUG: print("DEBUG: AFTER-block_level:", block_level) - if block_level == "": - print("WARNING: block_level is empty, domain:", domain) + logger.debug("rows[%s]()=%d", type(rows), len(rows)) + if len(rows) > 0: + logger.debug("Checking %d entries from domain='%s' ...", len(rows), domain) + for block in rows: + # Check type + logger.debug("block[]='%s'", type(block)) + if not isinstance(block, dict): + logger.debug("block[]='%s' is of type 'dict' - SKIPPED!", type(block)) + continue + elif "domain" not in block: + logger.debug("block='%s'", block) + logger.warning("block()=%d does not contain element 'domain' - SKIPPED!", len(block)) + continue + elif not domain_helper.is_wanted(block["domain"]): + logger.debug("block[domain]='%s' is not wanted - SKIPPED!", block["domain"]) + continue + elif "severity" not in block: + logger.warning("block()=%d does not contain element 'severity' - SKIPPED!", len(block)) + continue + elif block["severity"] in ["accept", "accepted"]: + logger.debug("block[domain]='%s' has unwanted severity level '%s' - SKIPPED!", block["domain"], block["severity"]) + continue + elif "digest" in block and not validators.hashes.sha256(block["digest"]): + logger.warning("block[domain]='%s' has invalid block[digest]='%s' - SKIPPED!", block["domain"], block["digest"]) continue - # DEBUG: print(f"DEBUG: Checking {len(blocklist)} entries from domain='{domain}',software='mastodon',block_level='{block_level}' ...") - for block in blocklist: - # DEBUG: print(f"DEBUG: block[]='{type(block)}'") - blocked, blocked_hash, reason = block.values() - # DEBUG: print(f"DEBUG: blocked='{blocked}',blocked_hash='{blocked_hash}',reason='{reason}':") - blocked = tidyup.domain(blocked) - reason = tidyup.reason(reason) if reason is not None and reason != "" else None - # DEBUG: print(f"DEBUG: blocked='{blocked}',reason='{reason}' - AFTER!") - - if blocked == "": - print("WARNING: blocked is empty:", domain) - continue - elif blacklist.is_blacklisted(blocked): - # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!") - continue - elif blocked.count("*") > 0: - # Doing the hash search for instance names as well to tidy up DB - fba.cursor.execute( - "SELECT domain, origin, nodeinfo_url FROM instances WHERE hash = ? LIMIT 1", [blocked_hash] - ) - searchres = fba.cursor.fetchone() - - if searchres is None: - print(f"WARNING: Cannot deobsfucate blocked='{blocked}',blocked_hash='{blocked_hash}' - SKIPPED!") - continue - - # DEBUG: print("DEBUG: Updating domain: ", searchres[0]) - blocked = searchres[0] - origin = searchres[1] - nodeinfo_url = searchres[2] - - # DEBUG: print("DEBUG: Looking up instance by domain:", blocked) - if not validators.domain(blocked): - print(f"WARNING: blocked='{blocked}',software='mastodon' is not a valid domain name - skipped!") - continue - elif not instances.is_registered(blocked): - # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., domain='{domain}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'") - instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url) - elif not validators.domain(blocked): - print(f"WARNING: blocked='{blocked}',software='mastodon' is not a valid domain name - skipped!") - continue - - # DEBUG: print("DEBUG: Looking up instance by domain:", blocked) - if not validators.domain(blocked): - print(f"WARNING: blocked='{blocked}',software='mastodon' is not a valid domain name - skipped!") - continue - elif not instances.is_registered(blocked): - # DEBUG: print("DEBUG: Hash wasn't found, adding:", blocked, domain) - instances.add(blocked, domain, inspect.currentframe().f_code.co_name, nodeinfo_url) - - blocking = blocked if blocked.count("*") <= 1 else blocked_hash - # DEBUG: print(f"DEBUG: blocking='{blocking}',blocked='{blocked}',blocked_hash='{blocked_hash}'") - - if not blocks.is_instance_blocked(domain, blocked, block_level): - # DEBUG: print("DEBUG: Blocking:", domain, blocked, block_level) - blocks.add_instance(domain, blocking, reason, block_level) - - if block_level == "reject": - blockdict.append({ - "blocked": blocked, - "reason" : reason - }) - else: - # DEBUG: print(f"DEBUG: Updating block last seen and reason for domain='{domain}',blocking='{blocking}' ...") - blocks.update_last_seen(domain, blocking, block_level) - blocks.update_reason(reason, domain, blocking, block_level) - - # DEBUG: print("DEBUG: Committing changes ...") - fba.connection.commit() - except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as exception: - print(f"ERROR: domain='{domain}',software='mastodon',exception[{type(exception)}]:'{str(exception)}'") - - # DEBUG: print("DEBUG: EXIT!") + reason = tidyup.reason(block["comment"]) if "comment" in block and block["comment"] is not None and block["comment"] != "" else None + + logger.debug("Appending blocker='%s',blocked='%s',reason='%s',block_level='%s'", domain, block["domain"], reason, block["severity"]) + blocklist.append({ + "blocker" : domain, + "blocked" : block["domain"], + "digest" : block["digest"] if "digest" in block else None, + "reason" : reason, + "block_level": blocks.alias_block_level(block["severity"]), + }) + else: + logger.debug("domain='%s' has no block list", domain) + + logger.debug("blocklist()=%d - EXIT!", len(blocklist)) + return blocklist