X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;f=fetch_blocks.py;h=ebf3697eea7340935352befca654994bdbc540b0;hb=0fe16980ab997a2671416b0ac31c5326b368188c;hp=9b71b6a34a13bef522ba5f73a128686f990f3bbc;hpb=719397e7b2dbe2d2c2da96c868ec588754ffb490;p=fba.git diff --git a/fetch_blocks.py b/fetch_blocks.py old mode 100644 new mode 100755 index 9b71b6a..ebf3697 --- a/fetch_blocks.py +++ b/fetch_blocks.py @@ -1,367 +1,454 @@ -from requests import get -from requests import post -from hashlib import sha256 -import sqlite3 -from bs4 import BeautifulSoup -from json import dumps -import re +#!/usr/bin/python3 +# -*- coding: utf-8 -*- -headers = { - "user-agent": "Mozilla/5.0 (Windows NT 10.0; rv:102.0) Gecko/20100101 Firefox/102.0" -} - - -def get_mastodon_blocks(domain: str) -> dict: - blocks = { - "Suspended servers": [], - "Filtered media": [], - "Limited servers": [], - "Silenced servers": [], - } - - translations = { - "Silenced instances": "Silenced servers", - "Suspended instances": "Suspended servers", - "Gesperrte Server": "Suspended servers", - "Gefilterte Medien": "Filtered media", - "Stummgeschaltete Server": "Silenced servers", - "停止済みのサーバー": "Suspended servers", - "メディアを拒否しているサーバー": "Filtered media", - "サイレンス済みのサーバー": "Silenced servers", - "Serveurs suspendus": "Suspended servers", - "Médias filtrés": "Filtered media", - "Serveurs limités": "Silenced servers", - } - - try: - doc = BeautifulSoup( - get(f"https://{domain}/about/more", headers=headers, timeout=5).text, - "html.parser", - ) - except: - return {} - - for header in doc.find_all("h3"): - header_text = header.text - if header_text in translations: - header_text = translations[header_text] - if header_text in blocks: - # replaced find_next_siblings with find_all_next to account for instances that e.g. hide lists in dropdown menu - for line in header.find_all_next("table")[0].find_all("tr")[1:]: - blocks[header_text].append( - { - "domain": line.find("span").text, - "hash": line.find("span")["title"][9:], - "reason": line.find_all("td")[1].text.strip(), - } - ) - return { - "reject": blocks["Suspended servers"], - "media_removal": blocks["Filtered media"], - "federated_timeline_removal": blocks["Limited servers"] - + blocks["Silenced servers"], - } - -def get_friendica_blocks(domain: str) -> dict: - blocks = [] - - try: - doc = BeautifulSoup( - get(f"https://{domain}/friendica", headers=headers, timeout=5).text, - "html.parser", - ) - except: - return {} - - blocklist = doc.find(id="about_blocklist") - for line in blocklist.find("table").find_all("tr")[1:]: - blocks.append( - { - "domain": line.find_all("td")[0].text.strip(), - "reason": line.find_all("td")[1].text.strip() - } - ) - - return { - "reject": blocks - } - -def get_pisskey_blocks(domain: str) -> dict: - blocks = { - "suspended": [], - "blocked": [] - } - - try: - counter = 0 - step = 99 - while True: - # iterating through all "suspended" (follow-only in its terminology) instances page-by-page, since that troonware doesn't support sending them all at once - try: - if counter == 0: - doc = post(f"https://{domain}/api/federation/instances", data=dumps({"sort":"+caughtAt","host":None,"suspended":True,"limit":step}), headers=headers, timeout=5).json() - if doc == []: raise - else: - doc = post(f"https://{domain}/api/federation/instances", data=dumps({"sort":"+caughtAt","host":None,"suspended":True,"limit":step,"offset":counter-1}), headers=headers, timeout=5).json() - if doc == []: raise - for instance in doc: - # just in case - if instance["isSuspended"]: - blocks["suspended"].append( - { - "domain": instance["host"], - # no reason field, nothing - "reason": "" - } - ) - counter = counter + step - except: - counter = 0 - break +# Fedi API Block - An aggregator for fetching blocking data from fediverse nodes +# Copyright (C) 2023 Free Software Foundation +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . - while True: - # same shit, different asshole ("blocked" aka full suspend) - try: - if counter == 0: - doc = post(f"https://{domain}/api/federation/instances", data=dumps({"sort":"+caughtAt","host":None,"blocked":True,"limit":step}), headers=headers, timeout=5).json() - if doc == []: raise - else: - doc = post(f"https://{domain}/api/federation/instances", data=dumps({"sort":"+caughtAt","host":None,"blocked":True,"limit":step,"offset":counter-1}), headers=headers, timeout=5).json() - if doc == []: raise - for instance in doc: - if instance["isBlocked"]: - blocks["blocked"].append( - { - "domain": instance["host"], - "reason": "" - } - ) - counter = counter + step - except: - counter = 0 - break - - return { - "reject": blocks["blocked"], - "followers_only": blocks["suspended"] - } - - except: - return {} - -def get_hash(domain: str) -> str: - return sha256(domain.encode("utf-8")).hexdigest() - - -def get_type(domain: str) -> str: - try: - res = get(f"https://{domain}/nodeinfo/2.1.json", headers=headers, timeout=5) - if res.status_code == 404: - res = get(f"https://{domain}/nodeinfo/2.0", headers=headers, timeout=5) - if res.status_code == 404: - res = get(f"https://{domain}/nodeinfo/2.0.json", headers=headers, timeout=5) - if res.ok and "text/html" in res.headers["content-type"]: - res = get(f"https://{domain}/nodeinfo/2.1", headers=headers, timeout=5) - if res.ok: - if res.json()["software"]["name"] in ["akkoma", "rebased"]: - return "pleroma" - elif res.json()["software"]["name"] in ["hometown", "ecko"]: - return "mastodon" - elif res.json()["software"]["name"] in ["calckey", "groundpolis", "foundkey"]: - return "misskey" - else: - return res.json()["software"]["name"] - elif res.status_code == 404: - res = get(f"https://{domain}/api/v1/instance", headers=headers, timeout=5) - if res.ok: - return "mastodon" - except: - return None +import reqto +import time +import bs4 +import itertools +import re +import fba + +fba.cursor.execute( + "SELECT domain, software, origin, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial', 'bookwyrm', 'takahe') AND (last_blocked IS NULL OR last_blocked < ?) ORDER BY rowid DESC", [time.time() - fba.config["recheck_block"]] +) +rows = fba.cursor.fetchall() +print(f"INFO: Checking {len(rows)} entries ...") +for blocker, software, origin, nodeinfo_url in rows: + # NOISY-DEBUG: print("DEBUG: BEFORE blocker,software,origin,nodeinfo_url:", blocker, software, origin, nodeinfo_url) + blockdict = [] + blocker = fba.tidyup(blocker) + # NOISY-DEBUG: print("DEBUG: AFTER blocker,software:", blocker, software) -conn = sqlite3.connect("blocks.db") -c = conn.cursor() + if blocker == "": + print("WARNING: blocker is now empty!") + continue + elif fba.is_blacklisted(blocker): + print(f"WARNING: blocker='{blocker}' is blacklisted now!") + continue -c.execute( - "select domain, software from instances where software in ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial')" -) + # NOISY-DEBUG: print(f"DEBUG: blocker='{blocker}'") + fba.update_last_blocked(blocker) -for blocker, software in c.fetchall(): if software == "pleroma": - print(blocker) + print("INFO: blocker:", blocker) try: # Blocks - federation = get( - f"https://{blocker}/nodeinfo/2.1.json", headers=headers, timeout=5 - ).json()["metadata"]["federation"] + json = fba.fetch_nodeinfo(blocker, nodeinfo_url) + if json is None: + print("WARNING: Could not fetch nodeinfo from blocker:", blocker) + continue + + print("DEBUG: Updating nodeinfo:", blocker) + fba.update_last_nodeinfo(blocker) + + federation = json["metadata"]["federation"] + + if "enabled" in federation: + # NOISY-DEBUG: print("DEBUG: Instance has no block list to analyze:", blocker) + continue + if "mrf_simple" in federation: for block_level, blocks in ( {**federation["mrf_simple"], **{"quarantined_instances": federation["quarantined_instances"]}} ).items(): + # NOISY-DEBUG: print("DEBUG: block_level, blocks():", block_level, len(blocks)) + block_level = fba.tidyup(block_level) + # NOISY-DEBUG: print("DEBUG: BEFORE block_level:", block_level) + + if block_level == "": + print("WARNING: block_level is now empty!") + continue + for blocked in blocks: + # NOISY-DEBUG: print("DEBUG: BEFORE blocked:", blocked) + blocked = fba.tidyup(blocked) + # NOISY-DEBUG: print("DEBUG: AFTER blocked:", blocked) + if blocked == "": + print("WARNING: blocked is empty after fba.tidyup():", blocker, block_level) continue - blocked = blocked.lower() - blocker = blocker.lower() - c.execute( - "select domain from instances where domain = ?", (blocked,) - ) - if c.fetchone() == None: - c.execute( - "insert into instances select ?, ?, ?", - (blocked, get_hash(blocked), get_type(blocked)), + + if blocked.count("*") > 1: + # -ACK!-oma also started obscuring domains without hash + fba.cursor.execute( + "SELECT domain, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")] ) - c.execute( - "select * from blocks where blocker = ? and blocked = ? and block_level = ?", - (blocker, blocked, block_level), + searchres = fba.cursor.fetchone() + # NOISY-DEBUG: print("DEBUG: searchres[]:", type(searchres)) + if searchres != None: + blocked = searchres[0] + nodeinfo_url = searchres[1] + # NOISY-DEBUG: print("DEBUG: Looked up domain:", blocked) + + # NOISY-DEBUG: print("DEBUG: Looking up instance by domain:", blocked) + if not fba.is_instance_registered(blocked): + # NOISY-DEBUG: print("DEBUG: Domain wasn't found, adding:", blocked, blocker) + fba.add_instance(blocked, blocker, origin, nodeinfo_url) + + fba.cursor.execute( + "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ? LIMIT 1", + ( + blocker, + blocked, + block_level + ), ) - if c.fetchone() == None: - c.execute( - "insert into blocks select ?, ?, '', ?", - (blocker, blocked, block_level), - ) - conn.commit() + + if fba.cursor.fetchone() == None: + # NOISY-DEBUG: print("DEBUG: Blocking:", blocker, blocked, block_level) + fba.block_instance(blocker, blocked, "unknown", block_level) + + if block_level == "reject": + # NOISY-DEBUG: print("DEBUG: Adding to blockdict:", blocked) + blockdict.append( + { + "blocked": blocked, + "reason" : None + }) + else: + # NOISY-DEBUG: print("DEBUG: Updating last_seen:", blocker, blocked, block_level) + fba.update_last_seen(blocker, blocked, block_level) + + fba.connection.commit() + # Reasons if "mrf_simple_info" in federation: + # NOISY-DEBUG: print("DEBUG: Found mrf_simple_info:", blocker) for block_level, info in ( {**federation["mrf_simple_info"], **(federation["quarantined_instances_info"] if "quarantined_instances_info" in federation else {})} ).items(): + # NOISY-DEBUG: print("DEBUG: block_level, info.items():", block_level, len(info.items())) + block_level = fba.tidyup(block_level) + # NOISY-DEBUG: print("DEBUG: BEFORE block_level:", block_level) + + if block_level == "": + print("WARNING: block_level is now empty!") + continue + for blocked, reason in info.items(): - blocker = blocker.lower() - blocked = blocked.lower() - c.execute( - "update blocks set reason = ? where blocker = ? and blocked = ? and block_level = ?", - (reason["reason"], blocker, blocked, block_level), - ) - conn.commit() + # NOISY-DEBUG: print("DEBUG: BEFORE blocked:", blocked) + blocked = fba.tidyup(blocked) + # NOISY-DEBUG: print("DEBUG: AFTER blocked:", blocked) + + if blocked == "": + print("WARNING: blocked is empty after fba.tidyup():", blocker, block_level) + continue + elif blocked.count("*") > 1: + # same domain guess as above, but for reasons field + fba.cursor.execute( + "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")] + ) + searchres = fba.cursor.fetchone() + + if searchres != None: + blocked = searchres[0] + + # NOISY-DEBUG: print("DEBUG: Updating block reason:", blocker, blocked, reason["reason"]) + fba.update_block_reason(reason["reason"], blocker, blocked, block_level) + + for entry in blockdict: + if entry["blocked"] == blocked: + # NOISY-DEBUG: print("DEBUG: Updating entry reason:", blocked) + entry["reason"] = reason["reason"] + + fba.connection.commit() except Exception as e: - print("error:", e, blocker) + print(f"ERROR: blocker='{blocker}',software='{software}',exception[{type(e)}]:'{str(e)}'") elif software == "mastodon": - print(blocker) + print("INFO: blocker:", blocker) try: - json = get_mastodon_blocks(blocker) + # json endpoint for newer mastodongs + try: + json = { + "reject" : [], + "media_removal" : [], + "followers_only": [], + "report_removal": [] + } + + # handling CSRF, I've saw at least one server requiring it to access the endpoint + # NOISY-DEBUG: print("DEBUG: Fetching meta:", blocker) + meta = bs4.BeautifulSoup( + reqto.get(f"https://{blocker}/about", headers=fba.headers, timeout=(fba.config["connection_timeout"], config["read_timeout"])).text, + "html.parser", + ) + try: + csrf = meta.find("meta", attrs={"name": "csrf-token"})["content"] + # NOISY-DEBUG: print("DEBUG: Adding CSRF token:", blocker, csrf) + reqheaders = {**fba.api_headers, **{"X-CSRF-Token": csrf}} + except: + # NOISY-DEBUG: print("DEBUG: No CSRF token found, using normal headers:", blocker) + reqheaders = fba.api_headers + + # NOISY-DEBUG: print("DEBUG: Quering API domain_blocks:", blocker) + blocks = reqto.get(f"https://{blocker}/api/v1/instance/domain_blocks", headers=reqheaders, timeout=(fba.config["connection_timeout"], config["read_timeout"])).json() + + # NOISY-DEBUG: print("DEBUG: blocks():", len(blocks)) + for block in blocks: + entry = { + 'domain': block['domain'], + 'hash' : block['digest'], + 'reason': block['comment'] + } + + # NOISY-DEBUG: print("DEBUG: severity,domain,hash,comment:", block['severity'], block['domain'], block['digest'], block['comment']) + if block['severity'] == 'suspend': + json['reject'].append(entry) + elif block['severity'] == 'silence': + json['followers_only'].append(entry) + elif block['severity'] == 'reject_media': + json['media_removal'].append(entry) + elif block['severity'] == 'reject_reports': + json['report_removal'].append(entry) + else: + print("WARNING: Unknown severity:", block['severity'], block['domain']) + except: + # NOISY-DEBUG: print("DEBUG: Failed, Trying mastodon-specific fetches:", blocker) + json = fba.get_mastodon_blocks(blocker) + + # NOISY-DEBUG: print("DEBUG: json.items():", blocker, len(json.items())) for block_level, blocks in json.items(): + # NOISY-DEBUG: print("DEBUG: blocker,block_level,blocks():", blocker, block_level, len(blocks)) + block_level = fba.tidyup(block_level) + # NOISY-DEBUG: print("DEBUG: AFTER-block_level:", block_level) + if block_level == "": + print("WARNING: block_level is empty, blocker:", blocker) + continue + for instance in blocks: blocked, blocked_hash, reason = instance.values() - blocked = blocked.lower() - blocker = blocker.lower() - if blocked.count("*") <= 1: - c.execute( - "select hash from instances where hash = ?", (blocked_hash,) + # NOISY-DEBUG: print("DEBUG: blocked,hash,reason:", blocked, blocked_hash, reason) + blocked = fba.tidyup(blocked) + # NOISY-DEBUG: print("DEBUG: AFTER-blocked:", blocked) + + if blocked == "": + print("WARNING: blocked is empty:", blocker) + continue + elif blocked.count("*") < 1: + # No obsfucation for this instance + fba.cursor.execute( + "SELECT hash FROM instances WHERE domain = ? LIMIT 1", [blocked] ) - if c.fetchone() == None: - c.execute( - "insert into instances select ?, ?, ?", - (blocked, get_hash(blocked), get_type(blocked)), - ) - c.execute( - "select * from blocks where blocker = ? and blocked = ? and block_level = ?", - (blocker, blocked if blocked.count("*") <= 1 else blocked_hash, block_level), - ) - if c.fetchone() == None: - c.execute( - "insert into blocks select ?, ?, ?, ?", - ( - blocker, - blocked if blocked.count("*") <= 1 else blocked_hash, - reason, - block_level, - ), + + if fba.cursor.fetchone() == None: + # NOISY-DEBUG: print("DEBUG: Hash wasn't found, adding:", blocked, blocker) + fba.add_instance(blocked, blocker, origin) + else: + # Doing the hash search for instance names as well to tidy up DB + fba.cursor.execute( + "SELECT domain FROM instances WHERE hash = ? LIMIT 1", [blocked_hash] ) - conn.commit() + searchres = fba.cursor.fetchone() + + if searchres != None: + # NOISY-DEBUG: print("DEBUG: Updating domain: ", searchres[0]) + blocked = searchres[0] + + fba.cursor.execute( + "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ? LIMIT 1", + ( + blocker, + blocked if blocked.count("*") <= 1 else blocked_hash, + block_level + ), + ) + + if fba.cursor.fetchone() == None: + fba.block_instance(blocker, blocked if blocked.count("*") <= 1 else blocked_hash, reason, block_level) + + if block_level == "reject": + blockdict.append( + { + "blocked": blocked, + "reason" : reason + }) + else: + fba.update_last_seen(blocker, blocked if blocked.count("*") <= 1 else blocked_hash, block_level) + + if reason != "": + # NOISY-DEBUG: print("DEBUG: Updating block reason:", blocker, blocked, reason) + fba.update_block_reason(reason, blocker, blocked if blocked.count("*") <= 1 else blocked_hash, block_level) + + fba.connection.commit() except Exception as e: - print("error:", e, blocker) - elif software == "friendica" or software == "misskey": - print(blocker) + print(f"ERROR: blocker='{blocker}',software='{software}',exception[{type(e)}]:'{str(e)}'") + elif software == "friendica" or software == "misskey" or software == "bookwyrm" or software == "takahe": + print("INFO: blocker:", blocker) try: if software == "friendica": - json = get_friendica_blocks(blocker) + json = fba.get_friendica_blocks(blocker) elif software == "misskey": - json = get_pisskey_blocks(blocker) + json = fba.get_misskey_blocks(blocker) + elif software == "bookwyrm": + print("WARNING: bookwyrm is not fully supported for fetching blacklist!", blocker) + #json = fba.get_bookwyrm_blocks(blocker) + elif software == "takahe": + print("WARNING: takahe is not fully supported for fetching blacklist!", blocker) + #json = fba.get_takahe_blocks(blocker) + for block_level, blocks in json.items(): + # NOISY-DEBUG: print("DEBUG: blocker,block_level,blocks():", blocker, block_level, len(blocks)) + block_level = fba.tidyup(block_level) + # NOISY-DEBUG: print("DEBUG: AFTER-block_level:", block_level) + if block_level == "": + print("WARNING: block_level is empty, blocker:", blocker) + continue + for instance in blocks: blocked, reason = instance.values() - blocked = blocked.lower() - blocker = blocker.lower() - c.execute( - "select domain from instances where domain = ?", (blocked,) - ) - if c.fetchone() == None: - c.execute( - "insert into instances select ?, ?, ?", - (blocked, get_hash(blocked), get_type(blocked)), + # NOISY-DEBUG: print("DEBUG: BEFORE blocked:", blocked) + blocked = fba.tidyup(blocked) + # NOISY-DEBUG: print("DEBUG: AFTER blocked:", blocked) + + if blocked == "": + print("WARNING: blocked is empty:", blocker) + continue + if blocked.count("*") > 0: + # Some friendica servers also obscure domains without hash + fba.cursor.execute( + "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")] + ) + searchres = fba.cursor.fetchone() + if searchres != None: + blocked = searchres[0] + + if blocked.count("?") > 0: + # Some obscure them with question marks, not sure if that's dependent on version or not + fba.cursor.execute( + "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("?", "_")] ) - c.execute( - "select * from blocks where blocker = ? and blocked = ?", + searchres = fba.cursor.fetchone() + if searchres != None: + blocked = searchres[0] + origin = searchres[1] + nodeinfo_url = searchres[2] + + # NOISY-DEBUG: print("DEBUG: AFTER-blocked:", blocked) + if not fba.is_instance_registered(blocked): + # NOISY-DEBUG: print("DEBUG: Hash wasn't found, adding:", blocked, blocker) + fba.add_instance(blocked, blocker, origin, nodeinfo_url) + + fba.cursor.execute( + "SELECT * FROM blocks WHERE blocker = ? AND blocked = ?", (blocker, blocked), ) - if c.fetchone() == None: - c.execute( - "insert into blocks select ?, ?, ?, ?", - ( - blocker, - blocked, - reason, - block_level, - ), - ) - conn.commit() + + if fba.cursor.fetchone() == None: + fba.block_instance(blocker, blocked, reason, block_level) + + if block_level == "reject": + blockdict.append( + { + "blocked": blocked, + "reason" : reason + }) + else: + fba.update_last_seen(blocker, blocked, block_level) + + if reason != '': + # NOISY-DEBUG: print("DEBUG: Updating block reason:", blocker, blocked, reason) + fba.update_block_reason(reason, blocker, blocked, block_level) + + fba.connection.commit() except Exception as e: - print("error:", e, blocker) + print(f"ERROR: blocker='{blocker}',software='{software}',exception[{type(e)}]:'{str(e)}'") elif software == "gotosocial": - print(blocker) + print("INFO: blocker:", blocker) try: # Blocks - federation = get( - f"https://{blocker}/api/v1/instance/peers?filter=suspended", headers=headers, timeout=5 - ).json() - for peer in federation: - blocked = peer["domain"].lower() - blocker = blocker.lower() - - if blocked.count("*") > 0: - # GTS does not have hashes for obscured domains, so we have to guess it - c.execute( - "select domain from instances where domain like ? order by rowid limit 1", (blocked.replace("*", "_"),) - ) - searchres = c.fetchone() - if searchres != None: - blocked = searchres[0] + federation = reqto.get(f"https://{blocker}{get_peers_url}?filter=suspended", headers=fba.api_headers, timeout=(fba.config["connection_timeout"], config["read_timeout"])).json() - c.execute( - "select domain from instances where domain = ?", (blocked,) - ) - if c.fetchone() == None: - c.execute( - "insert into instances select ?, ?, ?", - (blocked, get_hash(blocked), get_type(blocked)), - ) - c.execute( - "select * from blocks where blocker = ? and blocked = ? and block_level = ?", - (blocker, blocked, "reject"), - ) - if c.fetchone() == None: - c.execute( - "insert into blocks select ?, ?, ?, ?", - (blocker, blocked, "", "reject"), - ) + if (federation == None): + print("WARNING: No valid response:", blocker); + elif "error" in federation: + print("WARNING: API returned error:", federation["error"]) + else: + # NOISY-DEBUG: print("DEBUG: Checking fenderation():", len(federation)) + for peer in federation: + blocked = peer["domain"].lower() + # NOISY-DEBUG: print("DEBUG: BEFORE blocked:", blocked) + blocked = fba.tidyup(blocked) + # NOISY-DEBUG: print("DEBUG: AFTER blocked:", blocked) - if "public_comment" in peer: - reason = peer["public_comment"] - c.execute( - "select * from blocks where blocker = ? and blocked = ? and reason != ? and block_level = ?", - (blocker, blocked, "", "reject"), - ) - if c.fetchone() == None: - c.execute( - "update blocks set reason = ? where blocker = ? and blocked = ? and block_level = ?", - (reason, blocker, blocked, "reject"), + if blocked == "": + print("WARNING: blocked is empty:", blocker) + continue + elif blocked.count("*") > 0: + # GTS does not have hashes for obscured domains, so we have to guess it + fba.cursor.execute( + "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")] ) - conn.commit() + searchres = fba.cursor.fetchone() + + if searchres != None: + blocked = searchres[0] + origin = searchres[1] + nodeinfo_url = searchres[2] + + if not fba.is_instance_registered(blocked): + # NOISY-DEBUG: print("DEBUG: Domain wasn't found, adding:", blocked, blocker) + fba.add_instance(blocked, blocker, origin, nodeinfo_url) + + fba.cursor.execute( + "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ? LIMIT 1", + ( + blocker, + blocked, + "reject" + ), + ) + + if fba.cursor.fetchone() == None: + # NOISY-DEBUG: print(f"DEBUG: blocker='{blocker}' is blocking '{blocked}' for unknown reason at this point") + fba.block_instance(blocker, blocked, "unknown", "reject") + + blockdict.append( + { + "blocked": blocked, + "reason" : None + }) + else: + fba.update_last_seen(blocker, blocked, "reject") + + if "public_comment" in peer: + # NOISY-DEBUG: print("DEBUG: Updating block reason:", blocker, blocked, peer["public_comment"]) + fba.update_block_reason(peer["public_comment"], blocker, blocked, "reject") + + for entry in blockdict: + if entry["blocked"] == blocked: + # NOISY-DEBUG: print(f"DEBUG: Setting block reason for blocked='{blocked}':'{peer['public_comment']}'") + entry["reason"] = peer["public_comment"] + + fba.connection.commit() except Exception as e: - print("error:", e, blocker) -conn.close() + print(f"ERROR: blocker='{blocker}',software='{software}',exception[{type(e)}]:'{str(e)}'") + else: + print("WARNING: Unknown software:", blocker, software) + + if fba.config["bot_enabled"] and len(blockdict) > 0: + send_bot_post(blocker, blockdict) + + blockdict = [] + +fba.connection.close()