From c6a87356eddeb5bb2a986d694a14d41746d02b58 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Thu, 22 Jun 2023 17:23:02 +0200 Subject: [PATCH] Continued: - added column has_obsfucation for tracking if instances uses obsfucation in their block list - please run this SQL: ALTER TABLE instances ADD has_obsfucation TINYINT(1) NULL DEFAULT NULL - sorted links --- blocks_empty.db | Bin 32768 -> 32768 bytes fba/commands.py | 6 ++++++ fba/models/instances.py | 19 +++++++++++++++---- fba/networks/mastodon.py | 6 ++++++ fba/networks/pleroma.py | 30 ++++++++++++++++++++++++++++++ fba/utils.py | 6 ++++++ templates/base.html | 1 - templates/widgets/links.tpl | 2 +- 8 files changed, 64 insertions(+), 6 deletions(-) diff --git a/blocks_empty.db b/blocks_empty.db index 7d4e8870814c368ac2ddb88ed6c07ad30560232d..3dec226bca7ff8820a0971f3b8ffba7165214b4b 100644 GIT binary patch delta 111 zcmZo@U}|V!njkIsgMop81BhY3dZLc8^bZC-?}@zJn;6*Hr!w%(<$cFKmFqXxbS@?S zsT&)m*|}P|1lYypVUp2N$Ve=X&rd2&D@{%;$;{7F2=Vlb^z;kSFx1@4 I&-Jks0MG#)^Z)<= delta 86 zcmZo@U}|V!njkIsoq>UY1BhY3YNC#@^mhh5?}@zJn;6*H4>R!1<$cF~nCmy!bS@?S ishb4_+}MFqOf4}2?BeqBjGgS8=d*_}ZC2#EUkU(DQx!4* diff --git a/fba/commands.py b/fba/commands.py index 1861506..f388f57 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -227,6 +227,9 @@ def fetch_blocks(args: argparse.Namespace): logger.warning("blocked is empty, blocker='%s'", blocker) continue elif blocked.count("*") > 0: + logger.debug("blocker='%s' uses obsfucated domains, marking ...", blocker) + instances.set_has_obsfucation(blocker, True) + # Some friendica servers also obscure domains without hash row = instances.deobscure("*", blocked) @@ -239,6 +242,9 @@ def fetch_blocks(args: argparse.Namespace): origin = row[1] nodeinfo_url = row[2] elif blocked.count("?") > 0: + logger.debug("blocker='%s' uses obsfucated domains, marking ...", blocker) + instances.set_has_obsfucation(blocker, True) + # Some obscure them with question marks, not sure if that's dependent on version or not row = instances.deobscure("?", blocked) diff --git a/fba/models/instances.py b/fba/models/instances.py index 12f5f04..2632f41 100644 --- a/fba/models/instances.py +++ b/fba/models/instances.py @@ -60,6 +60,8 @@ _pending = { "last_status_code" : {}, # Last error details "last_error_details" : {}, + # Wether obsfucation has been used + "has_obsfucation" : {}, } def _set_data(key: str, domain: str, value: any): @@ -213,10 +215,6 @@ def set_last_nodeinfo(domain: str): logger.debug("Updating last_nodeinfo for domain='%s'", domain) _set_data("last_nodeinfo", domain, time.time()) - # Running pending updated - logger.debug("Invoking update_data(%s) ...", domain) - update_data(domain) - logger.debug("EXIT!") def set_last_error(domain: str, error: dict): @@ -365,6 +363,7 @@ def set_total_peers(domain: str, peers: list): def set_nodeinfo_url(domain: str, url: str): logger.debug(f"domain='{domain}',url='{url}' - CALLED!") domain_helper.raise_on(domain) + if not isinstance(url, str): raise ValueError("Parameter url[]='{type(url)}' is not 'list'") elif url == "": @@ -377,6 +376,7 @@ def set_nodeinfo_url(domain: str, url: str): def set_detection_mode(domain: str, mode: str): logger.debug(f"domain='{domain}',mode='{mode}' - CALLED!") domain_helper.raise_on(domain) + if not isinstance(mode, str): raise ValueError("Parameter mode[]='{type(mode)}' is not 'list'") elif mode == "": @@ -385,3 +385,14 @@ def set_detection_mode(domain: str, mode: str): # Set timestamp _set_data("detection_mode", domain, mode) logger.debug("EXIT!") + +def set_has_obsfucation(domain: str, status: bool): + logger.debug("domain(%d)='%s',status='%s' - CALLED!", len(domain), domain) + domain_helper.raise_on(domain) + + if not isinstance(status, bool): + raise ValueError(f"Parameter status[]='{type(status)}' is not 'bool'") + + # Set timestamp + _set_data("has_obsfucation", domain, status) + logger.debug("EXIT!") diff --git a/fba/networks/mastodon.py b/fba/networks/mastodon.py index 4c92c1e..47aefd9 100644 --- a/fba/networks/mastodon.py +++ b/fba/networks/mastodon.py @@ -250,6 +250,9 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): logger.warning("blocked is empty, domain='%s'", domain) continue elif blocked.count("*") > 0: + logger.debug("domain='%s' uses obsfucated domains, marking ...", domain) + instances.set_has_obsfucation(domain, True) + # Doing the hash search for instance names as well to tidy up DB row = instances.deobscure("*", blocked, blocked_hash) @@ -263,6 +266,9 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): origin = row[1] nodeinfo_url = row[2] elif blocked.count("?") > 0: + logger.debug("domain='%s' uses obsfucated domains, marking ...", domain) + instances.set_has_obsfucation(domain, True) + # Doing the hash search for instance names as well to tidy up DB row = instances.deobscure("?", blocked, blocked_hash) diff --git a/fba/networks/pleroma.py b/fba/networks/pleroma.py index 1f38e35..09f8697 100644 --- a/fba/networks/pleroma.py +++ b/fba/networks/pleroma.py @@ -117,6 +117,9 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): logger.debug("blocked='%s' is a fake domain - SKIPPED!", blocked) continue elif blocked.count("*") > 0: + logger.debug("domain='%s' uses obsfucated domains, marking ...", domain) + instances.set_has_obsfucation(domain, True) + # Obscured domain name with no hash row = instances.deobscure("*", blocked) @@ -130,6 +133,9 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): origin = row[1] nodeinfo_url = row[2] elif blocked.count("?") > 0: + logger.debug("domain='%s' uses obsfucated domains, marking ...", domain) + instances.set_has_obsfucation(domain, True) + # Obscured domain name with no hash row = instances.deobscure("?", blocked) @@ -188,6 +194,9 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): logger.debug("blocked='%s' is a fake domain - SKIPPED!", blocked) continue elif blocked.count("*") > 0: + logger.debug("domain='%s' uses obsfucated domains, marking ...", domain) + instances.set_has_obsfucation(domain, True) + # Obscured domain name with no hash row = instances.deobscure("*", blocked) @@ -201,6 +210,9 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): origin = row[1] nodeinfo_url = row[2] elif blocked.count("?") > 0: + logger.debug("domain='%s' uses obsfucated domains, marking ...", domain) + instances.set_has_obsfucation(domain, True) + # Obscured domain name with no hash row = instances.deobscure("?", blocked) @@ -287,6 +299,9 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): logger.warning("blocked is empty after tidyup.domain(): domain='%s',block_level='%s'", domain, block_level) continue elif blocked.count("*") > 0: + logger.debug("domain='%s' uses obsfucated domains, marking ...", domain) + instances.set_has_obsfucation(domain, True) + # Obscured domain name with no hash row = instances.deobscure("*", blocked) @@ -300,6 +315,9 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): origin = row[1] nodeinfo_url = row[2] elif blocked.count("?") > 0: + logger.debug("domain='%s' uses obsfucated domains, marking ...", domain) + instances.set_has_obsfucation(domain, True) + # Obscured domain name with no hash row = instances.deobscure("?", blocked) @@ -353,6 +371,9 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): logger.warning("blocked is empty after tidyup.domain(): domain='%s',block_level='%s'", domain, block_level) continue elif blocked.count("*") > 0: + logger.debug("domain='%s' uses obsfucated domains, marking ...", domain) + instances.set_has_obsfucation(domain, True) + # Obscured domain name with no hash row = instances.deobscure("*", blocked) @@ -366,6 +387,9 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): origin = row[1] nodeinfo_url = row[2] elif blocked.count("?") > 0: + logger.debug("domain='%s' uses obsfucated domains, marking ...", domain) + instances.set_has_obsfucation(domain, True) + # Obscured domain name with no hash row = instances.deobscure("?", blocked) @@ -420,6 +444,9 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): logger.warning("blocked is empty after tidyup.domain(): domain='%s',block_level='%s'", domain, block_level) continue elif blocked.count("*") > 0: + logger.debug("domain='%s' uses obsfucated domains, marking ...", domain) + instances.set_has_obsfucation(domain, True) + # Obscured domain name with no hash row = instances.deobscure("*", blocked) @@ -433,6 +460,9 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): origin = row[1] nodeinfo_url = row[2] elif blocked.count("?") > 0: + logger.debug("domain='%s' uses obsfucated domains, marking ...", domain) + instances.set_has_obsfucation(domain, True) + # Obscured domain name with no hash row = instances.deobscure("?", blocked) diff --git a/fba/utils.py b/fba/utils.py index 2460cd5..a0ee299 100644 --- a/fba/utils.py +++ b/fba/utils.py @@ -82,6 +82,9 @@ def process_domain(domain: str, blocker: str, command: str) -> bool: raise ValueError("Parameter 'command' is empty") if domain.find("*") > 0: + logger.debug("blocker='%s' uses obsfucated domains, marking ...", blocker) + instances.set_has_obsfucation(blocker, True) + # Try to de-obscure it row = instances.deobscure("*", domain) @@ -93,6 +96,9 @@ def process_domain(domain: str, blocker: str, command: str) -> bool: logger.debug(f"domain='{domain}' de-obscured to '{row[0]}'") domain = row[0] elif domain.find("?") > 0: + logger.debug("blocker='%s' uses obsfucated domains, marking ...", blocker) + instances.set_has_obsfucation(blocker, True) + # Try to de-obscure it row = instances.deobscure("?", domain) diff --git a/templates/base.html b/templates/base.html index ab2c710..9d3e735 100644 --- a/templates/base.html +++ b/templates/base.html @@ -64,7 +64,6 @@ } a.listlink { text-decoration: none; - font-size: 0.8em; } a:hover { color: #f00000; diff --git a/templates/widgets/links.tpl b/templates/widgets/links.tpl index f0499fa..e946612 100644 --- a/templates/widgets/links.tpl +++ b/templates/widgets/links.tpl @@ -1,3 +1,3 @@ -{{domain}} [D] [R] +{{domain}} -- 2.39.5