From: Roland Häder Date: Thu, 29 Jun 2023 09:47:03 +0000 (+0200) Subject: Continued: X-Git-Url: https://git.mxchange.org/?a=commitdiff_plain;h=d48bd2a47a08d3a80f9678160e8803a06ed7072d;p=fba.git Continued: - introduced new view "/infos" - changed indexed row array to associative e.g. row[0] had become row['domain'] --- diff --git a/daemon.py b/daemon.py index 7ec401f..d46d4e0 100755 --- a/daemon.py +++ b/daemon.py @@ -316,6 +316,38 @@ def top(request: Request, mode: str, value: str, amount: int = config.get("api_l "theme" : config.get("theme"), }) +@router.get(config.get("base_url") + "/infos") +def rss(request: Request, domain: str): + # Tidy up domain name + domain = tidyup.domain(domain) + + if not utils.is_domain_wanted(domain): + raise HTTPException(status_code=500, detail=f"domain='{domain}' is not wanted") + + # Fetch domain data + database.cursor.execute("SELECT * FROM instances WHERE domain = ? LIMIT 1", [domain]) + domain_data = database.cursor.fetchone() + + # Format timestamps + format = config.get("timestamp_format") + instance = dict() + for key in domain_data.keys(): + if key in ["last_nodeinfo", "last_blocked", "first_seen", "last_updated", "last_instance_fetch"]: + # Timestamps + instance[key] = datetime.utcfromtimestamp(domain_data[key]).strftime(format) if isinstance(domain_data[key], float) else "-" + else: + # Generic + instance[key] = domain_data[key] + + print(domain_data.keys()) + return templates.TemplateResponse("views/infos.html", { + "request" : request, + "domain" : domain, + "instance": instance, + "theme" : config.get("theme"), + "slogan" : config.get("slogan"), + }) + @router.get(config.get("base_url") + "/rss") def rss(request: Request, domain: str = None): if domain is not None: diff --git a/fba/commands.py b/fba/commands.py index 0b5715e..309a6b1 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -307,9 +307,9 @@ def fetch_blocks(args: argparse.Namespace) -> int: instances.set_has_obfuscation(blocker, True) continue - block["blocked"] = row[0] - origin = row[1] - nodeinfo_url = row[2] + block["blocked"] = row["domain"] + origin = row["origin"] + nodeinfo_url = row["nodeinfo_url"] elif block["blocked"].find("?") >= 0: logger.debug("blocker='%s' uses obfuscated domains", blocker) @@ -322,9 +322,9 @@ def fetch_blocks(args: argparse.Namespace) -> int: instances.set_has_obfuscation(blocker, True) continue - block["blocked"] = row[0] - origin = row[1] - nodeinfo_url = row[2] + block["blocked"] = row["domain"] + origin = row["origin"] + nodeinfo_url = row["nodeinfo_url"] logger.debug("Looking up instance by domainm, blocked='%s'", block["blocked"]) if not utils.is_domain_wanted(block["blocked"]): @@ -726,17 +726,17 @@ def fetch_instances(args: argparse.Namespace) -> int: rows = database.cursor.fetchall() logger.info("Checking %d entries ...", len(rows)) for row in rows: - logger.debug("domain='%s'", row[0]) - if not utils.is_domain_wanted(row[0]): - logger.debug("Domain row[0]='%s' is not wanted - SKIPPED!", row[0]) + logger.debug("domain='%s'", row["domain"]) + if not utils.is_domain_wanted(row["domain"]): + logger.debug("Domain row[domain]='%s' is not wanted - SKIPPED!", row["domain"]) continue try: - logger.info("Fetching instances for domain='%s',origin='%s',software='%s',nodeinfo_url='%s'", row[0], row[1], row[2], row[3]) - federation.fetch_instances(row[0], row[1], row[2], inspect.currentframe().f_code.co_name, row[3]) + logger.info("Fetching instances for domain='%s',origin='%s',software='%s',nodeinfo_url='%s'", row["domain"], row["origin"], row["software"], row["nodeinfo_url"]) + federation.fetch_instances(row["domain"], row["origin"], row["software"], inspect.currentframe().f_code.co_name, row["nodeinfo_url"]) except network.exceptions as exception: - logger.warning("Exception '%s' during fetching instances (fetch_instances) from row[0]='%s'", type(exception), row[0]) - instances.set_last_error(row[0], exception) + logger.warning("Exception '%s' during fetching instances (fetch_instances) from row[domain]='%s'", type(exception), row["domain"]) + instances.set_last_error(row["domain"], exception) logger.debug("Success - EXIT!") return 0 @@ -1117,9 +1117,9 @@ def recheck_obfuscation(args: argparse.Namespace) -> int: locking.acquire() - if args.domain != "" and utils.is_domain_wanted(args.domain): + if isinstance(args.domain, str) and args.domain != "" and utils.is_domain_wanted(args.domain): database.cursor.execute("SELECT domain, software, nodeinfo_url FROM instances WHERE has_obfuscation = 1 AND domain = ?", [args.domain]) - elif args.domain != "" and validators.domain(args.software) == args.software: + elif isinstance(args.software, str) and args.software != "" and validators.domain(args.software) == args.software: database.cursor.execute("SELECT domain, software, nodeinfo_url FROM instances WHERE has_obfuscation = 1 AND software = ?", [args.software]) else: database.cursor.execute("SELECT domain, software, nodeinfo_url FROM instances WHERE has_obfuscation = 1") @@ -1127,28 +1127,28 @@ def recheck_obfuscation(args: argparse.Namespace) -> int: rows = database.cursor.fetchall() logger.info("Checking %d domains ...", len(rows)) for row in rows: - logger.debug("Fetching peers from domain='%s',software='%s',nodeinfo_url='%s' ...", row[0], row[1], row[2]) + logger.debug("Fetching peers from domain='%s',software='%s',nodeinfo_url='%s' ...", row["domain"], row["software"], row["nodeinfo_url"]) blocking = list() - if row[1] == "pleroma": - logger.debug("domain='%s',software='%s'", row[0], row[1]) - blocking = pleroma.fetch_blocks(row[0], row[2]) - elif row[1] == "mastodon": - logger.debug("domain='%s',software='%s'", row[0], row[1]) - blocking = mastodon.fetch_blocks(row[0], row[2]) - elif row[1] == "lemmy": - logger.debug("domain='%s',software='%s'", row[0], row[1]) - blocking = lemmy.fetch_blocks(row[0], row[2]) - elif row[1] == "friendica": - logger.debug("domain='%s',software='%s'", row[0], row[1]) - blocking = friendica.fetch_blocks(row[0]) - elif row[1] == "misskey": - logger.debug("domain='%s',software='%s'", row[0], row[1]) - blocking = misskey.fetch_blocks(row[0]) + if row["software"] == "pleroma": + logger.debug("domain='%s',software='%s'", row["domain"], row["software"]) + blocking = pleroma.fetch_blocks(row["domain"], row["nodeinfo_url"]) + elif row["software"] == "mastodon": + logger.debug("domain='%s',software='%s'", row["domain"], row["software"]) + blocking = mastodon.fetch_blocks(row["domain"], row["nodeinfo_url"]) + elif row["software"] == "lemmy": + logger.debug("domain='%s',software='%s'", row["domain"], row["software"]) + blocking = lemmy.fetch_blocks(row["domain"], row["nodeinfo_url"]) + elif row["software"] == "friendica": + logger.debug("domain='%s',software='%s'", row["domain"], row["software"]) + blocking = friendica.fetch_blocks(row["domain"]) + elif row["software"] == "misskey": + logger.debug("domain='%s',software='%s'", row["domain"], row["software"]) + blocking = misskey.fetch_blocks(row["domain"]) else: - logger.warning("Unknown sofware: domain='%s',software='%s'", row[0], row[1]) + logger.warning("Unknown sofware: domain='%s',software='%s'", row["domain"], row["software"]) - logger.info("Checking %d block(s) from domain='%s' ...", len(blocking), row[0]) + logger.info("Checking %d block(s) from domain='%s' ...", len(blocking), row["domain"]) obfuscated = 0 blockdict = list() for block in blocking: @@ -1167,11 +1167,11 @@ def recheck_obfuscation(args: argparse.Namespace) -> int: elif block["blocked"].find("*") >= 0 or block["blocked"].find("?") >= 0: logger.debug("block='%s' is obfuscated.", block["blocked"]) obfuscated = obfuscated + 1 - blocked = utils.deobfuscate_domain(block["blocked"], row[0], block["hash"] if "hash" in block else None) + blocked = utils.deobfuscate_domain(block["blocked"], row["domain"], block["hash"] if "hash" in block else None) elif not utils.is_domain_wanted(block["blocked"]): logger.debug("blocked='%s' is not wanted - SKIPPED!", block["blocked"]) continue - elif blocks.is_instance_blocked(row[0], block["blocked"]): + elif blocks.is_instance_blocked(row["domain"], block["blocked"]): logger.debug("blocked='%s' is already blocked - SKIPPED!", block["blocked"]) continue @@ -1179,35 +1179,35 @@ def recheck_obfuscation(args: argparse.Namespace) -> int: if blocked is not None and blocked != block["blocked"]: logger.debug("blocked='%s' was deobfuscated to blocked='%s'", block["blocked"], blocked) obfuscated = obfuscated - 1 - if blocks.is_instance_blocked(row[0], blocked): - logger.debug("blocked='%s' is already blocked by domain='%s' - SKIPPED!", blocked, row[0]) + if blocks.is_instance_blocked(row["domain"], blocked): + logger.debug("blocked='%s' is already blocked by domain='%s' - SKIPPED!", blocked, row["domain"]) continue block["block_level"] = utils.alias_block_level(block["block_level"]) logger.info("blocked='%s' has been deobfuscated to blocked='%s', adding ...", block["blocked"], blocked) - if utils.process_block(row[0], blocked, block["reason"], block["block_level"]) and block["block_level"] == "reject" and config.get("bot_enabled"): - logger.debug("Appending blocked='%s',reason='%s' for blocker='%s' ...", block["blocked"], block["block_level"], row[0]) + if utils.process_block(row["domain"], blocked, block["reason"], block["block_level"]) and block["block_level"] == "reject" and config.get("bot_enabled"): + logger.debug("Appending blocked='%s',reason='%s' for blocker='%s' ...", block["blocked"], block["block_level"], row["domain"]) blockdict.append({ "blocked": blocked, "reason" : block["reason"], }) - logger.info("domain='%s' has %d obfuscated domain(s)", row[0], obfuscated) + logger.info("domain='%s' has %d obfuscated domain(s)", row["domain"], obfuscated) if obfuscated == 0 and len(blocking) > 0: - logger.info("Block list from domain='%s' has been fully deobfuscated.", row[0]) - instances.set_has_obfuscation(row[0], False) + logger.info("Block list from domain='%s' has been fully deobfuscated.", row["domain"]) + instances.set_has_obfuscation(row["domain"], False) - if instances.has_pending(row[0]): - logger.debug("Flushing updates for blocker='%s' ...", row[0]) - instances.update_data(row[0]) + if instances.has_pending(row["domain"]): + logger.debug("Flushing updates for blocker='%s' ...", row["domain"]) + instances.update_data(row["domain"]) logger.debug("Invoking commit() ...") database.connection.commit() if config.get("bot_enabled") and len(blockdict) > 0: - logger.info("Sending bot POST for blocker='%s,blockdict()=%d ...", row[0], len(blockdict)) - network.send_bot_post(row[0], blockdict) + logger.info("Sending bot POST for blocker='%s,blockdict()=%d ...", row["domain"], len(blockdict)) + network.send_bot_post(row["domain"], blockdict) logger.debug("Success! - EXIT!") return 0 diff --git a/fba/database.py b/fba/database.py index 53b33bc..b87d94d 100644 --- a/fba/database.py +++ b/fba/database.py @@ -21,4 +21,9 @@ logger = logging.getLogger(__name__) # Connect to database connection = sqlite3.connect("blocks.db") + +# Init row factory +connection.row_factory = sqlite3.Row + +# Get cursor cursor = connection.cursor() diff --git a/fba/helpers/cache.py b/fba/helpers/cache.py index 05d3a80..7a20a41 100644 --- a/fba/helpers/cache.py +++ b/fba/helpers/cache.py @@ -15,6 +15,7 @@ # along with this program. If not, see . import logging +import sqlite3 logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -37,7 +38,7 @@ def set_all(key: str, rows: list, value: any): for sub in rows: logger.debug("Setting key='%s',sub[%s]='%s'", key, type(sub), sub) - if isinstance(sub, tuple): + if isinstance(sub, sqlite3.Row): logger.debug("Setting key='%s',sub[%s]='%s',value[]='%s'", key, type(sub), sub, type(value)) _cache[key][sub[0]] = value else: diff --git a/fba/http/network.py b/fba/http/network.py index e323300..20ae9ee 100644 --- a/fba/http/network.py +++ b/fba/http/network.py @@ -82,6 +82,7 @@ def post_json_api(domain: str, path: str, data: str = "", headers: dict = dict() cookies=cookies.get_all(domain) if cookies.has(domain) else dict() ) + logger.info("Parsing JSON response from domain='%s',path='%s' ...", domain, path) json_reply["json"] = json_from_response(response) logger.debug("response.ok='%s',response.status_code=%d,json_reply[]='%s'", response.ok, response.status_code, type(json_reply)) @@ -89,8 +90,8 @@ def post_json_api(domain: str, path: str, data: str = "", headers: dict = dict() logger.warning("Cannot query JSON API: domain='%s',path='%s',data()=%d,response.status_code=%d,json_reply[]='%s'", domain, path, len(data), response.status_code, type(json_reply)) json_reply["status_code"] = response.status_code json_reply["error_message"] = response.reason - del json_reply["json"] instances.set_last_error(domain, response) + del json_reply["json"] except exceptions as exception: logger.debug("Fetching path='%s' from domain='%s' failed. exception[%s]='%s'", path, domain, type(exception), str(exception)) @@ -120,6 +121,7 @@ def fetch_api_url(url: str, timeout: tuple) -> dict: logger.debug("Fetching url='%s' ...", url) response = utils.fetch_url(url, api_headers, timeout) + logger.info("Parsing JSON response from url='%s' ...", url) json_reply["json"] = json_from_response(response) logger.debug("response.ok='%s',response.status_code='%s',json_reply[]='%s'", response.ok, response.status_code, type(json_reply)) @@ -173,6 +175,7 @@ def get_json_api(domain: str, path: str, headers: dict, timeout: tuple) -> dict: instances.set_last_error(domain, exception) raise exception + logger.info("Parsing JSON response from domain='%s',path='%s' ...", domain, path) json_reply["json"] = json_from_response(response) logger.debug("response.ok='%s',response.status_code=%d,json_reply[]='%s'", response.ok, response.status_code, type(json_reply)) @@ -272,7 +275,7 @@ def json_from_response(response: requests.models.Response) -> list: try: data = response.json() except json.decoder.JSONDecodeError as exception: - logger.warning("Exception '%s' during decoding JSON", type(exception)) + logger.warning("Exception '%s' during decoding JSON from response.url='%s'", type(exception), response.url) logger.debug("data[]='%s' - EXIT!", type(data)) return data diff --git a/fba/utils.py b/fba/utils.py index 2befee3..f2ff076 100644 --- a/fba/utils.py +++ b/fba/utils.py @@ -197,8 +197,8 @@ def deobfuscate_domain(domain: str, blocker: str, domain_hash: str = None) -> st logger.debug("row[]='%s'", type(row)) if row is not None: - logger.debug("domain='%s' de-obscured to '%s'", domain, row[0]) - domain = row[0] + logger.debug("domain='%s' de-obscured to '%s'", domain, row["domain"]) + domain = row["domain"] else: logger.debug("blocker='%s' has domain that cannot be deobfuscated.", blocker) instances.set_has_obfuscation(blocker, True) @@ -210,8 +210,8 @@ def deobfuscate_domain(domain: str, blocker: str, domain_hash: str = None) -> st logger.debug("row[]='%s'", type(row)) if row is not None: - logger.debug("domain='%s' de-obscured to '%s'", domain, row[0]) - domain = row[0] + logger.debug("domain='%s' de-obscured to '%s'", domain, row["domain"]) + domain = row["domain"] else: logger.debug("blocker='%s' has domain that cannot be deobfuscated.", blocker) instances.set_has_obfuscation(blocker, True) diff --git a/static/css/light.css b/static/css/light.css index 47aea43..e46a4a7 100644 --- a/static/css/light.css +++ b/static/css/light.css @@ -33,11 +33,26 @@ th { padding-bottom: 4px; } -table.with-rows > tbody > tr:nth-of-type(2n), -table.with-rows > thead { +.index-table { + width: 700px; + margin-left: auto; + margin-right: auto; +} + +.table-with-rows > tbody > tr:nth-of-type(2n), +.table-with-rows > thead, +.index-table > thead { background-color: #eaeaea; } +.index-table h3 { + margin: 0px; +} + +.index-table form { + padding: 2px; +} + table td { padding: 4px; text-align: left; diff --git a/templates/base.html b/templates/base.html index 7191ba4..d67f14d 100644 --- a/templates/base.html +++ b/templates/base.html @@ -4,11 +4,12 @@ fedi-block-api - {% block title %}{% endblock %} + - + {% block rss %}{% endblock %} - + diff --git a/templates/views/index.html b/templates/views/index.html index 71c0e80..320c1c0 100644 --- a/templates/views/index.html +++ b/templates/views/index.html @@ -5,40 +5,65 @@ {% block header %}

Welcome to FBA

{% endblock %} {% block content %} -

Enter a Domain

-
- - - -
+ + + + -

Enter a Reason

- - - - - - -

Reverse search

- - - - - + + + + + + + + + + +
+

Choose an option

+
+

Enter a domain:

+
+ + + +
+ 		+

Enter a reason:

+
+ + + +
+
+

Reverse search:

+
+ + + +
+ 		+

Info of a domain:

+
+ + +
+

Scoreboards:

{% endblock %} {% block footer %} diff --git a/templates/views/infos.html b/templates/views/infos.html new file mode 100644 index 0000000..2764c72 --- /dev/null +++ b/templates/views/infos.html @@ -0,0 +1,104 @@ +{% extends "base.html" %} + +{% block title %}Infos on domain {{domain}}{% endblock %} + +{% block header %}

Infos on {{domain}}

{% endblock %} + +{% block content %} +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+

Instance information

+
Domain name:{% with domain=instance['domain'] %}{% include "widgets/links.html" %}{% endwith %}
Domain hash:{{instance['hash']}}
Software:{{instance['software']}}
Originating instance:{% if instance['origin'] != None %}{% with domain=instance['origin'] %}{% include "widgets/links.html" %}{% endwith %}{% else %}-{% endif %}
Command:{{instance['command']}}
Detection mode:{{instance['detection_mode']}}
NodeInfo URL:{{instance['nodeinfo_url']}}
Total peers:{{instance['total_peers']}}
Has obfuscated block list:{% if instance['has_obfuscation']%}Yes{%elif not instance['has_obfuscation']%}No{%else%}-{%endif%}
First seen:{{instance['first_seen']}}
Last updated:{{instance['last_updated']}}
Last nodeinfo fetched:{{instance['last_nodeinfo']}}
Last blocks fetched:{{instance['last_blocked']}}
Last instances fetched:{{instance['last_instance_fetch']}}
Last status code:{{instance['last_status_code']}}
Last error details:{{instance['last_error_details']}}
+
+{% endblock %} + +{% block footer %} + Index / + {{ super() }} +{% endblock %} diff --git a/templates/views/scoreboard.html b/templates/views/scoreboard.html index 51a7f4f..512e409 100644 --- a/templates/views/scoreboard.html +++ b/templates/views/scoreboard.html @@ -30,7 +30,7 @@ {% block content %}
- +
@@ -47,7 +47,7 @@ {% elif entry['domain'] == None %} - {% elif mode == 'block_level' %} - {{entry['domain']}} + {{entry['domain']}} {% else %} {% with domain=entry['domain'] %} {% include "widgets/links.html" %} diff --git a/templates/views/top.html b/templates/views/top.html index 204db6e..9b0d1c3 100644 --- a/templates/views/top.html +++ b/templates/views/top.html @@ -5,9 +5,9 @@ {% block rss %} {{ super() }} {% if mode == 'domain' %} - + {% elif mode == 'reverse' %} - + {% endif %} {% endblock %} @@ -35,7 +35,7 @@ {% for block_level in blocklist %}

{{block_level}} ({{blocklist[block_level]|length}})

-
№ {% if mode in ('software', 'avg_peers', 'obfuscator') %}Software{% elif mode == 'obfuscation' %}Obfuscation status{% elif mode == 'detection_mode' %}Detection mode{% elif mode == 'error_code' %}Error code{% else %}Instance{% endif %}
+
diff --git a/templates/widgets/links.html b/templates/widgets/links.html index 46b435a..e058c2e 100644 --- a/templates/widgets/links.html +++ b/templates/widgets/links.html @@ -1,3 +1,4 @@ -[D] -[R] +[D] +[R] +[I] {{domain}}
Blocker {% if block_level == 'accept' %}Accepted{% else %}Blocked{% endif %}