From 8face31692f55a59ac1a2ccb7b4e48663ae9f641 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Wed, 5 Jul 2023 22:15:19 +0200 Subject: [PATCH] Continued: - added view /list which lists domains by some criteria (mode/value) - renamed blocks.is_valid_level() to valid() but now requires 2nd parameter with column to check --- daemon.py | 73 ++++++++++++++++++++++++--------- fba/helpers/software.py | 2 +- fba/models/blocks.py | 18 ++++---- fba/models/instances.py | 23 ++++++++++- templates/views/list.html | 58 ++++++++++++++++++++++++++ templates/views/scoreboard.html | 6 ++- 6 files changed, 150 insertions(+), 30 deletions(-) create mode 100644 templates/views/list.html diff --git a/daemon.py b/daemon.py index 2f2e328..85aa1f9 100755 --- a/daemon.py +++ b/daemon.py @@ -41,6 +41,7 @@ from fba.helpers import json as json_helper from fba.helpers import tidyup from fba.models import blocks +from fba.models import instances router = fastapi.FastAPI(docs_url=config.get("base_url") + "/docs", redoc_url=config.get("base_url") + "/redoc") router.mount( @@ -103,6 +104,24 @@ def api_scoreboard(mode: str, amount: int): return scores +@router.get(config.get("base_url") + "/api/list.json", response_class=JSONResponse) +def api_index(request: Request, mode: str, value: str, amount: int): + if mode is None or value is None or amount is None: + raise HTTPException(status_code=500, detail="No filter specified") + elif amount > config.get("api_limit"): + raise HTTPException(status_code=500, detail=f"amount={amount} is to big") + + domain = wildchar = punycode = reason = None + + if mode == "detection_mode": + database.cursor.execute( + f"SELECT domain, origin, software, command, total_peers, total_blocks, first_seen, last_updated FROM instances WHERE {mode} = ? LIMIT ?", [value, amount] + ) + + domainlist = database.cursor.fetchall() + + return domainlist + @router.get(config.get("base_url") + "/api/top.json", response_class=JSONResponse) def api_index(request: Request, mode: str, value: str, amount: int): if mode is None or value is None or amount is None: @@ -290,36 +309,52 @@ def index(request: Request): "slogan" : config.get("slogan"), }) -@router.get(config.get("base_url") + "/top") -def top(request: Request, mode: str, value: str, amount: int = config.get("api_limit")): - response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/info.json") +@router.get(config.get("base_url") + "/list") +def list_domains(request: Request, mode: str, value: str, amount: int = config.get("api_limit")): + if mode == "detection_mode" and not instances.valid(value, "detection_mode"): + raise HTTPException(status_code=500, detail="Invalid detection mode provided") - if not response.ok: - raise HTTPException(status_code=response.status_code, detail=response.text) - elif mode == "" or value == "" or amount == 0: - raise HTTPException(status_code=500, detail="Parameter mode, value and amount must always be set") - elif amount > config.get("api_limit"): - raise HTTPException(status_code=500, detail=f"amount='{amount}' is to big") + response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/list.json?mode={mode}&value={value}&amount={amount}") - info = response.json() - blocklist = list() + domainlist = list() + if response is not None and response.ok: + domainlist = response.json() + format = config.get("timestamp_format") + for row in domainlist: + row["first_seen"] = datetime.utcfromtimestamp(row["first_seen"]).strftime(format) + row["last_updated"] = datetime.utcfromtimestamp(row["last_updated"]).strftime(format) - if mode == "block_level" and not blocks.is_valid_level(value): + return templates.TemplateResponse("views/list.html", { + "request" : request, + "mode" : mode if response is not None else None, + "value" : value if response is not None else None, + "amount" : amount if response is not None else None, + "found" : len(domainlist), + "domainlist": domainlist, + "slogan" : config.get("slogan"), + "theme" : config.get("theme"), + }) + +@router.get(config.get("base_url") + "/top") +def top(request: Request, mode: str, value: str, amount: int = config.get("api_limit")): + if mode == "block_level" and not blocks.valid(value, "block_level"): raise HTTPException(status_code=500, detail="Invalid block level provided") elif mode in ["domain", "reverse"] and not utils.is_domain_wanted(value): raise HTTPException(status_code=500, detail="Invalid or blocked domain specified") response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/top.json?mode={mode}&value={value}&amount={amount}") + found = 0 + blocklist = list() if response is not None and response.ok: blocklist = response.json() - found = 0 - for block_level in blocklist: - for block in blocklist[block_level]: - block["first_seen"] = datetime.utcfromtimestamp(block["first_seen"]).strftime(config.get("timestamp_format")) - block["last_seen"] = datetime.utcfromtimestamp(block["last_seen"]).strftime(config.get("timestamp_format")) - found = found + 1 + format = config.get("timestamp_format") + for block_level in blocklist: + for block in blocklist[block_level]: + block["first_seen"] = datetime.utcfromtimestamp(block["first_seen"]).strftime(format) + block["last_seen"] = datetime.utcfromtimestamp(block["last_seen"]).strftime(format) + found = found + 1 return templates.TemplateResponse("views/top.html", { "request" : request, @@ -328,7 +363,7 @@ def top(request: Request, mode: str, value: str, amount: int = config.get("api_l "amount" : amount if response is not None else None, "found" : found, "blocklist": blocklist, - "info" : info, + "slogan" : config.get("slogan"), "theme" : config.get("theme"), }) diff --git a/fba/helpers/software.py b/fba/helpers/software.py index f4e53ca..89a3894 100644 --- a/fba/helpers/software.py +++ b/fba/helpers/software.py @@ -40,7 +40,7 @@ def alias(software: str) -> str: elif software in ["slipfox calckey", "calckey", "groundpolis", "foundkey", "cherrypick", "meisskey", "magnetar", "keybump", "dolphin"]: logger.debug("Setting misskey: software='%s'", software) software = "misskey" - elif software == "runtube.re": + elif software in ["runtube.re", "islameye"]: logger.debug("Setting peertube: software='%s'", software) software = "peertube" elif software == "nextcloud social": diff --git a/fba/models/blocks.py b/fba/models/blocks.py index d7e7121..6b8edef 100644 --- a/fba/models/blocks.py +++ b/fba/models/blocks.py @@ -174,16 +174,20 @@ def add_instance(blocker: str, blocked: str, reason: str, block_level: str): logger.debug("EXIT!") -def is_valid_level(block_level: str) -> bool: - logger.debug("block_level='%s' - CALLED!", block_level) - if not isinstance(block_level, str): - raise ValueError(f"Parameter block_level[]='{type(block_level)}' is not of type 'str'") - elif block_level == "": - raise ValueError("Parameter 'block_level' is empty") +def valid(value: str, column: str) -> bool: + logger.debug("value='%s' - CALLED!", value) + if not isinstance(value, str): + raise ValueError(f"Parameter value[]='{type(value)}' is not of type 'str'") + elif value == "": + raise ValueError("Parameter 'value' is empty") + elif not isinstance(column, str): + raise columnError(f"Parameter column[]='{type(column)}' is not of type 'str'") + elif column == "": + raise columnError("Parameter 'column' is empty") # Query database database.cursor.execute( - "SELECT block_level FROM blocks WHERE block_level = ? LIMIT 1", [block_level] + f"SELECT {column} FROM blocks WHERE {column} = ? LIMIT 1", [value] ) valid = database.cursor.fetchone() is not None diff --git a/fba/models/instances.py b/fba/models/instances.py index 0c39953..3c8433d 100644 --- a/fba/models/instances.py +++ b/fba/models/instances.py @@ -294,7 +294,7 @@ def is_recent(domain: str, column: str = "last_instance_fetch") -> bool: if not isinstance(column, str): raise ValueError(f"Parameter column[]='{type(column)}' is not of type 'str'") - elif column not in ["last_instance_fetch", "last_blocked"]: + elif column not in ["last_instance_fetch", "last_blocked", "last_nodeinfo"]: raise ValueError(f"Parameter column='{column}' is not expected") elif not is_registered(domain): logger.debug("domain='%s' is not registered, returning False - EXIT!", domain) @@ -453,3 +453,24 @@ def set_software(domain: str, software: str): # Set timestamp _set_data("software", domain, software) logger.debug("EXIT!") + +def valid(value: str, column: str) -> bool: + logger.debug("value='%s' - CALLED!", value) + if not isinstance(value, str): + raise ValueError(f"Parameter value[]='{type(value)}' is not of type 'str'") + elif value == "": + raise ValueError("Parameter 'value' is empty") + elif not isinstance(column, str): + raise columnError(f"Parameter column[]='{type(column)}' is not of type 'str'") + elif column == "": + raise columnError("Parameter 'column' is empty") + + # Query database + database.cursor.execute( + f"SELECT {column} FROM instances WHERE {column} = ? LIMIT 1", [value] + ) + + valid = database.cursor.fetchone() is not None + + logger.debug("valid='%s' - EXIT!", valid) + return valid diff --git a/templates/views/list.html b/templates/views/list.html new file mode 100644 index 0000000..c6851db --- /dev/null +++ b/templates/views/list.html @@ -0,0 +1,58 @@ +{% extends "base.html" %} + +{% block title %}{% if mode == 'detection_mode' %} - Detection mode {{value}}{% endif %}{% endblock %} + +{% block header %} + {% if mode == 'detection_mode' %} +

Instances detected by method {{value}}

+ {% endif %} +{% endblock %} + +{% block content %} + {% if amount == found %} +
+

Maximum amount reached!

+
+ Please note that the maximum allowed amount is only returned, the blocker/blocked/reason might be more. + Paging support is not finished yet. +
+
+ {% endif %} + + + + + + + + + + + + + {% for row in domainlist %} + + + + + + + + + {% endfor %} + +
DomainOriginSoftwareCommandFirst addedLast seen
+ {% with domain=row['domain'] %} + {% include "widgets/links.html" %} + {% endwith %} + + {% with domain=row['origin'] %} + {% include "widgets/links.html" %} + {% endwith %} + {{row['software']}}{{row['command']}}{{row['first_seen']}}{{row['last_seen']}}
+{% endblock %} + +{% block footer %} + Index / + {{ super() }} +{% endblock %} diff --git a/templates/views/scoreboard.html b/templates/views/scoreboard.html index 7a030ef..d727ac6 100644 --- a/templates/views/scoreboard.html +++ b/templates/views/scoreboard.html @@ -42,12 +42,14 @@ {{loop.index}} - {% if mode in ('software', 'command', 'error_code', 'detection_mode', 'avg_peers', 'obfuscator', 'obfuscation') %} + {% if mode in ('software', 'command', 'error_code', 'avg_peers', 'obfuscator', 'obfuscation') %} {{entry['domain']}} {% elif entry['domain'] == None %} - {% elif mode == 'block_level' %} - {{entry['domain']}} + {{entry['domain']}} + {% elif mode == 'detection_mode' %} + {{entry['domain']}} {% else %} {% with domain=entry['domain'] %} {% include "widgets/links.html" %} -- 2.39.2