From a77771a3a225edde0022e9e6bc0a195debc91adb Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Thu, 24 Aug 2023 21:12:14 +0200 Subject: [PATCH] Continued: - added parameter --software3 which searchinf for a file 'software.txt' - you can generate this by running e.g. sqlite3 ./blocks.db "SELECT software FROM instances WHAT_EVER_PARAMETER;" > software.txt - reset nodeinfo_url if software is now None - always set complete URL, including domain --- fba/commands.py | 7 +++++- fba/http/nodeinfo.py | 4 ++-- nodeinfo.sh | 55 +++++++++++++++++++++++++++++++------------- 3 files changed, 47 insertions(+), 19 deletions(-) diff --git a/fba/commands.py b/fba/commands.py index a682df2..1802c0f 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -1720,7 +1720,7 @@ def update_nodeinfo(args: argparse.Namespace) -> int: database.cursor.execute("SELECT domain, software FROM instances WHERE domain = ?", [args.domain]) elif args.software is not None and args.software != "": logger.info("Fetching domains for args.software='%s'", args.software) - database.cursor.execute("SELECT domain, software FROM instances WHERE software = ?", [args.software]) + database.cursor.execute("SELECT domain, software FROM instances WHERE software = ? AND (last_nodeinfo < ? OR last_nodeinfo IS NULL)", [args.software.lower(), time.time() - config.get("recheck_nodeinfo")]) elif args.mode is not None and args.mode != "": logger.info("Fetching domains for args.mode='%s'", args.mode.upper()) database.cursor.execute("SELECT domain, software FROM instances WHERE detection_mode = ? AND (last_nodeinfo < ? OR last_nodeinfo IS NULL)", [args.mode.upper(), time.time() - config.get("recheck_nodeinfo")]) @@ -1747,6 +1747,11 @@ def update_nodeinfo(args: argparse.Namespace) -> int: logger.debug("Determined software='%s'", software) if (software != row["software"] and software is not None) or args.force is True: + logger.debug("software='%s'", software) + if software is None: + logger.debug("Setting nodeinfo_url to 'None' for row[domain]='%s' ...", row["domain"]) + instances.set_nodeinfo_url(row["domain"], None) + logger.warning("Software type for row[domain]='%s' has changed from '%s' to '%s'!", row["domain"], row["software"], software) instances.set_software(row["domain"], software) diff --git a/fba/http/nodeinfo.py b/fba/http/nodeinfo.py index 724c875..f879e81 100644 --- a/fba/http/nodeinfo.py +++ b/fba/http/nodeinfo.py @@ -42,7 +42,7 @@ def fetch(domain: str, path: str = None, nodeinfo_url: str = None) -> dict: raise ValueError(f"Parameter nodeinfo_url[]='{type(nodeinfo_url)}' is not of type 'str'") logger.debug("nodeinfo_url='%s'", nodeinfo_url) - is_url = nodeinfo_url is not None and validators.url(nodeinfo_url) + is_url = nodeinfo_url is not None and not nodeinfo_url.startswith("/") and validators.url(nodeinfo_url) logger.debug("is_url='%s'", is_url) if not is_url: @@ -106,7 +106,7 @@ def fetch(domain: str, path: str = None, nodeinfo_url: str = None) -> dict: logger.debug("Success: request='%s' - Setting detection_mode=STATIC_CHECK ...", request) instances.set_last_nodeinfo(domain) instances.set_detection_mode(domain, "STATIC_CHECK") - instances.set_nodeinfo_url(domain, request) + instances.set_nodeinfo_url(domain, "https://{domain}{request}") break logger.warning("Failed fetching nodeinfo from domain='%s',status_code='%s',error_message='%s'", domain, data['status_code'], data['error_message']) diff --git a/nodeinfo.sh b/nodeinfo.sh index 53888d0..edb2085 100755 --- a/nodeinfo.sh +++ b/nodeinfo.sh @@ -1,9 +1,12 @@ #!/bin/sh MODE="" +DOMAINS="" +SOFTWARE_LIST="" + if [ "$1" = "--help" ] then - echo "Usage: $0 [file|--software|--software2|--nodeinfo|--generator|--detection|--no-auto|--no-auto2|--timeout]" + echo "Usage: $0 [file|--software|--software2|--software3|--nodeinfo|--generator|--detection|--no-auto|--no-auto2|--timeout]" exit 255 elif [ -n "$1" -a -f "$1" ] then @@ -37,6 +40,10 @@ elif [ "$1" = "--software2" ] then DOMAINS=$(sqlite3 blocks.db "SELECT domain FROM instances WHERE software IS NOT NULL AND detection_mode IS NULL ORDER BY last_updated ASC;") MODE="software2" +elif [ "$1" = "--software3" -a -f "software.txt" ] +then + SOFTWARE_LIST=$(cat software.txt) + MODE="software3" elif [ "$1" = "--generator" ] then DOMAINS=$(sqlite3 blocks.db "SELECT domain FROM instances WHERE detection_mode='GENERATOR' ORDER BY last_updated ASC;") @@ -45,23 +52,39 @@ else DOMAINS=$(sqlite3 blocks.db "SELECT domain FROM instances WHERE software IS NULL AND nodeinfo_url IS NOT NULL ORDER BY last_updated ASC;") fi -if [ -z "${DOMAINS}" ] +if [ -z "${DOMAINS}" and -z "${SOFTWARE_LIST}" ] then - echo "$0: No domains found! MODE='${MODE}'" + echo "$0: No domains or software found! MODE='${MODE}'" exit 255 fi -for DOMAIN in ${DOMAINS}; -do - echo "$0: DOMAIN='${DOMAIN}'" - if [ -n "$1" -a -z "${MODE}" ] - then - ./fba.py update_nodeinfo --domain=${DOMAIN} "$1" - elif [ -n "$2" ] - then - ./fba.py update_nodeinfo --domain=${DOMAIN} "$2" - else - ./fba.py update_nodeinfo --domain=${DOMAIN} - fi -done +if [ -n "${DOMAINS}" ] +then + for DOMAIN in ${DOMAINS}; + do + echo "$0: DOMAIN='${DOMAIN}'" + if [ -n "$1" -a -z "${MODE}" ] + then + ./fba.py update_nodeinfo --domain=${DOMAIN} "$1" + elif [ -n "$2" ] + then + ./fba.py update_nodeinfo --domain=${DOMAIN} "$2" + else + ./fba.py update_nodeinfo --domain=${DOMAIN} + fi + done +elif [ -n "${SOFTWARE_LIST}" ] +then + for SOFTWARE in ${SOFTWARE_LIST}; + do + echo "$0: SOFTWARE='${SOFTWARE}'" + if [ -n "$2" ] + then + ./fba.py update_nodeinfo --software=${SOFTWARE} "$2" + else + ./fba.py update_nodeinfo --software=${SOFTWARE} + fi + done +fi + echo "$0: All done." -- 2.39.5