Continued:

- added parameter --software3 which searchinf for a file 'software.txt'
- you can generate this by running e.g.
  sqlite3 ./blocks.db "SELECT software FROM instances WHAT_EVER_PARAMETER;" > software.txt
- reset nodeinfo_url if software is now None
- always set complete URL, including domain

diff --git a/fba/commands.py b/fba/commands.py
index a682df2e3615a415d3538c13e7bee7cf8895eff3..1802c0fcff444c5251fbdbb7a2fe63ef090c0c97 100644 (file)
--- a/fba/commands.py
+++ b/fba/commands.py
@@ -1720,7 +1720,7 @@ def update_nodeinfo(args: argparse.Namespace) -> int:
         database.cursor.execute("SELECT domain, software FROM instances WHERE domain = ?", [args.domain])
     elif args.software is not None and args.software != "":
         logger.info("Fetching domains for args.software='%s'", args.software)
-        database.cursor.execute("SELECT domain, software FROM instances WHERE software = ?", [args.software])
+        database.cursor.execute("SELECT domain, software FROM instances WHERE software = ? AND (last_nodeinfo < ? OR last_nodeinfo IS NULL)", [args.software.lower(), time.time() - config.get("recheck_nodeinfo")])
     elif args.mode is not None and args.mode != "":
         logger.info("Fetching domains for args.mode='%s'", args.mode.upper())
         database.cursor.execute("SELECT domain, software FROM instances WHERE detection_mode = ? AND (last_nodeinfo < ? OR last_nodeinfo IS NULL)", [args.mode.upper(), time.time() - config.get("recheck_nodeinfo")])
@@ -1747,6 +1747,11 @@ def update_nodeinfo(args: argparse.Namespace) -> int:
 
             logger.debug("Determined software='%s'", software)
             if (software != row["software"] and software is not None) or args.force is True:
+                logger.debug("software='%s'", software)
+                if software is None:
+                    logger.debug("Setting nodeinfo_url to 'None' for row[domain]='%s' ...", row["domain"])
+                    instances.set_nodeinfo_url(row["domain"], None)
+
                 logger.warning("Software type for row[domain]='%s' has changed from '%s' to '%s'!", row["domain"], row["software"], software)
                 instances.set_software(row["domain"], software)
 

diff --git a/fba/http/nodeinfo.py b/fba/http/nodeinfo.py
index 724c87501d45d7784a1eafb761a5b66ad63a33d2..f879e81ac2bb5b5be8fda5aab947ea2b5da587df 100644 (file)
--- a/fba/http/nodeinfo.py
+++ b/fba/http/nodeinfo.py
@@ -42,7 +42,7 @@ def fetch(domain: str, path: str = None, nodeinfo_url: str = None) -> dict:
         raise ValueError(f"Parameter nodeinfo_url[]='{type(nodeinfo_url)}' is not of type 'str'")
 
     logger.debug("nodeinfo_url='%s'", nodeinfo_url)
-    is_url = nodeinfo_url is not None and validators.url(nodeinfo_url)
+    is_url = nodeinfo_url is not None and not nodeinfo_url.startswith("/") and validators.url(nodeinfo_url)
 
     logger.debug("is_url='%s'", is_url)
     if not is_url:
@@ -106,7 +106,7 @@ def fetch(domain: str, path: str = None, nodeinfo_url: str = None) -> dict:
                 logger.debug("Success: request='%s' - Setting detection_mode=STATIC_CHECK ...", request)
                 instances.set_last_nodeinfo(domain)
                 instances.set_detection_mode(domain, "STATIC_CHECK")
-                instances.set_nodeinfo_url(domain, request)
+                instances.set_nodeinfo_url(domain, "https://{domain}{request}")
                 break
 
             logger.warning("Failed fetching nodeinfo from domain='%s',status_code='%s',error_message='%s'", domain, data['status_code'], data['error_message'])

diff --git a/nodeinfo.sh b/nodeinfo.sh
index 53888d0dab0ed6fd49274b749c3393e330c80d72..edb20856bdae2e1062ab11a49546838b5bfe1fd6 100755 (executable)
--- a/nodeinfo.sh
+++ b/nodeinfo.sh
@@ -1,9 +1,12 @@
 #!/bin/sh
 
 MODE=""
+DOMAINS=""
+SOFTWARE_LIST=""
+
 if [ "$1" = "--help" ]
 then
-       echo "Usage: $0 [file|--software|--software2|--nodeinfo|--generator|--detection|--no-auto|--no-auto2|--timeout]"
+       echo "Usage: $0 [file|--software|--software2|--software3|--nodeinfo|--generator|--detection|--no-auto|--no-auto2|--timeout]"
        exit 255
 elif [ -n "$1" -a -f "$1" ]
 then
@@ -37,6 +40,10 @@ elif [ "$1" = "--software2" ]
 then
        DOMAINS=$(sqlite3 blocks.db "SELECT domain FROM instances WHERE software IS NOT NULL AND detection_mode IS NULL ORDER BY last_updated ASC;")
        MODE="software2"
+elif [ "$1" = "--software3" -a -f "software.txt" ]
+then
+       SOFTWARE_LIST=$(cat software.txt)
+       MODE="software3"
 elif [ "$1" = "--generator" ]
 then
        DOMAINS=$(sqlite3 blocks.db "SELECT domain FROM instances WHERE detection_mode='GENERATOR' ORDER BY last_updated ASC;")
@@ -45,23 +52,39 @@ else
        DOMAINS=$(sqlite3 blocks.db "SELECT domain FROM instances WHERE software IS NULL AND nodeinfo_url IS NOT NULL ORDER BY last_updated ASC;")
 fi
 
-if [ -z "${DOMAINS}" ]
+if [ -z "${DOMAINS}" and -z "${SOFTWARE_LIST}" ]
 then
-       echo "$0: No domains found! MODE='${MODE}'"
+       echo "$0: No domains or software found! MODE='${MODE}'"
        exit 255
 fi
 
-for DOMAIN in ${DOMAINS};
-do
-       echo "$0: DOMAIN='${DOMAIN}'"
-       if [ -n "$1" -a -z "${MODE}" ]
-       then
-               ./fba.py update_nodeinfo --domain=${DOMAIN} "$1"
-       elif [ -n "$2" ]
-       then
-               ./fba.py update_nodeinfo --domain=${DOMAIN} "$2"
-       else
-               ./fba.py update_nodeinfo --domain=${DOMAIN}
-       fi
-done
+if [ -n "${DOMAINS}" ]
+then
+       for DOMAIN in ${DOMAINS};
+       do
+               echo "$0: DOMAIN='${DOMAIN}'"
+               if [ -n "$1" -a -z "${MODE}" ]
+               then
+                       ./fba.py update_nodeinfo --domain=${DOMAIN} "$1"
+               elif [ -n "$2" ]
+               then
+                       ./fba.py update_nodeinfo --domain=${DOMAIN} "$2"
+               else
+                       ./fba.py update_nodeinfo --domain=${DOMAIN}
+               fi
+       done
+elif [ -n "${SOFTWARE_LIST}" ]
+then
+       for SOFTWARE in ${SOFTWARE_LIST};
+       do
+               echo "$0: SOFTWARE='${SOFTWARE}'"
+               if [ -n "$2" ]
+               then
+                       ./fba.py update_nodeinfo --software=${SOFTWARE} "$2"
+               else
+                       ./fba.py update_nodeinfo --software=${SOFTWARE}
+               fi
+       done
+fi
+
 echo "$0: All done."