Continued:

- also strip out " by " and " see " (self-advertisement)
- same with " version"
- some version numbers had uncommon long patch levels, e.g. 8.0.0000

diff --git a/fba.py b/fba.py
index 08e489fc521e18edd13c7034ebfa4d1808db0cdc..14caf6119a16dea824bb6475a18233b2f135bdff 100644 (file)
--- a/fba.py
+++ b/fba.py
@@ -92,9 +92,9 @@ cursor = connection.cursor()
 # Pattern instance for version numbers
 patterns = [
     # semantic version number (with v|V) prefix)
-    re.compile("^(?P<version>v|V{0,1})(\.{0,1})(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)(\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)?$"),
+    re.compile("^(?P<version>v|V{0,1})(\.{0,1})(?P<major>0|[1-9]\d*)\.(?P<minor>0+|[1-9]\d*)(\.(?P<patch>0+|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)?$"),
     # non-sematic, e.g. 1.2.3.4
-    re.compile("^(?P<version>v|V{0,1})(\.{0,1})(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)(\.(?P<patch>0|[1-9]\d*)(\.(?P<subpatch>0|[1-9]\d*))?)$"),
+    re.compile("^(?P<version>v|V{0,1})(\.{0,1})(?P<major>0|[1-9]\d*)\.(?P<minor>0+|[1-9]\d*)(\.(?P<patch>0+|[1-9]\d*)(\.(?P<subpatch>0|[1-9]\d*))?)$"),
     # non-sematic, e.g. 2023-05
     re.compile("^(?P<year>[1-9]{1}[0-9]{3})\.(?P<month>[0-9]{2})$")
 ]
@@ -136,13 +136,19 @@ def remove_version(software: str) -> str:
 
     # NOISY-DEBUG: print(f"DEBUG: end[{type(end)}]={end}")
     software = software[0:end].strip()
+    if " version" in software:
+        # NOISY-DEBUG: print(f"DEBUG: software='{software}' contains word ' version'")
+        software = strip_until(software, " version")
 
     # NOISY-DEBUG: print(f"DEBUG: software='{software}' - EXIT!")
     return software
 
 def strip_powered_by(software: str) -> str:
     # NOISY-DEBUG: print(f"DEBUG: software='{software}' - CALLED!")
-    if not "powered by" in software:
+    if software == "":
+        print(f"ERROR: Bad method call, 'software' is empty")
+        raise Exception("Parameter 'software' is empty")
+    elif not "powered by" in software:
         print(f"WARNING: Cannot find 'powered by' in '{software}'!")
         return software
 
@@ -152,8 +158,26 @@ def strip_powered_by(software: str) -> str:
     software = software[start + 11:].strip()
     # NOISY-DEBUG: print(f"DEBUG: software='{software}'")
 
-    # Next, strip of ' - ' part
-    end = software.find(" - ")
+    software = strip_until(software, " - ")
+
+    # NOISY-DEBUG: print(f"DEBUG: software='{software}' - EXIT!")
+    return software
+
+def strip_until(software: str, until: str) -> str:
+    # NOISY-DEBUG: print(f"DEBUG: software='{software}',until='{until}' - CALLED!")
+    if software == "":
+        print(f"ERROR: Bad method call, 'software' is empty")
+        raise Exception("Parameter 'software' is empty")
+    elif until == "":
+        print(f"ERROR: Bad method call, 'until' is empty")
+        raise Exception("Parameter 'until' is empty")
+    elif not until in software:
+        print(f"WARNING: Cannot find 'powered by' in '{software}'!")
+        return software
+
+    # Next, strip until part
+    end = software.find(until)
+
     # NOISY-DEBUG: print(f"DEBUG: end[{type(end)}]='{end}'")
     if end > 0:
         software = software[0:end].strip()
@@ -527,6 +551,12 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str:
     if type(software) is str and "powered by" in software:
         # NOISY-DEBUG: print(f"DEBUG: software='{software}' has 'powered by' in it")
         software = remove_version(strip_powered_by(software))
+    elif type(software) is str and " by " in software:
+        # NOISY-DEBUG: print(f"DEBUG: software='{software}' has ' by ' in it")
+        software = strip_until(software, " by ")
+    elif type(software) is str and " see " in software:
+        # NOISY-DEBUG: print(f"DEBUG: software='{software}' has ' see ' in it")
+        software = strip_until(software, " see ")
 
     # NOISY-DEBUG: print(f"DEBUG: software='{software}' - EXIT!")
     return software
@@ -574,8 +604,14 @@ def determine_software(domain: str) -> str:
         print("WARNING: Spliting of pipe:", software)
         software = tidyup(software.split("|")[0]);
     elif "powered by" in software:
-        print(f"DEBUG: software='{software}' has 'powered by' in it")
+        # NOISY-DEBUG: print(f"DEBUG: software='{software}' has 'powered by' in it")
         software = strip_powered_by(software)
+    elif type(software) is str and " by " in software:
+        # NOISY-DEBUG: print(f"DEBUG: software='{software}' has ' by ' in it")
+        software = strip_until(software, " by ")
+    elif type(software) is str and " see " in software:
+        # NOISY-DEBUG: print(f"DEBUG: software='{software}' has ' see ' in it")
+        software = strip_until(software, " see ")
 
     # NOISY-DEBUG: print(f"DEBUG: software[]={type(software)}")
     if software == "":