Continued:

- moved original_software next to 'software' column
- replaced own _cache dict with @lru_cache

diff --git a/blocks_empty.db b/blocks_empty.db
index dec6ada3e62839974aaa9f105364a7ed5bf80a1e..a3233785f9affff2dcf5ad32a4ce18dd5746d1b0 100644 (file)
Binary files a/blocks_empty.db and b/blocks_empty.db differ

diff --git a/fba/helpers/domain.py b/fba/helpers/domain.py
index 2df92e8f9733005978e0f9ce5b6ce951009fc1c2..39fbb7797acee74c7c5207ed0b3b60c507276b2f 100644 (file)
--- a/fba/helpers/domain.py
+++ b/fba/helpers/domain.py
@@ -16,6 +16,7 @@
 
 import logging
 
+from functools import lru_cache
 from urllib.parse import urlparse
 
 import validators
@@ -28,18 +29,6 @@ from fba.models import instances
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
-# In-function cache
-_cache = {
-    # Cache for function is_in_url()
-    "is_in_url": {},
-
-    # Cache for function is_wanted()
-    "is_wanted": {},
-
-    # Cache for function raise_on()
-    "raise_on": {},
-}
-
 def raise_on(domain: str):
     logger.debug("domain='%s' - CALLED!", domain)
 
@@ -47,9 +36,6 @@ def raise_on(domain: str):
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not of type 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
-    elif domain in _cache["raise_on"]:
-        logger.debug("Returning cached raised_on='%s' - EXIT!", _cache["raise_on"][domain])
-        return _cache["raise_on"][domain]
     elif domain.lower() != domain:
         raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
     elif not validators.domain(domain.split("/")[0]):
@@ -63,9 +49,9 @@ def raise_on(domain: str):
     elif domain.endswith(".tld"):
         raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
 
-    _cache["raise_on"][domain] = True
     logger.debug("EXIT!")
 
+@lru_cache
 def is_in_url(domain: str, url: str) -> bool:
     logger.debug("domain='%s',url='%s' - CALLED!", domain, url)
     raise_on(domain)
@@ -78,9 +64,6 @@ def is_in_url(domain: str, url: str) -> bool:
         raise ValueError("Parameter 'url' is empty")
     elif not validators.url(url):
         raise ValueError(f"Parameter url='{url}' is not a valid URL")
-    elif domain + url in _cache["is_in_url"]:
-        logger.debug("Returning cached is_in_url='%s' - EXIT!", _cache["is_in_url"][domain + url])
-        return _cache["is_in_url"][domain + url]
 
     punycode = domain.encode("idna").decode("utf-8")
 
@@ -89,12 +72,10 @@ def is_in_url(domain: str, url: str) -> bool:
 
     is_found = (punycode in [components.netloc, components.hostname])
 
-    # Set cache
-    _cache["is_in_url"][domain + url] = is_found
-
     logger.debug("is_found='%s' - EXIT!", is_found)
     return is_found
 
+@lru_cache
 def is_wanted(domain: str) -> bool:
     logger.debug("domain='%s' - CALLED!", domain)
 
@@ -102,9 +83,6 @@ def is_wanted(domain: str) -> bool:
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not of type 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
-    elif domain in _cache["is_wanted"]:
-        logger.debug("Returning cached is_found='%s' - EXIT!", _cache["is_wanted"][domain])
-        return _cache["is_wanted"][domain]
 
     wanted = True
     if domain.lower() != domain:
@@ -135,8 +113,5 @@ def is_wanted(domain: str) -> bool:
         logger.debug("domain='%s' is a tag", domain)
         wanted = False
 
-    # Set cache
-    _cache["is_wanted"][domain] = wanted
-
     logger.debug("wanted='%s' - EXIT!", wanted)
     return wanted

diff --git a/fba/helpers/software.py b/fba/helpers/software.py
index eff6eb58b8f53e2f9e87f51df25dfd6d8b97915f..821660d5cabd6f90b8c597f5751f5e76d63cbb78 100644 (file)
--- a/fba/helpers/software.py
+++ b/fba/helpers/software.py
@@ -16,6 +16,8 @@
 
 import logging
 
+from functools import lru_cache
+
 from fba.helpers import tidyup
 
 logging.basicConfig(level=logging.INFO)
@@ -29,12 +31,7 @@ relays = [
     "pub-relay"
 ]
 
-# In-function cache
-_cache = {
-    # Cache for function alias()
-    "alias" : {},
-}
-
+@lru_cache
 def alias(software: str) -> str:
     logger.debug("software='%s'- CALLED!", software)
 
@@ -42,9 +39,6 @@ def alias(software: str) -> str:
         raise ValueError(f"software[]='{type(software)}' is not type 'str'")
     elif software == "":
         raise ValueError("Parameter 'software' is empty")
-    elif software in _cache["alias"]:
-        logger.debug("Returning cached alias='%s' - EXIT!", _cache["alias"][software])
-        return _cache["alias"][software]
 
     key = software
 
@@ -137,9 +131,6 @@ def alias(software: str) -> str:
         logger.debug("software='%s' is being cleaned up further ...")
         software = software.rstrip("!").strip()
 
-    # Set cache
-    _cache["alias"][key] = software
-
     logger.debug("software[%s]='%s' - EXIT!", type(software), software)
     return software
 

diff --git a/fba/http/network.py b/fba/http/network.py
index ec548d8f9c7de407db989aae36db48947a10d399..5158c94e810e1deff26702ff9671bd968b59082f 100644 (file)
--- a/fba/http/network.py
+++ b/fba/http/network.py
@@ -20,6 +20,7 @@ import time
 import reqto
 import requests
 import urllib3
+import validators
 
 from fba import utils
 

diff --git a/fba/http/nodeinfo.py b/fba/http/nodeinfo.py
index f66afd25c32f4a27fb50ec87646277ce59e9a725..9c944cda5f3cf880b5ed9650040d5a6b4bcd7ac4 100644 (file)
--- a/fba/http/nodeinfo.py
+++ b/fba/http/nodeinfo.py
@@ -124,7 +124,7 @@ def fetch(domain: str, path: str = None, update_mode: bool = True) -> dict:
                     instances.set_detection_mode(domain, "STATIC_CHECK")
 
                     logger.debug("domain='%s',request='%s'", domain, request)
-                    instances.set_nodeinfo_url(domain, "https://{domain}{request}")
+                    instances.set_nodeinfo_url(domain, f"https://{domain}{request}")
 
                 logger.debug("BREAK!")
                 break

diff --git a/fba/models/instances.py b/fba/models/instances.py
index f01dd36fc887ca2a5e4185ec0f8365a54d640471..77b198e4c920d3fcbe0134c9db91cd241f77248f 100644 (file)
--- a/fba/models/instances.py
+++ b/fba/models/instances.py
@@ -483,7 +483,7 @@ def set_nodeinfo_url(domain: str, url: str):
         raise ValueError(f"Parameter url[]='{type(url)}' is not of type 'str'")
     elif url == "":
         raise ValueError("Parameter 'url' is empty")
-    elif not validators.url(url):
+    elif url is not None and not validators.url(url):
         raise ValueError(f"Parameter url='{url}' is not a valid URL")
 
     # Set timestamp