Continued:

[fba.git] / fba / fba.py

diff --git a/fba/fba.py b/fba/fba.py
index e4224623c85a1e98239d3aa0edaa2e39c291321e..b175e57547aa6ebcd47d221e5ab19e3643571e2e 100644 (file)
--- a/fba/fba.py
+++ b/fba/fba.py
@@ -14,9 +14,7 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 import hashlib
-import json
 import sqlite3
-import time
 
 from urllib.parse import urlparse
 
@@ -24,11 +22,11 @@ import requests
 import validators
 
 from fba import blacklist
-from fba import config
 from fba import federation
-from fba import instances
 from fba import network
 
+from fba.models import instances
+
 # Connect to database
 connection = sqlite3.connect("blocks.db")
 cursor = connection.cursor()
@@ -47,41 +45,6 @@ def get_hash(domain: str) -> str:
 
     return hashlib.sha256(domain.encode("utf-8")).hexdigest()
 
-def log_error(domain: str, error: dict):
-    # DEBUG: print("DEBUG: domain,error[]:", domain, type(error))
-    if not isinstance(domain, str):
-        raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
-    elif domain == "":
-        raise ValueError("Parameter 'domain' is empty")
-    elif config.get("write_error_log").lower() != "true":
-        # DEBUG: print(f"DEBUG: Writing to error_log is disabled in configuruation file - EXIT!")
-        return
-
-    # DEBUG: print("DEBUG: BEFORE error[]:", type(error))
-    if isinstance(error, BaseException, error, json.decoder.JSONDecodeError):
-        error = f"error[{type(error)}]='{str(error)}'"
-
-    # DEBUG: print("DEBUG: AFTER error[]:", type(error))
-    if isinstance(error, str):
-        cursor.execute("INSERT INTO error_log (domain, error_code, error_message, created) VALUES (?, 999, ?, ?)",[
-            domain,
-            error,
-            time.time()
-        ])
-    else:
-        cursor.execute("INSERT INTO error_log (domain, error_code, error_message, created) VALUES (?, ?, ?, ?)",[
-            domain,
-            error["status_code"],
-            error["error_message"],
-            time.time()
-        ])
-
-    # Cleanup old entries
-    # DEBUG: print(f"DEBUG: Purging old records (distance: {config.get('error_log_cleanup')})")
-    cursor.execute("DELETE FROM error_log WHERE created < ?", [time.time() - config.get("error_log_cleanup")])
-
-    # DEBUG: print("DEBUG: EXIT!")
-
 def fetch_url(url: str, headers: dict, timeout: tuple) -> requests.models.Response:
     # DEBUG: print(f"DEBUG: url='{url}',headers()={len(headers)},timeout={timeout} - CALLED!")
     if not isinstance(url, str):
@@ -99,9 +62,9 @@ def fetch_url(url: str, headers: dict, timeout: tuple) -> requests.models.Respon
     # Invoke other function, avoid trailing ?
     # DEBUG: print(f"DEBUG: components[{type(components)}]={components}")
     if components.query != "":
-        response = network.fetch_response(components.hostname, f"{components.path}?{components.query}", headers, timeout)
+        response = network.fetch_response(components.netloc, f"{components.path}?{components.query}", headers, timeout)
     else:
-        response = network.fetch_response(components.hostname, f"{components.path}", headers, timeout)
+        response = network.fetch_response(components.netloc, f"{components.path}", headers, timeout)
 
     # DEBUG: print(f"DEBUG: response[]='{type(response)}' - EXXIT!")
     return response
@@ -127,7 +90,7 @@ def process_domain(domain: str, blocker: str, command: str) -> bool:
 
         # DEBUG: print(f"DEBUG: row[{type(row)}]='{row}'")
         if row is None:
-            print(f"WARNING: Cannot de-obfucate domain='{domain}' - skipped!")
+            print(f"WARNING: Cannot de-obfucate domain='{domain}' - SKIPPED!")
             return False
 
         # DEBUG: print(f"DEBUG: domain='{domain}' de-obscured to '{row[0]}'")
@@ -138,32 +101,33 @@ def process_domain(domain: str, blocker: str, command: str) -> bool:
 
         # DEBUG: print(f"DEBUG: row[{type(row)}]='{row}'")
         if row is None:
-            print(f"WARNING: Cannot de-obfucate domain='{domain}' - skipped!")
+            print(f"WARNING: Cannot de-obfucate domain='{domain}' - SKIPPED!")
             return False
 
         # DEBUG: print(f"DEBUG: domain='{domain}' de-obscured to '{row[0]}'")
         domain = row[0]
 
     if not validators.domain(domain):
-        print(f"WARNING: domain='{domain}' is not a valid domain - skipped!")
+        print(f"WARNING: domain='{domain}' is not a valid domain - SKIPPED!")
         return False
-    elif domain.split(".")[-1] == "arpa":
+    elif domain.endswith(".arpa"):
         print(f"WARNING: domain='{domain}' is a reversed .arpa domain and should not be used generally.")
         return False
     elif blacklist.is_blacklisted(domain):
-        # DEBUG: print(f"DEBUG: domain='{domain}' is blacklisted - skipped!")
+        # DEBUG: print(f"DEBUG: domain='{domain}' is blacklisted - SKIPPED!")
         return False
     elif instances.is_recent(domain):
-        # DEBUG: print(f"DEBUG: domain='{domain}' has been recently checked - skipped!")
+        # DEBUG: print(f"DEBUG: domain='{domain}' has been recently checked - SKIPPED!")
         return False
 
+    processed = False
     try:
         print(f"INFO: Fetching instances for instane='{domain}',blocker='{blocker}',command='{command}' ...")
         federation.fetch_instances(domain, blocker, None, command)
+        processed = True
     except network.exceptions as exception:
         print(f"WARNING: Exception '{type(exception)}' during fetching instances (fetch_oliphant) from domain='{domain}'")
-        instances.update_last_error(domain, exception)
-        return False
+        instances.set_last_error(domain, exception)
 
-    # DEBUG: print(f"DEBUG: Success! - EXIT!")
-    return True
+    # DEBUG: print(f"DEBUG: processed='{processed}' - EXIT!")
+    return processed