X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;f=fba%2Finstances.py;h=988401474b0d73dd81c08fb7842a343a89de6bfb;hb=b4b7a362cf34916fccecded268a717f6263c2d84;hp=1271048913b147b44c7903f58bc7a1ba6f8953bb;hpb=84b4e8ca4a486f8c071f2364d7d2f39eebce1fee;p=fba.git

diff --git a/fba/instances.py b/fba/instances.py
index 1271048..9884014 100644
--- a/fba/instances.py
+++ b/fba/instances.py
@@ -14,13 +14,25 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-from fba import fba
+import json
 import sys
+import time
+
+import requests
+import validators
+
+from fba import blacklist
+from fba import config
+from fba import fba
+from fba import federation
+from fba import network
+
+from fba.helpers import cache
 
 # Found info from node, such as nodeinfo URL, detection mode that needs to be
 # written to database. Both arrays must be filled at the same time or else
-# update_instance_data() will fail
-instance_data = {
+# update_data() will fail
+_pending = {
     # Detection mode: 'AUTO_DISCOVERY', 'STATIC_CHECKS' or 'GENERATOR'
     # NULL means all detection methods have failed (maybe still reachable instance)
     "detection_mode"     : {},
@@ -42,69 +54,70 @@ instance_data = {
     "last_error_details" : {},
 }
 
-def set_instance_data(key: str, domain: str, value: any):
-    # NOISY-DEBUG: print(f"DEBUG: key='{key}',domain='{domain}',value[]='{type(value)}' - CALLED!")
-    if type(key) != str:
+def set_data(key: str, domain: str, value: any):
+    # DEBUG: print(f"DEBUG: key='{key}',domain='{domain}',value[]='{type(value)}' - CALLED!")
+    if not isinstance(key, str):
         raise ValueError("Parameter key[]='{type(key)}' is not 'str'")
     elif key == "":
-        raise ValueError(f"Parameter 'key' cannot be empty")
-    elif type(domain) != str:
-        raise ValueError("Parameter domain[]='{type(domain)}' is not 'str'")
+        raise ValueError("Parameter 'key' is empty")
+    elif not isinstance(domain, str):
+        raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
-        raise ValueError(f"Parameter 'domain' cannot be empty")
-    elif not key in instance_data:
-        raise ValueError(f"key='{key}' not found in instance_data")
+        raise ValueError("Parameter 'domain' is empty")
+    elif not key in _pending:
+        raise ValueError(f"key='{key}' not found in _pending")
     elif not fba.is_primitive(value):
         raise ValueError(f"value[]='{type(value)}' is not a primitive type")
 
     # Set it
-    instance_data[key][domain] = value
+    _pending[key][domain] = value
 
     # DEBUG: print("DEBUG: EXIT!")
 
-def has_pending_instance_data(domain: str) -> bool:
+def has_pending(domain: str) -> bool:
     # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
-    if type(domain) != str:
-        raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
+    if not isinstance(domain, str):
+        raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
-        raise ValueError(f"Parameter 'domain' cannot be empty")
+        raise ValueError("Parameter 'domain' is empty")
 
-    has_pending = False
-    for key in instance_data:
-        # DEBUG: print(f"DEBUG: key='{key}',domain='{domain}',instance_data[key]()='{len(instance_data[key])}'")
-        if domain in instance_data[key]:
-            has_pending = True
+    has = False
+    for key in _pending:
+        # DEBUG: print(f"DEBUG: key='{key}',domain='{domain}',_pending[key]()='{len(_pending[key])}'")
+        if domain in _pending[key]:
+            has = True
             break
 
-    # DEBUG: print(f"DEBUG: has_pending='{has_pending}' - EXIT!")
-    return has_pending
+    # DEBUG: print(f"DEBUG: has='{has}' - EXIT!")
+    return has
 
-def update_instance_data(domain: str):
-    # DEBUG: print(f"DEBUG: domain={domain} - CALLED!")
-    if type(domain) != str:
-        raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
+def update_data(domain: str):
+    # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
+    if not isinstance(domain, str):
+        raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
-        raise ValueError(f"Parameter 'domain' cannot be empty")
-    elif not has_pending_instance_data(domain):
+        raise ValueError("Parameter 'domain' is empty")
+    elif not has_pending(domain):
         raise Exception(f"Domain '{domain}' has no pending instance data, but function invoked")
 
-    # DEBUG: print(f"DEBUG: Updating nodeinfo for domain='{domain}' ...")
-    sql_string = ''
+    # DEBUG: print(f"DEBUG: Updating instance data for domain='{domain}' ...")
+    sql_string = ""
     fields = list()
-    for key in instance_data:
+    for key in _pending:
         # DEBUG: print("DEBUG: key:", key)
-        if domain in instance_data[key]:
-           # DEBUG: print(f"DEBUG: Adding '{instance_data[key][domain]}' for key='{key}' ...")
-           fields.append(instance_data[key][domain])
-           sql_string += f" {key} = ?,"
+        if domain in _pending[key]:
+            # DEBUG: print(f"DEBUG: Adding '{_pending[key][domain]}' for key='{key}' ...")
+            fields.append(_pending[key][domain])
+            sql_string += f" {key} = ?,"
 
+    fields.append(time.time())
     fields.append(domain)
 
-    if sql_string == '':
+    if sql_string == "":
         raise ValueError(f"No fields have been set, but method invoked, domain='{domain}'")
 
     # DEBUG: print(f"DEBUG: sql_string='{sql_string}',fields()={len(fields)}")
-    sql_string = "UPDATE instances SET" + sql_string + " last_updated = TIME() WHERE domain = ? LIMIT 1"
+    sql_string = "UPDATE instances SET" + sql_string + " last_updated = ? WHERE domain = ? LIMIT 1"
     # DEBUG: print("DEBUG: sql_string:", sql_string)
 
     try:
@@ -113,21 +126,244 @@ def update_instance_data(domain: str):
 
         # DEBUG: print(f"DEBUG: Success! (rowcount={fba.cursor.rowcount })")
         if fba.cursor.rowcount == 0:
-            print(f"WARNING: Did not update any rows: domain='{domain}',fields()={len(fields)} - EXIT!")
+            # DEBUG: print(f"DEBUG: Did not update any rows: domain='{domain}',fields()={len(fields)} - EXIT!")
             return
 
+        # DEBUG: print("DEBUG: Committing changes ...")
         fba.connection.commit()
 
-        # DEBUG: print("DEBUG: Deleting instance_data for domain:", domain)
-        for key in instance_data:
-            try:
-                # DEBUG: print("DEBUG: Deleting key:", key)
-                del instance_data[key][domain]
-            except:
-                pass
+        # DEBUG: print(f"DEBUG: Deleting _pending for domain='{domain}'")
+        for key in _pending:
+            # DEBUG: print(f"DEBUG: domain='{domain}',key='{key}'")
+            if domain in _pending[key]:
+                del _pending[key][domain]
 
-    except BaseException as e:
-        print(f"ERROR: failed SQL query: domain='{domain}',sql_string='{sql_string}',exception[{type(e)}]:'{str(e)}'")
+    except BaseException as exception:
+        print(f"ERROR: failed SQL query: domain='{domain}',sql_string='{sql_string}',exception[{type(exception)}]:'{str(exception)}'")
         sys.exit(255)
 
     # DEBUG: print("DEBUG: EXIT!")
+
+def update_last_instance_fetch(domain: str):
+    # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
+    if not isinstance(domain, str):
+        raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
+    elif domain == "":
+        raise ValueError("Parameter 'domain' is empty")
+
+    # DEBUG: print("DEBUG: Updating last_instance_fetch for domain:", domain)
+    set_data("last_instance_fetch", domain, time.time())
+
+    # Running pending updated
+    # DEBUG: print(f"DEBUG: Invoking update_data({domain}) ...")
+    update_data(domain)
+
+    # DEBUG: print("DEBUG: EXIT!")
+
+def update_last_blocked(domain: str):
+    # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
+    if not isinstance(domain, str):
+        raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
+    elif domain == "":
+        raise ValueError("Parameter 'domain' is empty")
+
+    # DEBUG: print("DEBUG: Updating last_blocked for domain", domain)
+    set_data("last_blocked", domain, time.time())
+
+    # Running pending updated
+    # DEBUG: print(f"DEBUG: Invoking update_data({domain}) ...")
+    update_data(domain)
+
+    # DEBUG: print("DEBUG: EXIT!")
+
+def add(domain: str, origin: str, command: str, path: str = None):
+    # DEBUG: print(f"DEBUG: domain='{domain}',origin='{origin}',command='{command}',path='{path}' - CALLED!")
+    if not isinstance(domain, str):
+        raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
+    elif domain == "":
+        raise ValueError("Parameter 'domain' is empty")
+    elif not isinstance(origin, str) and origin is not None:
+        raise ValueError(f"origin[]='{type(origin)}' is not 'str'")
+    elif origin == "":
+        raise ValueError("Parameter 'origin' is empty")
+    elif not isinstance(command, str):
+        raise ValueError(f"command[]='{type(command)}' is not 'str'")
+    elif command == "":
+        raise ValueError("Parameter 'command' is empty")
+    elif not validators.domain(domain.split("/")[0]) or domain.split(".")[-1] == "arpa":
+        raise ValueError(f"Bad domain name='{domain}'")
+    elif origin is not None and not validators.domain(origin.split("/")[0]):
+        raise ValueError(f"Bad origin name='{origin}'")
+    elif blacklist.is_blacklisted(domain):
+        raise Exception(f"domain='{domain}' is blacklisted, but method invoked")
+    elif domain.find("/profile/") > 0 or domain.find("/users/") > 0:
+        raise Exception(f"domain='{domain}' is a single user")
+
+    software = None
+    try:
+        # DEBUG: print("DEBUG: domain,origin,command,path:", domain, origin, command, path)
+        software = federation.determine_software(domain, path)
+    except network.exceptions as exception:
+        print(f"WARNING Exception '{type(exception)}' during determining software type")
+
+    # DEBUG: print("DEBUG: Determined software:", software)
+    if software == "lemmy" and domain.find("/c/") > 0:
+        domain = domain.split("/c/")[0]
+        if is_registered(domain):
+            print(f"WARNING: domain='{domain}' already registered after cutting off user part. - EXIT!")
+            return
+
+    print(f"INFO: Adding instance domain='{domain}' (origin='{origin}',software='{software}')")
+    fba.cursor.execute(
+        "INSERT INTO instances (domain, origin, command, hash, software, first_seen) VALUES (?, ?, ?, ?, ?, ?)",
+        (
+           domain,
+           origin,
+           command,
+           fba.get_hash(domain),
+           software,
+           time.time()
+        ),
+    )
+
+    cache.set_sub_key("is_registered", domain, True)
+
+    if has_pending(domain):
+        # DEBUG: print(f"DEBUG: domain='{domain}' has pending nodeinfo being updated ...")
+        set_data("last_status_code"  , domain, None)
+        set_data("last_error_details", domain, None)
+        update_data(domain)
+
+    # DEBUG: print(f"DEBUG: Updating nodeinfo for domain='{domain}'")
+    update_last_nodeinfo(domain)
+
+    # DEBUG: print("DEBUG: EXIT!")
+
+def update_last_nodeinfo(domain: str):
+    # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
+    if not isinstance(domain, str):
+        raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
+    elif domain == "":
+        raise ValueError("Parameter 'domain' is empty")
+
+    # DEBUG: print("DEBUG: Updating last_nodeinfo for domain:", domain)
+    set_data("last_nodeinfo", domain, time.time())
+    set_data("last_updated" , domain, time.time())
+
+    # Running pending updated
+    # DEBUG: print(f"DEBUG: Invoking update_data({domain}) ...")
+    update_data(domain)
+
+    # DEBUG: print("DEBUG: EXIT!")
+
+def update_last_error(domain: str, error: dict):
+    # DEBUG: print("DEBUG: domain,error[]:", domain, type(error))
+    if not isinstance(domain, str):
+        raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
+    elif domain == "":
+        raise ValueError("Parameter 'domain' is empty")
+
+    # DEBUG: print("DEBUG: BEFORE error[]:", type(error))
+    if isinstance(error, (BaseException, json.decoder.JSONDecodeError)):
+        error = f"error[{type(error)}]='{str(error)}'"
+    # DEBUG: print("DEBUG: AFTER error[]:", type(error))
+
+    if isinstance(error, str):
+        # DEBUG: print(f"DEBUG: Setting last_error_details='{error}'")
+        set_data("last_status_code"  , domain, 999)
+        set_data("last_error_details", domain, error if error != "" else None)
+    elif isinstance(error, requests.models.Response):
+        # DEBUG: print(f"DEBUG: Setting last_error_details='{error.reason}'")
+        set_data("last_status_code"  , domain, error.status_code)
+        set_data("last_error_details", domain, error.reason if error.reason != "" else None)
+    elif not isinstance(error, dict):
+        raise KeyError(f"Cannot handle keys in error[{type(error)}]='{error}'")
+    elif "status_code" in error and "error_message" in error:
+        # DEBUG: print(f"DEBUG: Setting last_error_details='{error['error_message']}'")
+        set_data("last_status_code"  , domain, error["status_code"])
+        set_data("last_error_details", domain, error["error_message"] if error["error_message"] != "" else None)
+    elif "json" in error and "error" in error["json"]:
+        set_data("last_status_code"  , domain, error["status_code"])
+        set_data("last_error_details", domain, error["json"]["error"] if error["json"]["error"] != "" else None)
+
+    # Running pending updated
+    # DEBUG: print(f"DEBUG: Invoking update_data({domain}) ...")
+    update_data(domain)
+
+    fba.log_error(domain, error)
+
+    # DEBUG: print("DEBUG: EXIT!")
+
+def is_registered(domain: str) -> bool:
+    # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
+    if not isinstance(domain, str):
+        raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
+    elif domain == "":
+        raise ValueError("Parameter 'domain' is empty")
+
+    # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
+    if not cache.key_exists("is_registered"):
+        # DEBUG: print("DEBUG: Cache for 'is_registered' not initialized, fetching all rows ...")
+        fba.cursor.execute("SELECT domain FROM instances")
+
+        # Check Set all
+        cache.set_all("is_registered", fba.cursor.fetchall(), True)
+
+    # Is cache found?
+    registered = cache.sub_key_exists("is_registered", domain)
+
+    # DEBUG: print(f"DEBUG: registered='{registered}' - EXIT!")
+    return registered
+
+def is_recent(domain: str) -> bool:
+    # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
+    if not isinstance(domain, str):
+        raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
+    elif domain == "":
+        raise ValueError("Parameter 'domain' is empty")
+    elif not is_registered(domain):
+        # DEBUG: print(f"DEBUG: domain='{domain}' is not registered, returning False - EXIT!")
+        return False
+
+    # Query database
+    fba.cursor.execute("SELECT last_instance_fetch FROM instances WHERE domain = ? LIMIT 1", [domain])
+
+    # Fetch row
+    fetched = fba.cursor.fetchone()[0]
+
+    # DEBUG: print(f"DEBUG: fetched[{type(fetched)}]='{fetched}'")
+    recently = isinstance(fetched, float) and time.time() - fetched <= config.get("recheck_instance")
+
+    # DEBUG: print(f"DEBUG: recently='{recently}' - EXIT!")
+    return recently
+
+def deobscure(char: str, domain: str, blocked_hash: str = None) -> tuple:
+    #print(f"DEBUG: char='{char}',domain='{domain}',blocked_hash='{blocked_hash}' - CALLED!")
+    if not isinstance(char, str):
+        raise ValueError(f"Parameter char[]='{type(char)}' is not 'str'")
+    elif char == "":
+        raise ValueError("Parameter 'char' is empty")
+    elif not isinstance(domain, str):
+        raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
+    elif domain == "":
+        raise ValueError("Parameter 'domain' is empty")
+    elif not isinstance(blocked_hash, str) and blocked_hash is not None:
+        raise ValueError(f"Parameter blocked_hash[]='{type(blocked_hash)}' is not 'str'")
+
+    if isinstance(blocked_hash, str):
+        fba.cursor.execute(
+            "SELECT domain, origin, nodeinfo_url FROM instances WHERE hash = ? LIMIT 1", [blocked_hash]
+        )
+
+        if fba.cursor.fetchone() is None:
+            #print(f"DEBUG: blocked_hash='{blocked_hash}' not found, trying domain='{domain}' ...")
+            return deobscure(char, domain)
+    else:
+        fba.cursor.execute(
+            "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [domain.replace(char, "_")]
+        )
+
+    row = fba.cursor.fetchone()
+
+    #print(f"DEBUG: row[]='{type(row)}' - EXIT!")
+    return row