Continued:

[fba.git] / fba / commands.py

diff --git a/fba/commands.py b/fba/commands.py
index 9dea3f88a193b8e90b05b96cf401443c06e13426..b021cabd4c5b41ae5d4c8e969961070ce164ddae 100644 (file)
--- a/fba/commands.py
+++ b/fba/commands.py
@@ -64,7 +64,7 @@ logger = logging.getLogger(__name__)
 def check_instance(args: argparse.Namespace) -> int:
     logger.debug("args.domain='%s' - CALLED!", args.domain)
     status = 0
-    if not validators.hostname(args.domain):
+    if not validators.domain(args.domain):
         logger.warning("args.domain='%s' is not valid", args.domain)
         status = 100
     elif blacklist.is_blacklisted(args.domain):
@@ -266,7 +266,7 @@ def fetch_blocks(args: argparse.Namespace) -> int:
     logger.debug("args[]='%s' - CALLED!", type(args))
     if args.domain is not None and args.domain != "":
         logger.debug("args.domain='%s' - checking ...", args.domain)
-        if not validators.hostname(args.domain):
+        if not validators.domain(args.domain):
             logger.warning("args.domain='%s' is not valid.", args.domain)
             return 100
         elif blacklist.is_blacklisted(args.domain):
@@ -321,7 +321,7 @@ def fetch_blocks(args: argparse.Namespace) -> int:
 
         blocking = list()
 
-        if not federation.is_excluded_blocklist(blocker):
+        if blocker != "chaos.social" and not blocklists.is_excluded(blocker):
             logger.debug("blocker='%s',software='%s'", blocker, software)
             if software == "pleroma":
                 logger.info("blocker='%s',software='%s'", blocker, software)
@@ -349,7 +349,7 @@ def fetch_blocks(args: argparse.Namespace) -> int:
             logger.debug("Invoking instances.set_total_blocks(%s, %d) ...", blocker, len(blocking))
             instances.set_total_blocks(blocker, blocking)
         else:
-            logger.debug("Skipping chaos.social, run ./fba.py fetch_cs instead!")
+            logger.debug("Skipping blocker='%s', run ./fba.py fetch_cs or fetch_oliphant instead!", blocker)
 
         logger.info("Checking %d entries from blocker='%s',software='%s' ...", len(blocking), blocker, software)
         blockdict = list()
@@ -914,7 +914,7 @@ def fetch_instances(args: argparse.Namespace) -> int:
     logger.debug("args[]='%s' - CALLED!", type(args))
 
     logger.debug("args.domain='%s' - checking ...", args.domain)
-    if not validators.hostname(args.domain):
+    if not validators.domain(args.domain):
         logger.warning("args.domain='%s' is not valid.", args.domain)
         return 100
     elif blacklist.is_blacklisted(args.domain):
@@ -924,10 +924,21 @@ def fetch_instances(args: argparse.Namespace) -> int:
     logger.debug("Invoking locking.acquire() ...")
     locking.acquire()
 
+    # Initialize values
+    domain = tidyup.domain(args.domain)
+    origin = software = None
+
+    # Fetch record
+    database.cursor.execute("SELECT origin, software FROM instances WHERE domain = ? LIMIT 1", [args.domain])
+    row = database.cursor.fetchone()
+    if row is not None:
+        origin = row["origin"]
+        software = row["software"]
+
     # Initial fetch
     try:
-        logger.info("Fetching instances from args.domain='%s' ...", args.domain)
-        federation.fetch_instances(args.domain, None, None, inspect.currentframe().f_code.co_name)
+        logger.info("Fetching instances from args.domain='%s',origin='%s',software='%s' ...", domain, origin, software)
+        federation.fetch_instances(domain, origin, software, inspect.currentframe().f_code.co_name)
     except network.exceptions as exception:
         logger.warning("Exception '%s' during fetching instances (fetch_instances) from args.domain='%s'", type(exception), args.domain)
         instances.set_last_error(args.domain, exception)
@@ -1065,7 +1076,7 @@ def fetch_oliphant(args: argparse.Namespace) -> int:
                 domain = utils.deobfuscate(domain, block["blocker"])
                 logger.debug("domain='%s' - AFTER!", domain)
 
-            if not validators.hostname(domain):
+            if not validators.domain(domain):
                 logger.debug("domain='%s' is not a valid domain - SKIPPED!")
                 continue
             elif blacklist.is_blacklisted(domain):
@@ -1371,7 +1382,7 @@ def fetch_joinfediverse(args: argparse.Namespace) -> int:
                     if not isinstance(text, str):
                         logger.debug("text[]='%s' is not of type 'str' - SKIPPED!", type(text))
                         continue
-                    elif validators.hostname(text.strip()):
+                    elif validators.domain(text.strip()):
                         logger.debug("text='%s' is a domain - SKIPPED!", text.strip())
                         continue
 
@@ -1506,7 +1517,7 @@ def recheck_obfuscation(args: argparse.Namespace) -> int:
 
     if isinstance(args.domain, str) and args.domain != "" and domain_helper.is_wanted(args.domain):
         database.cursor.execute("SELECT domain, software, nodeinfo_url FROM instances WHERE has_obfuscation = 1 AND domain = ?", [args.domain])
-    elif isinstance(args.software, str) and args.software != "" and validators.hostname(args.software) == args.software:
+    elif isinstance(args.software, str) and args.software != "" and validators.domain(args.software) == args.software:
         database.cursor.execute("SELECT domain, software, nodeinfo_url FROM instances WHERE has_obfuscation = 1 AND software = ?", [args.software])
     else:
         database.cursor.execute("SELECT domain, software, nodeinfo_url FROM instances WHERE has_obfuscation = 1")
@@ -1539,8 +1550,9 @@ def recheck_obfuscation(args: argparse.Namespace) -> int:
             logger.warning("Unknown software: domain='%s',software='%s'", row["domain"], row["software"])
 
         logger.debug("row[domain]='%s'", row["domain"])
+
         # chaos.social requires special care ...
-        if not blocklists.is_excluded(row["domain"]):
+        if row["domain"] != "chaos.social" and not blocklists.is_excluded(row["domain"]):
             logger.debug("Invoking instances.set_total_blocks(%s, %d) ...", row["domain"], len(blocking))
             instances.set_last_blocked(row["domain"])
             instances.set_total_blocks(row["domain"], blocking)
@@ -2011,3 +2023,29 @@ def convert_idna(args: argparse.Namespace) -> int:
 
     logger.debug("Success! - EXIT!")
     return 0
+
+def remove_invalid(args: argparse.Namespace) -> int:
+    logger.debug("args[]='%s' - CALLED!", type(args))
+
+    logger.debug("Invoking locking.acquire() ...")
+    locking.acquire()
+
+    database.cursor.execute("SELECT domain FROM instances ORDER BY domain ASC")
+    rows = database.cursor.fetchall()
+
+    logger.info("Checking %d domains ...", len(rows))
+    for row in rows:
+        logger.debug("row[domain]='%s'", row["domain"])
+        if not validators.domain(row["domain"].split("/")[0]):
+            logger.info("Invalid row[domain]='%s' found, removing ...", row["domain"])
+            database.cursor.execute("DELETE FROM blocks WHERE blocker = ? OR blocked = ?", [row["domain"], row["domain"]])
+            database.cursor.execute("DELETE FROM instances WHERE domain = ? LIMIT 1", [row["domain"]])
+
+    logger.debug("Invoking commit() ...")
+    database.connection.commit()
+
+    logger.info("Vaccum cleaning database ...")
+    database.cursor.execute("VACUUM")
+
+    logger.debug("Success! - EXIT!")
+    return 0