Continued:

[fba.git] / fba / commands.py

diff --git a/fba/commands.py b/fba/commands.py
index fc5de72c60f58a785b0cbce34fd261786db2605a..7e363115077cfa0a142d8ee898a0e63ba3a4d6e0 100644 (file)
--- a/fba/commands.py
+++ b/fba/commands.py
@@ -1279,3 +1279,47 @@ def recheck_obfuscation(args: argparse.Namespace) -> int:
 
     logger.debug("Success! - EXIT!")
     return 0
+
+def fetch_fedilist(args: argparse.Namespace) -> int:
+    logger.debug("args[]='%s' - CALLED!", type(args))
+
+    url = "http://demo.fedilist.com/instance/csv?onion=not"
+    if args.software is not None and args.software != "":
+        logger.debug("args.software='%s'", args.software)
+        url = f"http://demo.fedilist.com/instance/csv?software={args.software}&onion=not"
+
+    locking.acquire()
+
+    logger.info("Fetching url='%s' from fedilist.com ...", url)
+    response = reqto.get(
+        url,
+        headers=network.web_headers,
+        timeout=(config.get("connection_timeout"), config.get("read_timeout")),
+        allow_redirects=False
+    )
+
+    logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text))
+    reader = csv.DictReader(response.content.decode('utf-8').splitlines(), dialect="unix")
+
+    logger.debug("reader[]='%s'", type(reader))
+    blockdict = list()
+    for row in reader:
+        logger.debug("row[]='%s'", type(row))
+        domain = tidyup.domain(row["hostname"])
+        logger.debug("domain='%s' - AFTER!", domain)
+
+        if domain == "":
+            logger.debug("domain is empty after tidyup: row[hostname]='%s' - SKIPPED!", row["hostname"])
+            continue
+        elif not utils.is_domain_wanted(domain):
+            logger.warning("domain='%s' is not wanted - SKIPPED!", domain)
+            continue
+        elif instances.is_recent(domain):
+            logger.debug("domain='%s' has been recently crawled - SKIPPED!", domain)
+            continue
+
+        logger.info("Fetching instances from domain='%s' ...", domain)
+        federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
+
+    logger.debug("Success! - EXIT!")
+    return 0