From 7c8c2ed0849b435a0ef41ad7574ed6b5ebe4c811 Mon Sep 17 00:00:00 2001
From: =?utf8?q?Roland=20H=C3=A4der?= <roland@mxchange.org>
Date: Sat, 20 May 2023 08:19:22 +0200
Subject: [PATCH] WIP: - blocked spam domain - also discover more instances by
 quering already existing instances

---
 fba.py             |  5 ++--
 fetch_instances.py | 68 ++++++++++++++++++++++++++++++----------------
 2 files changed, 48 insertions(+), 25 deletions(-)

diff --git a/fba.py b/fba.py
index 9eefe74..aa0db40 100644
--- a/fba.py
+++ b/fba.py
@@ -15,7 +15,8 @@ blacklist = [
     "gab.best",
     "4chan.icu",
     "social.shrimpcam.pw",
-    "mastotroll.netz.org"
+    "mastotroll.netz.org",
+    "ngrok.io",
 ]
 
 headers = {
@@ -37,7 +38,7 @@ def get_peers(domain: str) -> str:
         res = reqto.get(f"https://{domain}/api/v1/instance/peers", headers=headers, timeout=5)
         peers = res.json()
     except:
-        print("WARNING: Cannot fetch peers:", domain, res.status_code)
+        print("WARNING: Cannot fetch peers:", domain)
 
     # NOISY-DEBUG: print("DEBUG: Returning peers[]:", type(peers))
     return peers
diff --git a/fetch_instances.py b/fetch_instances.py
index 5527f9c..fd13b25 100644
--- a/fetch_instances.py
+++ b/fetch_instances.py
@@ -3,41 +3,63 @@ import sys
 import json
 import fba
 
-domain = sys.argv[1]
+def fetch_instances(domain: str):
+    print("DEBUG: Fetching instances for domain:", domain)
+    peerlist = fba.get_peers(domain)
 
-peerlist = fba.get_peers(domain)
+    if (peerlist is None):
+        print("FATAL: CANNOT FETCH PEERS:", domain)
+        return
 
-if (peerlist is None):
-    print("FATAL: CANNOT FETCH PEERS:", domain)
-    sys.exit(255)
+    for instance in peerlist:
+        instance = instance.lower()
 
+        blacklisted = False
+        for domain in fba.blacklist:
+            if domain in instance:
+                blacklisted = True
+
+        if blacklisted:
+            # NOISY-DEBUG: print("DEBUG: domain is blacklisted:", domain)
+            continue
+
+        # NOISY-DEBUG: print("DEBUG: Handling instance:", instance)
+        try:
+            fba.c.execute(
+                "SELECT domain FROM instances WHERE domain = ? LIMIT 1", (instance,)
+            )
+
+            if fba.c.fetchone() == None:
+                # NOISY-DEBUG: print("DEBUG: Adding new instance:", instance)
+                fba.add_instance(instance)
+
+            fba.conn.commit()
+
+        except Exception as e:
+            print("ERROR:", e, instance)
+            continue
+
+instance = sys.argv[1]
+
+# Initial fetch
+fetch_instances(instance)
+
+# Loop through some instances
 fba.c.execute(
-    "SELECT domain FROM instances WHERE 1"
+    "SELECT domain FROM instances WHERE software IS NOT NULL ORDER BY rowid DESC"
 )
 
-for instance in peerlist:
-    instance = instance.lower()
-
+for instance in fba.c.fetchall():
     blacklisted = False
     for domain in fba.blacklist:
-        if domain in instance:
+        if domain in instance[0]:
             blacklisted = True
 
     if blacklisted:
-        print("WARNING: domain is blacklisted:", domain)
+        # NOISY-DEBUG: print("DEBUG: domain is blacklisted:", instance)
         continue
 
-    print("INFO: Handling instance:", instance)
-    try:
-        fba.c.execute(
-            "SELECT domain FROM instances WHERE domain = ? LIMIT 1", (instance,)
-        )
-
-        if fba.c.fetchone() == None:
-            fba.add_instance(instance)
-
-        fba.conn.commit()
-    except Exception as e:
-        print("ERROR:", e, instance)
+    print("DEBUG: Fetching instances for instance:", instance[0])
+    fetch_instances(instance[0])
 
 fba.conn.close()
-- 
2.39.5