WIP:

- blocked spam domain
- also discover more instances by quering already existing instances

diff --git a/fba.py b/fba.py
index 9eefe74dac19bdfc63701f07c115124188e98e7b..aa0db400685aa595974316739fad9282baed643a 100644 (file)
--- a/fba.py
+++ b/fba.py
@@ -15,7 +15,8 @@ blacklist = [
     "gab.best",
     "4chan.icu",
     "social.shrimpcam.pw",
-    "mastotroll.netz.org"
+    "mastotroll.netz.org",
+    "ngrok.io",
 ]
 
 headers = {
@@ -37,7 +38,7 @@ def get_peers(domain: str) -> str:
         res = reqto.get(f"https://{domain}/api/v1/instance/peers", headers=headers, timeout=5)
         peers = res.json()
     except:
-        print("WARNING: Cannot fetch peers:", domain, res.status_code)
+        print("WARNING: Cannot fetch peers:", domain)
 
     # NOISY-DEBUG: print("DEBUG: Returning peers[]:", type(peers))
     return peers

diff --git a/fetch_instances.py b/fetch_instances.py
index 5527f9c7e82934c1ba7f2a6a692ff76687e5ce1c..fd13b25a1daf352523801fad8c1e52ddafbac405 100644 (file)
--- a/fetch_instances.py
+++ b/fetch_instances.py
@@ -3,41 +3,63 @@ import sys
 import json
 import fba
 
-domain = sys.argv[1]
+def fetch_instances(domain: str):
+    print("DEBUG: Fetching instances for domain:", domain)
+    peerlist = fba.get_peers(domain)
 
-peerlist = fba.get_peers(domain)
+    if (peerlist is None):
+        print("FATAL: CANNOT FETCH PEERS:", domain)
+        return
 
-if (peerlist is None):
-    print("FATAL: CANNOT FETCH PEERS:", domain)
-    sys.exit(255)
+    for instance in peerlist:
+        instance = instance.lower()
 
+        blacklisted = False
+        for domain in fba.blacklist:
+            if domain in instance:
+                blacklisted = True
+
+        if blacklisted:
+            # NOISY-DEBUG: print("DEBUG: domain is blacklisted:", domain)
+            continue
+
+        # NOISY-DEBUG: print("DEBUG: Handling instance:", instance)
+        try:
+            fba.c.execute(
+                "SELECT domain FROM instances WHERE domain = ? LIMIT 1", (instance,)
+            )
+
+            if fba.c.fetchone() == None:
+                # NOISY-DEBUG: print("DEBUG: Adding new instance:", instance)
+                fba.add_instance(instance)
+
+            fba.conn.commit()
+
+        except Exception as e:
+            print("ERROR:", e, instance)
+            continue
+
+instance = sys.argv[1]
+
+# Initial fetch
+fetch_instances(instance)
+
+# Loop through some instances
 fba.c.execute(
-    "SELECT domain FROM instances WHERE 1"
+    "SELECT domain FROM instances WHERE software IS NOT NULL ORDER BY rowid DESC"
 )
 
-for instance in peerlist:
-    instance = instance.lower()
-
+for instance in fba.c.fetchall():
     blacklisted = False
     for domain in fba.blacklist:
-        if domain in instance:
+        if domain in instance[0]:
             blacklisted = True
 
     if blacklisted:
-        print("WARNING: domain is blacklisted:", domain)
+        # NOISY-DEBUG: print("DEBUG: domain is blacklisted:", instance)
         continue
 
-    print("INFO: Handling instance:", instance)
-    try:
-        fba.c.execute(
-            "SELECT domain FROM instances WHERE domain = ? LIMIT 1", (instance,)
-        )
-
-        if fba.c.fetchone() == None:
-            fba.add_instance(instance)
-
-        fba.conn.commit()
-    except Exception as e:
-        print("ERROR:", e, instance)
+    print("DEBUG: Fetching instances for instance:", instance[0])
+    fetch_instances(instance[0])
 
 fba.conn.close()