Continued:

[fba.git] / fetch_instances.py

diff --git a/fetch_instances.py b/fetch_instances.py
index d20bd317347b41d2d962954f85c3d29ef1760b0d..0975ff4001ddc8f40586bf2c576586bc0fc55539 100644 (file)
--- a/fetch_instances.py
+++ b/fetch_instances.py
@@ -1,90 +1,69 @@
-from requests import get
-from hashlib import sha256
 import sqlite3
 import sys
 import json
+import time
+import fba
 
-domain = sys.argv[1]
-
-blacklist = [
-    "activitypub-troll.cf",
-    "gab.best",
-    "4chan.icu"
-]
-
-headers = {
-    "user-agent": "Mozilla/5.0 (Windows NT 10.0; rv:102.0) Gecko/20100101 Firefox/102.0"
-}
-
-
-def get_hash(domain: str) -> str:
-    return sha256(domain.encode("utf-8")).hexdigest()
-
-
-def get_peers(domain: str) -> str:
-    try:
-        res = get(f"https://{domain}/api/v1/instance/peers", headers=headers, timeout=5)
-        return res.json()
-    except:
-        return None
-
-peerlist = get_peers(domain)
-
-def get_type(instdomain: str) -> str:
-    try:
-        res = get(f"https://{instdomain}/nodeinfo/2.1.json", headers=headers, timeout=5)
-        if res.status_code == 404:
-            res = get(f"https://{instdomain}/nodeinfo/2.0", headers=headers, timeout=5)
-        if res.status_code == 404:
-            res = get(f"https://{instdomain}/nodeinfo/2.0.json", headers=headers, timeout=5)
-        if res.ok and "text/html" in res.headers["content-type"]:
-            res = get(f"https://{instdomain}/nodeinfo/2.1", headers=headers, timeout=5)
-        if res.ok:
-            if res.json()["software"]["name"] in ["akkoma", "rebased"]:
-                return "pleroma"
-            elif res.json()["software"]["name"] in ["hometown", "ecko"]:
-                return "mastodon"
-            elif res.json()["software"]["name"] in ["calckey", "groundpolis", "foundkey", "cherrypick"]:
-                return "misskey"
-            else:
-                return res.json()["software"]["name"]
-        elif res.status_code == 404:
-            res = get(f"https://{instdomain}/api/v1/instance", headers=headers, timeout=5)
-        if res.ok:
-            return "mastodon"
-    except:
-        return None
-
-
-conn = sqlite3.connect("blocks.db")
-c = conn.cursor()
-
-c.execute(
-    "select domain from instances where 1"
-)
+def fetch_instances(domain: str, origin: str):
+    # NOISY-DEBUG: print("DEBUG: Fetching instances for domain:", domain, origin)
+    peerlist = fba.get_peers(domain)
 
-for instance in peerlist:
-    instance = instance.lower()
+    if (peerlist is None):
+        print("ERROR: Cannot fetch peers:", domain)
+        return
 
-    blacklisted = False
-    for domain in blacklist:
-        if domain in instance:
-            blacklisted = True
+    fba.cursor.execute(
+        "SELECT domain FROM instances WHERE domain = ? LIMIT 1", [domain]
+    )
 
-    if blacklisted:
-        continue
+    if fba.cursor.fetchone() == None:
+        # NOISY-DEBUG: print("DEBUG: Adding new domain:", domain, origin)
+        fba.add_instance(domain, origin, sys.argv[0])
+
+    print(f"INFO: Checking {len(peerlist)} instances from {domain} ...")
+    for instance in peerlist:
+        instance = instance.lower()
+
+        if instance.find("@") > 0:
+            print("WARNING: Bad instance name,domain:", instance, domain)
+            continue
 
-    print(instance)
-    try:
-        c.execute(
-            "select domain from instances where domain = ?", (instance,)
-        )
-        if c.fetchone() == None:
-            c.execute(
-                "insert into instances select ?, ?, ?",
-                (instance, get_hash(instance), get_type(instance)),
+        if fba.is_blacklisted(instance):
+            # NOISY-DEBUG: print("DEBUG: instance is blacklisted:", instance)
+            continue
+
+        # NOISY-DEBUG: print("DEBUG: Handling instance:", instance)
+        try:
+            fba.cursor.execute(
+                "SELECT domain FROM instances WHERE domain = ? LIMIT 1", [instance]
             )
-        conn.commit()
-    except Exception as e:
-        print("error:", e, instance)
-conn.close()
+
+            if fba.cursor.fetchone() == None:
+                # NOISY-DEBUG: print("DEBUG: Adding new instance:", instance, domain)
+                fba.add_instance(instance, domain, sys.argv[0])
+
+            fba.connection.commit()
+
+        except Exception as e:
+            print("ERROR:", e, instance)
+            continue
+
+instance = sys.argv[1]
+
+# Initial fetch
+fetch_instances(instance, None)
+
+# Loop through some instances
+fba.cursor.execute(
+    "SELECT domain FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial') AND (last_nodeinfo IS NULL OR last_nodeinfo < ?) ORDER BY rowid DESC", [time.time() - fba.config["recheck_instance"]]
+)
+
+for instance in fba.cursor.fetchall():
+    if fba.is_blacklisted(instance[0]):
+        # NOISY-DEBUG: print("DEBUG: domain is blacklisted:", instance)
+        continue
+
+    print("INFO: Fetching instances for instance:", instance[0])
+    fetch_instances(instance[0], None)
+
+fba.connection.close()