Continued:

- PeerTube has discoverable instances lists, so let us browse them

diff --git a/api.py b/api.py
index 131ea389be623e313510e02263de770cd825b8d0..d94d75ed2f43d38cfafae189547eab3fe15097cc 100644 (file)
--- a/api.py
+++ b/api.py
@@ -14,7 +14,7 @@ templates = Jinja2Templates(directory=".")
 
 @app.get(fba.config["base_url"] + "/api/info")
 def info():
-    fba.cursor.execute("SELECT (SELECT COUNT(domain) FROM instances), (SELECT COUNT(domain) FROM instances WHERE software IN ('pleroma', 'mastodon', 'misskey', 'gotosocial', 'friendica', 'bookwyrm', 'takahe')), (SELECT COUNT(blocker) FROM blocks), (SELECT COUNT(domain) FROM instances WHERE last_status_code IS NOT NULL)")
+    fba.cursor.execute("SELECT (SELECT COUNT(domain) FROM instances), (SELECT COUNT(domain) FROM instances WHERE software IN ('pleroma', 'mastodon', 'misskey', 'gotosocial', 'friendica', 'bookwyrm', 'takahe', 'peertube')), (SELECT COUNT(blocker) FROM blocks), (SELECT COUNT(domain) FROM instances WHERE last_status_code IS NOT NULL)")
     known, indexed, blocks, errorous = fba.cursor.fetchone()
 
     return {

diff --git a/fba.py b/fba.py
index a4d3b737bad4cd248e4cbc929b3823344426d5a7..e8885a9829f880f4fccfccc9179d34be49f92072 100644 (file)
--- a/fba.py
+++ b/fba.py
@@ -186,13 +186,14 @@ def update_last_nodeinfo(domain: str):
 
 def get_peers(domain: str, software: str) -> list:
     # NOISY-DEBUG: print("DEBUG: Getting peers for domain:", domain, software)
-    peers = None
+    peers = list()
 
     if software == "lemmy":
-        # NOISY-DEBUG: print(f"DEBUG: domain='{domain}' is Lemmy. fetching JSON ...")
+        # NOISY-DEBUG: print(f"DEBUG: domain='{domain}' is Lemmy, fetching JSON ...")
         try:
             res = reqto.get(f"https://{domain}/api/v3/site", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"]))
 
+            # NOISY-DEBUG: print(f"DEBUG: res.ok={res.ok},res.json[]={type(res.json())}")
             if res.ok and isinstance(res.json(), dict):
                 # NOISY-DEBUG: print("DEBUG: Success, res.json():", len(res.json()))
                 json = res.json()
@@ -208,9 +209,48 @@ def get_peers(domain: str, software: str) -> list:
 
         # NOISY-DEBUG: print("DEBUG: Returning peers[]:", type(peers))
         return peers
+    elif software == "peertube":
+        # NOISY-DEBUG: print(f"DEBUG: domain='{domain}' is a PeerTube, fetching JSON ...")
 
+        start = 0
+        while True:
+            try:
+                res = reqto.get(f"https://{domain}/api/v1/server/followers?start={start}&count=100", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"]))
+
+                # NOISY-DEBUG: print(f"DEBUG: res.ok={res.ok},res.json[]={type(res.json())}")
+                if res.ok and isinstance(res.json(), dict):
+                    # NOISY-DEBUG: uccess, res.json():", len(res.json()))
+                    json = res.json()
+                    if start == 0:
+                        # NOISY-DEBUG: print(f"DEBUG: Setting get_peers_url for '{domain}' ...")
+                        nodeinfos["get_peers_url"][domain] = f"https://{domain}/api/v1/server/followers?start={start}&count=100"
+
+                    if "data" in json:
+                        # NOISY-DEBUG: print(f"DEBUG: Found {len(json['data'])} record(s).")
+                        for record in json["data"]:
+                            # NOISY-DEBUG: print(f"DEBUG: record()={len(record)}")
+                            if "follower" in record and "host" in record["follower"]:
+                                # NOISY-DEBUG: print(f"DEBUG: Found host={record['follower']['host']}, adding ...")
+                                peers.append(record["follower"]["host"])
+                            else:
+                                print(f"WARNING: record from '{domain}' has no 'follower' or 'host' record: {record}")
+
+                        if len(json["data"]) < 100:
+                            # NOISY-DEBUG: print("DEBUG: Reached end of JSON response:", domain)
+                            break
+
+                    # Continue with next row
+                    start = start + 100
+
+            except BaseException as e:
+                print("WARNING: Exception during fetching JSON:", domain, e)
+
+        update_last_nodeinfo(domain)
+
+        # NOISY-DEBUG: print("DEBUG: Returning peers[]:", type(peers))
+        return peers
     try:
-        res = reqto.get(f"https://{domain}/api/v1/instance/peers", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"]))
+        res = reqto.get(f"https://{domain}{get_peers_url}", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"]))
 
         if not res.ok or res.status_code >= 400:
             res = reqto.get(f"https://{domain}/api/v3/site", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"]))

diff --git a/fetch_blocks.py b/fetch_blocks.py
index 4b4aba4dc1c19a09323e49dffadea5aa8b1ec6e4..a5aa9ea76e014e462c36dacd9875803e9d436049 100644 (file)
--- a/fetch_blocks.py
+++ b/fetch_blocks.py
@@ -360,7 +360,7 @@ for blocker, software in fba.cursor.fetchall():
         print("INFO: blocker:", blocker)
         try:
             # Blocks
-            federation = reqto.get(f"https://{blocker}/api/v1/instance/peers?filter=suspended", headers=fba.headers, timeout=(fba.config["connection_timeout"], config["read_timeout"])).json()
+            federation = reqto.get(f"https://{blocker}{get_peers_url}?filter=suspended", headers=fba.headers, timeout=(fba.config["connection_timeout"], config["read_timeout"])).json()
 
             if (federation == None):
                 print("WARNING: No valid response:", blocker);