Continued:

[fba.git] / fetch_instances.py

diff --git a/fetch_instances.py b/fetch_instances.py
old mode 100644 (file)
new mode 100755 (executable)
index 5a9f534..08b49e8
--- a/fetch_instances.py
+++ b/fetch_instances.py
@@ -1,69 +1,50 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+# Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
+# Copyright (C) 2023 Free Software Foundation
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
 import sqlite3
 import sys
 import json
 import time
-import fba
-
-def fetch_instances(domain: str, origin: str):
-    # NOISY-DEBUG: print("DEBUG: Fetching instances for domain:", domain, origin)
-    peerlist = fba.get_peers(domain)
-
-    if (peerlist is None):
-        print("ERROR: Cannot fetch peers:", domain)
-        return
-
-    fba.c.execute(
-        "SELECT domain FROM instances WHERE domain = ? LIMIT 1", [domain]
-    )
-
-    if fba.c.fetchone() == None:
-        # NOISY-DEBUG: print("DEBUG: Adding new domain:", domain, origin)
-        fba.add_instance(domain, origin, sys.argv[0])
-
-    print(f"INFO: Checking {len(peerlist)} instances from {domain} ...")
-    for instance in peerlist:
-        instance = instance.lower()
-
-        if instance.find("@") > 0:
-            print("WARNING: Bad instance name,domain:", instance, domain)
-            continue
-
-        if fba.is_blacklisted(instance):
-            # NOISY-DEBUG: print("DEBUG: instance is blacklisted:", instance)
-            continue
-
-        # NOISY-DEBUG: print("DEBUG: Handling instance:", instance)
-        try:
-            fba.c.execute(
-                "SELECT domain FROM instances WHERE domain = ? LIMIT 1", [instance]
-            )
-
-            if fba.c.fetchone() == None:
-                # NOISY-DEBUG: print("DEBUG: Adding new instance:", instance, domain)
-                fba.add_instance(instance, domain, sys.argv[0])
-
-            fba.conn.commit()
+import validators
+from fba import *
 
-        except Exception as e:
-            print("ERROR:", e, instance)
-            continue
+boot.acquire_lock()
 
 instance = sys.argv[1]
 
 # Initial fetch
-fetch_instances(instance, None)
+fba.fetch_instances(instance, None, None, sys.argv[0])
 
 # Loop through some instances
-fba.c.execute(
-    "SELECT domain FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial') AND (last_nodeinfo IS NULL OR last_nodeinfo < ?) ORDER BY rowid DESC", [time.time() - fba.config["recheck_instance"]]
+fba.cursor.execute(
+    "SELECT domain, origin, software, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial', 'bookwyrm', 'takahe', 'lemmy') AND (last_instance_fetch IS NULL OR last_instance_fetch < ?) ORDER BY rowid DESC", [time.time() - fba.config["recheck_instance"]]
 )
 
-for instance in fba.c.fetchall():
-    if fba.is_blacklisted(instance[0]):
-        # NOISY-DEBUG: print("DEBUG: domain is blacklisted:", instance)
+rows = fba.cursor.fetchall()
+print(f"INFO: Checking {len(rows)} entries ...")
+for row in rows:
+    # DEBUG: print("DEBUG: domain:", row[0])
+    if fba.is_blacklisted(row[0]):
+        print("WARNING: domain is blacklisted:", row[0])
         continue
 
-    print("INFO: Fetching instances for instance:", instance[0])
-    fetch_instances(instance[0], None)
+    print(f"INFO: Fetching instances for instance '{row[0]}' ('{row[2]}') of origin='{row[1]}',nodeinfo_url='{row[3]}'")
+    fba.fetch_instances(row[0], row[1], row[2], sys.argv[0], row[3])
 
-fba.conn.close()
+boot.shutdown()