Continued:

[fba.git] / fetch_instances.py

diff --git a/fetch_instances.py b/fetch_instances.py
old mode 100644 (file)
new mode 100755 (executable)
index fbd113a..63a8fa7
--- a/fetch_instances.py
+++ b/fetch_instances.py
@@ -1,3 +1,22 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+# Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
+# Copyright (C) 2023 Free Software Foundation
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
 import sqlite3
 import sys
 import json
@@ -5,11 +24,11 @@ import time
 import validators
 import fba
 
-def fetch_instances(domain: str, origin: str, software: str):
-    # NOISY-DEBUG: print("DEBUG: domain,origin,software:", domain, origin, software)
+def fetch_instances(domain: str, origin: str, software: str, path: str = None):
+    # NOISY-DEBUG: print("DEBUG: domain,origin,software,path:", domain, origin, software, path)
     if not fba.is_instance_registered(domain):
         # NOISY-DEBUG: print("DEBUG: Adding new domain:", domain, origin)
-        fba.add_instance(domain, origin, sys.argv[0])
+        fba.add_instance(domain, origin, sys.argv[0], path)
 
     # NOISY-DEBUG: print("DEBUG: Fetching instances for domain:", domain, software)
     peerlist = fba.get_peers(domain, software)
@@ -17,7 +36,7 @@ def fetch_instances(domain: str, origin: str, software: str):
     if (peerlist is None):
         print("ERROR: Cannot fetch peers:", domain)
         return
-    elif domain in fba.nodeinfos["get_peers_url"]:
+    elif fba.has_pending_nodeinfos(domain):
         # NOISY-DEBUG: print(f"DEBUG: domain='{domain}' has pending nodeinfo data, flushing ...")
         fba.update_nodeinfos(domain)
 
@@ -43,7 +62,7 @@ def fetch_instances(domain: str, origin: str, software: str):
                 # NOISY-DEBUG: print("DEBUG: Adding new instance:", instance, domain)
                 fba.add_instance(instance, domain, sys.argv[0])
         except BaseException as e:
-            print(f"ERROR: instance='{instance}',exception:'{e}'")
+            print(f"ERROR: instance='{instance}',exception:'{str(e)}'")
             continue
 
 instance = sys.argv[1]
@@ -53,7 +72,7 @@ fetch_instances(instance, None, None)
 
 # Loop through some instances
 fba.cursor.execute(
-    "SELECT domain,origin,software FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial', 'bookwyrm', 'takahe', 'lemmy') AND (last_nodeinfo IS NULL OR last_nodeinfo < ?) ORDER BY rowid DESC", [time.time() - fba.config["recheck_instance"]]
+    "SELECT domain, origin, software, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial', 'bookwyrm', 'takahe', 'lemmy') AND (last_instance_fetch IS NULL OR last_instance_fetch < ?) ORDER BY rowid DESC", [time.time() - fba.config["recheck_instance"]]
 )
 
 rows = fba.cursor.fetchall()
@@ -64,7 +83,7 @@ for row in rows:
         print("WARNING: domain is blacklisted:", row[0])
         continue
 
-    print(f"INFO: Fetching instances for instance '{row[0]}'('{row[2]}') of origin '{row[1]}'")
-    fetch_instances(row[0], row[1], row[2])
+    print(f"INFO: Fetching instances for instance '{row[0]}'('{row[2]}') of origin '{row[1]}',nodeinfo_url='{row[3]}'")
+    fetch_instances(row[0], row[1], row[2], row[3])
 
 fba.connection.close()