+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+# Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
+# Copyright (C) 2023 Free Software Foundation
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
import sqlite3
import sys
import json
import time
+import validators
import fba
-def fetch_instances(domain: str):
- print("DEBUG: Fetching instances for domain:", domain)
- peerlist = fba.get_peers(domain)
+def fetch_instances(domain: str, origin: str, software: str, path: str = None):
+ # NOISY-DEBUG: print("DEBUG: domain,origin,software,path:", domain, origin, software, path)
+ if not fba.is_instance_registered(domain):
+ # NOISY-DEBUG: print("DEBUG: Adding new domain:", domain, origin)
+ fba.add_instance(domain, origin, sys.argv[0], path)
+
+ # NOISY-DEBUG: print("DEBUG: Fetching instances for domain:", domain, software)
+ peerlist = fba.get_peers(domain, software)
if (peerlist is None):
print("ERROR: Cannot fetch peers:", domain)
-
- fba.c.execute(
- "SELECT domain FROM instances WHERE domain = ? LIMIT 1", (domain,)
- )
-
- if fba.c.fetchone() == None:
- # NOISY-DEBUG: print("DEBUG: Adding new domain:", domain)
- fba.add_instance(domain)
-
- fba.conn.commit()
return
+ elif fba.has_pending_nodeinfos(domain):
+ # NOISY-DEBUG: print(f"DEBUG: domain='{domain}' has pending nodeinfo data, flushing ...")
+ fba.update_nodeinfos(domain)
+ print(f"INFO: Checking {len(peerlist)} instances from {domain} ...")
for instance in peerlist:
- instance = instance.lower()
- if instance.find("@") > 0:
- print("WARNING: Bad instance name,domain:", instance, domain)
- continue
-
- blacklisted = False
- for peer in fba.blacklist:
- if peer in instance:
- blacklisted = True
+ # NOISY-DEBUG: print("DEBUG: BEFORE instance:", instance)
+ instance = fba.tidyup(instance)
+ # NOISY-DEBUG: print("DEBUG: AFTER instance:", instance)
- if blacklisted:
- # NOISY-DEBUG: print("DEBUG: domain is blacklisted:", domain)
+ if instance == "":
+ print("WARNING: Empty instance after tidyup(), domain:", domain)
+ continue
+ elif not validators.domain(instance.split("/")[0]):
+ print(f"WARNING: Bad instance='{instance}' from domain='{domain}',origin='{origin}',software='{software}'")
+ continue
+ elif fba.is_blacklisted(instance):
+ # NOISY-DEBUG: print("DEBUG: instance is blacklisted:", instance)
continue
# NOISY-DEBUG: print("DEBUG: Handling instance:", instance)
try:
- fba.c.execute(
- "SELECT domain FROM instances WHERE domain = ? LIMIT 1", (instance,)
- )
-
- if fba.c.fetchone() == None:
- # NOISY-DEBUG: print("DEBUG: Adding new instance:", instance)
- fba.add_instance(instance)
-
- fba.conn.commit()
-
- except Exception as e:
- print("ERROR:", e, instance)
+ if not fba.is_instance_registered(instance):
+ # NOISY-DEBUG: print("DEBUG: Adding new instance:", instance, domain)
+ fba.add_instance(instance, domain, sys.argv[0])
+ except BaseException as e:
+ print(f"ERROR: instance='{instance}',exception:'{str(e)}'")
continue
instance = sys.argv[1]
# Initial fetch
-fetch_instances(instance)
+fetch_instances(instance, None, None)
# Loop through some instances
-fba.c.execute(
- "SELECT domain FROM instances WHERE software IS NOT NULL AND (last_nodeinfo IS NULL OR last_nodeinfo < ?) ORDER BY rowid DESC", [time.time() - fba.config["recheck_instance"]]
+fba.cursor.execute(
+ "SELECT domain, origin, software, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial', 'bookwyrm', 'takahe', 'lemmy') AND (last_instance_fetch IS NULL OR last_instance_fetch < ?) ORDER BY rowid DESC", [time.time() - fba.config["recheck_instance"]]
)
-for instance in fba.c.fetchall():
- blacklisted = False
- for domain in fba.blacklist:
- if domain in instance[0]:
- blacklisted = True
-
- if blacklisted:
- # NOISY-DEBUG: print("DEBUG: domain is blacklisted:", instance)
+rows = fba.cursor.fetchall()
+print(f"INFO: Checking {len(rows)} entries ...")
+for row in rows:
+ # NOISY-DEBUG: print("DEBUG: domain:", row[0])
+ if fba.is_blacklisted(row[0]):
+ print("WARNING: domain is blacklisted:", row[0])
continue
- print("DEBUG: Fetching instances for instance:", instance[0])
- fetch_instances(instance[0])
+ print(f"INFO: Fetching instances for instance '{row[0]}'('{row[2]}') of origin '{row[1]}',nodeinfo_url='{row[3]}'")
+ fetch_instances(row[0], row[1], row[2], row[3])
-fba.conn.close()
+fba.connection.close()