Continued:

- reformatted config.defaults.json
- also store script name

diff --git a/blocks_empty.db b/blocks_empty.db
index 457b338488c069c6b7d032bd44fbf7f8a64f55ff..93efe69100d170780db531eec9a331f466abdfae 100644 (file)
Binary files a/blocks_empty.db and b/blocks_empty.db differ

diff --git a/config.defaults.json b/config.defaults.json
index 87f5b65eb28082daab258643484a29f9acc3aefa..d2a1a42f838da41766c7b62a310744632ccb7a61 100644 (file)
--- a/config.defaults.json
+++ b/config.defaults.json
@@ -1,13 +1,14 @@
 {
-    "base_url": "",
-    "log_level": "info",
-    "port": 8069,
-    "useragent": "Mozilla/5.0 (Windows NT 10.0; rv:102.0) Gecko/20100101 Firefox/102.0",
-    "bot_enabled": false,
-    "bot_instance": "https://example.com",
-    "bot_token": "",
-    "bot_visibility": "unlisted",
-    "slogan": "### Your footer slogan ###",
+    "base_url"        : "",
+    "log_level"       : "info",
+    "port"            : 8069,
+    "useragent"       : "Mozilla/5.0 (Windows NT 10.0; rv:102.0) Gecko/20100101 Firefox/102.0",
+    "timeout"         : 5,
+    "bot_enabled"     : false,
+    "bot_instance"    : "https://example.com",
+    "bot_token"       : "",
+    "bot_visibility"  : "unlisted",
+    "slogan"          : "### Your footer slogan ###",
     "recheck_instance": 3600,
-    "recheck_block": 3600
+    "recheck_block"   : 3600
 }

diff --git a/fba.py b/fba.py
index 4468b804e793910571172aaac1f2293ef55c61a3..3136ea235d64fb157b3c3e1b80d9148c6a80f38d 100644 (file)
--- a/fba.py
+++ b/fba.py
@@ -75,7 +75,7 @@ def get_peers(domain: str) -> list:
     peers = None
 
     try:
-        res = reqto.get(f"https://{domain}/api/v1/instance/peers", headers=headers, timeout=5)
+        res = reqto.get(f"https://{domain}/api/v1/instance/peers", headers=headers, timeout=config["timeout"])
 
         if not res.ok or res.status_code >= 400:
             print("WARNING: Cannot fetch peers:", domain)
@@ -95,7 +95,7 @@ def get_peers(domain: str) -> list:
 def post_json_api(domain: str, path: str, data: str) -> list:
     try:
         # NOISY-DEBUG: print("DEBUG: Sending POST to domain,path,data:", domain, path, data)
-        res = reqto.post(f"https://{domain}{path}", data=data, headers=headers, timeout=5)
+        res = reqto.post(f"https://{domain}{path}", data=data, headers=headers, timeout=config["timeout"])
 
         if not res.ok or res.status_code >= 400:
             print("WARNING: Cannot query JSON API:", domain, path, data, res.status_code)
@@ -124,7 +124,7 @@ def fetch_nodeinfo(domain: str) -> list:
     json = None
     for request in requests:
         # NOISY-DEBUG: print("DEBUG: Fetching request:", request)
-        res = reqto.get(request, headers=headers, timeout=5)
+        res = reqto.get(request, headers=headers, timeout=config["timeout"])
 
         # NOISY-DEBUG: print("DEBUG: res.ok,res.json[]:", res.ok, type(res.json()))
         if res.ok and res.json() is not None:
@@ -218,7 +218,7 @@ def block_instance(blocker: str, blocked: str, reason: str, block_level: str, fi
     print("--- New block:", blocker, blocked, reason, block_level, first_added, last_seen)
     try:
         c.execute(
-            "INSERT INTO blocks SELECT ?, ?, ?, ?, ?, ?",
+            "INSERT INTO blocks (blocker, blocked, reason, block_level, first_added, last_seen) VALUES(?, ?, ?, ?, ?, ?)",
              (
                  blocker,
                  blocked,
@@ -233,21 +233,22 @@ def block_instance(blocker: str, blocked: str, reason: str, block_level: str, fi
         print("ERROR: failed SQL query:", blocker, blocked, reason, block_level, first_added, last_seen)
         sys.exit(255)
 
-def add_instance(domain: str, originator: str):
-    # NOISY-DEBUG: print("DEBUG: domain,originator:", domain, originator)
+def add_instance(domain: str, origin: str, originator: str):
+    # NOISY-DEBUG: print("DEBUG: domain,origin:", domain, origin, originator)
     if domain.find("@") > 0:
         print("WARNING: Bad domain name:", domain)
         raise
-    elif originator is not None and originator.find("@") > 0:
-        print("WARNING: Bad originator name:", originator)
+    elif origin is not None and origin.find("@") > 0:
+        print("WARNING: Bad origin name:", origin)
         raise
 
-    print("--- Adding new instance:", domain, originator)
+    print(f"--- Adding new instance {domain} (origin: {origin})")
     try:
         c.execute(
-            "INSERT INTO instances (domain,originator,hash,software) VALUES (?, ?, ?, ?)",
+            "INSERT INTO instances (domain,origin,originator,hash,software) VALUES (?, ?, ?, ?, ?)",
             (
                domain,
+               origin,
                originator,
                get_hash(domain),
                determine_software(domain)
@@ -314,7 +315,7 @@ def get_mastodon_blocks(domain: str) -> dict:
 
     try:
         doc = BeautifulSoup(
-            reqto.get(f"https://{domain}/about/more", headers=headers, timeout=5).text,
+            reqto.get(f"https://{domain}/about/more", headers=headers, timeout=config["timeout"]).text,
             "html.parser",
         )
     except:
@@ -351,7 +352,7 @@ def get_friendica_blocks(domain: str) -> dict:
 
     try:
         doc = BeautifulSoup(
-            reqto.get(f"https://{domain}/friendica", headers=headers, timeout=5).text,
+            reqto.get(f"https://{domain}/friendica", headers=headers, timeout=config["timeout"]).text,
             "html.parser",
         )
     except:

diff --git a/fetch_blocks.py b/fetch_blocks.py
index cc2b0ec2543434550138cc155a924898837bff34..a4248ac3c0e1dc74713904049422964496ed3d82 100644 (file)
--- a/fetch_blocks.py
+++ b/fetch_blocks.py
@@ -76,7 +76,7 @@ for blocker, software in fba.c.fetchall():
 
                         if fba.c.fetchone() == None:
                             # NOISY-DEBUG: print("DEBUG: Domain wasn't found, adding:", blocked, blocker)
-                            fba.add_instance(blocked, blocker)
+                            fba.add_instance(blocked, blocker, argv[0])
 
                         timestamp = int(time.time())
                         fba.c.execute(
@@ -166,7 +166,7 @@ for blocker, software in fba.c.fetchall():
                 # handling CSRF, I've saw at least one server requiring it to access the endpoint
                 # NOISY-DEBUG: print("DEBUG: Fetching meta:", blocker)
                 meta = bs4.BeautifulSoup(
-                    reqto.get(f"https://{blocker}/about", headers=fba.headers, timeout=5).text,
+                    reqto.get(f"https://{blocker}/about", headers=fba.headers, timeout=fba.config["timeout"]).text,
                     "html.parser",
                 )
                 try:
@@ -178,7 +178,7 @@ for blocker, software in fba.c.fetchall():
                     reqheaders = fba.headers
 
                 # NOISY-DEBUG: print("DEBUG: Quering API domain_blocks:", blocker)
-                blocks = reqto.get(f"https://{blocker}/api/v1/instance/domain_blocks", headers=reqheaders, timeout=5).json()
+                blocks = reqto.get(f"https://{blocker}/api/v1/instance/domain_blocks", headers=reqheaders, timeout=fba.config["timeout"]).json()
 
                 # NOISY-DEBUG: print("DEBUG: blocks():", len(blocks))
                 for block in blocks:
@@ -229,7 +229,7 @@ for blocker, software in fba.c.fetchall():
 
                         if fba.c.fetchone() == None:
                             # NOISY-DEBUG: print("DEBUG: Hash wasn't found, adding:", blocked, blocker)
-                            fba.add_instance(blocked, blocker)
+                            fba.add_instance(blocked, blocker, argv[0])
                     else:
                         # Doing the hash search for instance names as well to tidy up DB
                         fba.c.execute(
@@ -350,7 +350,7 @@ for blocker, software in fba.c.fetchall():
         print("INFO: blocker:", blocker)
         try:
             # Blocks
-            federation = reqto.get(f"https://{blocker}/api/v1/instance/peers?filter=suspended", headers=fba.headers, timeout=5).json()
+            federation = reqto.get(f"https://{blocker}/api/v1/instance/peers?filter=suspended", headers=fba.headers, timeout=fba.config["timeout"]).json()
 
             if (federation == None):
                 print("WARNING: No valid response:", blocker);

diff --git a/fetch_instances.py b/fetch_instances.py
index e4a9f5a28d18bf319ee97066e0f8b2a1371cf6a6..8a4a45c8ee33a256759a89f0252a0294f32f4917 100644 (file)
--- a/fetch_instances.py
+++ b/fetch_instances.py
@@ -4,8 +4,8 @@ import json
 import time
 import fba
 
-def fetch_instances(domain: str, originator: str):
-    # NOISY-DEBUG: print("DEBUG: Fetching instances for domain:", domain, originator)
+def fetch_instances(domain: str, origin: str):
+    # NOISY-DEBUG: print("DEBUG: Fetching instances for domain:", domain, origin)
     peerlist = fba.get_peers(domain)
 
     if (peerlist is None):
@@ -16,8 +16,8 @@ def fetch_instances(domain: str, originator: str):
         )
 
         if fba.c.fetchone() == None:
-            # NOISY-DEBUG: print("DEBUG: Adding new domain:", domain, originator)
-            fba.add_instance(domain, originator)
+            # NOISY-DEBUG: print("DEBUG: Adding new domain:", domain, origin)
+            fba.add_instance(domain, origin, sys.argv[0])
 
         fba.conn.commit()
         return
@@ -45,7 +45,7 @@ def fetch_instances(domain: str, originator: str):
 
             if fba.c.fetchone() == None:
                 # NOISY-DEBUG: print("DEBUG: Adding new instance:", instance, domain)
-                fba.add_instance(instance, domain)
+                fba.add_instance(instance, domain, sys.argv[0])
 
             fba.conn.commit()