Continued:

- more rewrites away from single-function imports
- added more debug lines
- maybe fixed adding duplicate instances

diff --git a/fetch_blocks.py b/fetch_blocks.py
index bcf4576314d028f886d897c8ef299e88aa9bc55f..040eddcea157e656588a582a6b58180f53a45701 100644 (file)
--- a/fetch_blocks.py
+++ b/fetch_blocks.py
@@ -1,13 +1,8 @@
-from reqto import get
-from reqto import post
-from hashlib import sha256
-from bs4 import BeautifulSoup
-from json import dumps
-from json import loads
-import re
-from time import time
-import itertools
+import time
+import bs4
 import fba
+import itertools
+import re
 
 fba.c.execute(
     "SELECT domain, software FROM instances WHERE domain='tooting.intensifi.es'"
@@ -15,13 +10,14 @@ fba.c.execute(
 )
 
 for blocker, software in fba.c.fetchall():
+    print("DEBUG: blocker,software:", blocker, software)
     blockdict = []
     blocker = fba.tidyup(blocker)
     if software == "pleroma":
-        print(blocker)
+        print("DEBUG: blocker:", blocker)
         try:
             # Blocks
-            federation = get(
+            federation = reqto.get(
                 f"https://{blocker}/nodeinfo/2.1.json", headers=headers, timeout=5
             ).json()["metadata"]["federation"]
             if "mrf_simple" in federation:
@@ -50,9 +46,10 @@ for blocker, software in fba.c.fetchall():
                         )
 
                         if fba.c.fetchone() == None:
+                            print("DEBUG: Hash wasn't found, adding:", blocked)
                             fba.add_instance(blocked)
 
-                        timestamp = int(time())
+                        timestamp = int(time.time())
                         fba.c.execute(
                             "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ?",
                             (
@@ -62,7 +59,7 @@ for blocker, software in fba.c.fetchall():
                            ),
                         )
                         if fba.c.fetchone() == None:
-                            block_instance(blocker, blocked, reason, block_level, timestamp, timestamp)
+                            fba.block_instance(blocker, blocked, reason, block_level, timestamp, timestamp)
 
                             if block_level == "reject":
                                 blockdict.append(
@@ -107,9 +104,9 @@ for blocker, software in fba.c.fetchall():
 
             fba.conn.commit()
         except Exception as e:
-            print("error:", e, blocker)
+            print("error:", e, blocker, software)
     elif software == "mastodon":
-        print(blocker)
+        print("DEBUG: blocker:", blocker)
         try:
             # json endpoint for newer mastodongs
             try:
@@ -121,8 +118,8 @@ for blocker, software in fba.c.fetchall():
                 }
 
                 # handling CSRF, I've saw at least one server requiring it to access the endpoint
-                meta = BeautifulSoup(
-                    get(f"https://{blocker}/about", headers=headers, timeout=5).text,
+                meta = bs4.BeautifulSoup(
+                    reqto.get(f"https://{blocker}/about", headers=headers, timeout=5).text,
                     "html.parser",
                 )
                 try:
@@ -131,12 +128,15 @@ for blocker, software in fba.c.fetchall():
                 except:
                     reqheaders = headers
 
-                blocks = get(
+                blocks = reqto.get(
                     f"https://{blocker}/api/v1/instance/domain_blocks", headers=reqheaders, timeout=5
                 ).json()
 
+                print("DEBUG: blocks():", len(blocks))
                 for block in blocks:
                     entry = {'domain': block['domain'], 'hash': block['digest'], 'reason': block['comment']}
+
+                    print("DEBUG: severity,domain,hash,comment:", block['severity'], block['domain'], block['digest'], block['comment'])
                     if block['severity'] == 'suspend':
                         json['reject'].append(entry)
                     elif block['severity'] == 'silence':
@@ -153,25 +153,32 @@ for blocker, software in fba.c.fetchall():
             for block_level, blocks in json.items():
                 for instance in blocks:
                     blocked, blocked_hash, reason = instance.values()
+                    print("DEBUG: blocked,hash,reason:", blocked, blocked_hash, reason)
+
                     blocked = fba.tidyup(blocked)
+                    print("DEBUG: blocked:", blocked)
 
-                    if blocked.count("*") <= 1:
+                    if blocked.count("*") < 1:
+                        # No obsfucation for this instance
                         fba.c.execute(
-                            "SELECT hash FROM instances WHERE hash = ?", (blocked_hash,)
+                            "SELECT hash FROM instances WHERE domain = ? LIMIT 1", (blocked,)
                         )
 
                         if fba.c.fetchone() == None:
+                            print("DEBUG: Hash wasn't found, adding:", blocked)
                             fba.add_instance(blocked)
                     else:
                         # Doing the hash search for instance names as well to tidy up DB
                         fba.c.execute(
-                            "SELECT domain FROM instances WHERE hash = ?", (blocked_hash,)
+                            "SELECT domain FROM instances WHERE hash = ? LIMIT 1", (blocked_hash,)
                         )
                         searchres = fba.c.fetchone()
+
                         if searchres != None:
+                            print("DEBUG: Updating domain: ", searchres[0])
                             blocked = searchres[0]
 
-                    timestamp = int(time())
+                    timestamp = int(time.time())
                     fba.c.execute(
                         "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ?",
                         (
@@ -180,8 +187,9 @@ for blocker, software in fba.c.fetchall():
                             block_level
                         ),
                     )
+
                     if fba.c.fetchone() == None:
-                        block_instance(blocker, blocked if blocked.count("*") <= 1 else blocked_hash, reason, block_level, timestamp, timestamp)
+                        fba.block_instance(blocker, blocked if blocked.count("*") <= 1 else blocked_hash, reason, block_level, timestamp, timestamp)
 
                         if block_level == "reject":
                             blockdict.append(
@@ -197,9 +205,9 @@ for blocker, software in fba.c.fetchall():
 
             fba.conn.commit()
         except Exception as e:
-            print("error:", e, blocker)
+            print("error:", e, blocker, software)
     elif software == "friendica" or software == "misskey":
-        print(blocker)
+        print("DEBUG: blocker:", blocker)
         try:
             if software == "friendica":
                 json = fba.get_friendica_blocks(blocker)
@@ -235,15 +243,16 @@ for blocker, software in fba.c.fetchall():
                     )
 
                     if fba.c.fetchone() == None:
+                        print("DEBUG: Hash wasn't found, adding:", blocked)
                         fba.add_instance(blocked)
 
-                    timestamp = int(time())
+                    timestamp = int(time.time())
                     fba.c.execute(
                         "SELECT * FROM blocks WHERE blocker = ? AND blocked = ?",
                         (blocker, blocked),
                     )
                     if fba.c.fetchone() == None:
-                        block_instance(blocker, blocked, reason, block_level, timestamp, timestamp)
+                        fba.block_instance(blocker, blocked, reason, block_level, timestamp, timestamp)
 
                         if block_level == "reject":
                             blockdict.append(
@@ -259,12 +268,12 @@ for blocker, software in fba.c.fetchall():
 
             fba.conn.commit()
         except Exception as e:
-            print("error:", e, blocker)
+            print("error:", e, blocker, software)
     elif software == "gotosocial":
-        print(blocker)
+        print("DEBUG: blocker:", blocker)
         try:
             # Blocks
-            federation = get(
+            federation = reqto.get(
                 f"https://{blocker}/api/v1/instance/peers?filter=suspended", headers=headers, timeout=5
             ).json()
 
@@ -289,16 +298,21 @@ for blocker, software in fba.c.fetchall():
                     )
 
                     if fba.c.fetchone() == None:
+                        print("DEBUG: Hash wasn't found, adding:", blocked)
                         fba.add_instance(blocked)
 
                     fba.c.execute(
                         "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ?",
-                        (blocker, blocked, "reject"),
+                        (
+                            blocker,
+                            blocked,
+                            "reject"
+                        ),
                     )
-                    timestamp = int(time())
+                    timestamp = int(time.time())
 
                     if fba.c.fetchone() == None:
-                        block_instance(blocker, blocked, "", "reject", timestamp, timestamp)
+                        fba.block_instance(blocker, blocked, "", "reject", timestamp, timestamp)
 
                         blockdict.append(
                             {
@@ -317,12 +331,13 @@ for blocker, software in fba.c.fetchall():
                                 entry["reason"] = reason
                 fba.conn.commit()
         except Exception as e:
-            print("error:", e, blocker)
+            print("error:", e, blocker, software)
     else:
         print("WARNING: Unknown software:", software)
 
     if fba.config["bot_enabled"] and len(blockdict) > 0:
         send_bot_post(blocker, blockdict)
+
     blockdict = []
 
 fba.conn.close()