Continued:

- since functions had been moved to module "fba", also the names must be changed

diff --git a/api.py b/api.py
index 917101b5e3eed37983480dd5d03158853e39168d..02de37d4a47f3ad76e8d0d4884b5da5b82f90ebf 100644 (file)
--- a/api.py
+++ b/api.py
@@ -5,51 +5,40 @@ from hashlib import sha256
 from fastapi.templating import Jinja2Templates
 from requests import get
 from json import loads
-from re import sub
 from datetime import datetime
 from email import utils
+import re
+import fba
 
-with open("config.json") as f:
-    config = loads(f.read())
-    base_url = config["base_url"]
-    port = config["port"]
-    log_level = config["log_level"]
-app = FastAPI(docs_url=base_url+"/docs", redoc_url=base_url+"/redoc")
+app = FastAPI(docs_url=fba.config["base_url"] + "/docs", redoc_url=fba.config["base_url"] + "/redoc")
 templates = Jinja2Templates(directory=".")
 
-def get_hash(domain: str) -> str:
-    return sha256(domain.encode("utf-8")).hexdigest()
-
-@app.get(base_url+"/info")
+@app.get(fba.config["base_url"] + "/info")
 def info():
-    conn = sqlite3.connect("blocks.db")
-    c = conn.cursor()
-    c.execute("SELECT (SELECT count(domain) FROM instances), (SELECT count(domain) FROM instances WHERE software IN ('pleroma', 'mastodon', 'misskey', 'gotosocial', 'friendica')), (SELECT count(blocker) FROM blocks)")
-    known, indexed, blocks = c.fetchone()
-    c.close()
+    fba.c.execute("SELECT (SELECT count(domain) FROM instances), (SELECT count(domain) FROM instances WHERE software IN ('pleroma', 'mastodon', 'misskey', 'gotosocial', 'friendica')), (SELECT count(blocker) FROM blocks)")
+    known, indexed, blocks = fba.c.fetchone()
+    fba.c.close()
     return {
         "known_instances": known,
         "indexed_instances": indexed,
         "blocks_recorded": blocks,
-        "slogan": config["slogan"]
+        "slogan": fba.config["slogan"]
     }
 
-@app.get(base_url+"/top")
+@app.get(fba.config["base_url"] + "/top")
 def top(blocked: int = None, blockers: int = None):
-    conn = sqlite3.connect("blocks.db")
-    c = conn.cursor()
     if blocked == None and blockers == None:
         raise HTTPException(status_code=400, detail="No filter specified")
     elif blocked != None:
         if blocked > 500:
             raise HTTPException(status_code=400, detail="Too many results")
-        c.execute("SELECT blocked, count(blocked) FROM blocks WHERE block_level = 'reject' group by blocked ORDER BY count(blocked) DESC LIMIT ?", (blocked,))
+        fba.c.execute("SELECT blocked, count(blocked) FROM blocks WHERE block_level = 'reject' group by blocked ORDER BY count(blocked) DESC LIMIT ?", (blocked,))
     elif blockers != None:
         if blockers > 500:
             raise HTTPException(status_code=400, detail="Too many results")
-        c.execute("SELECT blocker, count(blocker) FROM blocks WHERE block_level = 'reject' group by blocker ORDER BY count(blocker) DESC LIMIT ?", (blockers,))
-    scores = c.fetchall()
-    c.close()
+        fba.c.execute("SELECT blocker, count(blocker) FROM blocks WHERE block_level = 'reject' group by blocker ORDER BY count(blocker) DESC LIMIT ?", (blockers,))
+    scores = fba.c.fetchall()
+    fba.c.close()
 
     scoreboard = []
     print(scores)
@@ -58,27 +47,25 @@ def top(blocked: int = None, blockers: int = None):
 
     return scoreboard
 
-@app.get(base_url+"/api")
+@app.get(fba.config["base_url"] + "/api")
 def blocked(domain: str = None, reason: str = None, reverse: str = None):
     if domain == None and reason == None and reverse == None:
         raise HTTPException(status_code=400, detail="No filter specified")
     if reason != None:
-        reason = sub("(%|_)", "", reason)
+        reason = re.sub("(%|_)", "", reason)
         if len(reason) < 3:
             raise HTTPException(status_code=400, detail="Keyword is shorter than three characters")
-    conn = sqlite3.connect("blocks.db")
-    c = conn.cursor()
     if domain != None:
         wildchar = "*." + ".".join(domain.split(".")[-domain.count("."):])
         punycode = domain.encode('idna').decode('utf-8')
-        c.execute("SELECT blocker, blocked, block_level, reason, first_added, last_seen FROM blocks WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? ORDER BY first_added asc",
-                  (domain, "*." + domain, wildchar, get_hash(domain), punycode, "*." + punycode))
+        fba.c.execute("SELECT blocker, blocked, block_level, reason, first_added, last_seen FROM blocks WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? ORDER BY first_added asc",
+                  (domain, "*." + domain, wildchar, fba.get_hash(domain), punycode, "*." + punycode))
     elif reverse != None:
-        c.execute("SELECT blocker, blocked, block_level, reason, first_added, last_seen FROM blocks WHERE blocker = ? ORDER BY first_added asc", (reverse,))
+        fba.c.execute("SELECT blocker, blocked, block_level, reason, first_added, last_seen FROM blocks WHERE blocker = ? ORDER BY first_added asc", (reverse,))
     else:
-        c.execute("SELECT blocker, blocked, block_level, reason, first_added, last_seen FROM blocks WHERE reason like ? AND reason != '' ORDER BY first_added asc", ("%"+reason+"%",))
-    blocks = c.fetchall()
-    c.close()
+        fba.c.execute("SELECT blocker, blocked, block_level, reason, first_added, last_seen FROM blocks WHERE reason like ? AND reason != '' ORDER BY first_added asc", ("%"+reason+"%",))
+    blocks = fba.c.fetchall()
+    fba.c.close()
 
     result = {}
     for blocker, blocked, block_level, reason, first_added, last_seen in blocks:
@@ -90,37 +77,37 @@ def blocked(domain: str = None, reason: str = None, reverse: str = None):
 
     return result
 
-@app.get(base_url+"/scoreboard")
+@app.get(fba.config["base_url"] + "/scoreboard")
 def index(request: Request, blockers: int = None, blocked: int = None):
     if blockers == None and blocked == None:
         raise HTTPException(status_code=400, detail="No filter specified")
     elif blockers != None:
-        scores = get(f"http://127.0.0.1:{port}{base_url}/top?blockers={blockers}")
+        scores = get(f"http://127.0.0.1:{fba.config['base_url']}{fba.config['base_url']}/top?blockers={blockers}")
     elif blocked != None:
-        scores = get(f"http://127.0.0.1:{port}{base_url}/top?blocked={blocked}")
+        scores = get(f"http://127.0.0.1:{fba.config['base_url']}{fba.config['base_url']}/top?blocked={blocked}")
     if scores != None:
         if not scores.ok:
             raise HTTPException(status_code=blocks.status_code, detail=blocks.text)
         scores = scores.json()
     return templates.TemplateResponse("index.html", {"request": request, "scoreboard": True, "blockers": blockers, "blocked": blocked, "scores": scores})
 
-@app.get(base_url+"/")
+@app.get(fba.config["base_url"] + "/")
 def index(request: Request, domain: str = None, reason: str = None, reverse: str = None):
     if domain == "" or reason == "" or reverse == "":
         return responses.RedirectResponse("/")
     info = None
     blocks = None
     if domain == None and reason == None and reverse == None:
-        info = get(f"http://127.0.0.1:{port}{base_url}/info")
+        info = get(f"http://127.0.0.1:{fba.config['base_url']}{fba.config['base_url']}/info")
         if not info.ok:
             raise HTTPException(status_code=info.status_code, detail=info.text)
         info = info.json()
     elif domain != None:
-        blocks = get(f"http://127.0.0.1:{port}{base_url}/api?domain={domain}")
+        blocks = get(f"http://127.0.0.1:{fba.config['base_url']}{fba.config['base_url']}/api?domain={domain}")
     elif reason != None:
-        blocks = get(f"http://127.0.0.1:{port}{base_url}/api?reason={reason}")
+        blocks = get(f"http://127.0.0.1:{fba.config['base_url']}{fba.config['base_url']}/api?reason={reason}")
     elif reverse != None:
-        blocks = get(f"http://127.0.0.1:{port}{base_url}/api?reverse={reverse}")
+        blocks = get(f"http://127.0.0.1:{fba.config['base_url']}{fba.config['base_url']}/api?reverse={reverse}")
     if blocks != None:
         if not blocks.ok:
             raise HTTPException(status_code=blocks.status_code, detail=blocks.text)
@@ -132,12 +119,10 @@ def index(request: Request, domain: str = None, reason: str = None, reverse: str
 
     return templates.TemplateResponse("index.html", {"request": request, "domain": domain, "blocks": blocks, "reason": reason, "reverse": reverse, "info": info})
 
-@app.get(base_url+"/api/mutual")
+@app.get(fba.config["base_url"] + "/api/mutual")
 def mutual(domains: list[str] = Query()):
     """Return 200 if federation is open between the two, 4xx otherwise"""
-    conn = sqlite3.connect('blocks.db')
-    c = conn.cursor()
-    c.execute(
+    fba.c.execute(
         "SELECT block_level FROM blocks " \
         "WHERE ((blocker = :a OR blocker = :b) AND (blocked = :b OR blocked = :a OR blocked = :aw OR blocked = :bw)) " \
         "AND block_level = 'reject' " \
@@ -149,27 +134,26 @@ def mutual(domains: list[str] = Query()):
             "bw": "*." + domains[1],
         },
     )
-    res = c.fetchone()
-    c.close()
+    res = fba.c.fetchone()
+    fba.c.close()
     if res is not None:
         # Blocks found
         return responses.JSONResponse(status_code=418, content={})
     # No known blocks
     return responses.JSONResponse(status_code=200, content={})
 
-@app.get(base_url+"/rss")
+@app.get(fba.config["base_url"] + "/rss")
 def rss(request: Request, domain: str = None):
-    conn = sqlite3.connect("blocks.db")
-    c = conn.cursor()
     if domain != None:
         wildchar = "*." + ".".join(domain.split(".")[-domain.count("."):])
         punycode = domain.encode('idna').decode('utf-8')
-        c.execute("SELECT blocker, blocked, block_level, reason, first_added, last_seen FROM blocks WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? ORDER BY first_added DESC LIMIT 50",
-                  (domain, "*." + domain, wildchar, get_hash(domain), punycode, "*." + punycode))
+        fba.c.execute("SELECT blocker, blocked, block_level, reason, first_added, last_seen FROM blocks WHERE blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? OR blocked = ? ORDER BY first_added DESC LIMIT 50",
+                  (domain, "*." + domain, wildchar, fba.get_hash(domain), punycode, "*." + punycode))
     else:
-        c.execute("SELECT blocker, blocked, block_level, reason, first_added, last_seen FROM blocks ORDER BY first_added DESC LIMIT 50")
-    blocks = c.fetchall()
-    c.close()
+        fba.c.execute("SELECT blocker, blocked, block_level, reason, first_added, last_seen FROM blocks ORDER BY first_added DESC LIMIT 50")
+
+    blocks = fba.c.fetchall()
+    fba.c.close()
 
     result = []
     for blocker, blocked, block_level, reason, first_added, last_seen in blocks:
@@ -178,11 +162,20 @@ def rss(request: Request, domain: str = None):
             reason = "No reason provided."
         else:
             reason = "Provided reason: '" + reason + "'"
-        result.append({"blocker": blocker, "blocked": blocked, "block_level": block_level, "reason": reason, "first_added": first_added})
+
+        result.append(
+            {
+            "blocker": blocker,
+            "blocked": blocked,
+            "block_level": block_level,
+            "reason": reason,
+            "first_added": first_added
+            }
+        )
 
     timestamp = utils.format_datetime(datetime.now())
 
     return templates.TemplateResponse("rss.xml", {"request": request, "timestamp": timestamp, "domain": domain, "blocks": result}, headers={"Content-Type": "application/rss+xml"})
 
 if __name__ == "__main__":
-    uvicorn.run("api:app", host="127.0.0.1", port=port, log_level=log_level)
+    uvicorn.run("api:app", host="127.0.0.1", port=fba.config["port"], log_level=fba.config["log_level"])

diff --git a/fba.py b/fba.py
index c93996dafdee9711ae3d711571957e770f9a83ed..8614e32c9f682ce475065b4d0de6db41de9aee37 100644 (file)
--- a/fba.py
+++ b/fba.py
@@ -3,6 +3,7 @@ from reqto import post
 from bs4 import BeautifulSoup
 from reqto import get
 from hashlib import sha256
+import re
 import sqlite3
 import json
 import sys
@@ -301,14 +302,20 @@ def get_misskey_blocks(domain: str) -> dict:
 def tidyup(domain: str) -> str:
     # some retards put their blocks in variable case
     domain = domain.lower()
+
     # other retards put the port
     domain = re.sub("\:\d+$", "", domain)
+
     # bigger retards put the schema in their blocklist, sometimes even without slashes
     domain = re.sub("^https?\:(\/*)", "", domain)
+
     # and trailing slash
     domain = re.sub("\/$", "", domain)
+
     # and the @
     domain = re.sub("^\@", "", domain)
+
     # the biggest retards of them all try to block individual users
     domain = re.sub("(.+)\@", "", domain)
+
     return domain

diff --git a/fetch_blocks.py b/fetch_blocks.py
index 5ded601ad7242fbeb4bd414eeaf59319514607fc..bcf4576314d028f886d897c8ef299e88aa9bc55f 100644 (file)
--- a/fetch_blocks.py
+++ b/fetch_blocks.py
@@ -7,15 +7,14 @@ from json import loads
 import re
 from time import time
 import itertools
-from fba import c
 import fba
 
-c.execute(
+fba.c.execute(
     "SELECT domain, software FROM instances WHERE domain='tooting.intensifi.es'"
     #"SELECT domain, software FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial')"
 )
 
-for blocker, software in c.fetchall():
+for blocker, software in fba.c.fetchall():
     blockdict = []
     blocker = fba.tidyup(blocker)
     if software == "pleroma":
@@ -39,22 +38,22 @@ for blocker, software in c.fetchall():
 
                         if blocked.count("*") > 1:
                             # -ACK!-oma also started obscuring domains without hash
-                            c.execute(
+                            fba.c.execute(
                                 "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", (blocked.replace("*", "_"),)
                             )
-                            searchres = c.fetchone()
+                            searchres = fba.c.fetchone()
                             if searchres != None:
                                 blocked = searchres[0]
 
-                        c.execute(
+                        fba.c.execute(
                             "SELECT domain FROM instances WHERE domain = ?", (blocked)
                         )
 
-                        if c.fetchone() == None:
-                            add_instance(blocked)
+                        if fba.c.fetchone() == None:
+                            fba.add_instance(blocked)
 
                         timestamp = int(time())
-                        c.execute(
+                        fba.c.execute(
                             "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ?",
                             (
                                blocker,
@@ -62,7 +61,7 @@ for blocker, software in c.fetchall():
                                block_level
                            ),
                         )
-                        if c.fetchone() == None:
+                        if fba.c.fetchone() == None:
                             block_instance(blocker, blocked, reason, block_level, timestamp, timestamp)
 
                             if block_level == "reject":
@@ -72,7 +71,8 @@ for blocker, software in c.fetchall():
                                         "reason": None
                                     })
                         else:
-                            update_last_seen(timestamp, blocker, blocked, block_level)
+                            fba.update_last_seen(timestamp, blocker, blocked, block_level)
+
             fba.conn.commit()
             # Reasons
             if "mrf_simple_info" in federation:
@@ -91,15 +91,15 @@ for blocker, software in c.fetchall():
 
                         if blocked.count("*") > 1:
                             # same domain guess as above, but for reasons field
-                            c.execute(
+                            fba.c.execute(
                                 "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", (blocked.replace("*", "_"),)
                             )
-                            searchres = c.fetchone()
+                            searchres = fba.c.fetchone()
 
                             if searchres != None:
                                 blocked = searchres[0]
 
-                        update_block_reason(reason["reason"], blocker, blocked, block_level)
+                        fba.update_block_reason(reason["reason"], blocker, blocked, block_level)
 
                         for entry in blockdict:
                             if entry["blocked"] == blocked:
@@ -156,23 +156,23 @@ for blocker, software in c.fetchall():
                     blocked = fba.tidyup(blocked)
 
                     if blocked.count("*") <= 1:
-                        c.execute(
+                        fba.c.execute(
                             "SELECT hash FROM instances WHERE hash = ?", (blocked_hash,)
                         )
 
-                        if c.fetchone() == None:
-                            add_instance(blocked)
+                        if fba.c.fetchone() == None:
+                            fba.add_instance(blocked)
                     else:
                         # Doing the hash search for instance names as well to tidy up DB
-                        c.execute(
+                        fba.c.execute(
                             "SELECT domain FROM instances WHERE hash = ?", (blocked_hash,)
                         )
-                        searchres = c.fetchone()
+                        searchres = fba.c.fetchone()
                         if searchres != None:
                             blocked = searchres[0]
 
                     timestamp = int(time())
-                    c.execute(
+                    fba.c.execute(
                         "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ?",
                         (
                             blocker,
@@ -180,7 +180,7 @@ for blocker, software in c.fetchall():
                             block_level
                         ),
                     )
-                    if c.fetchone() == None:
+                    if fba.c.fetchone() == None:
                         block_instance(blocker, blocked if blocked.count("*") <= 1 else blocked_hash, reason, block_level, timestamp, timestamp)
 
                         if block_level == "reject":
@@ -190,10 +190,10 @@ for blocker, software in c.fetchall():
                                     "reason": reason
                                 })
                     else:
-                        update_last_seen(timestamp, blocker, blocked if blocked.count("*") <= 1 else blocked_hash, block_level)
+                        fba.update_last_seen(timestamp, blocker, blocked if blocked.count("*") <= 1 else blocked_hash, block_level)
 
                     if reason != '':
-                        update_block_reason(reason, blocker, blocked if blocked.count("*") <= 1 else blocked_hash, block_level)
+                        fba.update_block_reason(reason, blocker, blocked if blocked.count("*") <= 1 else blocked_hash, block_level)
 
             fba.conn.commit()
         except Exception as e:
@@ -213,36 +213,36 @@ for blocker, software in c.fetchall():
                     print("BEFORE-blocked:", blocked)
                     if blocked.count("*") > 0:
                         # Some friendica servers also obscure domains without hash
-                        c.execute(
+                        fba.c.execute(
                             "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", (blocked.replace("*", "_"),)
                         )
-                        searchres = c.fetchone()
+                        searchres = fba.c.fetchone()
                         if searchres != None:
                             blocked = searchres[0]
 
                     if blocked.count("?") > 0:
                         # Some obscure them with question marks, not sure if that's dependent on version or not
-                        c.execute(
+                        fba.c.execute(
                             "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", (blocked.replace("?", "_"),)
                         )
-                        searchres = c.fetchone()
+                        searchres = fba.c.fetchone()
                         if searchres != None:
                             blocked = searchres[0]
 
                     print("AFTER-blocked:", blocked)
-                    c.execute(
+                    fba.c.execute(
                         "SELECT domain FROM instances WHERE domain = ?", (blocked,)
                     )
 
-                    if c.fetchone() == None:
-                        add_instance(blocked)
+                    if fba.c.fetchone() == None:
+                        fba.add_instance(blocked)
 
                     timestamp = int(time())
-                    c.execute(
+                    fba.c.execute(
                         "SELECT * FROM blocks WHERE blocker = ? AND blocked = ?",
                         (blocker, blocked),
                     )
-                    if c.fetchone() == None:
+                    if fba.c.fetchone() == None:
                         block_instance(blocker, blocked, reason, block_level, timestamp, timestamp)
 
                         if block_level == "reject":
@@ -252,10 +252,10 @@ for blocker, software in c.fetchall():
                                     "reason": reason
                                 })
                     else:
-                        update_last_seen(timestamp, blocker, blocked, block_level)
+                        fba.update_last_seen(timestamp, blocker, blocked, block_level)
 
                     if reason != '':
-                        update_block_reason(reason, blocker, blocked, block_level)
+                        fba.update_block_reason(reason, blocker, blocked, block_level)
 
             fba.conn.commit()
         except Exception as e:
@@ -276,28 +276,28 @@ for blocker, software in c.fetchall():
 
                     if blocked.count("*") > 0:
                         # GTS does not have hashes for obscured domains, so we have to guess it
-                        c.execute(
+                        fba.c.execute(
                             "SELECT domain FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", (blocked.replace("*", "_"),)
                         )
-                        searchres = c.fetchone()
+                        searchres = fba.c.fetchone()
 
                         if searchres != None:
                             blocked = searchres[0]
 
-                    c.execute(
+                    fba.c.execute(
                         "SELECT domain FROM instances WHERE domain = ?", (blocked,)
                     )
 
-                    if c.fetchone() == None:
-                        add_instance(blocked)
+                    if fba.c.fetchone() == None:
+                        fba.add_instance(blocked)
 
-                    c.execute(
+                    fba.c.execute(
                         "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ?",
                         (blocker, blocked, "reject"),
                     )
                     timestamp = int(time())
 
-                    if c.fetchone() == None:
+                    if fba.c.fetchone() == None:
                         block_instance(blocker, blocked, "", "reject", timestamp, timestamp)
 
                         blockdict.append(
@@ -306,11 +306,11 @@ for blocker, software in c.fetchall():
                                 "reason": None
                             })
                     else:
-                        update_last_seen(timestamp, blocker, blocked, "reject")
+                        fba.update_last_seen(timestamp, blocker, blocked, "reject")
 
                     if "public_comment" in peer:
                         reason = peer["public_comment"]
-                        update_block_reason(reason, blocker, blocked, "reject")
+                        fba.update_block_reason(reason, blocker, blocked, "reject")
 
                         for entry in blockdict:
                             if entry["blocked"] == blocked:
@@ -321,7 +321,7 @@ for blocker, software in c.fetchall():
     else:
         print("WARNING: Unknown software:", software)
 
-    if config["bot_enabled"] and len(blockdict) > 0:
+    if fba.config["bot_enabled"] and len(blockdict) > 0:
         send_bot_post(blocker, blockdict)
     blockdict = []
 

diff --git a/fetch_instances.py b/fetch_instances.py
index f25b44045c9a76ecc0d4a9b34524bacf7b13cc39..346ece0c4a5a43d801f5c2f5743283b9d44532a0 100644 (file)
--- a/fetch_instances.py
+++ b/fetch_instances.py
@@ -1,7 +1,6 @@
 import sqlite3
 import sys
 import json
-from fba import c
 import fba
 
 domain = sys.argv[1]
@@ -12,7 +11,7 @@ if (peerlist is None):
     print("FATAL: CANNOT FETCH PEERS:", domain)
     sys.exit(255)
 
-c.execute(
+fba.c.execute(
     "SELECT domain FROM instances WHERE 1"
 )
 
@@ -30,11 +29,11 @@ for instance in peerlist:
 
     print("Handling instance:", instance)
     try:
-        c.execute(
+        fba.c.execute(
             "SELECT domain FROM instances WHERE domain = ?", (instance,)
         )
 
-        if c.fetchone() == None:
+        if fba.c.fetchone() == None:
             fba.add_instance(instance)
 
         fba.conn.commit()