From: Roland Häder Date: Wed, 17 May 2023 16:03:38 +0000 (+0200) Subject: Continued: X-Git-Url: https://git.mxchange.org/?a=commitdiff_plain;h=e995ace6d361e0b2a17f6b630b17b626a3f217fa;p=fba.git Continued: - more rewrites away from single-function imports - added more debug lines - maybe fixed adding duplicate instances --- diff --git a/fetch_blocks.py b/fetch_blocks.py index bcf4576..040eddc 100644 --- a/fetch_blocks.py +++ b/fetch_blocks.py @@ -1,13 +1,8 @@ -from reqto import get -from reqto import post -from hashlib import sha256 -from bs4 import BeautifulSoup -from json import dumps -from json import loads -import re -from time import time -import itertools +import time +import bs4 import fba +import itertools +import re fba.c.execute( "SELECT domain, software FROM instances WHERE domain='tooting.intensifi.es'" @@ -15,13 +10,14 @@ fba.c.execute( ) for blocker, software in fba.c.fetchall(): + print("DEBUG: blocker,software:", blocker, software) blockdict = [] blocker = fba.tidyup(blocker) if software == "pleroma": - print(blocker) + print("DEBUG: blocker:", blocker) try: # Blocks - federation = get( + federation = reqto.get( f"https://{blocker}/nodeinfo/2.1.json", headers=headers, timeout=5 ).json()["metadata"]["federation"] if "mrf_simple" in federation: @@ -50,9 +46,10 @@ for blocker, software in fba.c.fetchall(): ) if fba.c.fetchone() == None: + print("DEBUG: Hash wasn't found, adding:", blocked) fba.add_instance(blocked) - timestamp = int(time()) + timestamp = int(time.time()) fba.c.execute( "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ?", ( @@ -62,7 +59,7 @@ for blocker, software in fba.c.fetchall(): ), ) if fba.c.fetchone() == None: - block_instance(blocker, blocked, reason, block_level, timestamp, timestamp) + fba.block_instance(blocker, blocked, reason, block_level, timestamp, timestamp) if block_level == "reject": blockdict.append( @@ -107,9 +104,9 @@ for blocker, software in fba.c.fetchall(): fba.conn.commit() except Exception as e: - print("error:", e, blocker) + print("error:", e, blocker, software) elif software == "mastodon": - print(blocker) + print("DEBUG: blocker:", blocker) try: # json endpoint for newer mastodongs try: @@ -121,8 +118,8 @@ for blocker, software in fba.c.fetchall(): } # handling CSRF, I've saw at least one server requiring it to access the endpoint - meta = BeautifulSoup( - get(f"https://{blocker}/about", headers=headers, timeout=5).text, + meta = bs4.BeautifulSoup( + reqto.get(f"https://{blocker}/about", headers=headers, timeout=5).text, "html.parser", ) try: @@ -131,12 +128,15 @@ for blocker, software in fba.c.fetchall(): except: reqheaders = headers - blocks = get( + blocks = reqto.get( f"https://{blocker}/api/v1/instance/domain_blocks", headers=reqheaders, timeout=5 ).json() + print("DEBUG: blocks():", len(blocks)) for block in blocks: entry = {'domain': block['domain'], 'hash': block['digest'], 'reason': block['comment']} + + print("DEBUG: severity,domain,hash,comment:", block['severity'], block['domain'], block['digest'], block['comment']) if block['severity'] == 'suspend': json['reject'].append(entry) elif block['severity'] == 'silence': @@ -153,25 +153,32 @@ for blocker, software in fba.c.fetchall(): for block_level, blocks in json.items(): for instance in blocks: blocked, blocked_hash, reason = instance.values() + print("DEBUG: blocked,hash,reason:", blocked, blocked_hash, reason) + blocked = fba.tidyup(blocked) + print("DEBUG: blocked:", blocked) - if blocked.count("*") <= 1: + if blocked.count("*") < 1: + # No obsfucation for this instance fba.c.execute( - "SELECT hash FROM instances WHERE hash = ?", (blocked_hash,) + "SELECT hash FROM instances WHERE domain = ? LIMIT 1", (blocked,) ) if fba.c.fetchone() == None: + print("DEBUG: Hash wasn't found, adding:", blocked) fba.add_instance(blocked) else: # Doing the hash search for instance names as well to tidy up DB fba.c.execute( - "SELECT domain FROM instances WHERE hash = ?", (blocked_hash,) + "SELECT domain FROM instances WHERE hash = ? LIMIT 1", (blocked_hash,) ) searchres = fba.c.fetchone() + if searchres != None: + print("DEBUG: Updating domain: ", searchres[0]) blocked = searchres[0] - timestamp = int(time()) + timestamp = int(time.time()) fba.c.execute( "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ?", ( @@ -180,8 +187,9 @@ for blocker, software in fba.c.fetchall(): block_level ), ) + if fba.c.fetchone() == None: - block_instance(blocker, blocked if blocked.count("*") <= 1 else blocked_hash, reason, block_level, timestamp, timestamp) + fba.block_instance(blocker, blocked if blocked.count("*") <= 1 else blocked_hash, reason, block_level, timestamp, timestamp) if block_level == "reject": blockdict.append( @@ -197,9 +205,9 @@ for blocker, software in fba.c.fetchall(): fba.conn.commit() except Exception as e: - print("error:", e, blocker) + print("error:", e, blocker, software) elif software == "friendica" or software == "misskey": - print(blocker) + print("DEBUG: blocker:", blocker) try: if software == "friendica": json = fba.get_friendica_blocks(blocker) @@ -235,15 +243,16 @@ for blocker, software in fba.c.fetchall(): ) if fba.c.fetchone() == None: + print("DEBUG: Hash wasn't found, adding:", blocked) fba.add_instance(blocked) - timestamp = int(time()) + timestamp = int(time.time()) fba.c.execute( "SELECT * FROM blocks WHERE blocker = ? AND blocked = ?", (blocker, blocked), ) if fba.c.fetchone() == None: - block_instance(blocker, blocked, reason, block_level, timestamp, timestamp) + fba.block_instance(blocker, blocked, reason, block_level, timestamp, timestamp) if block_level == "reject": blockdict.append( @@ -259,12 +268,12 @@ for blocker, software in fba.c.fetchall(): fba.conn.commit() except Exception as e: - print("error:", e, blocker) + print("error:", e, blocker, software) elif software == "gotosocial": - print(blocker) + print("DEBUG: blocker:", blocker) try: # Blocks - federation = get( + federation = reqto.get( f"https://{blocker}/api/v1/instance/peers?filter=suspended", headers=headers, timeout=5 ).json() @@ -289,16 +298,21 @@ for blocker, software in fba.c.fetchall(): ) if fba.c.fetchone() == None: + print("DEBUG: Hash wasn't found, adding:", blocked) fba.add_instance(blocked) fba.c.execute( "SELECT * FROM blocks WHERE blocker = ? AND blocked = ? AND block_level = ?", - (blocker, blocked, "reject"), + ( + blocker, + blocked, + "reject" + ), ) - timestamp = int(time()) + timestamp = int(time.time()) if fba.c.fetchone() == None: - block_instance(blocker, blocked, "", "reject", timestamp, timestamp) + fba.block_instance(blocker, blocked, "", "reject", timestamp, timestamp) blockdict.append( { @@ -317,12 +331,13 @@ for blocker, software in fba.c.fetchall(): entry["reason"] = reason fba.conn.commit() except Exception as e: - print("error:", e, blocker) + print("error:", e, blocker, software) else: print("WARNING: Unknown software:", software) if fba.config["bot_enabled"] and len(blockdict) > 0: send_bot_post(blocker, blockdict) + blockdict = [] fba.conn.close()