From 0f30f29d0d3ef5492ef121453a13be39e1c851ad Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Tue, 25 Jul 2023 02:44:31 +0200 Subject: [PATCH] Continued: - fixed instances.is_recent() - blacklisted misskey-forkbomb.cf (the name says it) - logger level changed to debug --- fba/commands.py | 4 ++-- fba/helpers/blacklist.py | 2 ++ fba/models/instances.py | 8 +++++--- fba/utils.py | 10 +++++----- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/fba/commands.py b/fba/commands.py index c381f8d..f153b56 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -1270,7 +1270,7 @@ def fetch_joinmobilizon(args: argparse.Namespace) -> int: return 0 else: logger.debug("source_domain='%s' has not been recently used, marking ...", source_domain) - #sources.update(source_domain) + sources.update(source_domain) logger.info("Fetching instances from source_domain='%s' ...", source_domain) raw = utils.fetch_url( @@ -1489,7 +1489,7 @@ def fetch_joinfediverse(args: argparse.Namespace) -> int: logger.debug("block[blocked]='%s' has been recently checked - SKIPPED!", block["blocked"]) continue - logger.info("Proccessing blocked='%s' ...", block["blocked"]) + logger.debug("Proccessing blocked='%s' ...", block["blocked"]) processing.domain(block["blocked"], "climatejustice.social", inspect.currentframe().f_code.co_name) blockdict = list() diff --git a/fba/helpers/blacklist.py b/fba/helpers/blacklist.py index 02bd319..c96012e 100644 --- a/fba/helpers/blacklist.py +++ b/fba/helpers/blacklist.py @@ -20,11 +20,13 @@ from fba.helpers import domain as domain_helper logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) +#logger.setLevel(logging.DEBUG) # Don't check these, known trolls/flooders/testing/developing _blacklist = { "activitypub-troll.cf": "Floods federation with fake nodes as \"research\" project", "activitypub-proxy.cf": "Floods federation with fake nodes as \"research\" project", + "misskey-forkbomb.cf" : "Floods federation tables with fake nodes", "gab.best" : "Floods federation tables with fake nodes", "4chan.icu" : "Floods federation tables with fake nodes", "social.shrimpcam.pw" : "Floods federation tables with fake nodes", diff --git a/fba/models/instances.py b/fba/models/instances.py index c7db652..f27d90e 100644 --- a/fba/models/instances.py +++ b/fba/models/instances.py @@ -37,6 +37,7 @@ from fba.models import error_log logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) +#logger.setLevel(logging.DEBUG) # Found info from node, such as nodeinfo URL, detection mode that needs to be # written to database. Both arrays must be filled at the same time or else @@ -314,10 +315,11 @@ def is_recent(domain: str, column: str = "last_instance_fetch") -> bool: database.cursor.execute(f"SELECT {column} FROM instances WHERE domain = ? LIMIT 1", [domain]) # Fetch row - fetched = database.cursor.fetchone()[column] + fetched = float(database.cursor.fetchone()[column]) + diff = (time.time() - fetched) - logger.debug("fetched[%s]='%s',key='%s'", type(fetched), fetched, key) - recently = isinstance(fetched, float) and (time.time() - fetched) >= config.get(key) + logger.debug("fetched[%s]='%s',key='%s',diff=%f", type(fetched), fetched, key, diff) + recently = bool(diff < config.get(key)) logger.debug("recently='%s' - EXIT!", recently) return recently diff --git a/fba/utils.py b/fba/utils.py index d1cf021..ee2376b 100644 --- a/fba/utils.py +++ b/fba/utils.py @@ -120,19 +120,19 @@ def is_domain_wanted(domain: str) -> bool: elif domain.lower() != domain: wanted = False elif not validators.domain(domain.split("/")[0]): - logger.debug("domain='%s' is not a valid domain name - settings False ...", domain) + logger.debug("domain='%s' is not a valid domain name - setting False ...", domain) wanted = False elif domain.endswith(".arpa"): - logger.debug("domain='%s' is a domain for reversed IP addresses - settings False ...", domain) + logger.debug("domain='%s' is a domain for reversed IP addresses - setting False ...", domain) wanted = False elif domain.endswith(".onion"): - logger.debug("domain='%s' is a TOR .onion domain - settings False ...", domain) + logger.debug("domain='%s' is a TOR .onion domain - setting False ...", domain) wanted = False elif domain.endswith(".tld"): - logger.debug("domain='%s' is a fake domain - settings False ...", domain) + logger.debug("domain='%s' is a fake domain - setting False ...", domain) wanted = False elif blacklist.is_blacklisted(domain): - logger.debug("domain='%s' is blacklisted - settings False ...", domain) + logger.debug("domain='%s' is blacklisted - setting False ...", domain) wanted = False elif domain.find("/profile/") > 0 or domain.find("/users/") > 0 or (instances.is_registered(domain.split("/")[0]) and domain.find("/c/") > 0): logger.debug("domain='%s' is a single user", domain) -- 2.39.5