From e11b64e32d745c2bee579e7dd10a6f3d0fcec80d Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Sat, 24 Jun 2023 23:27:46 +0200 Subject: [PATCH] Continued: - clear cookies after federation.fetch_instances() has finished or failed its job - logger messages a bit rewritten --- fba/commands.py | 34 +++++++--------------------------- fba/http/federation.py | 16 ++++++++++++---- fba/models/instances.py | 3 ++- fba/utils.py | 9 +++------ 4 files changed, 24 insertions(+), 38 deletions(-) diff --git a/fba/commands.py b/fba/commands.py index 47a4c5a..a4bccbc 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -180,9 +180,6 @@ def fetch_bkali(args: argparse.Namespace) -> int: try: logger.info("Fetching instances from domain='%s' ...", domain) federation.fetch_instances(domain, 'tak.teleyal.blog', None, inspect.currentframe().f_code.co_name) - - logger.debug("Invoking cookies.clear(%s) ...", domain) - cookies.clear(domain) except network.exceptions as exception: logger.warning("Exception '%s' during fetching instances (fetch_bkali) from domain='%s'", type(exception), domain) instances.set_last_error(domain, exception) @@ -349,8 +346,9 @@ def fetch_blocks(args: argparse.Namespace) -> int: logger.debug("Invoking cookies.clear(%s) ...", block["blocked"]) cookies.clear(block["blocked"]) + logger.debug("Checking if blocker='%s' has pending updates ...", blocker) if instances.has_pending(blocker): - logger.debug("Invoking instances.update_data(%s) ...", blocker) + logger.debug("Flushing updates for blocker='%s' ...", blocker) instances.update_data(blocker) logger.debug("Invoking commit() ...") @@ -436,13 +434,13 @@ def fetch_observer(args: argparse.Namespace) -> int: elif instances.is_registered(domain): logger.debug("domain='%s' is already registered - SKIPPED!", domain) continue + elif instance.is_recent(domain): + logger.debug("domain='%s' is recently being handled - SKIPPED!", domain) + continue logger.info("Fetching instances for domain='%s',software='%s'", domain, software) federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) - logger.debug("Invoking cookies.clear(%s) ...", domain) - cookies.clear(domain) - logger.debug("Success! - EXIT!") return 0 @@ -480,9 +478,6 @@ def fetch_todon_wiki(args: argparse.Namespace) -> int: try: logger.info("Fetching instances from domain='%s' ...", blocked) federation.fetch_instances(blocked, 'chaos.social', None, inspect.currentframe().f_code.co_name) - - logger.debug("Invoking cookies.clear(%s) ...", blocked) - cookies.clear(blocked) except network.exceptions as exception: logger.warning("Exception '%s' during fetching instances (fetch_cs) from blocked='%s'", type(exception), blocked) instances.set_last_error(blocked, exception) @@ -554,9 +549,6 @@ def fetch_cs(args: argparse.Namespace): try: logger.info("Fetching instances from domain='%s' ...", row["domain"]) federation.fetch_instances(row["domain"], 'chaos.social', None, inspect.currentframe().f_code.co_name) - - logger.debug("Invoking cookies.clear(%s) ...", row["domain"]) - cookies.clear(row["domain"]) except network.exceptions as exception: logger.warning("Exception '%s' during fetching instances (fetch_cs) from row[domain]='%s'", type(exception), row["domain"]) instances.set_last_error(row["domain"], exception) @@ -610,9 +602,6 @@ def fetch_fba_rss(args: argparse.Namespace) -> int: try: logger.info("Fetching instances from domain='%s' ...", domain) federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) - - logger.debug("Invoking cookies.clear(%s) ...", domain) - cookies.clear(domain) except network.exceptions as exception: logger.warning("Exception '%s' during fetching instances (fetch_fba_rss) from domain='%s'", type(exception), domain) instances.set_last_error(domain, exception) @@ -667,9 +656,6 @@ def fetch_fbabot_atom(args: argparse.Namespace) -> int: try: logger.info("Fetching instances from domain='%s' ...", domain) federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) - - logger.debug("Invoking cookies.clear(%s) ...", domain) - cookies.clear(domain) except network.exceptions as exception: logger.warning("Exception '%s' during fetching instances (fetch_fbabot_atom) from domain='%s'", type(exception), domain) instances.set_last_error(domain, exception) @@ -685,9 +671,6 @@ def fetch_instances(args: argparse.Namespace) -> int: try: logger.info("Fetching instances from args.domain='%s' ...", args.domain) federation.fetch_instances(args.domain, None, None, inspect.currentframe().f_code.co_name) - - logger.debug("Invoking cookies.clear(%s) ...", args.domain) - cookies.clear(args.domain) except network.exceptions as exception: logger.warning("Exception '%s' during fetching instances (fetch_instances) from args.domain='%s'", type(exception), args.domain) instances.set_last_error(args.domain, exception) @@ -706,16 +689,13 @@ def fetch_instances(args: argparse.Namespace) -> int: logger.info("Checking %d entries ...", len(rows)) for row in rows: logger.debug("domain='%s'", row[0]) - if blacklist.is_blacklisted(row[0]): - logger.warning("Domain is blacklisted: row[0]='%s'", row[0]) + if not utils.is_domain_wanted(row[0]): + logger.debug("Domain is not wanted: row[0]='%s'", row[0]) continue try: logger.info("Fetching instances for domain='%s',software='%s',origin='%s',nodeinfo_url='%s'", row[0], row[2], row[1], row[3]) federation.fetch_instances(row[0], row[1], row[2], inspect.currentframe().f_code.co_name, row[3]) - - logger.debug("Invoking cookies.clear(%s) ...", row[0]) - cookies.clear(row[0]) except network.exceptions as exception: logger.warning("Exception '%s' during fetching instances (fetch_instances) from row[0]='%s'", type(exception), row[0]) instances.set_last_error(row[0], exception) diff --git a/fba/http/federation.py b/fba/http/federation.py index 84a0071..0dcf180 100644 --- a/fba/http/federation.py +++ b/fba/http/federation.py @@ -24,6 +24,7 @@ from fba import csrf from fba import utils from fba.helpers import config +from fba.helpers import cookies from fba.helpers import domain as domain_helper from fba.helpers import tidyup from fba.helpers import version @@ -62,9 +63,6 @@ def fetch_instances(domain: str, origin: str, software: str, command: str, path: elif command == "": raise ValueError("Parameter 'command' is empty") elif software is None: - logger.debug("Updating last_instance_fetch for domain='%s' ...", domain) - instances.set_last_instance_fetch(domain) - try: logger.debug("Software for domain='%s' is not set, determining ...", domain) software = determine_software(domain, path) @@ -91,12 +89,19 @@ def fetch_instances(domain: str, origin: str, software: str, command: str, path: logger.debug("Invoking instances.set_total_peerlist(%s,%d) ...", domain, len(peerlist)) instances.set_total_peers(domain, peerlist) + logger.debug("Checking if domain='%s' has pending updates ...", domain) if instances.has_pending(domain): - logger.debug("domain='%s' has pending nodeinfo data, flushing ...", domain) + logger.debug("Flushing updates for domain='%s' ...", domain) instances.update_data(domain) + logger.debug("peerlist[]='%s'", type(peerlist)) if peerlist is None: logger.warning("Cannot fetch peers: domain='%s'", domain) + + logger.debug("Invoking cookies.clear(%s) ...", domain) + cookies.clear(domain) + + logger.debug("EXIT!") return logger.info("Checking %d instance(s) from domain='%s',software='%s' ...", len(peerlist), domain, software) @@ -123,6 +128,9 @@ def fetch_instances(domain: str, origin: str, software: str, command: str, path: logger.debug("Adding new instance='%s',domain='%s',command='%s'", instance, domain, command) instances.add(instance, domain, command) + logger.debug("Invoking cookies.clear(%s) ...", domain) + cookies.clear(domain) + logger.debug("EXIT!") def fetch_peers(domain: str, software: str) -> list: diff --git a/fba/models/instances.py b/fba/models/instances.py index 8c977fe..8746d23 100644 --- a/fba/models/instances.py +++ b/fba/models/instances.py @@ -203,8 +203,9 @@ def add(domain: str, origin: str, command: str, path: str = None, software: str logger.debug("Marking domain='%s' as registered.", domain) cache.set_sub_key("is_registered", domain, True) + logger.debug("Checking if domain='%s' has pending updates ...", domain) if has_pending(domain): - logger.debug("domain='%s' has pending nodeinfo being updated ...", domain) + logger.debug("Flushing updates for domain='%s' ...", domain) update_data(domain) logger.debug("EXIT!") diff --git a/fba/utils.py b/fba/utils.py index 6f12fb7..4cc5f18 100644 --- a/fba/utils.py +++ b/fba/utils.py @@ -23,7 +23,6 @@ import requests import validators from fba.helpers import blacklist -from fba.helpers import cookies from fba.helpers import domain as domain_helper from fba.helpers import tidyup @@ -116,7 +115,7 @@ def process_domain(domain: str, blocker: str, command: str) -> bool: logger.debug("domain='%s' - DEOBFUSCATED!", domain) if instances.has_pending(blocker): - logger.debug("Invoking instances.update_data(%s) ...", blocker) + logger.debug("Flushing updates for blocker='%s' ...", blocker) instances.update_data(blocker) if not is_domain_wanted(domain): @@ -131,15 +130,13 @@ def process_domain(domain: str, blocker: str, command: str) -> bool: logger.info("Fetching instances for domain='%s',blocker='%s',command='%s' ...", domain, blocker, command) federation.fetch_instances(domain, blocker, None, command) processed = True - - logger.debug("Invoking cookies.clear(%s) ...", domain) - cookies.clear(domain) except network.exceptions as exception: logger.warning("Exception '%s' during fetching instances (fetch_oliphant) from domain='%s'", type(exception), domain) instances.set_last_error(domain, exception) + logger.debug("Checking if domain='%s' has pending updates ...") if instances.has_pending(domain): - logger.debug("domain='%s' has pending data updates") + logger.debug("Flushing updates for domain='%s' ...") instances.update_data(domain) logger.debug("processed='%s' - EXIT!", processed) -- 2.39.5