Continued:

- clear cookies after federation.fetch_instances() has finished or failed
  its job
- logger messages a bit rewritten

diff --git a/fba/commands.py b/fba/commands.py
index 47a4c5ac405ee83ef235a5e7f7b1fb4afd72f80c..a4bccbc1f8bcdad4e610d21013ac06c2602b65a6 100644 (file)
--- a/fba/commands.py
+++ b/fba/commands.py
@@ -180,9 +180,6 @@ def fetch_bkali(args: argparse.Namespace) -> int:
             try:
                 logger.info("Fetching instances from domain='%s' ...", domain)
                 federation.fetch_instances(domain, 'tak.teleyal.blog', None, inspect.currentframe().f_code.co_name)
-
-                logger.debug("Invoking cookies.clear(%s) ...", domain)
-                cookies.clear(domain)
             except network.exceptions as exception:
                 logger.warning("Exception '%s' during fetching instances (fetch_bkali) from domain='%s'", type(exception), domain)
                 instances.set_last_error(domain, exception)
@@ -349,8 +346,9 @@ def fetch_blocks(args: argparse.Namespace) -> int:
             logger.debug("Invoking cookies.clear(%s) ...", block["blocked"])
             cookies.clear(block["blocked"])
 
+        logger.debug("Checking if blocker='%s' has pending updates ...", blocker)
         if instances.has_pending(blocker):
-            logger.debug("Invoking instances.update_data(%s) ...", blocker)
+            logger.debug("Flushing updates for blocker='%s' ...", blocker)
             instances.update_data(blocker)
 
         logger.debug("Invoking commit() ...")
@@ -436,13 +434,13 @@ def fetch_observer(args: argparse.Namespace) -> int:
             elif instances.is_registered(domain):
                 logger.debug("domain='%s' is already registered - SKIPPED!", domain)
                 continue
+            elif instance.is_recent(domain):
+                logger.debug("domain='%s' is recently being handled - SKIPPED!", domain)
+                continue
 
             logger.info("Fetching instances for domain='%s',software='%s'", domain, software)
             federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
 
-            logger.debug("Invoking cookies.clear(%s) ...", domain)
-            cookies.clear(domain)
-
     logger.debug("Success! - EXIT!")
     return 0
 
@@ -480,9 +478,6 @@ def fetch_todon_wiki(args: argparse.Namespace) -> int:
                 try:
                     logger.info("Fetching instances from domain='%s' ...", blocked)
                     federation.fetch_instances(blocked, 'chaos.social', None, inspect.currentframe().f_code.co_name)
-
-                    logger.debug("Invoking cookies.clear(%s) ...", blocked)
-                    cookies.clear(blocked)
                 except network.exceptions as exception:
                     logger.warning("Exception '%s' during fetching instances (fetch_cs) from blocked='%s'", type(exception), blocked)
                     instances.set_last_error(blocked, exception)
@@ -554,9 +549,6 @@ def fetch_cs(args: argparse.Namespace):
                     try:
                         logger.info("Fetching instances from domain='%s' ...", row["domain"])
                         federation.fetch_instances(row["domain"], 'chaos.social', None, inspect.currentframe().f_code.co_name)
-
-                        logger.debug("Invoking cookies.clear(%s) ...", row["domain"])
-                        cookies.clear(row["domain"])
                     except network.exceptions as exception:
                         logger.warning("Exception '%s' during fetching instances (fetch_cs) from row[domain]='%s'", type(exception), row["domain"])
                         instances.set_last_error(row["domain"], exception)
@@ -610,9 +602,6 @@ def fetch_fba_rss(args: argparse.Namespace) -> int:
             try:
                 logger.info("Fetching instances from domain='%s' ...", domain)
                 federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
-
-                logger.debug("Invoking cookies.clear(%s) ...", domain)
-                cookies.clear(domain)
             except network.exceptions as exception:
                 logger.warning("Exception '%s' during fetching instances (fetch_fba_rss) from domain='%s'", type(exception), domain)
                 instances.set_last_error(domain, exception)
@@ -667,9 +656,6 @@ def fetch_fbabot_atom(args: argparse.Namespace) -> int:
             try:
                 logger.info("Fetching instances from domain='%s' ...", domain)
                 federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
-
-                logger.debug("Invoking cookies.clear(%s) ...", domain)
-                cookies.clear(domain)
             except network.exceptions as exception:
                 logger.warning("Exception '%s' during fetching instances (fetch_fbabot_atom) from domain='%s'", type(exception), domain)
                 instances.set_last_error(domain, exception)
@@ -685,9 +671,6 @@ def fetch_instances(args: argparse.Namespace) -> int:
     try:
         logger.info("Fetching instances from args.domain='%s' ...", args.domain)
         federation.fetch_instances(args.domain, None, None, inspect.currentframe().f_code.co_name)
-
-        logger.debug("Invoking cookies.clear(%s) ...", args.domain)
-        cookies.clear(args.domain)
     except network.exceptions as exception:
         logger.warning("Exception '%s' during fetching instances (fetch_instances) from args.domain='%s'", type(exception), args.domain)
         instances.set_last_error(args.domain, exception)
@@ -706,16 +689,13 @@ def fetch_instances(args: argparse.Namespace) -> int:
     logger.info("Checking %d entries ...", len(rows))
     for row in rows:
         logger.debug("domain='%s'", row[0])
-        if blacklist.is_blacklisted(row[0]):
-            logger.warning("Domain is blacklisted: row[0]='%s'", row[0])
+        if not utils.is_domain_wanted(row[0]):
+            logger.debug("Domain is not wanted: row[0]='%s'", row[0])
             continue
 
         try:
             logger.info("Fetching instances for domain='%s',software='%s',origin='%s',nodeinfo_url='%s'", row[0], row[2], row[1], row[3])
             federation.fetch_instances(row[0], row[1], row[2], inspect.currentframe().f_code.co_name, row[3])
-
-            logger.debug("Invoking cookies.clear(%s) ...", row[0])
-            cookies.clear(row[0])
         except network.exceptions as exception:
             logger.warning("Exception '%s' during fetching instances (fetch_instances) from row[0]='%s'", type(exception), row[0])
             instances.set_last_error(row[0], exception)

diff --git a/fba/http/federation.py b/fba/http/federation.py
index 84a007105a79201f0b6bcdcc11e2d5b0ffcfcfcf..0dcf1807b7dc9ff333f6f52b88fe46ed496f51d2 100644 (file)
--- a/fba/http/federation.py
+++ b/fba/http/federation.py
@@ -24,6 +24,7 @@ from fba import csrf
 from fba import utils
 
 from fba.helpers import config
+from fba.helpers import cookies
 from fba.helpers import domain as domain_helper
 from fba.helpers import tidyup
 from fba.helpers import version
@@ -62,9 +63,6 @@ def fetch_instances(domain: str, origin: str, software: str, command: str, path:
     elif command == "":
         raise ValueError("Parameter 'command' is empty")
     elif software is None:
-        logger.debug("Updating last_instance_fetch for domain='%s' ...", domain)
-        instances.set_last_instance_fetch(domain)
-
         try:
             logger.debug("Software for domain='%s' is not set, determining ...", domain)
             software = determine_software(domain, path)
@@ -91,12 +89,19 @@ def fetch_instances(domain: str, origin: str, software: str, command: str, path:
         logger.debug("Invoking instances.set_total_peerlist(%s,%d) ...", domain, len(peerlist))
         instances.set_total_peers(domain, peerlist)
 
+    logger.debug("Checking if domain='%s' has pending updates ...", domain)
     if instances.has_pending(domain):
-        logger.debug("domain='%s' has pending nodeinfo data, flushing ...", domain)
+        logger.debug("Flushing updates for domain='%s' ...", domain)
         instances.update_data(domain)
 
+    logger.debug("peerlist[]='%s'", type(peerlist))
     if peerlist is None:
         logger.warning("Cannot fetch peers: domain='%s'", domain)
+
+        logger.debug("Invoking cookies.clear(%s) ...", domain)
+        cookies.clear(domain)
+
+        logger.debug("EXIT!")
         return
 
     logger.info("Checking %d instance(s) from domain='%s',software='%s' ...", len(peerlist), domain, software)
@@ -123,6 +128,9 @@ def fetch_instances(domain: str, origin: str, software: str, command: str, path:
             logger.debug("Adding new instance='%s',domain='%s',command='%s'", instance, domain, command)
             instances.add(instance, domain, command)
 
+    logger.debug("Invoking cookies.clear(%s) ...", domain)
+    cookies.clear(domain)
+
     logger.debug("EXIT!")
 
 def fetch_peers(domain: str, software: str) -> list:

diff --git a/fba/models/instances.py b/fba/models/instances.py
index 8c977fe51e5fa9ece7d2c5264fabd21fe5ad2b22..8746d2376419745429c6aa67ba216c2ed8358ec9 100644 (file)
--- a/fba/models/instances.py
+++ b/fba/models/instances.py
@@ -203,8 +203,9 @@ def add(domain: str, origin: str, command: str, path: str = None, software: str
     logger.debug("Marking domain='%s' as registered.", domain)
     cache.set_sub_key("is_registered", domain, True)
 
+    logger.debug("Checking if domain='%s' has pending updates ...", domain)
     if has_pending(domain):
-        logger.debug("domain='%s' has pending nodeinfo being updated ...", domain)
+        logger.debug("Flushing updates for domain='%s' ...", domain)
         update_data(domain)
 
     logger.debug("EXIT!")

diff --git a/fba/utils.py b/fba/utils.py
index 6f12fb75a0a554a8992e3db31d5810f676d88eff..4cc5f18b3d0d51759b711906614a09ff51684c9f 100644 (file)
--- a/fba/utils.py
+++ b/fba/utils.py
@@ -23,7 +23,6 @@ import requests
 import validators
 
 from fba.helpers import blacklist
-from fba.helpers import cookies
 from fba.helpers import domain as domain_helper
 from fba.helpers import tidyup
 
@@ -116,7 +115,7 @@ def process_domain(domain: str, blocker: str, command: str) -> bool:
 
     logger.debug("domain='%s' - DEOBFUSCATED!", domain)
     if instances.has_pending(blocker):
-        logger.debug("Invoking instances.update_data(%s) ...", blocker)
+        logger.debug("Flushing updates for blocker='%s' ...", blocker)
         instances.update_data(blocker)
 
     if not is_domain_wanted(domain):
@@ -131,15 +130,13 @@ def process_domain(domain: str, blocker: str, command: str) -> bool:
         logger.info("Fetching instances for domain='%s',blocker='%s',command='%s' ...", domain, blocker, command)
         federation.fetch_instances(domain, blocker, None, command)
         processed = True
-
-        logger.debug("Invoking cookies.clear(%s) ...", domain)
-        cookies.clear(domain)
     except network.exceptions as exception:
         logger.warning("Exception '%s' during fetching instances (fetch_oliphant) from domain='%s'", type(exception), domain)
         instances.set_last_error(domain, exception)
 
+    logger.debug("Checking if domain='%s' has pending updates ...")
     if instances.has_pending(domain):
-        logger.debug("domain='%s' has pending data updates")
+        logger.debug("Flushing updates for domain='%s' ...")
         instances.update_data(domain)
 
     logger.debug("processed='%s' - EXIT!", processed)