Continued:

- introduced new module "processing"
- renamed process_*() to *() ;-)

diff --git a/fba/commands.py b/fba/commands.py
index 12caa75ef01ded30fd793231ee898cde3bff0739..ffb4ae736ca6d7009c3a4c94efe0c35ca3e8969a 100644 (file)
--- a/fba/commands.py
+++ b/fba/commands.py
@@ -37,6 +37,7 @@ from fba.helpers import blacklist
 from fba.helpers import config
 from fba.helpers import cookies
 from fba.helpers import locking
+from fba.helpers import processing
 from fba.helpers import software as software_helper
 from fba.helpers import tidyup
 
@@ -417,7 +418,7 @@ def fetch_blocks(args: argparse.Namespace) -> int:
 
             block["block_level"] = utils.alias_block_level(block["block_level"])
 
-            if utils.process_block(blocker, block["blocked"], block["reason"], block["block_level"]) and block["block_level"] == "reject" and config.get("bot_enabled"):
+            if processing.block(blocker, block["blocked"], block["reason"], block["block_level"]) and block["block_level"] == "reject" and config.get("bot_enabled"):
                 logger.debug("Appending blocked='%s',reason='%s' for blocker='%s' ...", block["blocked"], block["block_level"], blocker)
                 blockdict.append({
                     "blocked": block["blocked"],
@@ -604,7 +605,7 @@ def fetch_todon_wiki(args: argparse.Namespace) -> int:
                 continue
 
             logger.info("Adding new block: blocked='%s',block_level='%s'", blocked, block_level)
-            if utils.process_block(blocker, blocked, None, block_level) and block_level == "reject" and config.get("bot_enabled"):
+            if processing.block(blocker, blocked, None, block_level) and block_level == "reject" and config.get("bot_enabled"):
                 logger.debug("Appending blocked='%s',reason='%s' for blocker='%s' ...", blocked, block_level, blocker)
                 blockdict.append({
                     "blocked": blocked,
@@ -708,7 +709,7 @@ def fetch_cs(args: argparse.Namespace):
                         logger.warning("Exception '%s' during fetching instances (fetch_cs) from row[domain]='%s'", type(exception), row["domain"])
                         instances.set_last_error(row["domain"], exception)
 
-                if utils.process_block(blocker, row["domain"], row["reason"], block_level) and block_level == "reject" and config.get("bot_enabled"):
+                if processing.block(blocker, row["domain"], row["reason"], block_level) and block_level == "reject" and config.get("bot_enabled"):
                     logger.debug("Appending blocked='%s',reason='%s' for blocker='%s' ...", row["domain"], block_level, blocker)
                     blockdict.append({
                         "blocked": row["domain"],
@@ -1091,10 +1092,10 @@ def fetch_oliphant(args: argparse.Namespace) -> int:
             domains.append(domain)
 
             logger.debug("Processing domain='%s' ...", domain)
-            processed = utils.process_domain(domain, block["blocker"], inspect.currentframe().f_code.co_name)
+            processed = processing.domain(domain, block["blocker"], inspect.currentframe().f_code.co_name)
             logger.debug("processed='%s'", processed)
 
-            if utils.process_block(block["blocker"], domain, None, severity) and config.get("bot_enabled"):
+            if processing.block(block["blocker"], domain, None, severity) and config.get("bot_enabled"):
                 logger.debug("Appending blocked='%s',reason='%s' for blocker='%s' ...", domain, block["block_level"], block["blocker"])
                 blockdict.append({
                     "blocked": domain,
@@ -1102,9 +1103,9 @@ def fetch_oliphant(args: argparse.Namespace) -> int:
                 })
 
             if reject_media:
-                utils.process_block(block["blocker"], domain, None, "reject_media")
+                processing.block(block["blocker"], domain, None, "reject_media")
             if reject_reports:
-                utils.process_block(block["blocker"], domain, None, "reject_reports")
+                processing.block(block["blocker"], domain, None, "reject_reports")
 
         logger.debug("block[blocker]='%s'", block["blocker"])
         if block["blocker"] != "chaos.social":
@@ -1166,7 +1167,7 @@ def fetch_txt(args: argparse.Namespace) -> int:
                     continue
 
                 logger.debug("Processing domain='%s',row[blocker]='%s'", domain, row["blocker"])
-                processed = utils.process_domain(domain, row["blocker"], inspect.currentframe().f_code.co_name)
+                processed = processing.domain(domain, row["blocker"], inspect.currentframe().f_code.co_name)
 
                 logger.debug("processed='%s'", processed)
                 if not processed:
@@ -1366,7 +1367,7 @@ def fetch_joinfediverse(args: argparse.Namespace) -> int:
             continue
 
         logger.info("Proccessing blocked='%s' ...", block["blocked"])
-        utils.process_domain(block["blocked"], "climatejustice.social", inspect.currentframe().f_code.co_name)
+        processing.domain(block["blocked"], "climatejustice.social", inspect.currentframe().f_code.co_name)
 
     blockdict = list()
     for blocker in domains:
@@ -1386,7 +1387,7 @@ def fetch_joinfediverse(args: argparse.Namespace) -> int:
                 continue
 
             logger.debug("blocked='%s',reason='%s'", block["blocked"], block["reason"])
-            if utils.process_block(blocker, block["blocked"], block["reason"], "reject") and config.get("bot_enabled"):
+            if processing.block(blocker, block["blocked"], block["reason"], "reject") and config.get("bot_enabled"):
                 logger.debug("Appending blocked='%s',reason='%s' for blocker='%s' ...", block["blocked"], block["block_level"], blocker)
                 blockdict.append({
                     "blocked": block["blocked"],
@@ -1494,7 +1495,7 @@ def recheck_obfuscation(args: argparse.Namespace) -> int:
                 block["block_level"] = utils.alias_block_level(block["block_level"])
 
                 logger.info("blocked='%s' has been deobfuscated to blocked='%s', adding ...", block["blocked"], blocked)
-                if utils.process_block(row["domain"], blocked, block["reason"], block["block_level"]) and block["block_level"] == "reject" and config.get("bot_enabled"):
+                if processing.block(row["domain"], blocked, block["reason"], block["block_level"]) and block["block_level"] == "reject" and config.get("bot_enabled"):
                     logger.debug("Appending blocked='%s',reason='%s' for blocker='%s' ...", block["blocked"], block["block_level"], row["domain"])
                     blockdict.append({
                         "blocked": blocked,

diff --git a/fba/helpers/__init__.py b/fba/helpers/__init__.py
index 4e68780b6f9d824ab6927f130cb99dac117861bf..ce9f5f8b1a8b30a6cbcaa6caff070fc3010de33a 100644 (file)
--- a/fba/helpers/__init__.py
+++ b/fba/helpers/__init__.py
@@ -22,6 +22,7 @@ __all__ = [
     'domain',
     'json',
     'locking',
+    'processing',
     'software',
     'tidyup',
     'version',

diff --git a/fba/helpers/processing.py b/fba/helpers/processing.py
new file mode 100644 (file)
index 0000000..426ee26
--- /dev/null
+++ b/fba/helpers/processing.py
@@ -0,0 +1,93 @@
+       # Copyright (C) 2023 Free Software Foundation
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import logging
+
+from fba.helpers import domain as domain_helper
+
+from fba.http import federation
+from fba.http import network
+
+from fba.models import blocks
+from fba.models import instances
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+def domain(domain: str, blocker: str, command: str) -> bool:
+    logger.debug("domain='%s',blocker='%s',command='%s' - CALLED!", domain, blocker, command)
+    domain_helper.raise_on(domain)
+    domain_helper.raise_on(blocker)
+
+    if not isinstance(command, str):
+        raise ValueError(f"Parameter command[]='{type(command)}' is not of type 'str'")
+    elif command == "":
+        raise ValueError("Parameter 'command' is empty")
+
+    logger.debug("domain='%s' - BEFORE!", domain)
+    domain = deobfuscate(domain, blocker)
+
+    logger.debug("domain='%s' - DEOBFUSCATED!", domain)
+    if instances.has_pending(blocker):
+        logger.debug("Flushing updates for blocker='%s' ...", blocker)
+        instances.update_data(blocker)
+
+    if not is_domain_wanted(domain):
+        logger.debug("domain='%s' is not wanted - SKIPPED!", domain)
+        return False
+    elif instances.is_recent(domain):
+        logger.debug("domain='%s' has been recently checked - SKIPPED!", domain)
+        return False
+
+    processed = False
+    try:
+        logger.info("Fetching instances for domain='%s',blocker='%s',command='%s' ...", domain, blocker, command)
+        federation.fetch_instances(domain, blocker, None, command)
+        processed = True
+    except network.exceptions as exception:
+        logger.warning("Exception '%s' during fetching instances (%s) from domain='%s'", type(exception), command, domain)
+        instances.set_last_error(domain, exception)
+
+    logger.debug("Checking if domain='%s' has pending updates ...", domain)
+    if instances.has_pending(domain):
+        logger.debug("Flushing updates for domain='%s' ...", domain)
+        instances.update_data(domain)
+
+    logger.debug("processed='%s' - EXIT!", processed)
+    return processed
+
+def block(blocker: str, blocked: str, reason: str, block_level: str) -> bool:
+    logger.debug("blocker='%s',blocked='%s',reason='%s',block_level='%s' - CALLED!", blocker, blocked, reason, block_level)
+    domain_helper.raise_on(blocker)
+    domain_helper.raise_on(blocked)
+
+    added = False
+    if not isinstance(reason, str) and reason is not None:
+        raise ValueError(f"Parameter reason[]='{type(reason)}' is not of type 'str'")
+    elif not isinstance(block_level, str):
+        raise ValueError(f"Parameter block_level[]='{type(block_level)}' is not of type 'str'")
+    elif block_level == "":
+        raise ValueError("Parameter block_level is empty")
+
+    if not blocks.is_instance_blocked(blocker, blocked, block_level):
+        logger.debug("Invoking blocks.add_instance(%s, %s, %s, %s) ...", blocker, blocked, reason, block_level)
+        blocks.add_instance(blocker, blocked, reason, block_level)
+        added = True
+    else:
+        logger.debug("Updating block last seen and reason for blocker='%s',blocked='%s' ...", blocker, blocked)
+        blocks.update_last_seen(blocker, blocked, block_level)
+
+    logger.debug("added='%s' - EXIT!", added)
+    return added

diff --git a/fba/http/network.py b/fba/http/network.py
index 8553afaa8f73425a07df6cd69690e7edeb618334..390ba9ff1eb38144d05b0f74b656f8e622e2d2b4 100644 (file)
--- a/fba/http/network.py
+++ b/fba/http/network.py
@@ -16,7 +16,6 @@
 
 import logging
 
-import json
 import reqto
 import requests
 import urllib3

diff --git a/fba/utils.py b/fba/utils.py
index 756dc5bf18428b69950afef719fd319058937b30..56a004364ca86177e568d7376566aaeb0d4dfb58 100644 (file)
--- a/fba/utils.py
+++ b/fba/utils.py
@@ -26,10 +26,8 @@ from fba.helpers import blacklist
 from fba.helpers import domain as domain_helper
 from fba.helpers import tidyup
 
-from fba.http import federation
 from fba.http import network
 
-from fba.models import blocks
 from fba.models import instances
 
 logging.basicConfig(level=logging.INFO)
@@ -71,48 +69,6 @@ def fetch_url(url: str, headers: dict, timeout: tuple) -> requests.models.Respon
     logger.debug("response[]='%s' - EXIT!", type(response))
     return response
 
-def process_domain(domain: str, blocker: str, command: str) -> bool:
-    logger.debug("domain='%s',blocker='%s',command='%s' - CALLED!", domain, blocker, command)
-    domain_helper.raise_on(domain)
-    domain_helper.raise_on(blocker)
-
-    if not isinstance(command, str):
-        raise ValueError(f"Parameter command[]='{type(command)}' is not of type 'str'")
-    elif command == "":
-        raise ValueError("Parameter 'command' is empty")
-
-    logger.debug("domain='%s' - BEFORE!", domain)
-    domain = deobfuscate(domain, blocker)
-
-    logger.debug("domain='%s' - DEOBFUSCATED!", domain)
-    if instances.has_pending(blocker):
-        logger.debug("Flushing updates for blocker='%s' ...", blocker)
-        instances.update_data(blocker)
-
-    if not is_domain_wanted(domain):
-        logger.debug("domain='%s' is not wanted - SKIPPED!", domain)
-        return False
-    elif instances.is_recent(domain):
-        logger.debug("domain='%s' has been recently checked - SKIPPED!", domain)
-        return False
-
-    processed = False
-    try:
-        logger.info("Fetching instances for domain='%s',blocker='%s',command='%s' ...", domain, blocker, command)
-        federation.fetch_instances(domain, blocker, None, command)
-        processed = True
-    except network.exceptions as exception:
-        logger.warning("Exception '%s' during fetching instances (%s) from domain='%s'", type(exception), command, domain)
-        instances.set_last_error(domain, exception)
-
-    logger.debug("Checking if domain='%s' has pending updates ...", domain)
-    if instances.has_pending(domain):
-        logger.debug("Flushing updates for domain='%s' ...", domain)
-        instances.update_data(domain)
-
-    logger.debug("processed='%s' - EXIT!", processed)
-    return processed
-
 def find_domains(tags: bs4.element.ResultSet, search: str) -> list:
     logger.debug("tags[%s]()=%d,search='%s' - CALLED!", type(tags), len(tags), search)
     if not isinstance(tags, bs4.element.ResultSet):
@@ -230,30 +186,6 @@ def deobfuscate(domain: str, blocker: str, domain_hash: str = None) -> str:
     logger.debug("domain='%s' - EXIT!", domain)
     return domain
 
-def process_block(blocker: str, blocked: str, reason: str, block_level: str) -> bool:
-    logger.debug("blocker='%s',blocked='%s',reason='%s',block_level='%s' - CALLED!", blocker, blocked, reason, block_level)
-    domain_helper.raise_on(blocker)
-    domain_helper.raise_on(blocked)
-
-    added = False
-    if not isinstance(reason, str) and reason is not None:
-        raise ValueError(f"Parameter reason[]='{type(reason)}' is not of type 'str'")
-    elif not isinstance(block_level, str):
-        raise ValueError(f"Parameter block_level[]='{type(block_level)}' is not of type 'str'")
-    elif block_level == "":
-        raise ValueError("Parameter block_level is empty")
-
-    if not blocks.is_instance_blocked(blocker, blocked, block_level):
-        logger.debug("Invoking blocks.add_instance(%s, %s, %s, %s) ...", blocker, blocked, reason, block_level)
-        blocks.add_instance(blocker, blocked, reason, block_level)
-        added = True
-    else:
-        logger.debug("Updating block last seen and reason for blocker='%s',blocked='%s' ...", blocker, blocked)
-        blocks.update_last_seen(blocker, blocked, block_level)
-
-    logger.debug("added='%s' - EXIT!", added)
-    return added
-
 def alias_block_level(block_level: str) -> str:
     logger.debug("block_level='%s' - CALLED!", block_level)
     if not isinstance(block_level, str):