Continued:

- you can now optionally allow I2P domains being crawled (default: forbidden =
  clear-net)

diff --git a/config.defaults.json b/config.defaults.json
index 49c4742b9413d502911501370d5cbf431248d782..8b728462d490af4b2be0a8f1671ec31de1e9c2cf 100644 (file)
--- a/config.defaults.json
+++ b/config.defaults.json
@@ -26,6 +26,7 @@
     "rss_limit"               : 50,
     "api_limit"               : 500,
     "theme"                   : "light",
+    "allow_i2p_domain"        : "false",
     "instances_social_api_key": "",
     "max_crawl_depth"         : 2000,
     "min_peers_length"        : 1000

diff --git a/fba/commands.py b/fba/commands.py
index 4b41ac47d6b5e7207012e7a90fcac46666ad7284..7fc983652881fda72fd4b9d8d64620436089be6c 100644 (file)
--- a/fba/commands.py
+++ b/fba/commands.py
@@ -371,6 +371,9 @@ def fetch_blocks(args: argparse.Namespace) -> int:
             elif block["blocked"].endswith(".onion"):
                 logger.debug("blocked='%s' is a TOR .onion domain - SKIPPED", block["blocked"])
                 continue
+            elif block["blocked"].endswith(".i2p") and config.get("allow_i2p_domain"):
+                logger.debug("blocked='%s' is an I2P .onion domain - SKIPPED", block["blocked"])
+                continue
             elif block["blocked"].endswith(".arpa"):
                 logger.debug("blocked='%s' is a reverse IP address - SKIPPED", block["blocked"])
                 continue
@@ -1315,15 +1318,18 @@ def recheck_obfuscation(args: argparse.Namespace) -> int:
             if block["blocked"] == "":
                 logger.debug("block[blocked] is empty - SKIPPED!")
                 continue
+            elif block["blocked"].endswith(".onion"):
+                logger.debug("blocked='%s' is a TOR onion domain name - SKIPPED!", block["blocked"])
+                continue
+            elif block["blocked"].endswith(".i2p") and config.get("allow_i2p_domain"):
+                logger.debug("blocked='%s' is an I2P onion domain name - SKIPPED!", block["blocked"])
+                continue
             elif block["blocked"].endswith(".arpa"):
                 logger.debug("blocked='%s' is a reversed IP address - SKIPPED!", block["blocked"])
                 continue
             elif block["blocked"].endswith(".tld"):
                 logger.debug("blocked='%s' is a fake domain name - SKIPPED!", block["blocked"])
                 continue
-            elif block["blocked"].endswith(".onion"):
-                logger.debug("blocked='%s' is a TOR onion domain name - SKIPPED!", block["blocked"])
-                continue
             elif block["blocked"].find("*") >= 0 or block["blocked"].find("?") >= 0:
                 logger.debug("block='%s' is obfuscated.", block["blocked"])
                 obfuscated = obfuscated + 1

diff --git a/fba/helpers/domain.py b/fba/helpers/domain.py
index 84c9bcb3c48dc07a30aa3390408e7be061154993..d006cb1bed1f6639d564e98af9025903bdc449d3 100644 (file)
--- a/fba/helpers/domain.py
+++ b/fba/helpers/domain.py
@@ -21,6 +21,7 @@ from urllib.parse import urlparse
 import validators
 
 from fba.helpers import blacklist
+from fba.helpers import config
 
 from fba.models import instances
 
@@ -38,10 +39,12 @@ def raise_on(domain: str):
         raise ValueError(f"Parameter domain='{domain}' must be all lower-case")
     elif not validators.domain(domain.split("/")[0]):
         raise ValueError(f"domain='{domain}' is not a valid domain")
-    elif domain.endswith(".arpa"):
-        raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!")
     elif domain.endswith(".onion"):
         raise ValueError(f"domain='{domain}' is a TOR, please don't crawl them!")
+    elif domain.endswith(".i2p") and config.get("allow_i2p_domain"):
+        raise ValueError(f"domain='{domain}' is an I2P, please don't crawl them!")
+    elif domain.endswith(".arpa"):
+        raise ValueError(f"domain='{domain}' is a domain for reversed IP addresses, please don't crawl them!")
     elif domain.endswith(".tld"):
         raise ValueError(f"domain='{domain}' is a fake domain, please don't crawl them!")
 
@@ -76,6 +79,7 @@ def is_wanted(domain: str) -> bool:
 
     wanted = True
     if domain.lower() != domain:
+       logger.debug("domain='%s' is not all-lowercase - setting False ...", domain)
         wanted = False
     elif not validators.domain(domain.split("/")[0]):
         logger.debug("domain='%s' is not a valid domain name - setting False ...", domain)
@@ -86,6 +90,9 @@ def is_wanted(domain: str) -> bool:
     elif domain.endswith(".onion"):
         logger.debug("domain='%s' is a TOR .onion domain - setting False ...", domain)
         wanted = False
+    elif domain.endswith(".i2p") and config.get("allow_i2p_domain"):
+        logger.debug("domain='%s' is an I2P domain - setting False ...", domain)
+        wanted = False
     elif domain.endswith(".tld"):
         logger.debug("domain='%s' is a fake domain - setting False ...", domain)
         wanted = False

diff --git a/fba/helpers/processing.py b/fba/helpers/processing.py
index 00f6ae073d036c1d2102051e5853d4cd4e3e1a6f..dfb6388ff192f3a38e091558aadc2c6bb380ca82 100644 (file)
--- a/fba/helpers/processing.py
+++ b/fba/helpers/processing.py
@@ -199,6 +199,9 @@ def csv_block(blocker: str, url: str, command: str):
         elif domain.endswith(".onion"):
             logger.debug("domain='%s' is a TOR .onion domain - SKIPPED", domain)
             continue
+        elif domain.endswith(".i2p") and config.get("allow_i2p_domain"):
+            logger.debug("domain='%s' is an I2P .onion domain - SKIPPED", domain)
+            continue
         elif domain.endswith(".arpa"):
             logger.debug("domain='%s' is a reverse IP address - SKIPPED", domain)
             continue