# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
+import csv
import logging
+import validators
+
+from fba import database
from fba import utils
from fba.helpers import blacklist
+from fba.helpers import blocklists
+from fba.helpers import config
from fba.helpers import domain as domain_helper
+from fba.helpers import tidyup
from fba.http import federation
from fba.http import network
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
-def domain(name: str, blocker: str, command: str) -> bool:
- logger.debug("name='%s',blocker='%s',command='%s' - CALLED!", name, blocker, command)
- domain_helper.raise_on(name)
+def instance(blocked: str, blocker: str, command: str) -> bool:
+ logger.debug("blocked='%s',blocker='%s',command='%s' - CALLED!", blocked, blocker, command)
+ domain_helper.raise_on(blocked)
domain_helper.raise_on(blocker)
if not isinstance(command, str):
elif command == "":
raise ValueError("Parameter 'command' is empty")
- logger.debug("name='%s' - BEFORE!", name)
- name = utils.deobfuscate(name, blocker)
+ logger.debug("blocked='%s' - BEFORE!", blocked)
+ blocked = utils.deobfuscate(blocked, blocker)
- logger.debug("name='%s' - DEOBFUSCATED!", name)
+ logger.debug("blocked='%s' - DEOBFUSCATED!", blocked)
if instances.has_pending(blocker):
logger.debug("Flushing updates for blocker='%s' ...", blocker)
- instances.update_data(blocker)
+ instances.update(blocker)
- if not domain_helper.is_wanted(name):
- logger.debug("name='%s' is not wanted - SKIPPED!", name)
+ if not domain_helper.is_wanted(blocked):
+ logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
return False
- elif instances.is_recent(name):
- logger.debug("name='%s' has been recently checked - SKIPPED!", name)
+ elif instances.is_recent(blocked):
+ logger.debug("blocked='%s' has been recently checked - SKIPPED!", blocked)
return False
processed = False
try:
- logger.info("Fetching instances for name='%s',blocker='%s',command='%s' ...", name, blocker, command)
- federation.fetch_instances(name, blocker, None, command)
+ logger.info("Fetching instances for blocked='%s',blocker='%s',command='%s' ...", blocked, blocker, command)
+ federation.fetch_instances(blocked, blocker, None, command)
processed = True
except network.exceptions as exception:
- logger.warning("Exception '%s' during fetching instances (%s) from name='%s'", type(exception), command, name)
- instances.set_last_error(name, exception)
+ logger.warning("Exception '%s' during fetching instances (%s) from blocked='%s'", type(exception), command, blocked)
+ instances.set_last_error(blocked, exception)
- logger.debug("Checking if name='%s' has pending updates ...", name)
- if instances.has_pending(name):
- logger.debug("Flushing updates for name='%s' ...", name)
- instances.update_data(name)
+ logger.debug("Checking if blocked='%s' has pending updates ...", blocked)
+ if instances.has_pending(blocked):
+ logger.debug("Flushing updates for blocked='%s' ...", blocked)
+ instances.update(blocked)
logger.debug("processed='%s' - EXIT!", processed)
return processed
domain_helper.raise_on(blocker)
domain_helper.raise_on(blocked)
- added = False
if not isinstance(reason, str) and reason is not None:
raise ValueError(f"Parameter reason[]='{type(reason)}' is not of type 'str'")
elif not isinstance(block_level, str):
elif blacklist.is_blacklisted(blocked):
raise ValueError(f"blocked='{blocked}' is blacklisted but function was invoked")
+ added = False
if not blocks.is_instance_blocked(blocker, blocked, block_level):
logger.debug("Invoking blocks.add(%s, %s, %s, %s) ...", blocker, blocked, reason, block_level)
blocks.add(blocker, blocked, reason, block_level)
added = True
else:
- logger.debug("Updating block last seen and reason for blocker='%s',blocked='%s' ...", blocker, blocked)
+ logger.debug("Updating last_seen for blocker='%s',blocked='%s',block_level='%s' ...", blocker, blocked, block_level)
blocks.update_last_seen(blocker, blocked, block_level)
logger.debug("added='%s' - EXIT!", added)
return added
+
+def csv_block(blocker: str, url: str, command: str):
+ logger.debug("blocker='%s',url='%s',command='%s' - CALLED!", blocker, url, command)
+ domain_helper.raise_on(blocker)
+
+ if not isinstance(url, str):
+ raise ValueError(f"url[]='{url}' is not of type 'str'")
+ elif url == "":
+ raise ValueError("Parameter 'url' is empty")
+ elif not isinstance(command, str):
+ raise ValueError(f"command[]='{command}' is not of type 'str'")
+ elif command == "":
+ raise ValueError("Parameter 'command' is empty")
+
+ logger.debug("Setting last_blocked for blocker='%s' ...", blocker)
+ instances.set_last_blocked(blocker)
+
+ domains = list()
+
+ # Fetch this URL
+ logger.info("Fetching url='%s' for blocker='%s' ...", url, blocker)
+ response = utils.fetch_url(
+ url,
+ network.web_headers,
+ (config.get("connection_timeout"), config.get("read_timeout"))
+ )
+
+ logger.debug("response.ok='%s',response.status_code=%d,response.content()=%d", response.ok, response.status_code, len(response.content))
+ if not response.ok or response.status_code > 200 or response.content == "":
+ logger.warning("Could not fetch url='%s' for blocker='%s' - EXIT!", url, blocker)
+ return
+
+ logger.debug("Fetched %d Bytes, parsing CSV ...", len(response.content))
+ reader = csv.DictReader(response.content.decode("utf-8").splitlines(), dialect="unix")
+
+ blockdict = list()
+
+ cnt = 0
+ for row in reader:
+ logger.debug("row[%s]='%s'", type(row), row)
+ domain = severity = reason = None
+ reject_media = reject_reports = False
+
+ if "#domain" in row:
+ domain = tidyup.domain(row["#domain"]) if row["#domain"] != None and row["#domain"] != "" else None
+ elif "domain" in row:
+ domain = tidyup.domain(row["domain"]) if row["domain"] != None and row["domain"] != "" else None
+ elif "Domain" in row:
+ domain = tidyup.domain(row["Domain"]) if row["Domain"] != None and row["Domain"] != "" else None
+ else:
+ logger.warning("row='%s' does not contain domain column - SKIPPED!", row)
+ continue
+
+ if "#severity" in row:
+ severity = blocks.alias_block_level(row["#severity"])
+ elif "severity" in row:
+ severity = blocks.alias_block_level(row["severity"])
+ else:
+ logger.debug("row='%s' does not contain severity column, setting 'reject'", row)
+ severity = "reject"
+
+ if "reason" in row:
+ reason = tidup.reason(row["reason"]) if row["reason"] != None and row["reason"] != "" else None
+ elif "comment" in row:
+ reason = tidup.reason(row["comment"]) if row["comment"] != None and row["comment"] != "" else None
+ elif "Comment" in row:
+ reason = tidup.reason(row["Comment"]) if row["Comment"] != None and row["Comment"] != "" else None
+ else:
+ logger.debug("row='%s' has no reason/comment key provided", row)
+
+ if "#reject_media" in row and row["#reject_media"].lower() == "true":
+ reject_media = True
+ elif "reject_media" in row and row["reject_media"].lower() == "true":
+ reject_media = True
+
+ if "#reject_reports" in row and row["#reject_reports"].lower() == "true":
+ reject_reports = True
+ elif "reject_reports" in row and row["reject_reports"].lower() == "true":
+ reject_reports = True
+
+ cnt = cnt + 1
+ logger.debug("domain='%s',severity='%s',reject_media='%s',reject_reports='%s'", domain, severity, reject_media, reject_reports)
+ if domain is None or domain == "":
+ logger.debug("domain='%s' is empty - SKIPPED!", domain)
+ continue
+ elif domain.endswith(".onion"):
+ logger.debug("domain='%s' is a TOR .onion domain - SKIPPED", domain)
+ continue
+ elif domain.endswith(".arpa"):
+ logger.debug("domain='%s' is a reverse IP address - SKIPPED", domain)
+ continue
+ elif domain.endswith(".tld"):
+ logger.debug("domain='%s' is a fake domain - SKIPPED", domain)
+ continue
+ elif domain.find("*") >= 0 or domain.find("?") >= 0:
+ logger.debug("domain='%s' is obfuscated - Invoking utils.deobfuscate(%s, %s) ...", domain, domain, blocker)
+ domain = utils.deobfuscate(domain, blocker)
+ logger.debug("domain='%s' - AFTER!", domain)
+
+ if not validators.domain(domain):
+ logger.debug("domain='%s' is not a valid domain - SKIPPED!")
+ continue
+ elif blacklist.is_blacklisted(domain):
+ logger.warning("domain='%s' is blacklisted - SKIPPED!", domain)
+ continue
+ elif blocks.is_instance_blocked(blocker, domain, severity):
+ logger.debug("blocker='%s' has already blocked domain='%s' with severity='%s' - SKIPPED!", blocker, domain, severity)
+ continue
+
+ logger.debug("Marking domain='%s' as handled", domain)
+ domains.append(domain)
+
+ logger.debug("Processing domain='%s',blocker='%s',command='%s' ...", domain, blocker, command)
+ processed = instance(domain, blocker, command)
+ logger.debug("processed='%s'", processed)
+
+ if block(blocker, domain, reason, severity) and config.get("bot_enabled"):
+ logger.debug("Appending blocked='%s',reason='%s' for blocker='%s' ...", domain, reason, blocker)
+ blockdict.append({
+ "blocked": domain,
+ "reason" : reason,
+ })
+
+ if reject_media:
+ block(blocker, domain, None, "reject_media")
+ if reject_reports:
+ block(blocker, domain, None, "reject_reports")
+
+ logger.debug("blocker='%s'", blocker)
+ if not blocklists.has(blocker):
+ logger.debug("Invoking instances.set_total_blocks(%s, domains()=%d) ...", blocker, len(domains))
+ instances.set_total_blocks(blocker, domains)
+
+ logger.debug("Checking if blocker='%s' has pending updates ...", blocker)
+ if instances.has_pending(blocker):
+ logger.debug("Flushing updates for blocker='%s' ...", blocker)
+ instances.update(blocker)
+
+ logger.debug("Invoking commit() ...")
+ database.connection.commit()
+
+ logger.debug("config.get(bot_enabled)='%s',blockdict()=%d", config.get("bot_enabled"), len(blockdict))
+ if config.get("bot_enabled") and len(blockdict) > 0:
+ logger.info("Sending bot POST for blocker='%s',blockdict()=%d ...", blocker, len(blockdict))
+ network.send_bot_post(blocker, blockdict)
+
+ logger.debug("EXIT!")
projects / fba.git / blobdiff