From 61df799036bafa3cf39d1fab39d7ad5110e67c12 Mon Sep 17 00:00:00 2001 From: Brian Olsen Date: Tue, 15 May 2018 04:17:18 +0200 Subject: [PATCH] dovecot: Add spam filter traning using imapsieve --- mail-server/dovecot.nix | 47 +++++++++++++ .../dovecot/imap_sieve/report-ham.sieve | 15 ++++ .../dovecot/imap_sieve/report-spam.sieve | 7 ++ mail-server/dovecot/pipe_bin/sa-learn-ham.sh | 3 + mail-server/dovecot/pipe_bin/sa-learn-spam.sh | 3 + mail-server/rspamd.nix | 10 +++ tests/extern.nix | 69 ++++++++++++++++++- 7 files changed, 153 insertions(+), 1 deletion(-) create mode 100644 mail-server/dovecot/imap_sieve/report-ham.sieve create mode 100644 mail-server/dovecot/imap_sieve/report-spam.sieve create mode 100755 mail-server/dovecot/pipe_bin/sa-learn-ham.sh create mode 100755 mail-server/dovecot/pipe_bin/sa-learn-spam.sh diff --git a/mail-server/dovecot.nix b/mail-server/dovecot.nix index 604a62e..2be417d 100644 --- a/mail-server/dovecot.nix +++ b/mail-server/dovecot.nix @@ -27,6 +27,26 @@ let dovecotMaildir = "maildir:${cfg.mailDirectory}/%d/%n${maildirLayoutAppendix}"; postfixCfg = config.services.postfix; + dovecot2Cfg = config.services.dovecot2; + + stateDir = "/var/lib/dovecot"; + + pipeBin = pkgs.stdenv.mkDerivation { + name = "pipe_bin"; + src = ./dovecot/pipe_bin; + buildInputs = with pkgs; [ makeWrapper coreutils bash rspamd ]; + buildCommand = '' + mkdir -p $out/pipe/bin + cp $src/* $out/pipe/bin/ + chmod a+x $out/pipe/bin/* + patchShebangs $out/pipe/bin + + for file in $out/pipe/bin/*; do + wrapProgram $file \ + --set PATH "${pkgs.coreutils}/bin:${pkgs.rspamd}/bin" + done + ''; + }; in { config = with cfg; lib.mkIf enable { @@ -68,6 +88,7 @@ in protocol imap { mail_max_userip_connections = ${toString cfg.maxConnectionsPerUser} + mail_plugins = $mail_plugins imap_sieve } protocol pop3 { @@ -118,14 +139,40 @@ in } plugin { + sieve_plugins = sieve_imapsieve sieve_extprograms sieve = file:/var/sieve/%u/scripts;active=/var/sieve/%u/active.sieve sieve_default = file:/var/sieve/%u/default.sieve sieve_default_name = default + + # From elsewhere to Spam folder + imapsieve_mailbox1_name = Junk + imapsieve_mailbox1_causes = COPY + imapsieve_mailbox1_before = file:${stateDir}/imap_sieve/report-spam.sieve + + # From Spam folder to elsewhere + imapsieve_mailbox2_name = * + imapsieve_mailbox2_from = Junk + imapsieve_mailbox2_causes = COPY + imapsieve_mailbox2_before = file:${stateDir}/imap_sieve/report-ham.sieve + + sieve_pipe_bin_dir = ${pipeBin}/pipe/bin + + sieve_global_extensions = +vnd.dovecot.pipe +vnd.dovecot.environment } lda_mailbox_autosubscribe = yes lda_mailbox_autocreate = yes ''; }; + + systemd.services.dovecot2.preStart = '' + rm -rf '${stateDir}/imap_sieve' + mkdir '${stateDir}/imap_sieve' + cp -p "${./dovecot/imap_sieve}"/*.sieve '${stateDir}/imap_sieve/' + for k in "${stateDir}/imap_sieve"/*.sieve ; do + ${pkgs.dovecot_pigeonhole}/bin/sievec "$k" + done + chown -R '${dovecot2Cfg.mailUser}:${dovecot2Cfg.mailGroup}' '${stateDir}/imap_sieve' + ''; }; } diff --git a/mail-server/dovecot/imap_sieve/report-ham.sieve b/mail-server/dovecot/imap_sieve/report-ham.sieve new file mode 100644 index 0000000..da74b34 --- /dev/null +++ b/mail-server/dovecot/imap_sieve/report-ham.sieve @@ -0,0 +1,15 @@ +require ["vnd.dovecot.pipe", "copy", "imapsieve", "environment", "variables"]; + +if environment :matches "imap.mailbox" "*" { + set "mailbox" "${1}"; +} + +if string "${mailbox}" "Trash" { + stop; +} + +if environment :matches "imap.user" "*" { + set "username" "${1}"; +} + +pipe :copy "sa-learn-ham.sh" [ "${username}" ]; \ No newline at end of file diff --git a/mail-server/dovecot/imap_sieve/report-spam.sieve b/mail-server/dovecot/imap_sieve/report-spam.sieve new file mode 100644 index 0000000..4024b7a --- /dev/null +++ b/mail-server/dovecot/imap_sieve/report-spam.sieve @@ -0,0 +1,7 @@ +require ["vnd.dovecot.pipe", "copy", "imapsieve", "environment", "variables"]; + +if environment :matches "imap.user" "*" { + set "username" "${1}"; +} + +pipe :copy "sa-learn-spam.sh" [ "${username}" ]; \ No newline at end of file diff --git a/mail-server/dovecot/pipe_bin/sa-learn-ham.sh b/mail-server/dovecot/pipe_bin/sa-learn-ham.sh new file mode 100755 index 0000000..76fc4ed --- /dev/null +++ b/mail-server/dovecot/pipe_bin/sa-learn-ham.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -o errexit +exec rspamc -h /run/rspamd/worker-controller.sock learn_ham \ No newline at end of file diff --git a/mail-server/dovecot/pipe_bin/sa-learn-spam.sh b/mail-server/dovecot/pipe_bin/sa-learn-spam.sh new file mode 100755 index 0000000..2a2f766 --- /dev/null +++ b/mail-server/dovecot/pipe_bin/sa-learn-spam.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -o errexit +exec rspamc -h /run/rspamd/worker-controller.sock learn_spam \ No newline at end of file diff --git a/mail-server/rspamd.nix b/mail-server/rspamd.nix index e7e80a8..950ae56 100644 --- a/mail-server/rspamd.nix +++ b/mail-server/rspamd.nix @@ -61,6 +61,16 @@ in } ''; }; + workers.controller = { + type = "controller"; + count = 1; + bindSockets = [{ + socket = "/run/rspamd/worker-controller.sock"; + mode = "0666"; + }]; + includes = []; + }; + }; systemd.services.rspamd = { requires = (lib.optional cfg.virusScanning "clamav-daemon.service"); diff --git a/tests/extern.nix b/tests/extern.nix index 4ad2ff7..7ccd9ca 100644 --- a/tests/extern.nix +++ b/tests/extern.nix @@ -64,6 +64,7 @@ import { }; enableImap = true; + enableImapSsl = true; }; }; client = { nodes, config, pkgs, ... }: let @@ -79,9 +80,63 @@ import { echo grep '^Message-ID:.*@mail.example.com>$' "$@" >&2 exec grep '^Message-ID:.*@mail.example.com>$' "$@" ''; + test-imap-spam = pkgs.writeScriptBin "imap-mark-spam" '' + #!${pkgs.python3.interpreter} + import imaplib + + with imaplib.IMAP4_SSL('${serverIP}') as imap: + imap.login('user1@example.com', 'user1') + imap.select() + status, [response] = imap.search(None, 'ALL') + msg_ids = response.decode("utf-8").split(' ') + print(msg_ids) + assert status == 'OK' + assert len(msg_ids) == 1 + + imap.copy(','.join(msg_ids), 'Junk') + for num in msg_ids: + imap.store(num, '+FLAGS', '\\Deleted') + imap.expunge() + + imap.select('Junk') + status, [response] = imap.search(None, 'ALL') + msg_ids = response.decode("utf-8").split(' ') + print(msg_ids) + assert status == 'OK' + assert len(msg_ids) == 1 + + imap.close() + ''; + test-imap-ham = pkgs.writeScriptBin "imap-mark-ham" '' + #!${pkgs.python3.interpreter} + import imaplib + + with imaplib.IMAP4_SSL('${serverIP}') as imap: + imap.login('user1@example.com', 'user1') + imap.select('Junk') + status, [response] = imap.search(None, 'ALL') + msg_ids = response.decode("utf-8").split(' ') + print(msg_ids) + assert status == 'OK' + assert len(msg_ids) == 1 + + imap.copy(','.join(msg_ids), 'INBOX') + for num in msg_ids: + imap.store(num, '+FLAGS', '\\Deleted') + imap.expunge() + + imap.select('INBOX') + status, [response] = imap.search(None, 'ALL') + msg_ids = response.decode("utf-8").split(' ') + print(msg_ids) + assert status == 'OK' + assert len(msg_ids) == 1 + + imap.close() + ''; in { environment.systemPackages = with pkgs; [ - fetchmail msmtp procmail findutils grep-ip check-mail-id + fetchmail msmtp procmail findutils grep-ip check-mail-id test-imap-spam test-imap-ham ]; environment.etc = { "root/.fetchmailrc" = { @@ -325,6 +380,18 @@ import { }; + subtest "imap sieve junk trainer", sub { + # send email from user2 to user1 + $client->succeed("msmtp -a test --tls=on --tls-certcheck=off --auth=on user1\@example.com < /etc/root/email1 >&2"); + # give the mail server some time to process the mail + $server->waitUntilFails('[ "$(postqueue -p)" != "Mail queue is empty" ]'); + + $client->succeed("imap-mark-spam >&2"); + $server->waitUntilSucceeds("journalctl -u dovecot2 | grep -i sa-learn-spam.sh >&2"); + $client->succeed("imap-mark-ham >&2"); + $server->waitUntilSucceeds("journalctl -u dovecot2 | grep -i sa-learn-ham.sh >&2"); + }; + subtest "no warnings or errors", sub { $server->fail("journalctl -u postfix | grep -i error >&2"); $server->fail("journalctl -u postfix | grep -i warning >&2");