Move tests to the Python framework

This commit is contained in:
Antoine Eiche 2020-07-27 22:50:27 +02:00
parent 6ad2004ed1
commit 4008d0cb53
5 changed files with 189 additions and 178 deletions

View file

@ -5,10 +5,10 @@
"homepage": "https://github.com/NixOS/nixpkgs", "homepage": "https://github.com/NixOS/nixpkgs",
"owner": "NixOS", "owner": "NixOS",
"repo": "nixpkgs-channels", "repo": "nixpkgs-channels",
"rev": "29eddfc36d720dcc4822581175217543b387b1e8", "rev": "274831839c4f2622019a35d773d4ee588a08c311",
"sha256": "1gqv2m7plkladd3va664xyqb962pqs4pizzibvkm1nh0f4rfpxvy", "sha256": "1dn0cv8ciq83c75hjqlqljjmsns60wqyijx9hzm3j8jvfxm4gzgk",
"type": "tarball", "type": "tarball",
"url": "https://github.com/NixOS/nixpkgs-channels/archive/29eddfc36d720dcc4822581175217543b387b1e8.tar.gz", "url": "https://github.com/NixOS/nixpkgs-channels/archive/274831839c4f2622019a35d773d4ee588a08c311.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz" "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
}, },
"nixpkgs-unstable": { "nixpkgs-unstable": {
@ -17,10 +17,10 @@
"homepage": "https://github.com/NixOS/nixpkgs", "homepage": "https://github.com/NixOS/nixpkgs",
"owner": "NixOS", "owner": "NixOS",
"repo": "nixpkgs-channels", "repo": "nixpkgs-channels",
"rev": "fce7562cf46727fdaf801b232116bc9ce0512049", "rev": "5717d9d2f7ca0662291910c52f1d7b95b568fec2",
"sha256": "14rvi69ji61x3z88vbn17rg5vxrnw2wbnanxb7y0qzyqrj7spapx", "sha256": "17gxd2f622pyss3r6cjngdav6wzdbr31d7bqx9z2lawxg47mmk1l",
"type": "tarball", "type": "tarball",
"url": "https://github.com/NixOS/nixpkgs-channels/archive/fce7562cf46727fdaf801b232116bc9ce0512049.tar.gz", "url": "https://github.com/NixOS/nixpkgs-channels/archive/5717d9d2f7ca0662291910c52f1d7b95b568fec2.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz" "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
} }
} }

View file

@ -16,8 +16,8 @@
{ pkgs ? import <nixpkgs> {}}: { pkgs ? import <nixpkgs> {}}:
import (pkgs.path + "/nixos/tests/make-test.nix") { pkgs.nixosTest {
name = "clamav";
nodes = { nodes = {
server = { config, pkgs, lib, ... }: server = { config, pkgs, lib, ... }:
let let
@ -73,7 +73,6 @@ import (pkgs.path + "/nixos/tests/make-test.nix") {
mailserver = { mailserver = {
enable = true; enable = true;
debug = true;
fqdn = "mail.example.com"; fqdn = "mail.example.com";
domains = [ "example.com" "example2.com" ]; domains = [ "example.com" "example2.com" ];
virusScanning = true; virusScanning = true;
@ -194,52 +193,56 @@ import (pkgs.path + "/nixos/tests/make-test.nix") {
testScript = { nodes, ... }: testScript = { nodes, ... }:
'' ''
startAll; start_all()
$server->waitForUnit("multi-user.target"); server.wait_for_unit("multi-user.target")
$client->waitForUnit("multi-user.target"); client.wait_for_unit("multi-user.target")
# TODO put this blocking into the systemd units? I am not sure if rspamd already waits for the clamd socket. # TODO put this blocking into the systemd units? I am not sure if rspamd already waits for the clamd socket.
$server->waitUntilSucceeds("timeout 1 ${nodes.server.pkgs.netcat}/bin/nc -U /run/rspamd/rspamd-milter.sock < /dev/null; [ \$? -eq 124 ]"); server.wait_until_succeeds(
$server->waitUntilSucceeds("timeout 1 ${nodes.server.pkgs.netcat}/bin/nc -U /run/clamav/clamd.ctl < /dev/null; [ \$? -eq 124 ]"); "timeout 1 ${nodes.server.pkgs.netcat}/bin/nc -U /run/rspamd/rspamd-milter.sock < /dev/null; [ $? -eq 124 ]"
)
server.wait_until_succeeds(
"timeout 1 ${nodes.server.pkgs.netcat}/bin/nc -U /run/clamav/clamd.ctl < /dev/null; [ $? -eq 124 ]"
)
$client->execute("cp -p /etc/root/.* ~/"); client.execute("cp -p /etc/root/.* ~/")
$client->succeed("mkdir -p ~/mail"); client.succeed("mkdir -p ~/mail")
$client->succeed("ls -la ~/ >&2"); client.succeed("ls -la ~/ >&2")
$client->succeed("cat ~/.fetchmailrc >&2"); client.succeed("cat ~/.fetchmailrc >&2")
$client->succeed("cat ~/.procmailrc >&2"); client.succeed("cat ~/.procmailrc >&2")
$client->succeed("cat ~/.msmtprc >&2"); client.succeed("cat ~/.msmtprc >&2")
# fetchmail returns EXIT_CODE 1 when no new mail # fetchmail returns EXIT_CODE 1 when no new mail
$client->succeed("fetchmail --nosslcertck -v || [ \$? -eq 1 ] >&2"); client.succeed("fetchmail --nosslcertck -v || [ $? -eq 1 ] >&2")
# Verify that mail can be sent and received before testing virus scanner # Verify that mail can be sent and received before testing virus scanner
$client->execute("rm ~/mail/*"); client.execute("rm ~/mail/*")
$client->succeed("msmtp -a user2 user1\@example.com < /etc/root/safe-email >&2"); client.succeed("msmtp -a user2 user1@example.com < /etc/root/safe-email >&2")
# give the mail server some time to process the mail # give the mail server some time to process the mail
$server->waitUntilFails('[ "$(postqueue -p)" != "Mail queue is empty" ]'); server.wait_until_fails('[ "$(postqueue -p)" != "Mail queue is empty" ]')
$client->execute("rm ~/mail/*"); client.execute("rm ~/mail/*")
# fetchmail returns EXIT_CODE 0 when it retrieves mail # fetchmail returns EXIT_CODE 0 when it retrieves mail
$client->succeed("fetchmail --nosslcertck -v >&2"); client.succeed("fetchmail --nosslcertck -v >&2")
$client->execute("rm ~/mail/*"); client.execute("rm ~/mail/*")
subtest "virus scan file", sub { with subtest("virus scan file"):
$server->succeed("clamdscan \$(readlink -f /etc/root/eicar.com.txt) | grep \"Txt\\.Malware\\.Agent-1787597 FOUND\" >&2"); server.succeed(
}; 'clamdscan $(readlink -f /etc/root/eicar.com.txt) | grep "Txt\\.Malware\\.Agent-1787597 FOUND" >&2'
)
subtest "virus scan email", sub { with subtest("virus scan email"):
$client->succeed("msmtp -a user2 user1\@example.com < /etc/root/virus-email 2>&1 | tee /dev/stderr | grep \"server message: 554 5\\.7\\.1\" >&2"); client.succeed(
$server->succeed("journalctl -u rspamd | grep -i eicar"); 'msmtp -a user2 user1\@example.com < /etc/root/virus-email 2>&1 | tee /dev/stderr | grep "server message: 554 5\\.7\\.1" >&2'
)
server.succeed("journalctl -u rspamd | grep -i eicar")
# give the mail server some time to process the mail # give the mail server some time to process the mail
$server->waitUntilFails('[ "$(postqueue -p)" != "Mail queue is empty" ]'); server.wait_until_fails('[ "$(postqueue -p)" != "Mail queue is empty" ]')
};
subtest "no warnings or errors", sub {
$server->fail("journalctl -u postfix | grep -i error >&2");
$server->fail("journalctl -u postfix | grep -i warning >&2");
$server->fail("journalctl -u dovecot2 | grep -i error >&2");
$server->fail("journalctl -u dovecot2 | grep -i warning >&2");
};
with subtest("no warnings or errors"):
server.fail("journalctl -u postfix | grep -i error >&2")
server.fail("journalctl -u postfix | grep -i warning >&2")
server.fail("journalctl -u dovecot2 | grep -i error >&2")
server.fail("journalctl -u dovecot2 | grep -i warning >&2")
''; '';
} }

View file

@ -23,7 +23,7 @@ let
test = pkgs.callPackage (./. + "/${testName}.nix") { }; test = pkgs.callPackage (./. + "/${testName}.nix") { };
in { in {
"name"= builtins.replaceStrings ["." "-"] ["_" "_"] release; "name"= builtins.replaceStrings ["." "-"] ["_" "_"] release;
"value"= test { inherit pkgs; }; "value"= test;
}; };
releaseNames = [ releaseNames = [

View file

@ -16,8 +16,8 @@
{ pkgs ? import <nixpkgs> {}}: { pkgs ? import <nixpkgs> {}}:
import (pkgs.path + "/nixos/tests/make-test.nix") { pkgs.nixosTest {
name = "extern";
nodes = { nodes = {
server = { config, pkgs, ... }: server = { config, pkgs, ... }:
{ {
@ -282,137 +282,144 @@ import (pkgs.path + "/nixos/tests/make-test.nix") {
testScript = { nodes, ... }: testScript = { nodes, ... }:
'' ''
startAll; start_all()
$server->waitForUnit("multi-user.target"); server.wait_for_unit("multi-user.target")
$client->waitForUnit("multi-user.target"); client.wait_for_unit("multi-user.target")
# TODO put this blocking into the systemd units? # TODO put this blocking into the systemd units?
$server->waitUntilSucceeds("timeout 1 ${nodes.server.pkgs.netcat}/bin/nc -U /run/rspamd/rspamd-milter.sock < /dev/null; [ \$? -eq 124 ]"); server.wait_until_succeeds(
"timeout 1 ${nodes.server.pkgs.netcat}/bin/nc -U /run/rspamd/rspamd-milter.sock < /dev/null; [ $? -eq 124 ]"
)
$client->execute("cp -p /etc/root/.* ~/"); client.execute("cp -p /etc/root/.* ~/")
$client->succeed("mkdir -p ~/mail"); client.succeed("mkdir -p ~/mail")
$client->succeed("ls -la ~/ >&2"); client.succeed("ls -la ~/ >&2")
$client->succeed("cat ~/.fetchmailrc >&2"); client.succeed("cat ~/.fetchmailrc >&2")
$client->succeed("cat ~/.procmailrc >&2"); client.succeed("cat ~/.procmailrc >&2")
$client->succeed("cat ~/.msmtprc >&2"); client.succeed("cat ~/.msmtprc >&2")
subtest "imap retrieving mail", sub { with subtest("imap retrieving mail"):
# fetchmail returns EXIT_CODE 1 when no new mail # fetchmail returns EXIT_CODE 1 when no new mail
$client->succeed("fetchmail --nosslcertck -v || [ \$? -eq 1 ] >&2"); client.succeed("fetchmail --nosslcertck -v || [ $? -eq 1 ] >&2")
};
subtest "submission port send mail", sub { with subtest("submission port send mail"):
# send email from user2 to user1 # send email from user2 to user1
$client->succeed("msmtp -a test --tls=on --tls-certcheck=off --auth=on user1\@example.com < /etc/root/email1 >&2"); client.succeed(
"msmtp -a test --tls=on --tls-certcheck=off --auth=on user1\@example.com < /etc/root/email1 >&2"
)
# give the mail server some time to process the mail # give the mail server some time to process the mail
$server->waitUntilFails('[ "$(postqueue -p)" != "Mail queue is empty" ]'); server.wait_until_fails('[ "$(postqueue -p)" != "Mail queue is empty" ]')
};
subtest "imap retrieving mail 2", sub { with subtest("imap retrieving mail 2"):
$client->execute("rm ~/mail/*"); client.execute("rm ~/mail/*")
# fetchmail returns EXIT_CODE 0 when it retrieves mail # fetchmail returns EXIT_CODE 0 when it retrieves mail
$client->succeed("fetchmail --nosslcertck -v >&2"); client.succeed("fetchmail --nosslcertck -v >&2")
};
subtest "remove sensitive information on submission port", sub { with subtest("remove sensitive information on submission port"):
$client->succeed("cat ~/mail/* >&2"); client.succeed("cat ~/mail/* >&2")
## make sure our IP is _not_ in the email header ## make sure our IP is _not_ in the email header
$client->fail("grep-ip ~/mail/*"); client.fail("grep-ip ~/mail/*")
$client->succeed("check-mail-id ~/mail/*"); client.succeed("check-mail-id ~/mail/*")
};
subtest "have correct fqdn as sender", sub { with subtest("have correct fqdn as sender"):
$client->succeed("grep 'Received: from mail.example.com' ~/mail/*"); client.succeed("grep 'Received: from mail.example.com' ~/mail/*")
};
subtest "dkim has user-specified size", sub { with subtest("dkim has user-specified size"):
$server->succeed("openssl rsa -in /var/dkim/example.com.mail.key -text -noout | grep 'Private-Key: (1535 bit'"); server.succeed(
}; "openssl rsa -in /var/dkim/example.com.mail.key -text -noout | grep 'Private-Key: (1535 bit'"
)
subtest "dkim singing, multiple domains", sub { with subtest("dkim singing, multiple domains"):
$client->execute("rm ~/mail/*"); client.execute("rm ~/mail/*")
# send email from user2 to user1 # send email from user2 to user1
$client->succeed("msmtp -a test2 --tls=on --tls-certcheck=off --auth=on user1\@example.com < /etc/root/email2 >&2"); client.succeed(
$server->waitUntilFails('[ "$(postqueue -p)" != "Mail queue is empty" ]'); "msmtp -a test2 --tls=on --tls-certcheck=off --auth=on user1\@example.com < /etc/root/email2 >&2"
)
server.wait_until_fails('[ "$(postqueue -p)" != "Mail queue is empty" ]')
# fetchmail returns EXIT_CODE 0 when it retrieves mail # fetchmail returns EXIT_CODE 0 when it retrieves mail
$client->succeed("fetchmail --nosslcertck -v"); client.succeed("fetchmail --nosslcertck -v")
$client->succeed("cat ~/mail/* >&2"); client.succeed("cat ~/mail/* >&2")
# make sure it is dkim signed # make sure it is dkim signed
$client->succeed("grep DKIM ~/mail/*"); client.succeed("grep DKIM ~/mail/*")
};
subtest "aliases", sub { with subtest("aliases"):
$client->execute("rm ~/mail/*"); client.execute("rm ~/mail/*")
# send email from chuck to postmaster # send email from chuck to postmaster
$client->succeed("msmtp -a test3 --tls=on --tls-certcheck=off --auth=on postmaster\@example.com < /etc/root/email2 >&2"); client.succeed(
$server->waitUntilFails('[ "$(postqueue -p)" != "Mail queue is empty" ]'); "msmtp -a test3 --tls=on --tls-certcheck=off --auth=on postmaster\@example.com < /etc/root/email2 >&2"
)
server.wait_until_fails('[ "$(postqueue -p)" != "Mail queue is empty" ]')
# fetchmail returns EXIT_CODE 0 when it retrieves mail # fetchmail returns EXIT_CODE 0 when it retrieves mail
$client->succeed("fetchmail --nosslcertck -v"); client.succeed("fetchmail --nosslcertck -v")
};
subtest "catchAlls", sub { with subtest("catchAlls"):
$client->execute("rm ~/mail/*"); client.execute("rm ~/mail/*")
# send email from chuck to non exsitent account # send email from chuck to non exsitent account
$client->succeed("msmtp -a test3 --tls=on --tls-certcheck=off --auth=on lol\@example.com < /etc/root/email2 >&2"); client.succeed(
$server->waitUntilFails('[ "$(postqueue -p)" != "Mail queue is empty" ]'); "msmtp -a test3 --tls=on --tls-certcheck=off --auth=on lol\@example.com < /etc/root/email2 >&2"
)
server.wait_until_fails('[ "$(postqueue -p)" != "Mail queue is empty" ]')
# fetchmail returns EXIT_CODE 0 when it retrieves mail # fetchmail returns EXIT_CODE 0 when it retrieves mail
$client->succeed("fetchmail --nosslcertck -v"); client.succeed("fetchmail --nosslcertck -v")
$client->execute("rm ~/mail/*"); client.execute("rm ~/mail/*")
# send email from user1 to chuck # send email from user1 to chuck
$client->succeed("msmtp -a test4 --tls=on --tls-certcheck=off --auth=on chuck\@example.com < /etc/root/email2 >&2"); client.succeed(
$server->waitUntilFails('[ "$(postqueue -p)" != "Mail queue is empty" ]'); "msmtp -a test4 --tls=on --tls-certcheck=off --auth=on chuck\@example.com < /etc/root/email2 >&2"
)
server.wait_until_fails('[ "$(postqueue -p)" != "Mail queue is empty" ]')
# fetchmail returns EXIT_CODE 1 when no new mail # fetchmail returns EXIT_CODE 1 when no new mail
# if this succeeds, it means that user1 recieved the mail that was intended for chuck. # if this succeeds, it means that user1 recieved the mail that was intended for chuck.
$client->fail("fetchmail --nosslcertck -v"); client.fail("fetchmail --nosslcertck -v")
};
subtest "extraVirtualAliases", sub { with subtest("extraVirtualAliases"):
$client->execute("rm ~/mail/*"); client.execute("rm ~/mail/*")
# send email from single-alias to user1 # send email from single-alias to user1
$client->succeed("msmtp -a test5 --tls=on --tls-certcheck=off --auth=on user1\@example.com < /etc/root/email4 >&2"); client.succeed(
$server->waitUntilFails('[ "$(postqueue -p)" != "Mail queue is empty" ]'); "msmtp -a test5 --tls=on --tls-certcheck=off --auth=on user1\@example.com < /etc/root/email4 >&2"
)
server.wait_until_fails('[ "$(postqueue -p)" != "Mail queue is empty" ]')
# fetchmail returns EXIT_CODE 0 when it retrieves mail # fetchmail returns EXIT_CODE 0 when it retrieves mail
$client->succeed("fetchmail --nosslcertck -v"); client.succeed("fetchmail --nosslcertck -v")
$client->execute("rm ~/mail/*"); client.execute("rm ~/mail/*")
# send email from user1 to multi-alias (user{1,2}@example.com) # send email from user1 to multi-alias (user{1,2}@example.com)
$client->succeed("msmtp -a test --tls=on --tls-certcheck=off --auth=on multi-alias\@example.com < /etc/root/email5 >&2"); client.succeed(
$server->waitUntilFails('[ "$(postqueue -p)" != "Mail queue is empty" ]'); "msmtp -a test --tls=on --tls-certcheck=off --auth=on multi-alias\@example.com < /etc/root/email5 >&2"
)
server.wait_until_fails('[ "$(postqueue -p)" != "Mail queue is empty" ]')
# fetchmail returns EXIT_CODE 0 when it retrieves mail # fetchmail returns EXIT_CODE 0 when it retrieves mail
$client->succeed("fetchmail --nosslcertck -v"); client.succeed("fetchmail --nosslcertck -v")
};
subtest "quota", sub { with subtest("quota"):
$client->execute("rm ~/mail/*"); client.execute("rm ~/mail/*")
$client->execute("mv ~/.fetchmailRcLowQuota ~/.fetchmailrc"); client.execute("mv ~/.fetchmailRcLowQuota ~/.fetchmailrc")
$client->succeed("msmtp -a test3 --tls=on --tls-certcheck=off --auth=on lowquota\@example.com < /etc/root/email2 >&2"); client.succeed(
$server->waitUntilFails('[ "$(postqueue -p)" != "Mail queue is empty" ]'); "msmtp -a test3 --tls=on --tls-certcheck=off --auth=on lowquota\@example.com < /etc/root/email2 >&2"
)
server.wait_until_fails('[ "$(postqueue -p)" != "Mail queue is empty" ]')
# fetchmail returns EXIT_CODE 0 when it retrieves mail # fetchmail returns EXIT_CODE 0 when it retrieves mail
$client->fail("fetchmail --nosslcertck -v"); client.fail("fetchmail --nosslcertck -v")
}; with subtest("imap sieve junk trainer"):
subtest "imap sieve junk trainer", sub {
# send email from user2 to user1 # send email from user2 to user1
$client->succeed("msmtp -a test --tls=on --tls-certcheck=off --auth=on user1\@example.com < /etc/root/email1 >&2"); client.succeed(
"msmtp -a test --tls=on --tls-certcheck=off --auth=on user1\@example.com < /etc/root/email1 >&2"
)
# give the mail server some time to process the mail # give the mail server some time to process the mail
$server->waitUntilFails('[ "$(postqueue -p)" != "Mail queue is empty" ]'); server.wait_until_fails('[ "$(postqueue -p)" != "Mail queue is empty" ]')
$client->succeed("imap-mark-spam >&2"); client.succeed("imap-mark-spam >&2")
$server->waitUntilSucceeds("journalctl -u dovecot2 | grep -i sa-learn-spam.sh >&2"); server.wait_until_succeeds("journalctl -u dovecot2 | grep -i sa-learn-spam.sh >&2")
$client->succeed("imap-mark-ham >&2"); client.succeed("imap-mark-ham >&2")
$server->waitUntilSucceeds("journalctl -u dovecot2 | grep -i sa-learn-ham.sh >&2"); server.wait_until_succeeds("journalctl -u dovecot2 | grep -i sa-learn-ham.sh >&2")
};
subtest "no warnings or errors", sub {
$server->fail("journalctl -u postfix | grep -i error >&2");
$server->fail("journalctl -u postfix | grep -i warning >&2");
$server->fail("journalctl -u dovecot2 | grep -i error >&2");
$server->fail("journalctl -u dovecot2 | grep -i warning >&2");
};
with subtest("no warnings or errors"):
server.fail("journalctl -u postfix | grep -i error >&2")
server.fail("journalctl -u postfix | grep -i warning >&2")
server.fail("journalctl -u dovecot2 | grep -i error >&2")
server.fail("journalctl -u dovecot2 | grep -i warning >&2")
''; '';
} }

View file

@ -34,11 +34,10 @@ let
''; '';
in in
import (pkgs.path + "/nixos/tests/make-test.nix") { pkgs.nixosTest {
name = "intern";
machine = nodes = {
{ config, pkgs, ... }: machine = { config, pkgs, ... }: {
{
imports = [ imports = [
./../default.nix ./../default.nix
./lib/config.nix ./lib/config.nix
@ -65,25 +64,27 @@ import (pkgs.path + "/nixos/tests/make-test.nix") {
vmailUID = 5000; vmailUID = 5000;
}; };
}; };
testScript =
''
$machine->start;
$machine->waitForUnit("multi-user.target");
subtest "vmail gid is set correctly", sub {
$machine->succeed("getent group vmail | grep 5000");
}; };
testScript = ''
machine.start()
machine.wait_for_unit("multi-user.target")
subtest "mail to send only accounts is rejected", sub { with subtest("vmail gid is set correctly"):
$machine->waitForOpenPort(25); machine.succeed("getent group vmail | grep 5000")
# TODO put this blocking into the systemd units?
$machine->waitUntilSucceeds("timeout 1 ${pkgs.netcat}/bin/nc -U /run/rspamd/rspamd-milter.sock < /dev/null; [ \$? -eq 124 ]");
$machine->succeed("cat ${sendMail} | ${pkgs.netcat-gnu}/bin/nc localhost 25 | grep -q 'This account cannot receive emails'" );
};
subtest "rspamd controller serves web ui", sub { with subtest("mail to send only accounts is rejected"):
$machine->succeed("${pkgs.curl}/bin/curl --unix-socket /run/rspamd/worker-controller.sock http://localhost/ | grep -q '<body>'" ); machine.wait_for_open_port(25)
}; # TODO put this blocking into the systemd units
machine.wait_until_succeeds(
"timeout 1 ${pkgs.netcat}/bin/nc -U /run/rspamd/rspamd-milter.sock < /dev/null; [ $? -eq 124 ]"
)
machine.succeed(
"cat ${sendMail} | ${pkgs.netcat-gnu}/bin/nc localhost 25 | grep -q 'This account cannot receive emails'"
)
with subtest("rspamd controller serves web ui"):
machine.succeed(
"${pkgs.curl}/bin/curl --unix-socket /run/rspamd/worker-controller.sock http://localhost/ | grep -q '<body>'"
)
''; '';
} }