tests: also test client submission over smtps://

instead of just smtp:// with STARTTLS.

Opted to call the flag --ssl and not --tls to keep it consistent with
the module option (mailserver.enableSubmissionSsl), dovecot internals
and smtplib in mail-check.py.

scripts/mail-check.py

@@ -12,7 +12,15 @@ RETRY = 100
 
 
 def _send_mail(
-    smtp_host, smtp_port, smtp_username, from_addr, from_pwd, to_addr, subject, starttls
+    smtp_host,
+    smtp_port,
+    smtp_username,
+    from_addr,
+    from_pwd,
+    to_addr,
+    subject,
+    starttls,
+    ssl,
 ):
     print(f"Sending mail with subject '{subject}'")
     message = "\n".join(
@@ -28,9 +36,10 @@ def _send_mail(
     )
 
     retry = RETRY
+    smtp_class = smtplib.SMTP_SSL if ssl else smtplib.SMTP
     while True:
         try:
-            with smtplib.SMTP(smtp_host, port=smtp_port) as smtp:
+            with smtp_class(smtp_host, port=smtp_port) as smtp:
                 try:
                     if starttls:
                         smtp.starttls()
@@ -171,6 +180,7 @@ def send_and_read(args):
         to_addr=args.to_addr,
         subject=subject,
         starttls=args.smtp_starttls,
+        ssl=args.smtp_ssl,
     )
 
     _read_mail(
@@ -206,6 +216,7 @@ parser_send_and_read = subparsers.add_parser(
 parser_send_and_read.add_argument("--smtp-host", type=str)
 parser_send_and_read.add_argument("--smtp-port", type=str, default=25)
 parser_send_and_read.add_argument("--smtp-starttls", action="store_true")
+parser_send_and_read.add_argument("--smtp-ssl", action="store_true")
 parser_send_and_read.add_argument(
     "--smtp-username",
     type=str,

tests/internal.nix

@@ -116,7 +116,7 @@ in
 
       # Regression test for https://gitlab.com/simple-nixos-mailserver/nixos-mailserver/-/issues/205
       with subtest("mail forwarded can are locally kept"):
-          # A mail sent to user2@example.com is in the user1@example.com mailbox
+          # A mail sent to user2@example.com via explicit TLS is in the user1@example.com mailbox
           machine.succeed(
               " ".join(
                   [
@@ -134,13 +134,13 @@ in
                   ]
               )
           )
-          # A mail sent to user2@example.com is in the user2@example.com mailbox
+          # A mail sent to user2@example.com via implicit TLS is in the user2@example.com mailbox
           machine.succeed(
               " ".join(
                   [
                       "mail-check send-and-read",
-                      "--smtp-port 587",
+                      "--smtp-port 465",
-                      "--smtp-starttls",
+                      "--smtp-ssl",
                       "--smtp-host localhost",
                       "--imap-host localhost",
                       "--imap-username user2@example.com",
@@ -154,7 +154,7 @@ in
           )
 
       with subtest("regex email alias are received"):
-          # A mail sent to user2-regex-alias@domain.com is in the user2@example.com mailbox
+          # A mail sent to user2-regex-alias@domain.com via explicit TLS is in the user2@example.com mailbox
           machine.succeed(
               " ".join(
                   [
@@ -174,13 +174,14 @@ in
           )
 
       with subtest("user can send from regex email alias"):
-          # A mail sent from user2-regex-alias@domain.com, using user2@example.com credentials is received
+          # A mail sent to user1@example.com from user2-regex-alias@domain.com by
+          # user2@example.com via implicit TLS is in the user1@example.com mailbox
           machine.succeed(
               " ".join(
                   [
                       "mail-check send-and-read",
-                      "--smtp-port 587",
+                      "--smtp-port 465",
-                      "--smtp-starttls",
+                      "--smtp-ssl",
                       "--smtp-host localhost",
                       "--imap-host localhost",
                       "--smtp-username user2@example.com",

tests/ldap.nix

@@ -157,7 +157,7 @@ in
         machine.succeed("ls -l /run/postfix/*.cf | grep -e '-rw------- 1 root root'")
         machine.succeed("ls -l /run/dovecot2/dovecot-ldap.conf.ext | grep -e '-rw------- 1 root root'")
 
-      with subtest("Test account/mail address binding"):
+      with subtest("Test account/mail address binding via explicit TLS"):
         machine.fail(" ".join([
           "mail-check send-and-read",
           "--smtp-port 587",
@@ -174,11 +174,11 @@ in
         ]))
         machine.succeed("journalctl -u postfix | grep -q 'Sender address rejected: not owned by user alice@example.com'")
 
-      with subtest("Test mail delivery"):
+      with subtest("Test mail delivery via implicit TLS"):
         machine.succeed(" ".join([
           "mail-check send-and-read",
-          "--smtp-port 587",
+          "--smtp-port 465",
-          "--smtp-starttls",
+          "--smtp-ssl",
           "--smtp-host localhost",
           "--smtp-username alice@example.com",
           "--imap-host localhost",
@@ -190,7 +190,7 @@ in
           "--ignore-dkim-spf"
         ]))
 
-      with subtest("Test mail forwarding works"):
+      with subtest("Test mail forwarding via explicit TLS works"):
         machine.succeed(" ".join([
           "mail-check send-and-read",
           "--smtp-port 587",
@@ -206,11 +206,11 @@ in
           "--ignore-dkim-spf"
         ]))
 
-      with subtest("Test cannot send mail from forwarded address"):
+      with subtest("Test cannot send mail via implicit TLS from forwarded address"):
         machine.fail(" ".join([
           "mail-check send-and-read",
-          "--smtp-port 587",
+          "--smtp-port 465",
-          "--smtp-starttls",
+          "--smtp-ssl",
           "--smtp-host localhost",
           "--smtp-username bob@example.com",
           "--imap-host localhost",

tests/multiple.nix

@@ -100,14 +100,14 @@ in
         "set +e; timeout 1 nc -U /run/rspamd/rspamd-milter.sock < /dev/null; [ $? -eq 124 ]"
     )
 
-    # user@domain1.com sends a mail to user@domain2.com
+    # user@domain1.com sends a mail to user@domain2.com via explicit TLS
     client.succeed(
         "mail-check send-and-read --smtp-port 587 --smtp-starttls --smtp-host domain1 --from-addr user@domain1.com --imap-host domain2 --to-addr user@domain2.com --src-password-file ${password} --dst-password-file ${password} --ignore-dkim-spf"
     )
 
-    # Send a mail to the address forwarded and check it is in the recipient mailbox
+    # Send a mail to the address forwarded via implicit TLS and check it is in the recipient mailbox
     client.succeed(
-        "mail-check send-and-read --smtp-port 587 --smtp-starttls --smtp-host domain1 --from-addr user@domain1.com --imap-host domain2 --to-addr non-local@domain1.com --imap-username user@domain2.com --src-password-file ${password} --dst-password-file ${password} --ignore-dkim-spf"
+        "mail-check send-and-read --smtp-port 465 --smtp-ssl --smtp-host domain1 --from-addr user@domain1.com --imap-host domain2 --to-addr non-local@domain1.com --imap-username user@domain2.com --src-password-file ${password} --dst-password-file ${password} --ignore-dkim-spf"
     )
   '';
 }