Merge branch 'dovecot-home-mail-migration' into 'master'

dovecot: migrate to dedicated homedir and separate maildir paths

Closes #324

See merge request simple-nixos-mailserver/nixos-mailserver!408
This commit is contained in:
Martin Weinelt 2025-06-21 10:23:58 +00:00
commit 9d8caf5944
7 changed files with 441 additions and 198 deletions

View file

@ -13,6 +13,75 @@ to your setup.
NixOS 25.11
-----------
#3 Dovecot mail directory migration
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
The way the Dovecot home directory for login accounts were previously set up
resulted in shared home directories for all those users. This is not a
supported Dovecot configuration.
To resolve this we migrated the home directory into the individual
`domain/localpart` subdirectory below the `mailserver.mailDirectory`.
But since this now overlaps with the location of the Maildir, it must be
migrated into the `mail/` directory below the home directory.
And while the LDAP home directory is not affected we use this migration to
keep the Maildir configurations of LDAP users in sync with those of local
accounts.
This is a big step forward, since we can now more cleanly colocate other
data directories, like sieve in the home directory, which in turn simplifies
backups.
This migration is required for every configuration.
For remediating this issue the following steps are required:
1. Copy the `migration script <https://gitlab.com/simple-nixos-mailserver/nixos-mailserver/-/blob/master/migrations/nixos-mailserver-migration-03.py>`_ script to your mailserver
and make it executable:
.. code-block:: bash
wget https://gitlab.com/simple-nixos-mailserver/nixos-mailserver/-/raw/master/migrations/nixos-mailserver-migration-03.py
chmod +x nixos-mailserver-migration-03.py
2. Stop the ``dovecot2.service``.
.. code-block:: bash
systemctl stop dovecot2.service
3. Create a backup or snapshot of your ``mailserver.mailDirectory``, so you can restore
should anything go wrong.
4. Run the migration script under your virtual mail user with the following arguments:
- ``--layout default`` unless ``useFSLayout`` is enabled, then ``--layout folder``
- The value of ``mailserver.mailDirectory``, which defaults to ``/var/vmail``
The script will not modify your data unless called with ``--execute``.
Example:
.. code-block:: bash
sudo -u virtualMail ./nixos-mailserver-migration-03.py --layout default /var/vmail
5. Review the commands. They should be
- create a ``mail`` directory for each accounnt,
- move maildir contents from the parent directory into it,
- suggest removal of files that do not belong to the maildir
- their removal is not mandatory and the script **will not** remove them when called with ``--execute``
- review these items carefully if you want to remove them yourself
- remove obsolete files from the old home directory location
6. Rerun the command with ``--execute`` or run the commands manually.
7. Update the ``mailserver.stateVersion`` to ``3``.
#2 Dovecot LDAP home directory migration
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

View file

@ -72,7 +72,7 @@ common ones.
mailserver = {
enable = true;
stateVersion = 2;
stateVersion = 3;
fqdn = "mail.example.com";
domains = [ "example.com" ];

View file

@ -38,6 +38,16 @@
'';
}
]
++ [
{
assertion = config.mailserver.stateVersion >= 3;
message = ''
Issue: The dovecot mail location for all users has changed and need to be migrated.
Check https://nixos-mailserver.readthedocs.io/en/latest/migrations.html#dovecot-mail-directory-migration for the required remediation steps.
'';
}
]
++ lib.optionals (config.mailserver.certificateScheme != "acme") [
{
assertion = config.mailserver.acmeCertificateName == config.mailserver.fqdn;

View file

@ -45,9 +45,10 @@ let
maildirLayoutAppendix = lib.optionalString cfg.useFsLayout ":LAYOUT=fs";
maildirUTF8FolderNames = lib.optionalString cfg.useUTF8FolderNames ":UTF-8";
# maildir in format "/${domain}/${user}"
# https://doc.dovecot.org/2.3/configuration_manual/home_directories_for_virtual_users/#ways-to-set-up-home-directory
# Mail directory below the home directory
dovecotMaildir =
"maildir:${cfg.mailDirectory}/%{domain}/%{username}${maildirLayoutAppendix}${maildirUTF8FolderNames}"
"maildir:~/mail${maildirLayoutAppendix}${maildirUTF8FolderNames}"
+ (lib.optionalString (cfg.indexDir != null) ":INDEX=${cfg.indexDir}/%{domain}/%{username}");
postfixCfg = config.services.postfix;
@ -386,7 +387,10 @@ in
userdb {
driver = passwd-file
args = ${userdbFile}
default_fields = uid=${builtins.toString cfg.vmailUID} gid=${builtins.toString cfg.vmailUID} home=${cfg.mailDirectory}
default_fields = \
home=${cfg.mailDirectory}/%{domain}/%{username} \
uid=${builtins.toString cfg.vmailUID} \
gid=${builtins.toString cfg.vmailUID}
}
${lib.optionalString cfg.ldap.enable ''
@ -398,7 +402,14 @@ in
userdb {
driver = ldap
args = ${ldapConfFile}
default_fields = home=${cfg.mailDirectory}/ldap/%{user} uid=${toString cfg.vmailUID} gid=${toString cfg.vmailUID}
default_fields = \
home=${cfg.mailDirectory}/ldap/%{user} \
uid=${toString cfg.vmailUID} \
gid=${toString cfg.vmailUID} \
mail=maildir:~/mail${maildirLayoutAppendix}${maildirUTF8FolderNames}${
lib.optionalString (cfg.indexDir != null) ":INDEX=${cfg.indexDir}/ldap/%{user}"
}
}
''}

View file

@ -0,0 +1,132 @@
#!/usr/bin/env nix-shell
#!nix-shell -i python3 -p python3
import argparse
import os
import shutil
import sys
from enum import Enum
from pathlib import Path
from pwd import getpwnam
class FolderLayout(Enum):
Default = 1
Folder = 2
def check_user(vmail_root: Path):
owner = vmail_root.owner()
owner_uid = getpwnam(owner).pw_uid
if os.geteuid() == owner_uid:
return
try:
print(
f"Trying to switch effective user id to {owner_uid} ({owner})",
file=sys.stderr,
)
os.seteuid(owner_uid)
return
except PermissionError:
print(
f"Failed switching to virtual mail user. Please run this script under it, for example by using `sudo -u {owner}`)",
file=sys.stderr,
)
sys.exit(1)
def is_maildir_related(path: Path, layout: FolderLayout) -> bool:
if not path.is_dir():
return False
if path.name in ["cur", "new", "tmp"]:
return True
if layout is FolderLayout.Default and path.name.startswith("."):
return True
if layout is FolderLayout.Folder:
return True
return False
def mkdir(dst: Path, dry_run: bool = True):
print(f'mkdir "{dst}"')
if not dry_run:
# u+rwx, setgid
dst.mkdir(mode=0o2700)
def move(src: Path, dst: Path, dry_run: bool = True):
print(f'mv "{src}" "{dst}"')
if not dry_run:
src.rename(dst)
def delete(dst: Path, dry_run: bool = True):
if not dst.exists():
return
if dst.is_dir():
print(f'rm --recursive "{dst}"')
if not dry_run:
shutil.rmtree(dst)
else:
print(f'rm "{dst}"')
if not dry_run:
dst.unlink()
def main(vmail_root: Path, layout: FolderLayout, dry_run: bool = True):
maildirs = {path.parent for path in vmail_root.glob("*/*/cur")}
maybe_delete = []
# The old maildir will be the new home directory
for homedir in maildirs:
maildir = homedir / "mail"
mkdir(maildir, dry_run)
for path in homedir.iterdir():
if is_maildir_related(path, layout):
move(path, maildir / path.name, dry_run)
else:
maybe_delete.append(path)
# Files that are part of the previous home directory, but now obsolete
for path in [
vmail_root / ".dovecot.lda-dupes",
vmail_root / ".dovecot.lda-dupes.locks",
]:
delete(path, dry_run)
# The remaining files are likely obsolete, but should still be checked with care
for path in maybe_delete:
print(f"# rm {str(path)}")
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="""
NixOS Mailserver Migration #3: Dovecot mail directory migration
(https://nixos-mailserver.readthedocs.io/en/latest/migrations.html#dovecot-mail-directory-migration)
"""
)
parser.add_argument(
"vmail_root", type=Path, help="Path to the `mailserver.mailDirectory`"
)
parser.add_argument(
"--layout",
choices=["default", "folder"],
required=True,
help="Folder layout: 'default' unless `mailserver.useFsLayout` was enabled, then'folder'",
)
parser.add_argument(
"--execute", action="store_true", help="Actually perform changes"
)
args = parser.parse_args()
layout = FolderLayout.Default if args.layout == "default" else FolderLayout.Folder
check_user(args.vmail_root)
main(args.vmail_root, layout, not args.execute)

View file

@ -100,117 +100,128 @@ in
vmailGroupName = "vmail";
vmailUID = 5000;
indexDir = "/var/lib/dovecot/indices";
enableImap = false;
};
};
};
testScript = ''
machine.start()
machine.wait_for_unit("multi-user.target")
testScript =
{
nodes,
...
}:
''
machine.start()
machine.wait_for_unit("multi-user.target")
# Regression test for https://gitlab.com/simple-nixos-mailserver/nixos-mailserver/-/issues/205
with subtest("mail forwarded can are locally kept"):
# A mail sent to user2@example.com is in the user1@example.com mailbox
machine.succeed(
" ".join(
[
"mail-check send-and-read",
"--smtp-port 587",
"--smtp-starttls",
"--smtp-host localhost",
"--imap-host localhost",
"--imap-username user1@example.com",
"--from-addr user1@example.com",
"--to-addr user2@example.com",
"--src-password-file ${passwordFile}",
"--dst-password-file ${passwordFile}",
"--ignore-dkim-spf",
]
)
)
# A mail sent to user2@example.com is in the user2@example.com mailbox
machine.succeed(
" ".join(
[
"mail-check send-and-read",
"--smtp-port 587",
"--smtp-starttls",
"--smtp-host localhost",
"--imap-host localhost",
"--imap-username user2@example.com",
"--from-addr user1@example.com",
"--to-addr user2@example.com",
"--src-password-file ${passwordFile}",
"--dst-password-file ${passwordFile}",
"--ignore-dkim-spf",
]
)
)
# Regression test for https://gitlab.com/simple-nixos-mailserver/nixos-mailserver/-/issues/205
with subtest("mail forwarded can are locally kept"):
# A mail sent to user2@example.com is in the user1@example.com mailbox
machine.succeed(
" ".join(
[
"mail-check send-and-read",
"--smtp-port 587",
"--smtp-starttls",
"--smtp-host localhost",
"--imap-host localhost",
"--imap-username user1@example.com",
"--from-addr user1@example.com",
"--to-addr user2@example.com",
"--src-password-file ${passwordFile}",
"--dst-password-file ${passwordFile}",
"--ignore-dkim-spf",
]
)
)
# A mail sent to user2@example.com is in the user2@example.com mailbox
machine.succeed(
" ".join(
[
"mail-check send-and-read",
"--smtp-port 587",
"--smtp-starttls",
"--smtp-host localhost",
"--imap-host localhost",
"--imap-username user2@example.com",
"--from-addr user1@example.com",
"--to-addr user2@example.com",
"--src-password-file ${passwordFile}",
"--dst-password-file ${passwordFile}",
"--ignore-dkim-spf",
]
)
)
with subtest("regex email alias are received"):
# A mail sent to user2-regex-alias@domain.com is in the user2@example.com mailbox
machine.succeed(
" ".join(
[
"mail-check send-and-read",
"--smtp-port 587",
"--smtp-starttls",
"--smtp-host localhost",
"--imap-host localhost",
"--imap-username user2@example.com",
"--from-addr user1@example.com",
"--to-addr user2-regex-alias@domain.com",
"--src-password-file ${passwordFile}",
"--dst-password-file ${passwordFile}",
"--ignore-dkim-spf",
]
)
)
with subtest("regex email alias are received"):
# A mail sent to user2-regex-alias@domain.com is in the user2@example.com mailbox
machine.succeed(
" ".join(
[
"mail-check send-and-read",
"--smtp-port 587",
"--smtp-starttls",
"--smtp-host localhost",
"--imap-host localhost",
"--imap-username user2@example.com",
"--from-addr user1@example.com",
"--to-addr user2-regex-alias@domain.com",
"--src-password-file ${passwordFile}",
"--dst-password-file ${passwordFile}",
"--ignore-dkim-spf",
]
)
)
with subtest("user can send from regex email alias"):
# A mail sent from user2-regex-alias@domain.com, using user2@example.com credentials is received
machine.succeed(
" ".join(
[
"mail-check send-and-read",
"--smtp-port 587",
"--smtp-starttls",
"--smtp-host localhost",
"--imap-host localhost",
"--smtp-username user2@example.com",
"--from-addr user2-regex-alias@domain.com",
"--to-addr user1@example.com",
"--src-password-file ${passwordFile}",
"--dst-password-file ${passwordFile}",
"--ignore-dkim-spf",
]
)
)
with subtest("user can send from regex email alias"):
# A mail sent from user2-regex-alias@domain.com, using user2@example.com credentials is received
machine.succeed(
" ".join(
[
"mail-check send-and-read",
"--smtp-port 587",
"--smtp-starttls",
"--smtp-host localhost",
"--imap-host localhost",
"--smtp-username user2@example.com",
"--from-addr user2-regex-alias@domain.com",
"--to-addr user1@example.com",
"--src-password-file ${passwordFile}",
"--dst-password-file ${passwordFile}",
"--ignore-dkim-spf",
]
)
)
with subtest("vmail gid is set correctly"):
machine.succeed("getent group vmail | grep 5000")
with subtest("vmail gid is set correctly"):
machine.succeed("getent group vmail | grep 5000")
with subtest("mail to send only accounts is rejected"):
machine.wait_for_open_port(25)
# TODO put this blocking into the systemd units
machine.wait_until_succeeds(
"set +e; timeout 1 nc -U /run/rspamd/rspamd-milter.sock < /dev/null; [ $? -eq 124 ]"
)
machine.succeed(
"cat ${sendMail} | nc localhost 25 | grep -q '554 5.5.0 Error'"
)
with subtest("Check dovecot maildir and index locations"):
# If these paths change we need a migration
machine.succeed("doveadm user -f home user1@example.com | grep ${nodes.machine.config.mailserver.mailDirectory}/example.com/user1")
machine.succeed("doveadm user -f mail user1@example.com | grep 'maildir:~/mail:INDEX=${nodes.machine.config.mailserver.indexDir}/example.com/user1'")
with subtest("rspamd controller serves web ui"):
machine.succeed(
"set +o pipefail; curl --unix-socket /run/rspamd/worker-controller.sock http://localhost/ | grep -q '<body>'"
)
with subtest("mail to send only accounts is rejected"):
machine.wait_for_open_port(25)
# TODO put this blocking into the systemd units
machine.wait_until_succeeds(
"set +e; timeout 1 nc -U /run/rspamd/rspamd-milter.sock < /dev/null; [ $? -eq 124 ]"
)
machine.succeed(
"cat ${sendMail} | nc localhost 25 | grep -q '554 5.5.0 Error'"
)
with subtest("imap port 143 is closed and imaps is serving SSL"):
machine.wait_for_closed_port(143)
machine.wait_for_open_port(993)
machine.succeed(
"echo | openssl s_client -connect localhost:993 | grep 'New, TLS'"
)
'';
with subtest("rspamd controller serves web ui"):
machine.succeed(
"set +o pipefail; curl --unix-socket /run/rspamd/worker-controller.sock http://localhost/ | grep -q '<body>'"
)
with subtest("imap port 143 is closed and imaps is serving SSL"):
machine.wait_for_closed_port(143)
machine.wait_for_open_port(993)
machine.succeed(
"echo | openssl s_client -connect localhost:993 | grep 'New, TLS'"
)
'';
}

View file

@ -90,6 +90,7 @@ in
fqdn = "mail.example.com";
domains = [ "example.com" ];
localDnsResolver = false;
indexDir = "/var/lib/dovecot/indices";
ldap = {
enable = true;
@ -115,107 +116,116 @@ in
};
};
};
testScript = ''
import sys
import re
testScript =
{
nodes,
...
}:
''
import sys
import re
machine.start()
machine.wait_for_unit("multi-user.target")
machine.start()
machine.wait_for_unit("multi-user.target")
# This function retrieves the ldap table file from a postconf
# command.
# A key lookup is achived and the returned value is compared
# to the expected value.
def test_lookup(postconf_cmdline, key, expected):
conf = machine.succeed(postconf_cmdline).rstrip()
ldap_table_path = re.match('.* =.*ldap:(.*)', conf).group(1)
value = machine.succeed(f"postmap -q {key} ldap:{ldap_table_path}").rstrip()
try:
assert value == expected
except AssertionError:
print(f"Expected {conf} lookup for key '{key}' to return '{expected}, but got '{value}'", file=sys.stderr)
raise
# This function retrieves the ldap table file from a postconf
# command.
# A key lookup is achived and the returned value is compared
# to the expected value.
def test_lookup(postconf_cmdline, key, expected):
conf = machine.succeed(postconf_cmdline).rstrip()
ldap_table_path = re.match('.* =.*ldap:(.*)', conf).group(1)
value = machine.succeed(f"postmap -q {key} ldap:{ldap_table_path}").rstrip()
try:
assert value == expected
except AssertionError:
print(f"Expected {conf} lookup for key '{key}' to return '{expected}, but got '{value}'", file=sys.stderr)
raise
with subtest("Test postmap lookups"):
test_lookup("postconf virtual_mailbox_maps", "alice@example.com", "alice@example.com")
test_lookup("postconf -P submission/inet/smtpd_sender_login_maps", "alice@example.com", "alice@example.com")
with subtest("Test postmap lookups"):
test_lookup("postconf virtual_mailbox_maps", "alice@example.com", "alice@example.com")
test_lookup("postconf -P submission/inet/smtpd_sender_login_maps", "alice@example.com", "alice@example.com")
test_lookup("postconf virtual_mailbox_maps", "bob@example.com", "bob@example.com")
test_lookup("postconf -P submission/inet/smtpd_sender_login_maps", "bob@example.com", "bob@example.com")
test_lookup("postconf virtual_mailbox_maps", "bob@example.com", "bob@example.com")
test_lookup("postconf -P submission/inet/smtpd_sender_login_maps", "bob@example.com", "bob@example.com")
with subtest("Test doveadm lookups"):
machine.succeed("doveadm user -u alice@example.com")
machine.succeed("doveadm user -u bob@example.com")
with subtest("Test doveadm lookups"):
machine.succeed("doveadm user -u alice@example.com")
machine.succeed("doveadm user -u bob@example.com")
with subtest("Files containing secrets are only readable by root"):
machine.succeed("ls -l /run/postfix/*.cf | grep -e '-rw------- 1 root root'")
machine.succeed("ls -l /run/dovecot2/dovecot-ldap.conf.ext | grep -e '-rw------- 1 root root'")
with subtest("Files containing secrets are only readable by root"):
machine.succeed("ls -l /run/postfix/*.cf | grep -e '-rw------- 1 root root'")
machine.succeed("ls -l /run/dovecot2/dovecot-ldap.conf.ext | grep -e '-rw------- 1 root root'")
with subtest("Test account/mail address binding"):
machine.fail(" ".join([
"mail-check send-and-read",
"--smtp-port 587",
"--smtp-starttls",
"--smtp-host localhost",
"--smtp-username alice@example.com",
"--imap-host localhost",
"--imap-username bob@example.com",
"--from-addr bob@example.com",
"--to-addr aliceb@example.com",
"--src-password-file <(echo '${alicePassword}')",
"--dst-password-file <(echo '${bobPassword}')",
"--ignore-dkim-spf"
]))
machine.succeed("journalctl -u postfix | grep -q 'Sender address rejected: not owned by user alice@example.com'")
with subtest("Test account/mail address binding"):
machine.fail(" ".join([
"mail-check send-and-read",
"--smtp-port 587",
"--smtp-starttls",
"--smtp-host localhost",
"--smtp-username alice@example.com",
"--imap-host localhost",
"--imap-username bob@example.com",
"--from-addr bob@example.com",
"--to-addr aliceb@example.com",
"--src-password-file <(echo '${alicePassword}')",
"--dst-password-file <(echo '${bobPassword}')",
"--ignore-dkim-spf"
]))
machine.succeed("journalctl -u postfix | grep -q 'Sender address rejected: not owned by user alice@example.com'")
with subtest("Test mail delivery"):
machine.succeed(" ".join([
"mail-check send-and-read",
"--smtp-port 587",
"--smtp-starttls",
"--smtp-host localhost",
"--smtp-username alice@example.com",
"--imap-host localhost",
"--imap-username bob@example.com",
"--from-addr alice@example.com",
"--to-addr bob@example.com",
"--src-password-file <(echo '${alicePassword}')",
"--dst-password-file <(echo '${bobPassword}')",
"--ignore-dkim-spf"
]))
with subtest("Test mail delivery"):
machine.succeed(" ".join([
"mail-check send-and-read",
"--smtp-port 587",
"--smtp-starttls",
"--smtp-host localhost",
"--smtp-username alice@example.com",
"--imap-host localhost",
"--imap-username bob@example.com",
"--from-addr alice@example.com",
"--to-addr bob@example.com",
"--src-password-file <(echo '${alicePassword}')",
"--dst-password-file <(echo '${bobPassword}')",
"--ignore-dkim-spf"
]))
with subtest("Test mail forwarding works"):
machine.succeed(" ".join([
"mail-check send-and-read",
"--smtp-port 587",
"--smtp-starttls",
"--smtp-host localhost",
"--smtp-username alice@example.com",
"--imap-host localhost",
"--imap-username bob@example.com",
"--from-addr alice@example.com",
"--to-addr bob_fw@example.com",
"--src-password-file <(echo '${alicePassword}')",
"--dst-password-file <(echo '${bobPassword}')",
"--ignore-dkim-spf"
]))
with subtest("Test mail forwarding works"):
machine.succeed(" ".join([
"mail-check send-and-read",
"--smtp-port 587",
"--smtp-starttls",
"--smtp-host localhost",
"--smtp-username alice@example.com",
"--imap-host localhost",
"--imap-username bob@example.com",
"--from-addr alice@example.com",
"--to-addr bob_fw@example.com",
"--src-password-file <(echo '${alicePassword}')",
"--dst-password-file <(echo '${bobPassword}')",
"--ignore-dkim-spf"
]))
with subtest("Test cannot send mail from forwarded address"):
machine.fail(" ".join([
"mail-check send-and-read",
"--smtp-port 587",
"--smtp-starttls",
"--smtp-host localhost",
"--smtp-username bob@example.com",
"--imap-host localhost",
"--imap-username alice@example.com",
"--from-addr bob_fw@example.com",
"--to-addr alice@example.com",
"--src-password-file <(echo '${bobPassword}')",
"--dst-password-file <(echo '${alicePassword}')",
"--ignore-dkim-spf"
]))
machine.succeed("journalctl -u postfix | grep -q 'Sender address rejected: not owned by user bob@example.com'")
with subtest("Test cannot send mail from forwarded address"):
machine.fail(" ".join([
"mail-check send-and-read",
"--smtp-port 587",
"--smtp-starttls",
"--smtp-host localhost",
"--smtp-username bob@example.com",
"--imap-host localhost",
"--imap-username alice@example.com",
"--from-addr bob_fw@example.com",
"--to-addr alice@example.com",
"--src-password-file <(echo '${bobPassword}')",
"--dst-password-file <(echo '${alicePassword}')",
"--ignore-dkim-spf"
]))
machine.succeed("journalctl -u postfix | grep -q 'Sender address rejected: not owned by user bob@example.com'")
'';
with subtest("Check dovecot mail and index locations"):
# If these paths change we need a migration
machine.succeed("doveadm user -f home bob@example.com | grep ${nodes.machine.config.mailserver.mailDirectory}/ldap/bob@example.com")
machine.succeed("doveadm user -f mail bob@example.com | grep 'maildir:~/mail:INDEX=${nodes.machine.config.mailserver.indexDir}/ldap/bob@example.com'")
'';
}