Merge branch 'dovecot-home-mail-migration' into 'master'

dovecot: migrate to dedicated homedir and separate maildir paths

Closes #324

See merge request simple-nixos-mailserver/nixos-mailserver!408

docs/migrations.rst

@@ -13,6 +13,75 @@ to your setup.
 NixOS 25.11
 -----------
 
+#3 Dovecot mail directory migration
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The way the Dovecot home directory for login accounts were previously set up
+resulted in shared home directories for all those users. This is not a
+supported Dovecot configuration.
+
+To resolve this we migrated the home directory into the individual
+`domain/localpart` subdirectory below the `mailserver.mailDirectory`.
+
+But since this now overlaps with the location of the Maildir, it must be
+migrated into the `mail/` directory below the home directory.
+And while the LDAP home directory is not affected we use this migration to
+keep the Maildir configurations of LDAP users in sync with those of local
+accounts.
+
+This is a big step forward, since we can now more cleanly colocate other
+data directories, like sieve in the home directory, which in turn simplifies
+backups.
+
+This migration is required for every configuration.
+
+For remediating this issue the following steps are required:
+
+1. Copy the `migration script <https://gitlab.com/simple-nixos-mailserver/nixos-mailserver/-/blob/master/migrations/nixos-mailserver-migration-03.py>`_ script to your mailserver
+   and make it executable:
+
+   .. code-block:: bash
+
+      wget https://gitlab.com/simple-nixos-mailserver/nixos-mailserver/-/raw/master/migrations/nixos-mailserver-migration-03.py
+      chmod +x nixos-mailserver-migration-03.py
+
+2. Stop the ``dovecot2.service``.
+
+   .. code-block:: bash
+
+      systemctl stop dovecot2.service
+
+3. Create a backup or snapshot of your ``mailserver.mailDirectory``, so you can restore
+   should anything go wrong.
+
+4. Run the migration script under your virtual mail user with the following arguments:
+
+   - ``--layout default`` unless ``useFSLayout`` is enabled, then ``--layout folder``
+   - The value of ``mailserver.mailDirectory``, which defaults to ``/var/vmail``
+
+   The script will not modify your data unless called with ``--execute``.
+
+   Example:
+
+   .. code-block:: bash
+
+      sudo -u virtualMail ./nixos-mailserver-migration-03.py --layout default /var/vmail
+
+5. Review the commands. They should be
+
+   - create a ``mail`` directory for each accounnt,
+   - move maildir contents from the parent directory into it,
+   - suggest removal of files that do not belong to the maildir
+
+      - their removal is not mandatory and the script **will not** remove them when called with ``--execute``
+      - review these items carefully if you want to remove them yourself
+
+   - remove obsolete files from the old home directory location
+
+6. Rerun the command with ``--execute`` or run the commands manually.
+
+7. Update the ``mailserver.stateVersion`` to ``3``.
+
 #2 Dovecot LDAP home directory migration
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 

docs/setup-guide.rst

@@ -72,7 +72,7 @@ common ones.
 
      mailserver = {
        enable = true;
-       stateVersion = 2;
+       stateVersion = 3;
        fqdn = "mail.example.com";
        domains = [ "example.com" ];
 

mail-server/assertions.nix

@@ -38,6 +38,16 @@
             '';
           }
         ]
+    ++ [
+      {
+        assertion = config.mailserver.stateVersion >= 3;
+        message = ''
+          Issue: The dovecot mail location for all users has changed and need to be migrated.
+
+          Check https://nixos-mailserver.readthedocs.io/en/latest/migrations.html#dovecot-mail-directory-migration for the required remediation steps.
+        '';
+      }
+    ]
     ++ lib.optionals (config.mailserver.certificateScheme != "acme") [
       {
         assertion = config.mailserver.acmeCertificateName == config.mailserver.fqdn;

mail-server/dovecot.nix

@@ -45,9 +45,10 @@ let
   maildirLayoutAppendix = lib.optionalString cfg.useFsLayout ":LAYOUT=fs";
   maildirUTF8FolderNames = lib.optionalString cfg.useUTF8FolderNames ":UTF-8";
 
-  # maildir in format "/${domain}/${user}"
+  # https://doc.dovecot.org/2.3/configuration_manual/home_directories_for_virtual_users/#ways-to-set-up-home-directory
+  # Mail directory below the home directory
   dovecotMaildir =
-    "maildir:${cfg.mailDirectory}/%{domain}/%{username}${maildirLayoutAppendix}${maildirUTF8FolderNames}"
+    "maildir:~/mail${maildirLayoutAppendix}${maildirUTF8FolderNames}"
     + (lib.optionalString (cfg.indexDir != null) ":INDEX=${cfg.indexDir}/%{domain}/%{username}");
 
   postfixCfg = config.services.postfix;
@@ -386,7 +387,10 @@ in
         userdb {
           driver = passwd-file
           args = ${userdbFile}
-          default_fields = uid=${builtins.toString cfg.vmailUID} gid=${builtins.toString cfg.vmailUID} home=${cfg.mailDirectory}
+          default_fields = \
+            home=${cfg.mailDirectory}/%{domain}/%{username} \
+            uid=${builtins.toString cfg.vmailUID} \
+            gid=${builtins.toString cfg.vmailUID}
         }
 
         ${lib.optionalString cfg.ldap.enable ''
@@ -398,7 +402,14 @@ in
           userdb {
             driver = ldap
             args = ${ldapConfFile}
-            default_fields = home=${cfg.mailDirectory}/ldap/%{user} uid=${toString cfg.vmailUID} gid=${toString cfg.vmailUID}
+            default_fields = \
+              home=${cfg.mailDirectory}/ldap/%{user} \
+              uid=${toString cfg.vmailUID} \
+              gid=${toString cfg.vmailUID} \
+              mail=maildir:~/mail${maildirLayoutAppendix}${maildirUTF8FolderNames}${
+                lib.optionalString (cfg.indexDir != null) ":INDEX=${cfg.indexDir}/ldap/%{user}"
+              }
+
           }
         ''}
 

migrations/nixos-mailserver-migration-03.py

@@ -0,0 +1,132 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i python3 -p python3
+
+import argparse
+import os
+import shutil
+import sys
+from enum import Enum
+from pathlib import Path
+from pwd import getpwnam
+
+
+class FolderLayout(Enum):
+    Default = 1
+    Folder = 2
+
+
+def check_user(vmail_root: Path):
+    owner = vmail_root.owner()
+    owner_uid = getpwnam(owner).pw_uid
+
+    if os.geteuid() == owner_uid:
+        return
+
+    try:
+        print(
+            f"Trying to switch effective user id to {owner_uid} ({owner})",
+            file=sys.stderr,
+        )
+        os.seteuid(owner_uid)
+        return
+    except PermissionError:
+        print(
+            f"Failed switching to virtual mail user. Please run this script under it, for example by using `sudo -u {owner}`)",
+            file=sys.stderr,
+        )
+    sys.exit(1)
+
+
+def is_maildir_related(path: Path, layout: FolderLayout) -> bool:
+    if not path.is_dir():
+        return False
+    if path.name in ["cur", "new", "tmp"]:
+        return True
+    if layout is FolderLayout.Default and path.name.startswith("."):
+        return True
+    if layout is FolderLayout.Folder:
+        return True
+
+    return False
+
+
+def mkdir(dst: Path, dry_run: bool = True):
+    print(f'mkdir "{dst}"')
+    if not dry_run:
+        # u+rwx, setgid
+        dst.mkdir(mode=0o2700)
+
+
+def move(src: Path, dst: Path, dry_run: bool = True):
+    print(f'mv "{src}" "{dst}"')
+    if not dry_run:
+        src.rename(dst)
+
+
+def delete(dst: Path, dry_run: bool = True):
+    if not dst.exists():
+        return
+
+    if dst.is_dir():
+        print(f'rm --recursive "{dst}"')
+        if not dry_run:
+            shutil.rmtree(dst)
+    else:
+        print(f'rm "{dst}"')
+        if not dry_run:
+            dst.unlink()
+
+
+def main(vmail_root: Path, layout: FolderLayout, dry_run: bool = True):
+    maildirs = {path.parent for path in vmail_root.glob("*/*/cur")}
+    maybe_delete = []
+
+    # The old maildir will be the new home directory
+    for homedir in maildirs:
+        maildir = homedir / "mail"
+        mkdir(maildir, dry_run)
+
+        for path in homedir.iterdir():
+            if is_maildir_related(path, layout):
+                move(path, maildir / path.name, dry_run)
+            else:
+                maybe_delete.append(path)
+
+    # Files that are part of the previous home directory, but now obsolete
+    for path in [
+        vmail_root / ".dovecot.lda-dupes",
+        vmail_root / ".dovecot.lda-dupes.locks",
+    ]:
+        delete(path, dry_run)
+
+    # The remaining files are likely obsolete, but should still be checked with care
+    for path in maybe_delete:
+        print(f"# rm {str(path)}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="""
+        NixOS Mailserver Migration #3: Dovecot mail directory migration
+        (https://nixos-mailserver.readthedocs.io/en/latest/migrations.html#dovecot-mail-directory-migration)
+    """
+    )
+    parser.add_argument(
+        "vmail_root", type=Path, help="Path to the `mailserver.mailDirectory`"
+    )
+    parser.add_argument(
+        "--layout",
+        choices=["default", "folder"],
+        required=True,
+        help="Folder layout: 'default' unless `mailserver.useFsLayout` was enabled, then'folder'",
+    )
+    parser.add_argument(
+        "--execute", action="store_true", help="Actually perform changes"
+    )
+
+    args = parser.parse_args()
+
+    layout = FolderLayout.Default if args.layout == "default" else FolderLayout.Folder
+
+    check_user(args.vmail_root)
+    main(args.vmail_root, layout, not args.execute)

tests/internal.nix

@@ -100,12 +100,18 @@ in
 
           vmailGroupName = "vmail";
           vmailUID = 5000;
+          indexDir = "/var/lib/dovecot/indices";
 
           enableImap = false;
         };
       };
   };
-  testScript = ''
+  testScript =
+    {
+      nodes,
+      ...
+    }:
+    ''
       machine.start()
       machine.wait_for_unit("multi-user.target")
 
@@ -191,6 +197,11 @@ in
       with subtest("vmail gid is set correctly"):
           machine.succeed("getent group vmail | grep 5000")
 
+      with subtest("Check dovecot maildir and index locations"):
+          # If these paths change we need a migration
+          machine.succeed("doveadm user -f home user1@example.com | grep ${nodes.machine.config.mailserver.mailDirectory}/example.com/user1")
+          machine.succeed("doveadm user -f mail user1@example.com | grep 'maildir:~/mail:INDEX=${nodes.machine.config.mailserver.indexDir}/example.com/user1'")
+
       with subtest("mail to send only accounts is rejected"):
           machine.wait_for_open_port(25)
           # TODO put this blocking into the systemd units

tests/ldap.nix

@@ -90,6 +90,7 @@ in
           fqdn = "mail.example.com";
           domains = [ "example.com" ];
           localDnsResolver = false;
+          indexDir = "/var/lib/dovecot/indices";
 
           ldap = {
             enable = true;
@@ -115,7 +116,12 @@ in
         };
       };
   };
-  testScript = ''
+  testScript =
+    {
+      nodes,
+      ...
+    }:
+    ''
       import sys
       import re
 
@@ -217,5 +223,9 @@ in
         ]))
         machine.succeed("journalctl -u postfix | grep -q 'Sender address rejected: not owned by user bob@example.com'")
 
+      with subtest("Check dovecot mail and index locations"):
+        # If these paths change we need a migration
+        machine.succeed("doveadm user -f home bob@example.com | grep ${nodes.machine.config.mailserver.mailDirectory}/ldap/bob@example.com")
+        machine.succeed("doveadm user -f mail bob@example.com | grep 'maildir:~/mail:INDEX=${nodes.machine.config.mailserver.indexDir}/ldap/bob@example.com'")
     '';
 }