nixos/etc: optionally mount etc as an overlay

nixos/modules/system/etc/build-composefs-dump.py

@@ -0,0 +1,209 @@
+#!/usr/bin/env python3
+
+"""Build a composefs dump from a Json config
+
+See the man page of composefs-dump for details about the format:
+https://github.com/containers/composefs/blob/main/man/composefs-dump.md
+
+Ensure to check the file with the check script when you make changes to it:
+
+./check-build-composefs-dump.sh ./build-composefs_dump.py
+"""
+
+import glob
+import json
+import os
+import sys
+from enum import Enum
+from pathlib import Path
+from typing import Any
+
+Attrs = dict[str, Any]
+
+
+class FileType(Enum):
+    """The filetype as defined by the `st_mode` stat field in octal
+
+    You can check the st_mode stat field of a path in Python with
+    `oct(os.stat("/path/").st_mode)`
+    """
+
+    directory = "4"
+    file = "10"
+    symlink = "12"
+
+
+class ComposefsPath:
+    path: str
+    size: int
+    filetype: FileType
+    mode: str
+    uid: str
+    gid: str
+    payload: str
+    rdev: str = "0"
+    nlink: int = 1
+    mtime: str = "1.0"
+    content: str = "-"
+    digest: str = "-"
+
+    def __init__(
+        self,
+        attrs: Attrs,
+        size: int,
+        filetype: FileType,
+        mode: str,
+        payload: str,
+        path: str | None = None,
+    ):
+        if path is None:
+            path = attrs["target"]
+        self.path = "/" + path
+        self.size = size
+        self.filetype = filetype
+        self.mode = mode
+        self.uid = attrs["uid"]
+        self.gid = attrs["gid"]
+        self.payload = payload
+
+    def write_line(self) -> str:
+        line_list = [
+            str(self.path),
+            str(self.size),
+            f"{self.filetype.value}{self.mode}",
+            str(self.nlink),
+            str(self.uid),
+            str(self.gid),
+            str(self.rdev),
+            str(self.mtime),
+            str(self.payload),
+            str(self.content),
+            str(self.digest),
+        ]
+        return " ".join(line_list)
+
+
+def eprint(*args, **kwargs) -> None:
+    print(args, **kwargs, file=sys.stderr)
+
+
+def leading_directories(path: str) -> list[str]:
+    """Return the leading directories of path
+
+    Given the path "alsa/conf.d/50-pipewire.conf", for example, this function
+    returns `[ "alsa", "alsa/conf.d" ]`.
+    """
+    parents = list(Path(path).parents)
+    parents.reverse()
+    # remove the implicit `.` from the start of a relative path or `/` from an
+    # absolute path
+    del parents[0]
+    return [str(i) for i in parents]
+
+
+def add_leading_directories(
+    target: str, attrs: Attrs, paths: dict[str, ComposefsPath]
+) -> None:
+    """Add the leading directories of a target path to the composefs paths
+
+    mkcomposefs expects that all leading directories are explicitly listed in
+    the dump file. Given the path "alsa/conf.d/50-pipewire.conf", for example,
+    this function adds "alsa" and "alsa/conf.d" to the composefs paths.
+    """
+    path_components = leading_directories(target)
+    for component in path_components:
+        composefs_path = ComposefsPath(
+            attrs,
+            path=component,
+            size=4096,
+            filetype=FileType.directory,
+            mode="0755",
+            payload="-",
+        )
+        paths[component] = composefs_path
+
+
+def main() -> None:
+    """Build a composefs dump from a Json config
+
+    This config describes the files that the final composefs image is supposed
+    to contain.
+    """
+    config_file = sys.argv[1]
+    if not config_file:
+        eprint("No config file was supplied.")
+        sys.exit(1)
+
+    with open(config_file, "rb") as f:
+        config = json.load(f)
+
+    if not config:
+        eprint("Config is empty.")
+        sys.exit(1)
+
+    eprint("Building composefs dump...")
+
+    paths: dict[str, ComposefsPath] = {}
+    for attrs in config:
+        target = attrs["target"]
+        source = attrs["source"]
+        mode = attrs["mode"]
+
+        if "*" in source:  # Path with globbing
+            glob_sources = glob.glob(source)
+            for glob_source in glob_sources:
+                basename = os.path.basename(glob_source)
+                glob_target = f"{target}/{basename}"
+
+                composefs_path = ComposefsPath(
+                    attrs,
+                    path=glob_target,
+                    size=100,
+                    filetype=FileType.symlink,
+                    mode="0777",
+                    payload=glob_source,
+                )
+
+                paths[glob_target] = composefs_path
+                add_leading_directories(glob_target, attrs, paths)
+        else:  # Without globbing
+            if mode == "symlink":
+                composefs_path = ComposefsPath(
+                    attrs,
+                    # A high approximation of the size of a symlink
+                    size=100,
+                    filetype=FileType.symlink,
+                    mode="0777",
+                    payload=source,
+                )
+            else:
+                if os.path.isdir(source):
+                    composefs_path = ComposefsPath(
+                        attrs,
+                        size=4096,
+                        filetype=FileType.directory,
+                        mode=mode,
+                        payload=source,
+                    )
+                else:
+                    composefs_path = ComposefsPath(
+                        attrs,
+                        size=os.stat(source).st_size,
+                        filetype=FileType.file,
+                        mode=mode,
+                        payload=target,
+                    )
+            paths[target] = composefs_path
+            add_leading_directories(target, attrs, paths)
+
+    composefs_dump = ["/ 4096 40755 1 0 0 0 0.0 - - -"]  # Root directory
+    for key in sorted(paths):
+        composefs_path = paths[key]
+        eprint(composefs_path.path)
+        composefs_dump.append(composefs_path.write_line())
+
+    print("\n".join(composefs_dump))
+
+
+if __name__ == "__main__":
+    main()

nixos/modules/system/etc/check-build-composefs-dump.sh

@@ -0,0 +1,8 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -i bash -p black ruff mypy
+
+file=$1
+
+black --check --diff $file
+ruff --line-length 88 $file
+mypy --strict $file

nixos/modules/system/etc/etc-activation.nix

@@ -1,12 +1,96 @@
 { config, lib, ... }:
-let
-  inherit (lib) stringAfter;
-in {
+
+{
 
   imports = [ ./etc.nix ];
 
-  config = {
-    system.activationScripts.etc =
-      stringAfter [ "users" "groups" ] config.system.build.etcActivationCommands;
-  };
+  config = lib.mkMerge [
+
+    {
+      system.activationScripts.etc =
+        lib.stringAfter [ "users" "groups" ] config.system.build.etcActivationCommands;
+    }
+
+    (lib.mkIf config.system.etc.overlay.enable {
+
+      assertions = [
+        {
+          assertion = config.boot.initrd.systemd.enable;
+          message = "`system.etc.overlay.enable` requires `boot.initrd.systemd.enable`";
+        }
+        {
+          assertion = (!config.system.etc.overlay.mutable) -> config.systemd.sysusers.enable;
+          message = "`system.etc.overlay.mutable = false` requires `systemd.sysusers.enable`";
+        }
+        {
+          assertion = lib.versionAtLeast config.boot.kernelPackages.kernel.version "6.6";
+          message = "`system.etc.overlay.enable requires a newer kernel, at least version 6.6";
+        }
+        {
+          assertion = config.systemd.sysusers.enable -> (config.users.mutableUsers == config.system.etc.overlay.mutable);
+          message = ''
+            When using systemd-sysusers and mounting `/etc` via an overlay, users
+            can only be mutable when `/etc` is mutable and vice versa.
+          '';
+        }
+      ];
+
+      boot.initrd.availableKernelModules = [ "loop" "erofs" "overlay" ];
+
+      boot.initrd.systemd = {
+        mounts = [
+          {
+            where = "/run/etc-metadata";
+            what = "/sysroot${config.system.build.etcMetadataImage}";
+            type = "erofs";
+            options = "loop";
+            unitConfig.RequiresMountsFor = [
+              "/sysroot/nix/store"
+            ];
+          }
+          {
+            where = "/sysroot/etc";
+            what = "overlay";
+            type = "overlay";
+            options = lib.concatStringsSep "," ([
+              "relatime"
+              "redirect_dir=on"
+              "metacopy=on"
+              "lowerdir=/run/etc-metadata::/sysroot${config.system.build.etcBasedir}"
+            ] ++ lib.optionals config.system.etc.overlay.mutable [
+              "rw"
+              "upperdir=/sysroot/.rw-etc/upper"
+              "workdir=/sysroot/.rw-etc/work"
+            ] ++ lib.optionals (!config.system.etc.overlay.mutable) [
+              "ro"
+            ]);
+            wantedBy = [ "initrd-fs.target" ];
+            before = [ "initrd-fs.target" ];
+            requires = lib.mkIf config.system.etc.overlay.mutable [ "rw-etc.service" ];
+            after = lib.mkIf config.system.etc.overlay.mutable [ "rw-etc.service" ];
+            unitConfig.RequiresMountsFor = [
+              "/sysroot/nix/store"
+              "/run/etc-metadata"
+            ];
+          }
+        ];
+        services = lib.mkIf config.system.etc.overlay.mutable {
+          rw-etc = {
+            unitConfig = {
+              DefaultDependencies = false;
+              RequiresMountsFor = "/sysroot";
+            };
+            serviceConfig = {
+              Type = "oneshot";
+              ExecStart = ''
+                /bin/mkdir -p -m 0755 /sysroot/.rw-etc/upper /sysroot/.rw-etc/work
+              '';
+            };
+          };
+        };
+      };
+
+    })
+
+  ];
 }

nixos/modules/system/etc/etc.nix

@@ -62,6 +62,16 @@ let
     ]) etc'}
   '';
 
+  etcHardlinks = filter (f: f.mode != "symlink") etc';
+
+  build-composefs-dump = pkgs.runCommand "build-composefs-dump.py"
+    {
+      buildInputs = [ pkgs.python3 ];
+    } ''
+    install ${./build-composefs-dump.py} $out
+    patchShebangs --host $out
+  '';
+
 in
 
 {
@@ -72,6 +82,30 @@ in
 
   options = {
 
+    system.etc.overlay = {
+      enable = mkOption {
+        type = types.bool;
+        default = false;
+        description = lib.mdDoc ''
+          Mount `/etc` as an overlayfs instead of generating it via a perl script.
+
+          Note: This is currently experimental. Only enable this option if you're
+          confident that you can recover your system if it breaks.
+        '';
+      };
+
+      mutable = mkOption {
+        type = types.bool;
+        default = true;
+        description = lib.mdDoc ''
+          Whether to mount `/etc` mutably (i.e. read-write) or immutably (i.e. read-only).
+
+          If this is false, only the immutable lowerdir is mounted. If it is
+          true, a writable upperdir is mounted on top.
+        '';
+      };
+    };
+
     environment.etc = mkOption {
       default = {};
       example = literalExpression ''
@@ -190,12 +224,84 @@ in
   config = {
 
     system.build.etc = etc;
-    system.build.etcActivationCommands =
-      ''
-        # Set up the statically computed bits of /etc.
-        echo "setting up /etc..."
-        ${pkgs.perl.withPackages (p: [ p.FileSlurp ])}/bin/perl ${./setup-etc.pl} ${etc}/etc
+    system.build.etcActivationCommands = let
+      etcOverlayOptions = lib.concatStringsSep "," ([
+        "relatime"
+        "redirect_dir=on"
+        "metacopy=on"
+      ] ++ lib.optionals config.system.etc.overlay.mutable [
+        "upperdir=/.rw-etc/upper"
+        "workdir=/.rw-etc/work"
+      ]);
+    in if config.system.etc.overlay.enable then ''
+      # This script atomically remounts /etc when switching configuration. On a (re-)boot
+      # this should not run because /etc is mounted via a systemd mount unit
+      # instead. To a large extent this mimics what composefs does. Because
+      # it's relatively simple, however, we avoid the composefs dependency.
+      if [[ ! $IN_NIXOS_SYSTEMD_STAGE1 ]]; then
+        echo "remounting /etc..."
+
+        tmpMetadataMount=$(mktemp --directory)
+        mount --type erofs ${config.system.build.etcMetadataImage} $tmpMetadataMount
+
+        # Mount the new /etc overlay to a temporary private mount.
+        # This needs the indirection via a private bind mount because you
+        # cannot move shared mounts.
+        tmpEtcMount=$(mktemp --directory)
+        mount --bind --make-private $tmpEtcMount $tmpEtcMount
+        mount --type overlay overlay \
+          --options lowerdir=$tmpMetadataMount::${config.system.build.etcBasedir},${etcOverlayOptions} \
+          $tmpEtcMount
+
+        # Move the new temporary /etc mount underneath the current /etc mount.
+        #
+        # This should eventually use util-linux to perform this move beneath,
+        # however, this functionality is not yet in util-linux. See this
+        # tracking issue: https://github.com/util-linux/util-linux/issues/2604
+        ${pkgs.move-mount-beneath}/bin/move-mount --move --beneath $tmpEtcMount /etc
+
+        # Unmount the top /etc mount to atomically reveal the new mount.
+        umount /etc
+
+      fi
+    '' else ''
+      # Set up the statically computed bits of /etc.
+      echo "setting up /etc..."
+      ${pkgs.perl.withPackages (p: [ p.FileSlurp ])}/bin/perl ${./setup-etc.pl} ${etc}/etc
+    '';
+
+    system.build.etcBasedir = pkgs.runCommandLocal "etc-lowerdir" { } ''
+      set -euo pipefail
+
+      makeEtcEntry() {
+        src="$1"
+        target="$2"
+
+        mkdir -p "$out/$(dirname "$target")"
+        cp "$src" "$out/$target"
+      }
+
+      mkdir -p "$out"
+      ${concatMapStringsSep "\n" (etcEntry: escapeShellArgs [
+        "makeEtcEntry"
+        # Force local source paths to be added to the store
+        "${etcEntry.source}"
+        etcEntry.target
+      ]) etcHardlinks}
+    '';
+
+    system.build.etcMetadataImage =
+      let
+        etcJson = pkgs.writeText "etc-json" (builtins.toJSON etc');
+        etcDump = pkgs.runCommand "etc-dump" { } "${build-composefs-dump} ${etcJson} > $out";
+      in
+      pkgs.runCommand "etc-metadata.erofs" {
+        nativeBuildInputs = [ pkgs.composefs pkgs.erofs-utils ];
+      } ''
+        mkcomposefs --from-file ${etcDump} $out
+        fsck.erofs $out
       '';
+
   };
 
 }

nixos/tests/activation/etc-overlay-immutable.nix

@@ -0,0 +1,30 @@
+{ lib, ... }: {
+
+  name = "activation-etc-overlay-immutable";
+
+  meta.maintainers = with lib.maintainers; [ nikstur ];
+
+  nodes.machine = { pkgs, ... }: {
+    system.etc.overlay.enable = true;
+    system.etc.overlay.mutable = false;
+
+    # Prerequisites
+    systemd.sysusers.enable = true;
+    users.mutableUsers = false;
+    boot.initrd.systemd.enable = true;
+    boot.kernelPackages = pkgs.linuxPackages_latest;
+
+    specialisation.new-generation.configuration = {
+      environment.etc."newgen".text = "newgen";
+    };
+  };
+
+  testScript = ''
+    machine.succeed("findmnt --kernel --type overlay /etc")
+    machine.fail("stat /etc/newgen")
+
+    machine.succeed("/run/current-system/specialisation/new-generation/bin/switch-to-configuration switch")
+
+    assert machine.succeed("cat /etc/newgen") == "newgen"
+  '';
+}

nixos/tests/activation/etc-overlay-mutable.nix

@@ -0,0 +1,30 @@
+{ lib, ... }: {
+
+  name = "activation-etc-overlay-mutable";
+
+  meta.maintainers = with lib.maintainers; [ nikstur ];
+
+  nodes.machine = { pkgs, ... }: {
+    system.etc.overlay.enable = true;
+    system.etc.overlay.mutable = true;
+
+    # Prerequisites
+    boot.initrd.systemd.enable = true;
+    boot.kernelPackages = pkgs.linuxPackages_latest;
+
+    specialisation.new-generation.configuration = {
+      environment.etc."newgen".text = "newgen";
+    };
+  };
+
+  testScript = ''
+    machine.succeed("findmnt --kernel --type overlay /etc")
+    machine.fail("stat /etc/newgen")
+    machine.succeed("echo -n 'mutable' > /etc/mutable")
+
+    machine.succeed("/run/current-system/specialisation/new-generation/bin/switch-to-configuration switch")
+
+    assert machine.succeed("cat /etc/newgen") == "newgen"
+    assert machine.succeed("cat /etc/mutable") == "mutable"
+  '';
+}

nixos/tests/all-tests.nix

@@ -285,6 +285,8 @@ in {
   activation = pkgs.callPackage ../modules/system/activation/test.nix { };
   activation-var = runTest ./activation/var.nix;
   activation-nix-channel = runTest ./activation/nix-channel.nix;
+  activation-etc-overlay-mutable = runTest ./activation/etc-overlay-mutable.nix;
+  activation-etc-overlay-immutable = runTest ./activation/etc-overlay-immutable.nix;
   etcd = handleTestOn ["x86_64-linux"] ./etcd.nix {};
   etcd-cluster = handleTestOn ["x86_64-linux"] ./etcd-cluster.nix {};
   etebase-server = handleTest ./etebase-server.nix {};