nixos/k3s: add containerdConfigTemplate option

nixos/modules/services/cluster/k3s/default.nix

@@ -21,6 +21,7 @@ let
   manifestDir = "/var/lib/rancher/k3s/server/manifests";
   chartDir = "/var/lib/rancher/k3s/server/static/charts";
   imageDir = "/var/lib/rancher/k3s/agent/images";
+  containerdConfigTemplateFile = "/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl";
 
   manifestModule =
     let
@@ -119,6 +120,11 @@ let
     ${builtins.concatStringsSep "\n" (map linkManifestEntry enabledManifests)}
     ${builtins.concatStringsSep "\n" (lib.mapAttrsToList linkChartEntry cfg.charts)}
     ${builtins.concatStringsSep "\n" (map linkImageEntry cfg.images)}
+
+    ${lib.optionalString (cfg.containerdConfigTemplate != null) ''
+      mkdir -p $(dirname ${containerdConfigTemplateFile})
+      ${pkgs.coreutils-full}/bin/ln -sfn ${pkgs.writeText "config.toml.tmpl" cfg.containerdConfigTemplate} ${containerdConfigTemplateFile}
+    ''}
   '';
 in
 {
@@ -340,6 +346,26 @@ in
       '';
     };
 
+    containerdConfigTemplate = mkOption {
+      type = types.nullOr types.str;
+      default = null;
+      example = lib.literalExpression ''
+        # Base K3s config
+        {{ template "base" . }}
+
+        # Add a custom runtime
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes."custom"]
+          runtime_type = "io.containerd.runc.v2"
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes."custom".options]
+          BinaryName = "/path/to/custom-container-runtime"
+      '';
+      description = ''
+        Config template for containerd, to be placed at
+        `/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl`.
+        See the K3s docs on [configuring containerd](https://docs.k3s.io/advanced#configuring-containerd).
+      '';
+    };
+
     images = mkOption {
       type = with types; listOf package;
       default = [ ];

nixos/tests/k3s/containerd-config.nix

@@ -0,0 +1,52 @@
+# A test that containerdConfigTemplate settings get written to containerd/config.toml
+import ../make-test-python.nix (
+  { lib, k3s, ... }:
+  let
+    nodeName = "test";
+  in
+  {
+    name = "${k3s.name}-containerd-config";
+    nodes.machine =
+      { ... }:
+      {
+        # k3s uses enough resources the default vm fails.
+        virtualisation.memorySize = 1536;
+        virtualisation.diskSize = 4096;
+
+        services.k3s = {
+          enable = true;
+          package = k3s;
+          # Slightly reduce resource usage
+          extraFlags = [
+            "--disable coredns"
+            "--disable local-storage"
+            "--disable metrics-server"
+            "--disable servicelb"
+            "--disable traefik"
+            "--node-name ${nodeName}"
+          ];
+          containerdConfigTemplate = ''
+            # Base K3s config
+            {{ template "base" . }}
+
+            # MAGIC COMMENT
+          '';
+        };
+      };
+
+    testScript = ''
+      start_all()
+      machine.wait_for_unit("k3s")
+      # wait until the node is ready
+      machine.wait_until_succeeds(r"""kubectl wait --for='jsonpath={.status.conditions[?(@.type=="Ready")].status}=True' nodes/${nodeName}""")
+      # test whether the config template file contains the magic comment
+      out=machine.succeed("cat /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl")
+      assert "MAGIC COMMENT" in out, "the containerd config template does not contain the magic comment"
+      # test whether the config file contains the magic comment
+      out=machine.succeed("cat /var/lib/rancher/k3s/agent/etc/containerd/config.toml")
+      assert "MAGIC COMMENT" in out, "the containerd config does not contain the magic comment"
+    '';
+
+    meta.maintainers = lib.teams.k3s.members;
+  }
+)

nixos/tests/k3s/default.nix

@@ -11,6 +11,9 @@ in
     _: k3s: import ./airgap-images.nix { inherit system pkgs k3s; }
   ) allK3s;
   auto-deploy = lib.mapAttrs (_: k3s: import ./auto-deploy.nix { inherit system pkgs k3s; }) allK3s;
+  containerd-config = lib.mapAttrs (
+    _: k3s: import ./containerd-config.nix { inherit system pkgs k3s; }
+  ) allK3s;
   etcd = lib.mapAttrs (
     _: k3s:
     import ./etcd.nix {