nixos/rke2: add rke2 service

Set assertions to avoid obvious errors.

Eliminate the conflict between default CNI (`cana`) and `NetworkManager`.

Determine whether optional can be used for agent.

Add the option `cisHardening` to enable CIS Hardening.

Set kernel parameters by `boot.kernel.sysctl`.

Using `lib.escapeShellArgs` to make `ExecStart` more resilient to escaping issues.

Using a list of `str` to extra flags.

nixos/doc/manual/release-notes/rl-2405.section.md

@@ -215,6 +215,8 @@ The pre-existing [services.ankisyncd](#opt-services.ankisyncd.enable) has been m
 
 - [keto](https://www.ory.sh/keto/), a permission & access control server, the first open source implementation of ["Zanzibar: Google's Consistent, Global Authorization System"](https://research.google/pubs/zanzibar-googles-consistent-global-authorization-system/).
 
+- [RKE2](https://github.com/rancher/rke2), also known as RKE Government, is Rancher's next-generation Kubernetes distribution. Available as [services.rke2](#opt-services.rke2.enable).
+
 ## Backward Incompatibilities {#sec-release-24.05-incompatibilities}
 
 <!-- To avoid merge conflicts, consider adding your item at an arbitrary place in the list instead. -->

nixos/modules/module-list.nix

@@ -415,6 +415,7 @@
   ./services/cluster/kubernetes/scheduler.nix
   ./services/cluster/pacemaker/default.nix
   ./services/cluster/patroni/default.nix
+  ./services/cluster/rke2/default.nix
   ./services/cluster/spark/default.nix
   ./services/computing/boinc/client.nix
   ./services/computing/foldingathome/client.nix

nixos/modules/services/cluster/rke2/default.nix

@@ -0,0 +1,311 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+let
+  cfg = config.services.rke2;
+in
+{
+  imports = [ ];
+
+  options.services.rke2 = {
+    enable = mkEnableOption "rke2";
+
+    package = mkPackageOption pkgs "rke2" { };
+
+    role = mkOption {
+      type = types.enum [ "server" "agent" ];
+      description = ''
+        Whether rke2 should run as a server or agent.
+
+        If it's a server:
+
+        - By default it also runs workloads as an agent.
+        - any optionals is allowed.
+
+        If it's an agent:
+
+        - `serverAddr` is required.
+        - `token` or `tokenFile` is required.
+        - `agentToken` or `agentTokenFile` or `disable` or `cni` are not allowed.
+      '';
+      default = "server";
+    };
+
+    configPath = mkOption {
+      type = types.path;
+      description = "Load configuration from FILE.";
+      default = "/etc/rancher/rke2/config.yaml";
+    };
+
+    debug = mkOption {
+      type = types.bool;
+      description = "Turn on debug logs.";
+      default = false;
+    };
+
+    dataDir = mkOption {
+      type = types.path;
+      description = "The folder to hold state in.";
+      default = "/var/lib/rancher/rke2";
+    };
+
+    token = mkOption {
+      type = types.str;
+      description = ''
+        Shared secret used to join a server or agent to a cluster.
+
+        > WARNING: This option will expose store your token unencrypted world-readable in the nix store.
+        If this is undesired use the `tokenFile` option instead.
+      '';
+      default = "";
+    };
+
+    tokenFile = mkOption {
+      type = types.nullOr types.path;
+      description = "File path containing rke2 token to use when connecting to the server.";
+      default = null;
+    };
+
+    disable = mkOption {
+      type = types.listOf types.str;
+      description = "Do not deploy packaged components and delete any deployed components.";
+      default = [ ];
+    };
+
+    nodeName = mkOption {
+      type = types.nullOr types.str;
+      description = "Node name.";
+      default = null;
+    };
+
+    nodeLabel = mkOption {
+      type = types.listOf types.str;
+      description = "Registering and starting kubelet with set of labels.";
+      default = [ ];
+    };
+
+    nodeTaint = mkOption {
+      type = types.listOf types.str;
+      description = "Registering kubelet with set of taints.";
+      default = [ ];
+    };
+
+    nodeIP = mkOption {
+      type = types.nullOr types.str;
+      description = "IPv4/IPv6 addresses to advertise for node.";
+      default = null;
+    };
+
+    agentToken = mkOption {
+      type = types.str;
+      description = ''
+        Shared secret used to join agents to the cluster, but not servers.
+
+        > **WARNING**: This option will expose store your token unencrypted world-readable in the nix store.
+        If this is undesired use the `agentTokenFile` option instead.
+      '';
+      default = "";
+    };
+
+    agentTokenFile = mkOption {
+      type = types.nullOr types.path;
+      description = "File path containing rke2 agent token to use when connecting to the server.";
+      default = null;
+    };
+
+    serverAddr = mkOption {
+      type = types.str;
+      description = "The rke2 server to connect to, used to join a cluster.";
+      example = "https://10.0.0.10:6443";
+      default = "";
+    };
+
+    selinux = mkOption {
+      type = types.bool;
+      description = "Enable SELinux in containerd.";
+      default = false;
+    };
+
+    cni = mkOption {
+      type = types.enum [ "none" "canal" "cilium" "calico" "flannel" ];
+      description = ''
+        CNI Plugins to deploy, one of `none`, `calico`, `canal`, `cilium` or `flannel`.
+
+        All CNI plugins get installed via a helm chart after the main components are up and running
+        and can be [customized by modifying the helm chart options](https://docs.rke2.io/helm).
+
+        [Learn more about RKE2 and CNI plugins](https://docs.rke2.io/networking/basic_network_options)
+
+        > **WARNING**: Flannel support in RKE2 is currently experimental.
+      '';
+      default = "canal";
+    };
+
+    cisHardening = mkOption {
+      type = types.bool;
+      description = ''
+        Enable CIS Hardening for RKE2.
+
+        It will set the configurations and controls required to address Kubernetes benchmark controls
+        from the Center for Internet Security (CIS).
+
+        Learn more about [CIS Hardening for RKE2](https://docs.rke2.io/security/hardening_guide).
+
+        > **NOTICE**:
+        >
+        > You may need restart the `systemd-sysctl` muaually by:
+        >
+        > ```shell
+        > sudo systemctl restart systemd-sysctl
+        > ```
+      '';
+      default = false;
+    };
+
+    extraFlags = mkOption {
+      type = types.listOf types.str;
+      description = ''
+        Extra flags to pass to the rke2 service/agent.
+
+        Here you can find all the available flags:
+
+        - [Server Configuration Reference](https://docs.rke2.io/reference/server_config)
+        - [Agent Configuration Reference](https://docs.rke2.io/reference/linux_agent_config)
+      '';
+      example = [ "--disable-kube-proxy" "--cluster-cidr=10.24.0.0/16" ];
+      default = [ ];
+    };
+
+    environmentVars = mkOption {
+      type = types.attrsOf types.str;
+      description = ''
+        Environment variables for configuring the rke2 service/agent.
+
+        Here you can find all the available environment variables:
+
+        - [Server Configuration Reference](https://docs.rke2.io/reference/server_config)
+        - [Agent Configuration Reference](https://docs.rke2.io/reference/linux_agent_config)
+
+        Besides the options above, you can also active environment variables by edit/create those files:
+
+        - `/etc/default/rke2`
+        - `/etc/sysconfig/rke2`
+        - `/usr/local/lib/systemd/system/rke2.env`
+      '';
+      # See: https://github.com/rancher/rke2/blob/master/bundle/lib/systemd/system/rke2-server.env#L1
+      default = {
+        HOME = "/root";
+      };
+    };
+  };
+
+  config = mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = cfg.role == "agent" -> (builtins.pathExists cfg.configPath || cfg.serverAddr != "");
+        message = "serverAddr or configPath (with 'server' key) should be set if role is 'agent'";
+      }
+      {
+        assertion = cfg.role == "agent" -> (builtins.pathExists cfg.configPath || cfg.tokenFile != null || cfg.token != "");
+        message = "token or tokenFile or configPath (with 'token' or 'token-file' keys) should be set if role is 'agent'";
+      }
+      {
+        assertion = cfg.role == "agent" -> ! (cfg.agentTokenFile != null || cfg.agentToken != "");
+        message = "agentToken or agentTokenFile should be set if role is 'agent'";
+      }
+      {
+        assertion = cfg.role == "agent" -> ! (cfg.disable != [ ]);
+        message = "disable should not be set if role is 'agent'";
+      }
+      {
+        assertion = cfg.role == "agent" -> ! (cfg.cni != "canal");
+        message = "cni should not be set if role is 'agent'";
+      }
+    ];
+
+    environment.systemPackages = [ config.services.rke2.package ];
+    # To configure NetworkManager to ignore calico/flannel related network interfaces.
+    # See: https://docs.rke2.io/known_issues#networkmanager
+    environment.etc."NetworkManager/conf.d/rke2-canal.conf" = {
+      enable = config.networking.networkmanager.enable;
+      text = ''
+        [keyfile]
+        unmanaged-devices=interface-name:cali*;interface-name:flannel*
+      '';
+    };
+    # See: https://docs.rke2.io/security/hardening_guide#set-kernel-parameters
+    boot.kernel.sysctl = mkIf cfg.cisHardening {
+      "vm.panic_on_oom" = 0;
+      "vm.overcommit_memory" = 1;
+      "kernel.panic" = 10;
+      "kernel.panic_on_oops" = 1;
+    };
+
+    systemd.services.rke2 = {
+      description = "Rancher Kubernetes Engine v2";
+      documentation = [ "https://github.com/rancher/rke2#readme" ];
+      after = [ "network-online.target" ];
+      wants = [ "network-online.target" ];
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        Type = if cfg.role == "agent" then "exec" else "notify";
+        EnvironmentFile = [
+          "-/etc/default/%N"
+          "-/etc/sysconfig/%N"
+          "-/usr/local/lib/systemd/system/%N.env"
+        ];
+        Environment = mapAttrsToList (k: v: "${k}=${v}") cfg.environmentVars;
+        KillMode = "process";
+        Delegate = "yes";
+        LimitNOFILE = 1048576;
+        LimitNPROC = "infinity";
+        LimitCORE = "infinity";
+        TasksMax = "infinity";
+        TimeoutStartSec = 0;
+        Restart = "always";
+        RestartSec = "5s";
+        ExecStartPre = [
+          # There is a conflict between RKE2 and `nm-cloud-setup.service`. This service add a routing table that
+          # interfere with the CNI plugin's configuration. This script checks if the service is enabled and if so,
+          # failed the RKE2 start.
+          # See: https://github.com/rancher/rke2/issues/1053
+          (pkgs.writeScript "check-nm-cloud-setup.sh" ''
+            #! ${pkgs.runtimeShell}
+            set -x
+            ! /run/current-system/systemd/bin/systemctl is-enabled --quiet nm-cloud-setup.service
+          '')
+          "-${pkgs.kmod}/bin/modprobe br_netfilter"
+          "-${pkgs.kmod}/bin/modprobe overlay"
+        ];
+        ExecStart = "${cfg.package}/bin/rke2 '${cfg.role}' ${escapeShellArgs (
+             (optional (cfg.configPath != "/etc/rancher/rke2/config.yaml") "--config=${cfg.configPath}")
+          ++ (optional cfg.debug "--debug")
+          ++ (optional (cfg.dataDir != "/var/lib/rancher/rke2") "--data-dir=${cfg.dataDir}")
+          ++ (optional (cfg.token != "") "--token=${cfg.token}")
+          ++ (optional (cfg.tokenFile != null) "--token-file=${cfg.tokenFile}")
+          ++ (optionals (cfg.role == "server" && cfg.disable != [ ]) (map (d: "--disable=${d}") cfg.disable))
+          ++ (optional (cfg.nodeName != null) "--node-name=${cfg.nodeName}")
+          ++ (optionals (cfg.nodeLabel != [ ]) (map (l: "--node-label=${l}") cfg.nodeLabel))
+          ++ (optionals (cfg.nodeTaint != [ ]) (map (t: "--node-taint=${t}") cfg.nodeTaint))
+          ++ (optional (cfg.nodeIP != null) "--node-ip=${cfg.nodeIP}")
+          ++ (optional (cfg.role == "server" && cfg.agentToken != "") "--agent-token=${cfg.agentToken}")
+          ++ (optional (cfg.role == "server" && cfg.agentTokenFile != null) "--agent-token-file=${cfg.agentTokenFile}")
+          ++ (optional (cfg.serverAddr != "") "--server=${cfg.serverAddr}")
+          ++ (optional cfg.selinux "--selinux")
+          ++ (optional (cfg.role == "server" && cfg.cni != "canal") "--cni=${cfg.cni}")
+          ++ (optional cfg.cisHardening "--profile=${if cfg.package.version >= "1.25" then "cis-1.23" else "cis-1.6"}")
+          ++ cfg.extraFlags
+        )}";
+        ExecStopPost = let
+          killProcess = pkgs.writeScript "kill-process.sh" ''
+            #! ${pkgs.runtimeShell}
+            /run/current-system/systemd/bin/systemd-cgls /system.slice/$1 | \
+            ${pkgs.gnugrep}/bin/grep -Eo '[0-9]+ (containerd|kubelet)' | \
+            ${pkgs.gawk}/bin/awk '{print $1}' | \
+            ${pkgs.findutils}/bin/xargs -r ${pkgs.util-linux}/bin/kill
+          '';
+        in "-${killProcess} %n";
+      };
+    };
+  };
+}