mirror of
https://github.com/NixOS/nixpkgs.git
synced 2025-04-14 13:47:43 +00:00
Merge pull request #275180 from rorosen/extend-k3s-module
This commit is contained in:
Marcus Ramberg
GitHub
commit
72249a0d35
@ -17,6 +17,109 @@ let
|
||||
]
|
||||
++ config
|
||||
) instruction;
|
||||
|
||||
manifestDir = "/var/lib/rancher/k3s/server/manifests";
|
||||
chartDir = "/var/lib/rancher/k3s/server/static/charts";
|
||||
imageDir = "/var/lib/rancher/k3s/agent/images";
|
||||
|
||||
manifestModule =
|
||||
let
|
||||
mkTarget =
|
||||
name: if (lib.hasSuffix ".yaml" name || lib.hasSuffix ".yml" name) then name else name + ".yaml";
|
||||
in
|
||||
lib.types.submodule (
|
||||
{
|
||||
name,
|
||||
config,
|
||||
options,
|
||||
...
|
||||
}:
|
||||
{
|
||||
options = {
|
||||
enable = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = true;
|
||||
description = "Whether this manifest file should be generated.";
|
||||
};
|
||||
|
||||
target = lib.mkOption {
|
||||
type = lib.types.nonEmptyStr;
|
||||
example = lib.literalExpression "manifest.yaml";
|
||||
description = ''
|
||||
Name of the symlink (relative to {file}`${manifestDir}`).
|
||||
Defaults to the attribute name.
|
||||
'';
|
||||
};
|
||||
|
||||
content = lib.mkOption {
|
||||
type = with lib.types; nullOr (either attrs (listOf attrs));
|
||||
default = null;
|
||||
description = ''
|
||||
Content of the manifest file. A single attribute set will
|
||||
generate a single document YAML file. A list of attribute sets
|
||||
will generate multiple documents separated by `---` in a single
|
||||
YAML file.
|
||||
'';
|
||||
};
|
||||
|
||||
source = lib.mkOption {
|
||||
type = lib.types.path;
|
||||
example = lib.literalExpression "./manifests/app.yaml";
|
||||
description = ''
|
||||
Path of the source `.yaml` file.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
config = {
|
||||
target = lib.mkDefault (mkTarget name);
|
||||
source = lib.mkIf (config.content != null) (
|
||||
let
|
||||
name' = "k3s-manifest-" + builtins.baseNameOf name;
|
||||
docName = "k3s-manifest-doc-" + builtins.baseNameOf name;
|
||||
yamlDocSeparator = builtins.toFile "yaml-doc-separator" "\n---\n";
|
||||
mkYaml = name: x: (pkgs.formats.yaml { }).generate name x;
|
||||
mkSource =
|
||||
value:
|
||||
if builtins.isList value then
|
||||
pkgs.concatText name' (
|
||||
lib.concatMap (x: [
|
||||
yamlDocSeparator
|
||||
(mkYaml docName x)
|
||||
]) value
|
||||
)
|
||||
else
|
||||
mkYaml name' value;
|
||||
in
|
||||
lib.mkDerivedConfig options.content mkSource
|
||||
);
|
||||
};
|
||||
}
|
||||
);
|
||||
|
||||
enabledManifests = with builtins; filter (m: m.enable) (attrValues cfg.manifests);
|
||||
linkManifestEntry = m: "${pkgs.coreutils-full}/bin/ln -sfn ${m.source} ${manifestDir}/${m.target}";
|
||||
linkImageEntry = image: "${pkgs.coreutils-full}/bin/ln -sfn ${image} ${imageDir}/${image.name}";
|
||||
linkChartEntry =
|
||||
let
|
||||
mkTarget = name: if (lib.hasSuffix ".tgz" name) then name else name + ".tgz";
|
||||
in
|
||||
name: value:
|
||||
"${pkgs.coreutils-full}/bin/ln -sfn ${value} ${chartDir}/${mkTarget (builtins.baseNameOf name)}";
|
||||
|
||||
activateK3sContent = pkgs.writeShellScript "activate-k3s-content" ''
|
||||
${lib.optionalString (
|
||||
builtins.length enabledManifests > 0
|
||||
) "${pkgs.coreutils-full}/bin/mkdir -p ${manifestDir}"}
|
||||
${lib.optionalString (cfg.charts != { }) "${pkgs.coreutils-full}/bin/mkdir -p ${chartDir}"}
|
||||
${lib.optionalString (
|
||||
builtins.length cfg.images > 0
|
||||
) "${pkgs.coreutils-full}/bin/mkdir -p ${imageDir}"}
|
||||
|
||||
${builtins.concatStringsSep "\n" (map linkManifestEntry enabledManifests)}
|
||||
${builtins.concatStringsSep "\n" (lib.mapAttrsToList linkChartEntry cfg.charts)}
|
||||
${builtins.concatStringsSep "\n" (map linkImageEntry cfg.images)}
|
||||
'';
|
||||
in
|
||||
{
|
||||
imports = [ (removeOption [ "docker" ] "k3s docker option is no longer supported.") ];
|
||||
@ -127,11 +230,148 @@ in
|
||||
default = null;
|
||||
description = "File path containing the k3s YAML config. This is useful when the config is generated (for example on boot).";
|
||||
};
|
||||
|
||||
manifests = mkOption {
|
||||
type = types.attrsOf manifestModule;
|
||||
default = { };
|
||||
example = lib.literalExpression ''
|
||||
deployment.source = ../manifests/deployment.yaml;
|
||||
my-service = {
|
||||
enable = false;
|
||||
target = "app-service.yaml";
|
||||
content = {
|
||||
apiVersion = "v1";
|
||||
kind = "Service";
|
||||
metadata = {
|
||||
name = "app-service";
|
||||
};
|
||||
spec = {
|
||||
selector = {
|
||||
"app.kubernetes.io/name" = "MyApp";
|
||||
};
|
||||
ports = [
|
||||
{
|
||||
name = "name-of-service-port";
|
||||
protocol = "TCP";
|
||||
port = 80;
|
||||
targetPort = "http-web-svc";
|
||||
}
|
||||
];
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
nginx.content = [
|
||||
{
|
||||
apiVersion = "v1";
|
||||
kind = "Pod";
|
||||
metadata = {
|
||||
name = "nginx";
|
||||
labels = {
|
||||
"app.kubernetes.io/name" = "MyApp";
|
||||
};
|
||||
};
|
||||
spec = {
|
||||
containers = [
|
||||
{
|
||||
name = "nginx";
|
||||
image = "nginx:1.14.2";
|
||||
ports = [
|
||||
{
|
||||
containerPort = 80;
|
||||
name = "http-web-svc";
|
||||
}
|
||||
];
|
||||
}
|
||||
];
|
||||
};
|
||||
}
|
||||
{
|
||||
apiVersion = "v1";
|
||||
kind = "Service";
|
||||
metadata = {
|
||||
name = "nginx-service";
|
||||
};
|
||||
spec = {
|
||||
selector = {
|
||||
"app.kubernetes.io/name" = "MyApp";
|
||||
};
|
||||
ports = [
|
||||
{
|
||||
name = "name-of-service-port";
|
||||
protocol = "TCP";
|
||||
port = 80;
|
||||
targetPort = "http-web-svc";
|
||||
}
|
||||
];
|
||||
};
|
||||
}
|
||||
];
|
||||
'';
|
||||
description = ''
|
||||
Auto-deploying manifests that are linked to {file}`${manifestDir}` before k3s starts.
|
||||
Note that deleting manifest files will not remove or otherwise modify the resources
|
||||
it created. Please use the the `--disable` flag or `.skip` files to delete/disable AddOns,
|
||||
as mentioned in the [docs](https://docs.k3s.io/installation/packaged-components#disabling-manifests).
|
||||
This option only makes sense on server nodes (`role = server`).
|
||||
Read the [auto-deploying manifests docs](https://docs.k3s.io/installation/packaged-components#auto-deploying-manifests-addons)
|
||||
for further information.
|
||||
'';
|
||||
};
|
||||
|
||||
charts = mkOption {
|
||||
type = with types; attrsOf (either path package);
|
||||
default = { };
|
||||
example = lib.literalExpression ''
|
||||
nginx = ../charts/my-nginx-chart.tgz;
|
||||
redis = ../charts/my-redis-chart.tgz;
|
||||
'';
|
||||
description = ''
|
||||
Packaged Helm charts that are linked to {file}`${chartDir}` before k3s starts.
|
||||
The attribute name will be used as the link target (relative to {file}`${chartDir}`).
|
||||
The specified charts will only be placed on the file system and made available to the
|
||||
Kubernetes APIServer from within the cluster, you may use the
|
||||
[k3s Helm controller](https://docs.k3s.io/helm#using-the-helm-controller)
|
||||
to deploy the charts. This option only makes sense on server nodes
|
||||
(`role = server`).
|
||||
'';
|
||||
};
|
||||
|
||||
images = mkOption {
|
||||
type = with types; listOf package;
|
||||
default = [ ];
|
||||
example = lib.literalExpression ''
|
||||
[
|
||||
(pkgs.dockerTools.pullImage {
|
||||
imageName = "docker.io/bitnami/keycloak";
|
||||
imageDigest = "sha256:714dfadc66a8e3adea6609bda350345bd3711657b7ef3cf2e8015b526bac2d6b";
|
||||
sha256 = "0imblp0kw9vkcr7sp962jmj20fpmb3hvd3hmf4cs4x04klnq3k90";
|
||||
finalImageTag = "21.1.2-debian-11-r0";
|
||||
})
|
||||
]
|
||||
'';
|
||||
description = ''
|
||||
List of derivations that provide container images.
|
||||
All images are linked to {file}`${imageDir}` before k3s starts and consequently imported
|
||||
by the k3s agent. This option only makes sense on nodes with an enabled agent.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
# implementation
|
||||
|
||||
config = mkIf cfg.enable {
|
||||
warnings =
|
||||
(lib.optional (cfg.role != "server" && cfg.manifests != { })
|
||||
"k3s: Auto deploying manifests are only installed on server nodes (role == server), they will be ignored by this node."
|
||||
)
|
||||
++ (lib.optional (cfg.role != "server" && cfg.charts != { })
|
||||
"k3s: Helm charts are only made available to the cluster on server nodes (role == server), they will be ignored by this node."
|
||||
)
|
||||
++ (lib.optional (cfg.disableAgent && cfg.images != [ ])
|
||||
"k3s: Images are only imported on nodes with an enabled agent, they will be ignored by this node"
|
||||
);
|
||||
|
||||
assertions = [
|
||||
{
|
||||
assertion = cfg.role == "agent" -> (cfg.configPath != null || cfg.serverAddr != "");
|
||||
@ -178,6 +418,7 @@ in
|
||||
LimitCORE = "infinity";
|
||||
TasksMax = "infinity";
|
||||
EnvironmentFile = cfg.environmentFile;
|
||||
ExecStartPre = activateK3sContent;
|
||||
ExecStart = concatStringsSep " \\\n " (
|
||||
[ "${cfg.package}/bin/k3s ${cfg.role}" ]
|
||||
++ (optional cfg.clusterInit "--cluster-init")
|
||||
|
||||
122
nixos/tests/k3s/auto-deploy.nix
Normal file
122
nixos/tests/k3s/auto-deploy.nix
Normal file
@ -0,0 +1,122 @@
|
||||
import ../make-test-python.nix (
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
k3s,
|
||||
...
|
||||
}:
|
||||
let
|
||||
pauseImageEnv = pkgs.buildEnv {
|
||||
name = "k3s-pause-image-env";
|
||||
paths = with pkgs; [
|
||||
tini
|
||||
(hiPrio coreutils)
|
||||
busybox
|
||||
];
|
||||
};
|
||||
pauseImage = pkgs.dockerTools.buildImage {
|
||||
name = "test.local/pause";
|
||||
tag = "local";
|
||||
copyToRoot = pauseImageEnv;
|
||||
config.Entrypoint = [
|
||||
"/bin/tini"
|
||||
"--"
|
||||
"/bin/sleep"
|
||||
"inf"
|
||||
];
|
||||
};
|
||||
helloImage = pkgs.dockerTools.buildImage {
|
||||
name = "test.local/hello";
|
||||
tag = "local";
|
||||
copyToRoot = pkgs.hello;
|
||||
config.Entrypoint = [ "${pkgs.hello}/bin/hello" ];
|
||||
};
|
||||
in
|
||||
{
|
||||
name = "${k3s.name}-auto-deploy";
|
||||
|
||||
nodes.machine =
|
||||
{ pkgs, ... }:
|
||||
{
|
||||
environment.systemPackages = [ k3s ];
|
||||
|
||||
# k3s uses enough resources the default vm fails.
|
||||
virtualisation.memorySize = 1536;
|
||||
virtualisation.diskSize = 4096;
|
||||
|
||||
services.k3s.enable = true;
|
||||
services.k3s.role = "server";
|
||||
services.k3s.package = k3s;
|
||||
# Slightly reduce resource usage
|
||||
services.k3s.extraFlags = builtins.toString [
|
||||
"--disable coredns"
|
||||
"--disable local-storage"
|
||||
"--disable metrics-server"
|
||||
"--disable servicelb"
|
||||
"--disable traefik"
|
||||
"--pause-image test.local/pause:local"
|
||||
];
|
||||
services.k3s.images = [
|
||||
pauseImage
|
||||
helloImage
|
||||
];
|
||||
services.k3s.manifests = {
|
||||
absent = {
|
||||
enable = false;
|
||||
content = {
|
||||
apiVersion = "v1";
|
||||
kind = "Namespace";
|
||||
metadata.name = "absent";
|
||||
};
|
||||
};
|
||||
|
||||
present = {
|
||||
target = "foo-namespace.yaml";
|
||||
content = {
|
||||
apiVersion = "v1";
|
||||
kind = "Namespace";
|
||||
metadata.name = "foo";
|
||||
};
|
||||
};
|
||||
|
||||
hello.content = {
|
||||
apiVersion = "batch/v1";
|
||||
kind = "Job";
|
||||
metadata.name = "hello";
|
||||
spec = {
|
||||
template.spec = {
|
||||
containers = [
|
||||
{
|
||||
name = "hello";
|
||||
image = "test.local/hello:local";
|
||||
}
|
||||
];
|
||||
restartPolicy = "OnFailure";
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
testScript = ''
|
||||
start_all()
|
||||
|
||||
machine.wait_for_unit("k3s")
|
||||
# check existence of the manifest files
|
||||
machine.fail("ls /var/lib/rancher/k3s/server/manifests/absent.yaml")
|
||||
machine.succeed("ls /var/lib/rancher/k3s/server/manifests/foo-namespace.yaml")
|
||||
machine.succeed("ls /var/lib/rancher/k3s/server/manifests/hello.yaml")
|
||||
|
||||
# check if container images got imported
|
||||
machine.succeed("crictl img | grep 'test\.local/pause'")
|
||||
machine.succeed("crictl img | grep 'test\.local/hello'")
|
||||
|
||||
# check if resources of manifests got created
|
||||
machine.wait_until_succeeds("kubectl get ns foo")
|
||||
machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello")
|
||||
machine.fail("kubectl get ns absent")
|
||||
|
||||
machine.shutdown()
|
||||
'';
|
||||
}
|
||||
)
|
||||
@ -19,4 +19,6 @@ in
|
||||
single-node = lib.mapAttrs (_: k3s: import ./single-node.nix { inherit system pkgs k3s; }) allK3s;
|
||||
# Run a multi-node k3s cluster and verify pod networking works across nodes
|
||||
multi-node = lib.mapAttrs (_: k3s: import ./multi-node.nix { inherit system pkgs k3s; }) allK3s;
|
||||
# Test wether container images are imported and auto deploying manifests work
|
||||
auto-deploy = lib.mapAttrs (_: k3s: import ./auto-deploy.nix { inherit system pkgs k3s; }) allK3s;
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user