From 2b883410d4a31cbbd033c0ea92ce2e192193d3f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20Fern=C3=A1ndez=20L=C3=B3pez?= Date: Fri, 14 Jun 2024 09:57:34 +0200 Subject: [PATCH 1/7] nvidia-container-toolkit: only mount existing paths in the host --- .../nvidia-container-toolkit/cdi-generate.nix | 29 ++++++++++++++----- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/nixos/modules/services/hardware/nvidia-container-toolkit/cdi-generate.nix b/nixos/modules/services/hardware/nvidia-container-toolkit/cdi-generate.nix index 360a832e28cb..6a04fa0ff750 100644 --- a/nixos/modules/services/hardware/nvidia-container-toolkit/cdi-generate.nix +++ b/nixos/modules/services/hardware/nvidia-container-toolkit/cdi-generate.nix @@ -13,11 +13,14 @@ inherit hostPath containerPath; options = mountOptions; }; - jqAddMountExpression = ".containerEdits.mounts[.containerEdits.mounts | length] |= . +"; - allJqMounts = lib.concatMap - (mount: - ["${lib.getExe jq} '${jqAddMountExpression} ${builtins.toJSON (mkMount mount)}'"]) - mounts; + mountToCommand = mount: + "additionalMount \"${mount.hostPath}\" \"${mount.containerPath}\" '${builtins.toJSON mount.mountOptions}'"; + mountsToCommands = mounts: + if (builtins.length mounts) == 0 then + "cat" + else + (lib.strings.concatMapStringsSep " | \\\n" + mountToCommand mounts); in writeScriptBin "nvidia-cdi-generator" '' @@ -32,6 +35,18 @@ function cdiGenerate { --nvidia-ctk-path ${lib.getExe' nvidia-container-toolkit "nvidia-ctk"} } -cdiGenerate | \ - ${lib.concatStringsSep " | " allJqMounts} > $RUNTIME_DIRECTORY/nvidia-container-toolkit.json +function additionalMount { + local hostPath="$1" + local containerPath="$2" + local mountOptions="$3" + if [ -e "$hostPath" ]; then + ${lib.getExe jq} ".containerEdits.mounts[.containerEdits.mounts | length] = { \"hostPath\": \"$hostPath\", \"containerPath\": \"$containerPath\", \"options\": $mountOptions }" + else + echo "Mount $hostPath ignored: could not find path in the host machine" >&2 + cat + fi +} + +cdiGenerate | + ${mountsToCommands mounts} > $RUNTIME_DIRECTORY/nvidia-container-toolkit.json '' From d665ca4fb2c6c5814a763fa0cd861730d89ef609 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20Fern=C3=A1ndez=20L=C3=B3pez?= Date: Fri, 16 Aug 2024 20:33:50 +0200 Subject: [PATCH 2/7] nvidia-container-toolkit: add initial set of tests to check closures --- .../nvidia-container-toolkit/default.nix | 3 + nixos/tests/all-tests.nix | 1 + nixos/tests/nvidia-container-toolkit.nix | 177 ++++++++++++++++++ 3 files changed, 181 insertions(+) create mode 100644 nixos/tests/nvidia-container-toolkit.nix diff --git a/nixos/modules/services/hardware/nvidia-container-toolkit/default.nix b/nixos/modules/services/hardware/nvidia-container-toolkit/default.nix index fe4999bfbdbb..3936b2757c46 100644 --- a/nixos/modules/services/hardware/nvidia-container-toolkit/default.nix +++ b/nixos/modules/services/hardware/nvidia-container-toolkit/default.nix @@ -71,6 +71,8 @@ /usr/local/nvidia/lib64. ''; }; + + package = lib.mkPackageOption pkgs "nvidia-container-toolkit" { }; }; }; @@ -129,6 +131,7 @@ let script = pkgs.callPackage ./cdi-generate.nix { inherit (config.hardware.nvidia-container-toolkit) mounts; + nvidia-container-toolkit = config.hardware.nvidia-container-toolkit.package; nvidia-driver = config.hardware.nvidia.package; deviceNameStrategy = config.hardware.nvidia-container-toolkit.device-name-strategy; }; diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index e00c6b2b39d3..efe14ed699b6 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -699,6 +699,7 @@ in { ntfy-sh = handleTest ./ntfy-sh.nix {}; ntfy-sh-migration = handleTest ./ntfy-sh-migration.nix {}; ntpd-rs = handleTest ./ntpd-rs.nix {}; + nvidia-container-toolkit = handleTest ./nvidia-container-toolkit.nix {}; nvmetcfg = handleTest ./nvmetcfg.nix {}; nzbget = handleTest ./nzbget.nix {}; nzbhydra2 = handleTest ./nzbhydra2.nix {}; diff --git a/nixos/tests/nvidia-container-toolkit.nix b/nixos/tests/nvidia-container-toolkit.nix new file mode 100644 index 000000000000..78d3751eef4d --- /dev/null +++ b/nixos/tests/nvidia-container-toolkit.nix @@ -0,0 +1,177 @@ +import ./make-test-python.nix ( + { + pkgs, + lib, + system, + ... + }: + let + testContainerImage = + let + testCDIScript = pkgs.writeShellScriptBin "test-cdi" '' + die() { + echo "$1" + exit 1 + } + + check_file_referential_integrity() { + echo "checking $1 referential integrity" + ( ${pkgs.glibc.bin}/bin/ldd "$1" | ${lib.getExe pkgs.gnugrep} "not found" &> /dev/null ) && return 1 + return 0 + } + + check_directory_referential_integrity() { + ${lib.getExe pkgs.findutils} "$1" -type f -print0 | while read -d $'\0' file; do + if [[ $(${lib.getExe pkgs.file} "$file" | ${lib.getExe pkgs.gnugrep} ELF) ]]; then + check_file_referential_integrity "$file" || exit 1 + else + echo "skipping $file: not an ELF file" + fi + done + } + + check_directory_referential_integrity "/usr/bin" || exit 1 + check_directory_referential_integrity "${pkgs.addDriverRunpath.driverLink}" || exit 1 + check_directory_referential_integrity "/usr/local/nvidia" || exit 1 + ''; + in + pkgs.dockerTools.buildImage { + name = "cdi-test"; + tag = "latest"; + config = { + Cmd = [ "${testCDIScript}/bin/test-cdi" ]; + }; + copyToRoot = ( + with pkgs.dockerTools; + [ + usrBinEnv + binSh + ] + ); + }; + emptyCDISpec = '' + #! ${pkgs.runtimeShell} + cat < $out/bin/nvidia-ctk + ${emptyCDISpec} + EOF + chmod +x $out/bin/nvidia-ctk + ''; + }; + }; + in + { + name = "nvidia-container-toolkit"; + meta = with lib.maintainers; { + maintainers = [ ereslibre ]; + }; + nodes = { + no-nvidia-gpus = + { config, ... }: + { + environment.systemPackages = with pkgs; [ jq ]; + hardware = { + inherit nvidia-container-toolkit; + nvidia = { + open = true; + package = config.boot.kernelPackages.nvidiaPackages.stable.open; + }; + }; + }; + + nvidia-one-gpu = + { config, pkgs, ... }: + { + virtualisation.diskSize = 10240; + environment.systemPackages = with pkgs; [ + jq + podman + ]; + hardware = { + inherit nvidia-container-toolkit; + nvidia = { + open = true; + package = config.boot.kernelPackages.nvidiaPackages.stable.open; + }; + opengl.enable = true; + }; + virtualisation.containers.enable = true; + }; + + nvidia-one-gpu-invalid-host-paths = + { config, pkgs, ... }: + { + virtualisation.diskSize = 10240; + environment.systemPackages = with pkgs; [ jq ]; + hardware = { + nvidia-container-toolkit = nvidia-container-toolkit // { + mounts = [ + { + hostPath = "/non-existant-path"; + containerPath = "/some/path"; + } + ]; + }; + nvidia = { + open = true; + package = config.boot.kernelPackages.nvidiaPackages.stable.open; + }; + opengl.enable = true; + }; + virtualisation.containers.enable = true; + }; + }; + testScript = '' + start_all() + + with subtest("Generate an empty CDI spec for a machine with no Nvidia GPUs"): + no_nvidia_gpus.wait_for_unit("nvidia-container-toolkit-cdi-generator.service") + no_nvidia_gpus.succeed("cat /var/run/cdi/nvidia-container-toolkit.json | jq") + + with subtest("Podman loads the generated CDI spec for a machine with an Nvidia GPU"): + nvidia_one_gpu.wait_for_unit("nvidia-container-toolkit-cdi-generator.service") + nvidia_one_gpu.succeed("cat /var/run/cdi/nvidia-container-toolkit.json | jq") + nvidia_one_gpu.succeed("podman load < ${testContainerImage}") + print(nvidia_one_gpu.succeed("podman run --pull=never --device=nvidia.com/gpu=all -v /run/opengl-driver:/run/opengl-driver:ro cdi-test:latest")) + + # Issue: https://github.com/NixOS/nixpkgs/issues/319201 + with subtest("The generated CDI spec skips specified non-existant paths in the host"): + nvidia_one_gpu_invalid_host_paths.wait_for_unit("nvidia-container-toolkit-cdi-generator.service") + nvidia_one_gpu_invalid_host_paths.fail("grep 'non-existant-path' /var/run/cdi/nvidia-container-toolkit.json") + ''; + } +) From 1b7f2fe883b0846b7cc99f0da81d50054d175860 Mon Sep 17 00:00:00 2001 From: Someone Serge Date: Thu, 22 Aug 2024 17:09:56 +0000 Subject: [PATCH 3/7] tests/nvidia-container-toolkit: more getExe --- nixos/tests/nvidia-container-toolkit.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos/tests/nvidia-container-toolkit.nix b/nixos/tests/nvidia-container-toolkit.nix index 78d3751eef4d..aa174a8bd7cd 100644 --- a/nixos/tests/nvidia-container-toolkit.nix +++ b/nixos/tests/nvidia-container-toolkit.nix @@ -39,7 +39,7 @@ import ./make-test-python.nix ( name = "cdi-test"; tag = "latest"; config = { - Cmd = [ "${testCDIScript}/bin/test-cdi" ]; + Cmd = [ (lib.getExe testCDIScript) ]; }; copyToRoot = ( with pkgs.dockerTools; From 1e571aeab4c10daa086f1a8a78bb562b8e8ed245 Mon Sep 17 00:00:00 2001 From: Someone Serge Date: Thu, 22 Aug 2024 17:25:49 +0000 Subject: [PATCH 4/7] tests/nvidia-container-toolkit: less nesting --- nixos/tests/nvidia-container-toolkit.nix | 131 +++++++++++------------ 1 file changed, 65 insertions(+), 66 deletions(-) diff --git a/nixos/tests/nvidia-container-toolkit.nix b/nixos/tests/nvidia-container-toolkit.nix index aa174a8bd7cd..59105c5be15a 100644 --- a/nixos/tests/nvidia-container-toolkit.nix +++ b/nixos/tests/nvidia-container-toolkit.nix @@ -6,91 +6,90 @@ import ./make-test-python.nix ( ... }: let - testContainerImage = - let - testCDIScript = pkgs.writeShellScriptBin "test-cdi" '' - die() { - echo "$1" - exit 1 - } + testCDIScript = pkgs.writeShellScriptBin "test-cdi" '' + die() { + echo "$1" + exit 1 + } - check_file_referential_integrity() { - echo "checking $1 referential integrity" - ( ${pkgs.glibc.bin}/bin/ldd "$1" | ${lib.getExe pkgs.gnugrep} "not found" &> /dev/null ) && return 1 - return 0 - } + check_file_referential_integrity() { + echo "checking $1 referential integrity" + ( ${pkgs.glibc.bin}/bin/ldd "$1" | ${lib.getExe pkgs.gnugrep} "not found" &> /dev/null ) && return 1 + return 0 + } - check_directory_referential_integrity() { - ${lib.getExe pkgs.findutils} "$1" -type f -print0 | while read -d $'\0' file; do - if [[ $(${lib.getExe pkgs.file} "$file" | ${lib.getExe pkgs.gnugrep} ELF) ]]; then - check_file_referential_integrity "$file" || exit 1 - else - echo "skipping $file: not an ELF file" - fi - done - } + check_directory_referential_integrity() { + ${lib.getExe pkgs.findutils} "$1" -type f -print0 | while read -d $'\0' file; do + if [[ $(${lib.getExe pkgs.file} "$file" | ${lib.getExe pkgs.gnugrep} ELF) ]]; then + check_file_referential_integrity "$file" || exit 1 + else + echo "skipping $file: not an ELF file" + fi + done + } - check_directory_referential_integrity "/usr/bin" || exit 1 - check_directory_referential_integrity "${pkgs.addDriverRunpath.driverLink}" || exit 1 - check_directory_referential_integrity "/usr/local/nvidia" || exit 1 - ''; - in - pkgs.dockerTools.buildImage { - name = "cdi-test"; - tag = "latest"; - config = { - Cmd = [ (lib.getExe testCDIScript) ]; - }; - copyToRoot = ( - with pkgs.dockerTools; - [ - usrBinEnv - binSh - ] - ); + check_directory_referential_integrity "/usr/bin" || exit 1 + check_directory_referential_integrity "${pkgs.addDriverRunpath.driverLink}" || exit 1 + check_directory_referential_integrity "/usr/local/nvidia" || exit 1 + ''; + testContainerImage = pkgs.dockerTools.buildImage { + name = "cdi-test"; + tag = "latest"; + config = { + Cmd = [ (lib.getExe testCDIScript) ]; }; + copyToRoot = with pkgs.dockerTools; [ + usrBinEnv + binSh + ]; + }; emptyCDISpec = '' - #! ${pkgs.runtimeShell} - cat < $out/bin/nvidia-ctk - ${emptyCDISpec} + mkdir -p $out/bin $out/share/nvidia-container-toolkit + cp "$emptyCDISpecPath" "$out/share/nvidia-container-toolkit/spec.json" + echo -n "$emptyCDISpec" > "$out/bin/nvidia-ctk"; + cat << EOF > "$out/bin/nvidia-ctk" + #!${pkgs.runtimeShell} + cat "$out/share/nvidia-container-toolkit/spec.json" EOF chmod +x $out/bin/nvidia-ctk ''; + meta.mainProgram = "nvidia-ctk"; }; }; in From 0780a3ebc81069648e4ab57d3f5603d1d59d953c Mon Sep 17 00:00:00 2001 From: Someone Serge Date: Thu, 22 Aug 2024 17:34:48 +0000 Subject: [PATCH 5/7] tests/nvidia-container-toolkit: strip {no,one}-nvidia prefix as obvious from the context --- nixos/tests/nvidia-container-toolkit.nix | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/nixos/tests/nvidia-container-toolkit.nix b/nixos/tests/nvidia-container-toolkit.nix index 59105c5be15a..3cad0f6fc227 100644 --- a/nixos/tests/nvidia-container-toolkit.nix +++ b/nixos/tests/nvidia-container-toolkit.nix @@ -99,7 +99,7 @@ import ./make-test-python.nix ( maintainers = [ ereslibre ]; }; nodes = { - no-nvidia-gpus = + no-gpus = { config, ... }: { environment.systemPackages = with pkgs; [ jq ]; @@ -112,7 +112,7 @@ import ./make-test-python.nix ( }; }; - nvidia-one-gpu = + one-gpu = { config, pkgs, ... }: { virtualisation.diskSize = 10240; @@ -131,7 +131,7 @@ import ./make-test-python.nix ( virtualisation.containers.enable = true; }; - nvidia-one-gpu-invalid-host-paths = + one-gpu-invalid-host-paths = { config, pkgs, ... }: { virtualisation.diskSize = 10240; @@ -158,19 +158,19 @@ import ./make-test-python.nix ( start_all() with subtest("Generate an empty CDI spec for a machine with no Nvidia GPUs"): - no_nvidia_gpus.wait_for_unit("nvidia-container-toolkit-cdi-generator.service") - no_nvidia_gpus.succeed("cat /var/run/cdi/nvidia-container-toolkit.json | jq") + no_gpus.wait_for_unit("nvidia-container-toolkit-cdi-generator.service") + no_gpus.succeed("cat /var/run/cdi/nvidia-container-toolkit.json | jq") with subtest("Podman loads the generated CDI spec for a machine with an Nvidia GPU"): - nvidia_one_gpu.wait_for_unit("nvidia-container-toolkit-cdi-generator.service") - nvidia_one_gpu.succeed("cat /var/run/cdi/nvidia-container-toolkit.json | jq") - nvidia_one_gpu.succeed("podman load < ${testContainerImage}") - print(nvidia_one_gpu.succeed("podman run --pull=never --device=nvidia.com/gpu=all -v /run/opengl-driver:/run/opengl-driver:ro cdi-test:latest")) + one_gpu.wait_for_unit("nvidia-container-toolkit-cdi-generator.service") + one_gpu.succeed("cat /var/run/cdi/nvidia-container-toolkit.json | jq") + one_gpu.succeed("podman load < ${testContainerImage}") + print(one_gpu.succeed("podman run --pull=never --device=nvidia.com/gpu=all -v /run/opengl-driver:/run/opengl-driver:ro cdi-test:latest")) # Issue: https://github.com/NixOS/nixpkgs/issues/319201 with subtest("The generated CDI spec skips specified non-existant paths in the host"): - nvidia_one_gpu_invalid_host_paths.wait_for_unit("nvidia-container-toolkit-cdi-generator.service") - nvidia_one_gpu_invalid_host_paths.fail("grep 'non-existant-path' /var/run/cdi/nvidia-container-toolkit.json") + one_gpu_invalid_host_paths.wait_for_unit("nvidia-container-toolkit-cdi-generator.service") + one_gpu_invalid_host_paths.fail("grep 'non-existant-path' /var/run/cdi/nvidia-container-toolkit.json") ''; } ) From d970b4d6cd3c297865b4d2b083cd4b3f523ed973 Mon Sep 17 00:00:00 2001 From: Someone Serge Date: Thu, 22 Aug 2024 17:37:26 +0000 Subject: [PATCH 6/7] tests/nvidia-container-toolkit: hardware.opengl -> graphics --- nixos/tests/nvidia-container-toolkit.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nixos/tests/nvidia-container-toolkit.nix b/nixos/tests/nvidia-container-toolkit.nix index 3cad0f6fc227..8c4fa9b6e933 100644 --- a/nixos/tests/nvidia-container-toolkit.nix +++ b/nixos/tests/nvidia-container-toolkit.nix @@ -126,7 +126,7 @@ import ./make-test-python.nix ( open = true; package = config.boot.kernelPackages.nvidiaPackages.stable.open; }; - opengl.enable = true; + graphics.enable = true; }; virtualisation.containers.enable = true; }; @@ -149,7 +149,7 @@ import ./make-test-python.nix ( open = true; package = config.boot.kernelPackages.nvidiaPackages.stable.open; }; - opengl.enable = true; + graphics.enable = true; }; virtualisation.containers.enable = true; }; From f72b7b56fb6f8d6c1839c0dc70477ad4c8193e10 Mon Sep 17 00:00:00 2001 From: Someone Serge Date: Thu, 22 Aug 2024 17:51:54 +0000 Subject: [PATCH 7/7] tests/nvidia-container-toolkit: mv shared config to `defaults` --- nixos/tests/all-tests.nix | 2 +- nixos/tests/nvidia-container-toolkit.nix | 307 +++++++++++------------ 2 files changed, 141 insertions(+), 168 deletions(-) diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index efe14ed699b6..a6834bb197ed 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -699,7 +699,7 @@ in { ntfy-sh = handleTest ./ntfy-sh.nix {}; ntfy-sh-migration = handleTest ./ntfy-sh-migration.nix {}; ntpd-rs = handleTest ./ntpd-rs.nix {}; - nvidia-container-toolkit = handleTest ./nvidia-container-toolkit.nix {}; + nvidia-container-toolkit = runTest ./nvidia-container-toolkit.nix; nvmetcfg = handleTest ./nvmetcfg.nix {}; nzbget = handleTest ./nzbget.nix {}; nzbhydra2 = handleTest ./nzbhydra2.nix {}; diff --git a/nixos/tests/nvidia-container-toolkit.nix b/nixos/tests/nvidia-container-toolkit.nix index 8c4fa9b6e933..b22b989c0814 100644 --- a/nixos/tests/nvidia-container-toolkit.nix +++ b/nixos/tests/nvidia-container-toolkit.nix @@ -1,176 +1,149 @@ -import ./make-test-python.nix ( - { - pkgs, - lib, - system, - ... - }: - let - testCDIScript = pkgs.writeShellScriptBin "test-cdi" '' - die() { - echo "$1" - exit 1 - } +{ pkgs, lib, ... }: +let + testCDIScript = pkgs.writeShellScriptBin "test-cdi" '' + die() { + echo "$1" + exit 1 + } - check_file_referential_integrity() { - echo "checking $1 referential integrity" - ( ${pkgs.glibc.bin}/bin/ldd "$1" | ${lib.getExe pkgs.gnugrep} "not found" &> /dev/null ) && return 1 - return 0 - } + check_file_referential_integrity() { + echo "checking $1 referential integrity" + ( ${pkgs.glibc.bin}/bin/ldd "$1" | ${lib.getExe pkgs.gnugrep} "not found" &> /dev/null ) && return 1 + return 0 + } - check_directory_referential_integrity() { - ${lib.getExe pkgs.findutils} "$1" -type f -print0 | while read -d $'\0' file; do - if [[ $(${lib.getExe pkgs.file} "$file" | ${lib.getExe pkgs.gnugrep} ELF) ]]; then - check_file_referential_integrity "$file" || exit 1 - else - echo "skipping $file: not an ELF file" - fi - done - } + check_directory_referential_integrity() { + ${lib.getExe pkgs.findutils} "$1" -type f -print0 | while read -d $'\0' file; do + if [[ $(${lib.getExe pkgs.file} "$file" | ${lib.getExe pkgs.gnugrep} ELF) ]]; then + check_file_referential_integrity "$file" || exit 1 + else + echo "skipping $file: not an ELF file" + fi + done + } - check_directory_referential_integrity "/usr/bin" || exit 1 - check_directory_referential_integrity "${pkgs.addDriverRunpath.driverLink}" || exit 1 - check_directory_referential_integrity "/usr/local/nvidia" || exit 1 - ''; - testContainerImage = pkgs.dockerTools.buildImage { - name = "cdi-test"; - tag = "latest"; - config = { - Cmd = [ (lib.getExe testCDIScript) ]; + check_directory_referential_integrity "/usr/bin" || exit 1 + check_directory_referential_integrity "${pkgs.addDriverRunpath.driverLink}" || exit 1 + check_directory_referential_integrity "/usr/local/nvidia" || exit 1 + ''; + testContainerImage = pkgs.dockerTools.buildImage { + name = "cdi-test"; + tag = "latest"; + config = { + Cmd = [ (lib.getExe testCDIScript) ]; + }; + copyToRoot = with pkgs.dockerTools; [ + usrBinEnv + binSh + ]; + }; + emptyCDISpec = '' + { + "cdiVersion": "0.5.0", + "kind": "nvidia.com/gpu", + "devices": [ + { + "name": "all", + "containerEdits": { + "deviceNodes": [ + { + "path": "/dev/urandom" + } + ], + "hooks": [], + "mounts": [] + } + } + ], + "containerEdits": { + "deviceNodes": [], + "hooks": [], + "mounts": [] + } + } + ''; + nvidia-container-toolkit = { + enable = true; + package = pkgs.stdenv.mkDerivation { + pname = "nvidia-ctk-dummy"; + version = "1.0.0"; + dontUnpack = true; + dontBuild = true; + + inherit emptyCDISpec; + passAsFile = [ "emptyCDISpec" ]; + + installPhase = '' + mkdir -p $out/bin $out/share/nvidia-container-toolkit + cp "$emptyCDISpecPath" "$out/share/nvidia-container-toolkit/spec.json" + echo -n "$emptyCDISpec" > "$out/bin/nvidia-ctk"; + cat << EOF > "$out/bin/nvidia-ctk" + #!${pkgs.runtimeShell} + cat "$out/share/nvidia-container-toolkit/spec.json" + EOF + chmod +x $out/bin/nvidia-ctk + ''; + meta.mainProgram = "nvidia-ctk"; + }; + }; +in +{ + name = "nvidia-container-toolkit"; + meta = with lib.maintainers; { + maintainers = [ ereslibre ]; + }; + defaults = + { config, ... }: + { + environment.systemPackages = with pkgs; [ jq ]; + virtualisation.diskSize = lib.mkDefault 10240; + virtualisation.containers.enable = lib.mkDefault true; + hardware = { + inherit nvidia-container-toolkit; + nvidia = { + open = true; + package = config.boot.kernelPackages.nvidiaPackages.stable.open; + }; + graphics.enable = lib.mkDefault true; }; - copyToRoot = with pkgs.dockerTools; [ - usrBinEnv - binSh + }; + nodes = { + no-gpus = { + virtualisation.containers.enable = false; + hardware.graphics.enable = false; + }; + one-gpu = + { pkgs, ... }: + { + environment.systemPackages = with pkgs; [ podman ]; + hardware.graphics.enable = true; + }; + + one-gpu-invalid-host-paths = { + hardware.nvidia-container-toolkit.mounts = [ + { + hostPath = "/non-existant-path"; + containerPath = "/some/path"; + } ]; }; - emptyCDISpec = '' - { - "cdiVersion": "0.5.0", - "kind": "nvidia.com/gpu", - "devices": [ - { - "name": "all", - "containerEdits": { - "deviceNodes": [ - { - "path": "/dev/urandom" - } - ], - "hooks": [], - "mounts": [] - } - } - ], - "containerEdits": { - "deviceNodes": [], - "hooks": [], - "mounts": [] - } - } - ''; - nvidia-container-toolkit = { - enable = true; - package = pkgs.stdenv.mkDerivation { - pname = "nvidia-ctk-dummy"; - version = "1.0.0"; - dontUnpack = true; - dontBuild = true; + }; + testScript = '' + start_all() - inherit emptyCDISpec; - passAsFile = [ "emptyCDISpec" ]; + with subtest("Generate an empty CDI spec for a machine with no Nvidia GPUs"): + no_gpus.wait_for_unit("nvidia-container-toolkit-cdi-generator.service") + no_gpus.succeed("cat /var/run/cdi/nvidia-container-toolkit.json | jq") - installPhase = '' - mkdir -p $out/bin $out/share/nvidia-container-toolkit - cp "$emptyCDISpecPath" "$out/share/nvidia-container-toolkit/spec.json" - echo -n "$emptyCDISpec" > "$out/bin/nvidia-ctk"; - cat << EOF > "$out/bin/nvidia-ctk" - #!${pkgs.runtimeShell} - cat "$out/share/nvidia-container-toolkit/spec.json" - EOF - chmod +x $out/bin/nvidia-ctk - ''; - meta.mainProgram = "nvidia-ctk"; - }; - }; - in - { - name = "nvidia-container-toolkit"; - meta = with lib.maintainers; { - maintainers = [ ereslibre ]; - }; - nodes = { - no-gpus = - { config, ... }: - { - environment.systemPackages = with pkgs; [ jq ]; - hardware = { - inherit nvidia-container-toolkit; - nvidia = { - open = true; - package = config.boot.kernelPackages.nvidiaPackages.stable.open; - }; - }; - }; + with subtest("Podman loads the generated CDI spec for a machine with an Nvidia GPU"): + one_gpu.wait_for_unit("nvidia-container-toolkit-cdi-generator.service") + one_gpu.succeed("cat /var/run/cdi/nvidia-container-toolkit.json | jq") + one_gpu.succeed("podman load < ${testContainerImage}") + print(one_gpu.succeed("podman run --pull=never --device=nvidia.com/gpu=all -v /run/opengl-driver:/run/opengl-driver:ro cdi-test:latest")) - one-gpu = - { config, pkgs, ... }: - { - virtualisation.diskSize = 10240; - environment.systemPackages = with pkgs; [ - jq - podman - ]; - hardware = { - inherit nvidia-container-toolkit; - nvidia = { - open = true; - package = config.boot.kernelPackages.nvidiaPackages.stable.open; - }; - graphics.enable = true; - }; - virtualisation.containers.enable = true; - }; - - one-gpu-invalid-host-paths = - { config, pkgs, ... }: - { - virtualisation.diskSize = 10240; - environment.systemPackages = with pkgs; [ jq ]; - hardware = { - nvidia-container-toolkit = nvidia-container-toolkit // { - mounts = [ - { - hostPath = "/non-existant-path"; - containerPath = "/some/path"; - } - ]; - }; - nvidia = { - open = true; - package = config.boot.kernelPackages.nvidiaPackages.stable.open; - }; - graphics.enable = true; - }; - virtualisation.containers.enable = true; - }; - }; - testScript = '' - start_all() - - with subtest("Generate an empty CDI spec for a machine with no Nvidia GPUs"): - no_gpus.wait_for_unit("nvidia-container-toolkit-cdi-generator.service") - no_gpus.succeed("cat /var/run/cdi/nvidia-container-toolkit.json | jq") - - with subtest("Podman loads the generated CDI spec for a machine with an Nvidia GPU"): - one_gpu.wait_for_unit("nvidia-container-toolkit-cdi-generator.service") - one_gpu.succeed("cat /var/run/cdi/nvidia-container-toolkit.json | jq") - one_gpu.succeed("podman load < ${testContainerImage}") - print(one_gpu.succeed("podman run --pull=never --device=nvidia.com/gpu=all -v /run/opengl-driver:/run/opengl-driver:ro cdi-test:latest")) - - # Issue: https://github.com/NixOS/nixpkgs/issues/319201 - with subtest("The generated CDI spec skips specified non-existant paths in the host"): - one_gpu_invalid_host_paths.wait_for_unit("nvidia-container-toolkit-cdi-generator.service") - one_gpu_invalid_host_paths.fail("grep 'non-existant-path' /var/run/cdi/nvidia-container-toolkit.json") - ''; - } -) + # Issue: https://github.com/NixOS/nixpkgs/issues/319201 + with subtest("The generated CDI spec skips specified non-existant paths in the host"): + one_gpu_invalid_host_paths.wait_for_unit("nvidia-container-toolkit-cdi-generator.service") + one_gpu_invalid_host_paths.fail("grep 'non-existant-path' /var/run/cdi/nvidia-container-toolkit.json") + ''; +}