From d665ca4fb2c6c5814a763fa0cd861730d89ef609 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20Fern=C3=A1ndez=20L=C3=B3pez?= Date: Fri, 16 Aug 2024 20:33:50 +0200 Subject: [PATCH] nvidia-container-toolkit: add initial set of tests to check closures --- .../nvidia-container-toolkit/default.nix | 3 + nixos/tests/all-tests.nix | 1 + nixos/tests/nvidia-container-toolkit.nix | 177 ++++++++++++++++++ 3 files changed, 181 insertions(+) create mode 100644 nixos/tests/nvidia-container-toolkit.nix diff --git a/nixos/modules/services/hardware/nvidia-container-toolkit/default.nix b/nixos/modules/services/hardware/nvidia-container-toolkit/default.nix index fe4999bfbdbb..3936b2757c46 100644 --- a/nixos/modules/services/hardware/nvidia-container-toolkit/default.nix +++ b/nixos/modules/services/hardware/nvidia-container-toolkit/default.nix @@ -71,6 +71,8 @@ /usr/local/nvidia/lib64. ''; }; + + package = lib.mkPackageOption pkgs "nvidia-container-toolkit" { }; }; }; @@ -129,6 +131,7 @@ let script = pkgs.callPackage ./cdi-generate.nix { inherit (config.hardware.nvidia-container-toolkit) mounts; + nvidia-container-toolkit = config.hardware.nvidia-container-toolkit.package; nvidia-driver = config.hardware.nvidia.package; deviceNameStrategy = config.hardware.nvidia-container-toolkit.device-name-strategy; }; diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index e00c6b2b39d3..efe14ed699b6 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -699,6 +699,7 @@ in { ntfy-sh = handleTest ./ntfy-sh.nix {}; ntfy-sh-migration = handleTest ./ntfy-sh-migration.nix {}; ntpd-rs = handleTest ./ntpd-rs.nix {}; + nvidia-container-toolkit = handleTest ./nvidia-container-toolkit.nix {}; nvmetcfg = handleTest ./nvmetcfg.nix {}; nzbget = handleTest ./nzbget.nix {}; nzbhydra2 = handleTest ./nzbhydra2.nix {}; diff --git a/nixos/tests/nvidia-container-toolkit.nix b/nixos/tests/nvidia-container-toolkit.nix new file mode 100644 index 000000000000..78d3751eef4d --- /dev/null +++ b/nixos/tests/nvidia-container-toolkit.nix @@ -0,0 +1,177 @@ +import ./make-test-python.nix ( + { + pkgs, + lib, + system, + ... + }: + let + testContainerImage = + let + testCDIScript = pkgs.writeShellScriptBin "test-cdi" '' + die() { + echo "$1" + exit 1 + } + + check_file_referential_integrity() { + echo "checking $1 referential integrity" + ( ${pkgs.glibc.bin}/bin/ldd "$1" | ${lib.getExe pkgs.gnugrep} "not found" &> /dev/null ) && return 1 + return 0 + } + + check_directory_referential_integrity() { + ${lib.getExe pkgs.findutils} "$1" -type f -print0 | while read -d $'\0' file; do + if [[ $(${lib.getExe pkgs.file} "$file" | ${lib.getExe pkgs.gnugrep} ELF) ]]; then + check_file_referential_integrity "$file" || exit 1 + else + echo "skipping $file: not an ELF file" + fi + done + } + + check_directory_referential_integrity "/usr/bin" || exit 1 + check_directory_referential_integrity "${pkgs.addDriverRunpath.driverLink}" || exit 1 + check_directory_referential_integrity "/usr/local/nvidia" || exit 1 + ''; + in + pkgs.dockerTools.buildImage { + name = "cdi-test"; + tag = "latest"; + config = { + Cmd = [ "${testCDIScript}/bin/test-cdi" ]; + }; + copyToRoot = ( + with pkgs.dockerTools; + [ + usrBinEnv + binSh + ] + ); + }; + emptyCDISpec = '' + #! ${pkgs.runtimeShell} + cat < $out/bin/nvidia-ctk + ${emptyCDISpec} + EOF + chmod +x $out/bin/nvidia-ctk + ''; + }; + }; + in + { + name = "nvidia-container-toolkit"; + meta = with lib.maintainers; { + maintainers = [ ereslibre ]; + }; + nodes = { + no-nvidia-gpus = + { config, ... }: + { + environment.systemPackages = with pkgs; [ jq ]; + hardware = { + inherit nvidia-container-toolkit; + nvidia = { + open = true; + package = config.boot.kernelPackages.nvidiaPackages.stable.open; + }; + }; + }; + + nvidia-one-gpu = + { config, pkgs, ... }: + { + virtualisation.diskSize = 10240; + environment.systemPackages = with pkgs; [ + jq + podman + ]; + hardware = { + inherit nvidia-container-toolkit; + nvidia = { + open = true; + package = config.boot.kernelPackages.nvidiaPackages.stable.open; + }; + opengl.enable = true; + }; + virtualisation.containers.enable = true; + }; + + nvidia-one-gpu-invalid-host-paths = + { config, pkgs, ... }: + { + virtualisation.diskSize = 10240; + environment.systemPackages = with pkgs; [ jq ]; + hardware = { + nvidia-container-toolkit = nvidia-container-toolkit // { + mounts = [ + { + hostPath = "/non-existant-path"; + containerPath = "/some/path"; + } + ]; + }; + nvidia = { + open = true; + package = config.boot.kernelPackages.nvidiaPackages.stable.open; + }; + opengl.enable = true; + }; + virtualisation.containers.enable = true; + }; + }; + testScript = '' + start_all() + + with subtest("Generate an empty CDI spec for a machine with no Nvidia GPUs"): + no_nvidia_gpus.wait_for_unit("nvidia-container-toolkit-cdi-generator.service") + no_nvidia_gpus.succeed("cat /var/run/cdi/nvidia-container-toolkit.json | jq") + + with subtest("Podman loads the generated CDI spec for a machine with an Nvidia GPU"): + nvidia_one_gpu.wait_for_unit("nvidia-container-toolkit-cdi-generator.service") + nvidia_one_gpu.succeed("cat /var/run/cdi/nvidia-container-toolkit.json | jq") + nvidia_one_gpu.succeed("podman load < ${testContainerImage}") + print(nvidia_one_gpu.succeed("podman run --pull=never --device=nvidia.com/gpu=all -v /run/opengl-driver:/run/opengl-driver:ro cdi-test:latest")) + + # Issue: https://github.com/NixOS/nixpkgs/issues/319201 + with subtest("The generated CDI spec skips specified non-existant paths in the host"): + nvidia_one_gpu_invalid_host_paths.wait_for_unit("nvidia-container-toolkit-cdi-generator.service") + nvidia_one_gpu_invalid_host_paths.fail("grep 'non-existant-path' /var/run/cdi/nvidia-container-toolkit.json") + ''; + } +)