Merge pull request #324789 from NixOS/devShellTools-env

`devShellTools`: add environment functions

doc/build-helpers/dev-shell-tools.chapter.md

@@ -27,3 +27,49 @@ devShellTools.valueToString (builtins.toFile "foo" "bar")
 devShellTools.valueToString false
 => ""
 ```
+
+:::
+
+## `devShellTools.unstructuredDerivationInputEnv` {#sec-devShellTools-unstructuredDerivationInputEnv}
+
+Convert a set of derivation attributes (as would be passed to [`derivation`]) to a set of environment variables that can be used in a shell script.
+This function does not support `__structuredAttrs`, but does support `passAsFile`.
+
+:::{.example}
+## `unstructuredDerivationInputEnv` usage example
+
+```nix
+devShellTools.unstructuredDerivationInputEnv {
+  drvAttrs = {
+    name = "foo";
+    buildInputs = [ hello figlet ];
+    builder = bash;
+    args = [ "-c" "${./builder.sh}" ];
+  };
+}
+=> {
+  name = "foo";
+  buildInputs = "/nix/store/...-hello /nix/store/...-figlet";
+  builder = "/nix/store/...-bash";
+}
+```
+
+Note that `args` is not included, because Nix does not added it to the builder process environment.
+
+:::
+
+## `devShellTools.derivationOutputEnv` {#sec-devShellTools-derivationOutputEnv}
+
+Takes the relevant parts of a derivation and returns a set of environment variables, that would be present in the derivation.
+
+:::{.example}
+## `derivationOutputEnv` usage example
+
+```nix
+let
+  pkg = hello;
+in
+devShellTools.derivationOutputEnv { outputList = pkg.outputs; outputMap = pkg; }
+```
+
+:::

nixos/tests/all-tests.nix

@@ -264,6 +264,7 @@ in {
   docker-rootless = handleTestOn ["aarch64-linux" "x86_64-linux"] ./docker-rootless.nix {};
   docker-registry = handleTest ./docker-registry.nix {};
   docker-tools = handleTestOn ["x86_64-linux"] ./docker-tools.nix {};
+  docker-tools-nix-shell = runTest ./docker-tools-nix-shell.nix;
   docker-tools-cross = handleTestOn ["x86_64-linux" "aarch64-linux"] ./docker-tools-cross.nix {};
   docker-tools-overlay = handleTestOn ["x86_64-linux"] ./docker-tools-overlay.nix {};
   documize = handleTest ./documize.nix {};

nixos/tests/docker-tools-nix-shell.nix

@@ -0,0 +1,95 @@
+# nix-build -A nixosTests.docker-tools-nix-shell
+{ config, lib, ... }:
+let
+  inherit (config.node.pkgs.dockerTools) examples;
+in
+{
+  name = "docker-tools-nix-shell";
+  meta = with lib.maintainers; {
+    maintainers = [
+      infinisil
+      roberth
+    ];
+  };
+
+  nodes = {
+    docker =
+      { ... }:
+      {
+        virtualisation = {
+          diskSize = 3072;
+          docker.enable = true;
+        };
+      };
+  };
+
+  testScript = ''
+    docker.wait_for_unit("sockets.target")
+
+    with subtest("buildImageWithNixDB: Has a nix database"):
+        docker.succeed(
+            "docker load --input='${examples.nix}'",
+            "docker run --rm ${examples.nix.imageName} nix-store -q --references /bin/bash"
+        )
+
+    with subtest("buildNixShellImage: Can build a basic derivation"):
+        docker.succeed(
+            "${examples.nix-shell-basic} | docker load",
+            "docker run --rm nix-shell-basic bash -c 'buildDerivation && $out/bin/hello' | grep '^Hello, world!$'"
+        )
+
+    with subtest("buildNixShellImage: Runs the shell hook"):
+        docker.succeed(
+            "${examples.nix-shell-hook} | docker load",
+            "docker run --rm -it nix-shell-hook | grep 'This is the shell hook!'"
+        )
+
+    with subtest("buildNixShellImage: Sources stdenv, making build inputs available"):
+        docker.succeed(
+            "${examples.nix-shell-inputs} | docker load",
+            "docker run --rm -it nix-shell-inputs | grep 'Hello, world!'"
+        )
+
+    with subtest("buildNixShellImage: passAsFile works"):
+        docker.succeed(
+            "${examples.nix-shell-pass-as-file} | docker load",
+            "docker run --rm -it nix-shell-pass-as-file | grep 'this is a string'"
+        )
+
+    with subtest("buildNixShellImage: run argument works"):
+        docker.succeed(
+            "${examples.nix-shell-run} | docker load",
+            "docker run --rm -it nix-shell-run | grep 'This shell is not interactive'"
+        )
+
+    with subtest("buildNixShellImage: command argument works"):
+        docker.succeed(
+            "${examples.nix-shell-command} | docker load",
+            "docker run --rm -it nix-shell-command | grep 'This shell is interactive'"
+        )
+
+    with subtest("buildNixShellImage: home directory is writable by default"):
+        docker.succeed(
+            "${examples.nix-shell-writable-home} | docker load",
+            "docker run --rm -it nix-shell-writable-home"
+        )
+
+    with subtest("buildNixShellImage: home directory can be made non-existent"):
+        docker.succeed(
+            "${examples.nix-shell-nonexistent-home} | docker load",
+            "docker run --rm -it nix-shell-nonexistent-home"
+        )
+
+    with subtest("buildNixShellImage: can build derivations"):
+        docker.succeed(
+            "${examples.nix-shell-build-derivation} | docker load",
+            "docker run --rm -it nix-shell-build-derivation"
+        )
+
+    with subtest("streamLayeredImage: with nix db"):
+        docker.succeed(
+            "${examples.nix-layered} | docker load",
+            "docker run --rm ${examples.nix-layered.imageName} nix-store -q --references /bin/bash"
+        )
+  '';
+}

nixos/tests/docker-tools.nix

@@ -60,7 +60,7 @@ let
     };
 
   nonRootTestImage =
-    pkgs.dockerTools.streamLayeredImage rec {
+    pkgs.dockerTools.streamLayeredImage {
       name = "non-root-test";
       tag = "latest";
       uid = 1000;
@@ -567,66 +567,6 @@ in {
         docker.succeed("docker run --rm image-with-certs:latest test -r /etc/pki/tls/certs/ca-bundle.crt")
         docker.succeed("docker image rm image-with-certs:latest")
 
-    with subtest("buildImageWithNixDB: Has a nix database"):
-        docker.succeed(
-            "docker load --input='${examples.nix}'",
-            "docker run --rm ${examples.nix.imageName} nix-store -q --references /bin/bash"
-        )
-
-    with subtest("buildNixShellImage: Can build a basic derivation"):
-        docker.succeed(
-            "${examples.nix-shell-basic} | docker load",
-            "docker run --rm nix-shell-basic bash -c 'buildDerivation && $out/bin/hello' | grep '^Hello, world!$'"
-        )
-
-    with subtest("buildNixShellImage: Runs the shell hook"):
-        docker.succeed(
-            "${examples.nix-shell-hook} | docker load",
-            "docker run --rm -it nix-shell-hook | grep 'This is the shell hook!'"
-        )
-
-    with subtest("buildNixShellImage: Sources stdenv, making build inputs available"):
-        docker.succeed(
-            "${examples.nix-shell-inputs} | docker load",
-            "docker run --rm -it nix-shell-inputs | grep 'Hello, world!'"
-        )
-
-    with subtest("buildNixShellImage: passAsFile works"):
-        docker.succeed(
-            "${examples.nix-shell-pass-as-file} | docker load",
-            "docker run --rm -it nix-shell-pass-as-file | grep 'this is a string'"
-        )
-
-    with subtest("buildNixShellImage: run argument works"):
-        docker.succeed(
-            "${examples.nix-shell-run} | docker load",
-            "docker run --rm -it nix-shell-run | grep 'This shell is not interactive'"
-        )
-
-    with subtest("buildNixShellImage: command argument works"):
-        docker.succeed(
-            "${examples.nix-shell-command} | docker load",
-            "docker run --rm -it nix-shell-command | grep 'This shell is interactive'"
-        )
-
-    with subtest("buildNixShellImage: home directory is writable by default"):
-        docker.succeed(
-            "${examples.nix-shell-writable-home} | docker load",
-            "docker run --rm -it nix-shell-writable-home"
-        )
-
-    with subtest("buildNixShellImage: home directory can be made non-existent"):
-        docker.succeed(
-            "${examples.nix-shell-nonexistent-home} | docker load",
-            "docker run --rm -it nix-shell-nonexistent-home"
-        )
-
-    with subtest("buildNixShellImage: can build derivations"):
-        docker.succeed(
-            "${examples.nix-shell-build-derivation} | docker load",
-            "docker run --rm -it nix-shell-build-derivation"
-        )
-
     with subtest("streamLayeredImage: chown is persistent in fakeRootCommands"):
         docker.succeed(
             "${chownTestImage} | docker load",
@@ -638,11 +578,5 @@ in {
             "${nonRootTestImage} | docker load",
             "docker run --rm ${chownTestImage.imageName} | diff /dev/stdin <(echo 12345:12345)"
         )
-
-    with subtest("streamLayeredImage: with nix db"):
-        docker.succeed(
-            "${examples.nix-layered} | docker load",
-            "docker run --rm ${examples.nix-layered.imageName} nix-store -q --references /bin/bash"
-        )
   '';
 })

pkgs/build-support/dev-shell-tools/default.nix

@@ -1,8 +1,13 @@
-{ lib }:
+{
+  lib,
+  writeTextFile,
+}:
 let
   inherit (builtins) typeOf;
 in
 rec {
+  # Docs: doc/build-helpers/dev-shell-tools.chapter.md
+  # Tests: ./tests/default.nix
   # This function closely mirrors what this Nix code does:
   # https://github.com/NixOS/nix/blob/2.8.0/src/libexpr/primops.cc#L1102
   # https://github.com/NixOS/nix/blob/2.8.0/src/libexpr/eval.cc#L1981-L2036
@@ -13,4 +18,48 @@ rec {
     if typeOf value == "path" then "${value}"
     else if typeOf value == "list" then toString (map valueToString value)
     else toString value;
+
+
+  # Docs: doc/build-helpers/dev-shell-tools.chapter.md
+  # Tests: ./tests/default.nix
+  # https://github.com/NixOS/nix/blob/2.8.0/src/libstore/build/local-derivation-goal.cc#L992-L1004
+  unstructuredDerivationInputEnv = { drvAttrs }:
+    # FIXME: this should be `normalAttrs // passAsFileAttrs`
+    lib.mapAttrs'
+      (name: value:
+        let str = valueToString value;
+        in if lib.elem name (drvAttrs.passAsFile or [])
+        then
+          let
+            nameHash =
+              if builtins?convertHash
+              then builtins.convertHash {
+                hash = "sha256:" + builtins.hashString "sha256" name;
+                toHashFormat = "nix32";
+              }
+              else
+                builtins.hashString "sha256" name;
+            basename = ".attr-${nameHash}";
+          in
+            lib.nameValuePair "${name}Path" "${
+              writeTextFile {
+                name = "shell-passAsFile-${name}";
+                text = str;
+                destination = "/${basename}";
+              }
+            }/${basename}"
+        else lib.nameValuePair name str
+      )
+      (removeAttrs drvAttrs [
+        # TODO: there may be more of these
+        "args"
+      ]);
+
+  # Docs: doc/build-helpers/dev-shell-tools.chapter.md
+  # Tests: ./tests/default.nix
+  derivationOutputEnv = { outputList, outputMap }:
+    # A mapping from output name to the nix store path where they should end up
+    # https://github.com/NixOS/nix/blob/2.8.0/src/libexpr/primops.cc#L1253
+    lib.genAttrs outputList (output: builtins.unsafeDiscardStringContext outputMap.${output}.outPath);
+
 }

pkgs/build-support/dev-shell-tools/tests/default.nix

@@ -4,11 +4,23 @@
   lib,
   stdenv,
   hello,
+  writeText,
+  zlib,
+  nixosTests,
 }:
 let
-  inherit (lib) escapeShellArg;
+  inherit (lib)
+    concatLines
+    escapeShellArg
+    isString
+    mapAttrsToList
+    ;
 in
-{
+lib.recurseIntoAttrs {
+
+  # nix-build -A tests.devShellTools.nixos
+  nixos = nixosTests.docker-tools-nix-shell;
+
   # nix-build -A tests.devShellTools.valueToString
   valueToString =
     let inherit (devShellTools) valueToString; in
@@ -42,4 +54,112 @@ in
         ) >log 2>&1 || { cat log; exit 1; }
       '';
     };
+
+  # nix-build -A tests.devShellTools.valueToString
+  unstructuredDerivationInputEnv =
+    let
+      inherit (devShellTools) unstructuredDerivationInputEnv;
+
+      drvAttrs = {
+        one = 1;
+        boolTrue = true;
+        boolFalse = false;
+        foo = "foo";
+        list = [ 1 2 3 ];
+        pathDefaultNix = ./default.nix;
+        stringWithDep = "Exe: ${hello}/bin/hello";
+        aPackageAttrSet = hello;
+        anOutPath = hello.outPath;
+        anAnAlternateOutput = zlib.dev;
+        args = [ "args must not be added to the environment" "Nix doesn't do it." ];
+
+        passAsFile = [ "bar" ];
+        bar = ''
+          bar
+          ${writeText "qux" "yea"}
+        '';
+      };
+      result = unstructuredDerivationInputEnv { inherit drvAttrs; };
+    in
+    assert result // { barPath = "<check later>"; } == {
+      one = "1";
+      boolTrue = "1";
+      boolFalse = "";
+      foo = "foo";
+      list = "1 2 3";
+      pathDefaultNix = "${./default.nix}";
+      stringWithDep = "Exe: ${hello}/bin/hello";
+      aPackageAttrSet = "${hello}";
+      anOutPath = "${hello.outPath}";
+      anAnAlternateOutput = "${zlib.dev}";
+
+      passAsFile = "bar";
+      barPath = "<check later>";
+    };
+
+    # Not runCommand, because it alters `passAsFile`
+    stdenv.mkDerivation ({
+      name = "devShellTools-unstructuredDerivationInputEnv-built-tests";
+
+      exampleBarPathString =
+        assert isString result.barPath;
+        result.barPath;
+
+      dontUnpack = true;
+      dontBuild = true;
+      dontFixup = true;
+      doCheck = true;
+
+      installPhase = "touch $out";
+
+      checkPhase = ''
+        fail() {
+          echo "$@" >&2
+          exit 1
+        }
+        checkAttr() {
+          echo checking attribute $1...
+          if [[ "$2" != "$3" ]]; then
+            echo "expected: $3"
+            echo "actual: $2"
+            exit 1
+          fi
+        }
+        ${
+          concatLines
+            (mapAttrsToList
+              (name: value:
+                "checkAttr ${name} \"\$${name}\" ${escapeShellArg value}"
+              )
+              (removeAttrs
+                result
+                [
+                  "args"
+
+                  # Nix puts it in workdir, which is not a concept for
+                  # unstructuredDerivationInputEnv, so we have to put it in the
+                  # store instead. This means the full path won't match.
+                  "barPath"
+                ])
+            )
+        }
+        (
+          set -x
+
+          diff $exampleBarPathString $barPath
+
+          ${lib.optionalString (builtins?convertHash) ''
+            [[ "$(basename $exampleBarPathString)" = "$(basename $barPath)" ]]
+          ''}
+        )
+
+        ''${args:+fail "args should not be set by Nix. We don't expect it to and unstructuredDerivationInputEnv removes it."}
+        if [[ "''${builder:-x}" == x ]]; then
+          fail "builder should be set by Nix. We don't remove it in unstructuredDerivationInputEnv."
+        fi
+      '';
+    } // removeAttrs drvAttrs [
+      # This would break the derivation. Instead, we have a check in the derivation to make sure Nix doesn't set it.
+      "args"
+    ]);
 }

pkgs/build-support/docker/default.nix

@@ -1129,26 +1129,18 @@ rec {
   );
 
   # This function streams a docker image that behaves like a nix-shell for a derivation
+  # Docs: doc/build-helpers/images/dockertools.section.md
+  # Tests: nixos/tests/docker-tools-nix-shell.nix
   streamNixShellImage =
-    { # The derivation whose environment this docker image should be based on
+    { drv
-      drv
+    , name ? drv.name + "-env"
-    , # Image Name
+    , tag ? null
-      name ? drv.name + "-env"
+    , uid ? 1000
-    , # Image tag, the Nix's output hash will be used if null
+    , gid ? 1000
-      tag ? null
+    , homeDirectory ? "/build"
-    , # User id to run the container as. Defaults to 1000, because many
+    , shell ? bashInteractive + "/bin/bash"
-      # binaries don't like to be run as root
+    , command ? null
-      uid ? 1000
+    , run ? null
-    , # Group id to run the container as, see also uid
-      gid ? 1000
-    , # The home directory of the user
-      homeDirectory ? "/build"
-    , # The path to the bash binary to use as the shell. See `NIX_BUILD_SHELL` in `man nix-shell`
-      shell ? bashInteractive + "/bin/bash"
-    , # Run this command in the environment of the derivation, in an interactive shell. See `--command` in `man nix-shell`
-      command ? null
-    , # Same as `command`, but runs the command in a non-interactive shell instead. See `--run` in `man nix-shell`
-      run ? null
     }:
       assert lib.assertMsg (! (drv.drvAttrs.__structuredAttrs or false))
         "streamNixShellImage: Does not work with the derivation ${drv.name} because it uses __structuredAttrs";
@@ -1190,16 +1182,9 @@ rec {
         # https://github.com/NixOS/nix/blob/2.8.0/src/libstore/globals.hh#L464-L465
         sandboxBuildDir = "/build";
 
-        # https://github.com/NixOS/nix/blob/2.8.0/src/libstore/build/local-derivation-goal.cc#L992-L1004
+        drvEnv =
-        drvEnv = lib.mapAttrs' (name: value:
+          devShellTools.unstructuredDerivationInputEnv { inherit (drv) drvAttrs; }
-          let str = valueToString value;
+          // devShellTools.derivationOutputEnv { outputList = drv.outputs; outputMap = drv; };
-          in if lib.elem name (drv.drvAttrs.passAsFile or [])
-          then lib.nameValuePair "${name}Path" (writeText "pass-as-text-${name}" str)
-          else lib.nameValuePair name str
-        ) drv.drvAttrs //
-          # A mapping from output name to the nix store path where they should end up
-          # https://github.com/NixOS/nix/blob/2.8.0/src/libexpr/primops.cc#L1253
-          lib.genAttrs drv.outputs (output: builtins.unsafeDiscardStringContext drv.${output}.outPath);
 
         # Environment variables set in the image
         envVars = {
@@ -1291,6 +1276,8 @@ rec {
       };
 
   # Wrapper around streamNixShellImage to build an image from the result
+  # Docs: doc/build-helpers/images/dockertools.section.md
+  # Tests: nixos/tests/docker-tools-nix-shell.nix
   buildNixShellImage = { drv, compressor ? "gz", ... }@args:
     let
       stream = streamNixShellImage (builtins.removeAttrs args ["compressor"]);