Merge pull request #178717 from ShamrockLee/write-multiple-references

trivial-builders: replace writeReferencesToFile with writeClosure

doc/build-helpers/trivial-build-helpers.chapter.md

@@ -658,14 +658,18 @@ This creates a derivation with a directory structure like the following:
 
 ## `writeReferencesToFile` {#trivial-builder-writeReferencesToFile}
 
-Writes the closure of transitive dependencies to a file.
+Deprecated. Use [`writeClosure`](#trivial-builder-writeClosure) instead.
 
-This produces the equivalent of `nix-store -q --requisites`.
+## `writeClosure` {#trivial-builder-writeClosure}
+
+Given a list of [store paths](https://nixos.org/manual/nix/stable/glossary#gloss-store-path) (or string-like expressions coercible to store paths), write their collective [closure](https://nixos.org/manual/nix/stable/glossary#gloss-closure) to a text file.
+
+The result is equivalent to the output of `nix-store -q --requisites`.
 
 For example,
 
 ```nix
-writeReferencesToFile (writeScriptBin "hi" ''${hello}/bin/hello'')
+writeClosure [ (writeScriptBin "hi" ''${hello}/bin/hello'') ]
 ```
 
 produces an output path `/nix/store/<hash>-runtime-deps` containing

nixos/doc/manual/release-notes/rl-2405.section.md

@@ -171,6 +171,8 @@ The pre-existing [services.ankisyncd](#opt-services.ankisyncd.enable) has been m
 
 - Invidious has changed its default database username from `kemal` to `invidious`. Setups involving an externally provisioned database (i.e. `services.invidious.database.createLocally == false`) should adjust their configuration accordingly. The old `kemal` user will not be removed automatically even when the database is provisioned automatically.(https://github.com/NixOS/nixpkgs/pull/265857)
 
+- `writeReferencesToFile` is deprecated in favour of the new trivial build helper `writeClosure`. The latter accepts a list of paths and has an unambiguous name and cleaner implementation.
+
 - `inetutils` now has a lower priority to avoid shadowing the commonly used `util-linux`. If one wishes to restore the default priority, simply use `lib.setPrio 5 inetutils` or override with `meta.priority = 5`.
 
 - `paperless`' `services.paperless.extraConfig` setting has been removed and converted to the freeform type and option named `services.paperless.settings`.

nixos/tests/nixops/default.nix

@@ -93,23 +93,5 @@ let
 
   inherit (import ../ssh-keys.nix pkgs) snakeOilPrivateKey snakeOilPublicKey;
 
-  /*
-    Return a store path with a closure containing everything including
-    derivations and all build dependency outputs, all the way down.
-  */
-  allDrvOutputs = pkg:
-    let name = "allDrvOutputs-${pkg.pname or pkg.name or "unknown"}";
-    in
-    pkgs.runCommand name { refs = pkgs.writeReferencesToFile pkg.drvPath; } ''
-      touch $out
-      while read ref; do
-        case $ref in
-          *.drv)
-            cat $ref >>$out
-            ;;
-        esac
-      done <$refs
-    '';
-
 in
 tests

pkgs/build-support/docker/default.nix

@@ -29,7 +29,7 @@
 , tarsum
 , util-linux
 , vmTools
-, writeReferencesToFile
+, writeClosure
 , writeScript
 , writeShellScriptBin
 , writeText
@@ -630,7 +630,7 @@ rec {
           imageName = lib.toLower name;
           imageTag = lib.optionalString (tag != null) tag;
           inherit fromImage baseJson;
-          layerClosure = writeReferencesToFile layer;
+          layerClosure = writeClosure [ layer ];
           passthru.buildArgs = args;
           passthru.layer = layer;
           passthru.imageTag =

pkgs/build-support/oci-tools/default.nix

@@ -1,4 +1,4 @@
-{ lib, writeText, runCommand, writeReferencesToFile }:
+{ lib, writeText, runCommand, writeClosure }:
 
 {
   buildContainer =
@@ -72,7 +72,7 @@
       set -o pipefail
       mkdir -p $out/rootfs/{dev,proc,sys}
       cp ${config} $out/config.json
-      xargs tar c < ${writeReferencesToFile args} | tar -xC $out/rootfs/
+      xargs tar c < ${writeClosure args} | tar -xC $out/rootfs/
     '';
 }
 

pkgs/build-support/references-by-popularity/closure-graph.py

@@ -8,8 +8,8 @@
 # and how deep in the tree they live. Equally-"popular" paths are then
 # sorted by name.
 #
-# The existing writeReferencesToFile prints the paths in a simple
-# ascii-based sorting of the paths.
+# The existing writeClosure prints the paths in a simple ascii-based
+# sorting of the paths.
 #
 # Sorting the paths by graph improves the chances that the difference
 # between two builds appear near the end of the list, instead of near

pkgs/build-support/singularity-tools/default.nix

@@ -4,7 +4,7 @@
 , storeDir ? builtins.storeDir
 , writeScript
 , singularity
-, writeReferencesToFile
+, writeClosure
 , bash
 , vmTools
 , gawk
@@ -50,18 +50,13 @@ rec {
     }:
     let
       projectName = singularity.projectName or "singularity";
-      layer = mkLayer {
-        inherit name;
-        contents = contents ++ [ bash runScriptFile ];
-        inherit projectName;
-      };
       runAsRootFile = shellScript "run-as-root.sh" runAsRoot;
       runScriptFile = shellScript "run-script.sh" runScript;
       result = vmTools.runInLinuxVM (
         runCommand "${projectName}-image-${name}.img"
           {
             buildInputs = [ singularity e2fsprogs util-linux gawk ];
-            layerClosure = writeReferencesToFile layer;
+            layerClosure = writeClosure contents;
             preVM = vmTools.createEmptyImage {
               size = diskSize;
               fullName = "${projectName}-run-disk";

pkgs/build-support/trivial-builders/default.nix

@@ -1,4 +1,4 @@
-{ lib, stdenv, stdenvNoCC, lndir, runtimeShell, shellcheck-minimal }:
+{ lib, config, stdenv, stdenvNoCC, jq, lndir, runtimeShell, shellcheck-minimal }:
 
 let
   inherit (lib)
@@ -625,18 +625,22 @@ rec {
 
   # Docs in doc/build-helpers/trivial-build-helpers.chapter.md
   # See https://nixos.org/manual/nixpkgs/unstable/#trivial-builder-writeReferencesToFile
-  writeReferencesToFile = path: runCommand "runtime-deps"
+  # TODO: Convert to throw after Nixpkgs 24.05 branch-off.
+  writeReferencesToFile = (if config.allowAliases then lib.warn else throw)
+    "writeReferencesToFile is deprecated in favour of writeClosure"
+    (path: writeClosure [ path ]);
+
+  # Docs in doc/build-helpers/trivial-build-helpers.chapter.md
+  # See https://nixos.org/manual/nixpkgs/unstable/#trivial-builder-writeClosure
+  writeClosure = paths: runCommand "runtime-deps"
     {
-      exportReferencesGraph = [ "graph" path ];
+      # Get the cleaner exportReferencesGraph interface
+      __structuredAttrs = true;
+      exportReferencesGraph.graph = paths;
+      nativeBuildInputs = [ jq ];
     }
     ''
-      touch $out
-      while read path; do
-        echo $path >> $out
-        read dummy
-        read nrRefs
-        for ((i = 0; i < nrRefs; i++)); do read ref; done
-      done < graph
+      jq -r ".graph | map(.path) | sort | .[]" "$NIX_ATTRS_JSON_FILE" > "$out"
     '';
 
   # Docs in doc/build-helpers/trivial-build-helpers.chapter.md

pkgs/build-support/trivial-builders/test/default.nix

@@ -26,6 +26,9 @@ recurseIntoAttrs {
     then references
     else {};
   writeCBin = callPackage ./writeCBin.nix {};
+  writeClosure-union = callPackage ./writeClosure-union.nix {
+    inherit (references) samples;
+  };
   writeShellApplication = callPackage ./writeShellApplication.nix {};
   writeScriptBin = callPackage ./writeScriptBin.nix {};
   writeShellScript = callPackage ./write-shell-script.nix {};

pkgs/build-support/trivial-builders/test/references/default.nix

@@ -12,7 +12,7 @@
 , cleanSamples ? lib.filterAttrs (n: lib.isStringLike)
   # Test targets
 , writeDirectReferencesToFile
-, writeReferencesToFile
+, writeClosure
 }:
 
 # -------------------------------------------------------------------------- #
@@ -46,8 +46,9 @@ let
   samplesToString = attrs:
     lib.concatMapStringsSep " " (name: "[${name}]=${lib.escapeShellArg "${attrs.${name}}"}") (builtins.attrNames attrs);
 
-  references = lib.mapAttrs (n: v: writeReferencesToFile v) samples;
+  closures = lib.mapAttrs (n: v: writeClosure [ v ]) samples;
   directReferences = lib.mapAttrs (n: v: writeDirectReferencesToFile v) samples;
+  collectiveClosure = writeClosure (lib.attrValues samples);
 
   testScriptBin = stdenvNoCC.mkDerivation (finalAttrs: {
     name = "references-test";
@@ -61,8 +62,9 @@ let
       mkdir -p "$out/bin"
       substitute "$src" "$out/bin/${finalAttrs.meta.mainProgram}" \
         --replace "@SAMPLES@" ${lib.escapeShellArg (samplesToString samples)} \
-        --replace "@REFERENCES@" ${lib.escapeShellArg (samplesToString references)} \
-        --replace "@DIRECT_REFS@" ${lib.escapeShellArg (samplesToString directReferences)}
+        --replace "@CLOSURES@" ${lib.escapeShellArg (samplesToString closures)} \
+        --replace "@DIRECT_REFS@" ${lib.escapeShellArg (samplesToString directReferences)} \
+        --replace "@COLLECTIVE_CLOSURE@" ${lib.escapeShellArg collectiveClosure}
       runHook postInstall
       chmod +x "$out/bin/${finalAttrs.meta.mainProgram}"
     '';
@@ -79,8 +81,9 @@ let
 
     passthru = {
       inherit
+        collectiveClosure
         directReferences
-        references
+        closures
         samples
         ;
     };
@@ -109,8 +112,9 @@ testers.nixosTest {
     '';
   passthru = {
     inherit
+      collectiveClosure
       directReferences
-      references
+      closures
       samples
       testScriptBin
       ;

pkgs/build-support/trivial-builders/test/references/references-test.sh

@@ -33,16 +33,17 @@ set -euo pipefail
 
 cd "$(dirname "${BASH_SOURCE[0]}")"  # nixpkgs root
 
-  # Injected by Nix (to avoid evaluating in a derivation)
-  # turn them into arrays
-  # shellcheck disable=SC2206 # deliberately unquoted
-  declare -A samples=( @SAMPLES@ )
-  # shellcheck disable=SC2206 # deliberately unquoted
-  declare -A directRefs=( @DIRECT_REFS@ )
-  # shellcheck disable=SC2206 # deliberately unquoted
-  declare -A references=( @REFERENCES@ )
+  # Inject the path to compare from the Nix expression
 
-echo >&2 Testing direct references...
+  # Associative Arrays
+  declare -A samples=( @SAMPLES@ )
+  declare -A directRefs=( @DIRECT_REFS@ )
+  declare -A closures=( @CLOSURES@ )
+
+  # Path string
+  collectiveClosure=@COLLECTIVE_CLOSURE@
+
+echo >&2 Testing direct closures...
 for i in "${!samples[@]}"; do
   echo >&2 Checking "$i" "${samples[$i]}" "${directRefs[$i]}"
   diff -U3 \
@@ -52,10 +53,16 @@ done
 
 echo >&2 Testing closure...
 for i in "${!samples[@]}"; do
-  echo >&2 Checking "$i" "${samples[$i]}" "${references[$i]}"
+  echo >&2 Checking "$i" "${samples[$i]}" "${closures[$i]}"
   diff -U3 \
-    <(sort <"${references[$i]}") \
+    <(sort <"${closures[$i]}") \
     <(nix-store -q --requisites "${samples[$i]}" | sort)
 done
 
+echo >&2 Testing mixed closures...
+echo >&2 Checking all samples "(${samples[*]})" "$collectiveClosure"
+diff -U3 \
+  <(sort <"$collectiveClosure") \
+  <(nix-store -q --requisites "${samples[@]}" | sort)
+
 echo 'OK!'

pkgs/build-support/trivial-builders/test/writeClosure-union.nix

@@ -0,0 +1,23 @@
+{ lib
+, runCommandLocal
+  # Test targets
+, writeClosure
+, samples
+}:
+runCommandLocal "test-trivial-builders-writeClosure-union" {
+  __structuredAttrs = true;
+  closures = lib.mapAttrs (n: v: writeClosure [ v ]) samples;
+  collectiveClosure = writeClosure (lib.attrValues samples);
+  inherit samples;
+  meta.maintainers = with lib.maintainers; [
+    ShamrockLee
+  ];
+} ''
+  set -eu -o pipefail
+  echo >&2 Testing mixed closures...
+  echo >&2 Checking all samples "(''${samples[*]})" "$collectiveClosure"
+  diff -U3 \
+    <(sort <"$collectiveClosure") \
+    <(cat "''${closures[@]}" | sort | uniq)
+  touch "$out"
+''

pkgs/top-level/stage.nix

@@ -110,8 +110,9 @@ let
   trivialBuilders = self: super:
     import ../build-support/trivial-builders {
       inherit lib;
+      inherit (self) config;
       inherit (self) runtimeShell stdenv stdenvNoCC;
-      inherit (self.pkgsBuildHost) shellcheck-minimal;
+      inherit (self.pkgsBuildHost) jq shellcheck-minimal;
       inherit (self.pkgsBuildHost.xorg) lndir;
     };