testers.testBuildFailure: init

doc/builders/testers.chapter.md

@@ -35,6 +35,46 @@ passthru.tests.version = testers.testVersion {
 };
 ```
 
+## `testBuildFailure` {#tester-testBuildFailure}
+
+Make sure that a build does not succeed. This is useful for testing testers.
+
+This returns a derivation with an override on the builder, with the following effects:
+
+ - Fail the build when the original builder succeeds
+ - Move `$out` to `$out/result`, if it exists (assuming `out` is the default output)
+ - Save the build log to `$out/testBuildFailure.log` (same)
+
+Example:
+
+```nix
+runCommand "example" {
+  failed = testers.testBuildFailure (runCommand "fail" {} ''
+    echo ok-ish >$out
+    echo failing though
+    exit 3
+  '');
+} ''
+  grep -F 'ok-ish' $failed/result
+  grep -F 'failing though' $failed/testBuildFailure.log
+  [[ 3 = $(cat $failed/testBuildFailure.exit) ]]
+  touch $out
+'';
+```
+
+While `testBuildFailure` is designed to keep changes to the original builder's 
+environment to a minimum, some small changes are inevitable.
+
+ - The file `$TMPDIR/testBuildFailure.log` is present. It should not be deleted.
+ - `stdout` and `stderr` are a pipe instead of a tty. This could be improved.
+ - One or two extra processes are present in the sandbox during the original
+   builder's execution.
+ - The derivation and output hashes are different, but not unusual.
+ - The derivation includes a dependency on `buildPackages.bash` and
+   `expect-failure.sh`, which is built to include a transitive dependency on
+   `buildPackages.coreutils` and possibly more. These are not added to `PATH`
+   or any other environment variable, so they should be hard to observe.
+
 ## `testEqualDerivation` {#tester-testEqualDerivation}
 
 Checks that two packages produce the exact same build instructions.

pkgs/build-support/testers/default.nix

@@ -1,6 +1,16 @@
-{ pkgs, lib, callPackage, runCommand, stdenv }:
+{ pkgs, buildPackages, lib, callPackage, runCommand, stdenv, substituteAll, }:
 # Documentation is in doc/builders/testers.chapter.md
 {
+  # See https://nixos.org/manual/nixpkgs/unstable/#tester-testBuildFailure
+  # or doc/builders/testers.chapter.md
+  testBuildFailure = drv: drv.overrideAttrs (orig: {
+    builder = buildPackages.bash;
+    args = [
+      (substituteAll { coreutils = buildPackages.coreutils; src = ./expect-failure.sh; })
+      orig.realBuilder or stdenv.shell
+    ] ++ orig.args or ["-e" (orig.builder or ../../stdenv/generic/default-builder.sh)];
+  });
+
   testEqualDerivation = callPackage ./test-equal-derivation.nix { };
 
   testVersion =

pkgs/build-support/testers/expect-failure.sh

@@ -0,0 +1,62 @@
+# Run a builder, flip exit code, save log and fix outputs
+#
+# Sub-goals:
+# - Delegate to another original builder passed via args
+# - Save the build log to output for further checks
+# - Make the derivation succeed if the original builder fails
+# - Make the derivation fail if the original builder returns exit code 0
+#
+# Requirements:
+# This runs before, without and after stdenv. Do not modify the environment;
+# especially not before invoking the original builder. For example, use
+# "@" substitutions instead of PATH.
+# Do not export any variables.
+
+# Stricter bash
+set -eu
+
+# ------------------------
+# Run the original builder
+
+echo "testBuildFailure: Expecting non-zero exit from builder and args: ${*@Q}"
+
+("$@" 2>&1) | @coreutils@/bin/tee $TMPDIR/testBuildFailure.log \
+  | while read ln; do
+    echo "original builder: $ln"
+  done
+
+r=${PIPESTATUS[0]}
+if [[ $r = 0 ]]; then
+  echo "testBuildFailure: The builder did not fail, but a failure was expected!"
+  exit 1
+fi
+echo "testBuildFailure: Original builder produced exit code: $r"
+
+# -----------------------------------------
+# Write the build log to the default output
+
+outs=( $outputs )
+defOut=${outs[0]}
+defOutPath=${!defOut}
+
+if [[ ! -d $defOutPath ]]; then
+  if [[ -e $defOutPath ]]; then
+    @coreutils@/bin/mv $defOutPath $TMPDIR/out-node
+    @coreutils@/bin/mkdir $defOutPath
+    @coreutils@/bin/mv $TMPDIR/out-node $defOutPath/result
+  fi
+fi
+
+@coreutils@/bin/mkdir -p $defOutPath
+@coreutils@/bin/mv $TMPDIR/testBuildFailure.log $defOutPath/testBuildFailure.log
+echo $r >$defOutPath/testBuildFailure.exit
+
+# ------------------------------------------------------
+# Put empty directories in place for any missing outputs
+
+for outputName in ${outputs:-out}; do
+  outputPath="${!outputName}"
+  if [[ ! -e "${outputPath}" ]]; then
+    @coreutils@/bin/mkdir "${outputPath}";
+  fi
+done

pkgs/build-support/testers/test/default.nix

@@ -1,4 +1,4 @@
-{ testers, lib, pkgs, ... }:
+{ testers, lib, pkgs, hello, runCommand, ... }:
 let
   pkgs-with-overlay = pkgs.extend(final: prev: {
     proof-of-overlay-hello = prev.hello;
@@ -24,4 +24,53 @@ lib.recurseIntoAttrs {
       machine.succeed("hello | figlet >/dev/console")
     '';
   });
+
+  testBuildFailure = lib.recurseIntoAttrs {
+    happy = runCommand "testBuildFailure-happy" {
+      failed = testers.testBuildFailure (runCommand "fail" {} ''
+        echo ok-ish >$out
+        echo failing though
+        echo also stderr 1>&2
+        exit 3
+      '');
+    } ''
+      grep -F 'failing though' $failed/testBuildFailure.log
+      grep -F 'also stderr' $failed/testBuildFailure.log
+      grep -F 'ok-ish' $failed/result
+      [[ 3 = $(cat $failed/testBuildFailure.exit) ]]
+      touch $out
+    '';
+
+    helloDoesNotFail = runCommand "testBuildFailure-helloDoesNotFail" {
+      failed = testers.testBuildFailure (testers.testBuildFailure hello);
+
+      # Add hello itself as a prerequisite, so we don't try to run this test if
+      # there's an actual failure in hello.
+      inherit hello;
+    } ''
+      echo "Checking $failed/testBuildFailure.log"
+      grep -F 'testBuildFailure: The builder did not fail, but a failure was expected' $failed/testBuildFailure.log
+      [[ 1 = $(cat $failed/testBuildFailure.exit) ]]
+      touch $out
+    '';
+
+    multiOutput = runCommand "testBuildFailure-multiOutput" {
+      failed = testers.testBuildFailure (runCommand "fail" {
+        # dev will be the default output
+        outputs = ["dev" "doc" "out"];
+      } ''
+        echo i am failing
+        exit 1
+      '');
+    } ''
+      grep -F 'i am failing' $failed/testBuildFailure.log >/dev/null
+      [[ 1 = $(cat $failed/testBuildFailure.exit) ]]
+
+      # Checking our note that dev is the default output
+      echo $failed/_ | grep -- '-dev/_' >/dev/null
+
+      touch $out
+    '';
+  };
+
 }