From cee75fbf644abae923b8893a033c43f6d2d4736d Mon Sep 17 00:00:00 2001
From: "Shahar \"Dawn\" Or" <mightyiampresence@gmail.com>
Date: Fri, 21 Mar 2025 17:34:11 +0000
Subject: [PATCH] Add test case for catching caching of transient flake
 evaluation error

Co-authored-by: Robert Hensing <robert@roberthensing.nl>
---
 src/libexpr/primops.cc                | 20 ++++++++++++++++
 tests/functional/flakes/eval-cache.sh | 34 +++++++++++++++++++++++++++
 2 files changed, 54 insertions(+)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 54682ea31..499cc6ce8 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -4653,6 +4653,16 @@ static RegisterPrimOp primop_splitVersion({
     .fun = prim_splitVersion,
 });
 
+static void prim_testThrowErrorMaybe(EvalState & state, const PosIdx pos, Value * * args, Value & v)
+{
+    if (getEnv("_NIX_TEST_DO_THROW_ERROR") == "1") {
+        state.error<EvalError>("this is a dummy error").atPos(pos).debugThrow();
+    } else {
+        state.forceValue(*args[0], pos);
+        v = *args[0];
+    }
+}
+
 
 /*************************************************************
  * Primop registration
@@ -4880,6 +4890,16 @@ void EvalState::createBaseEnv()
         .fun = settings.traceVerbose ? prim_trace : prim_second,
     });
 
+    if (getEnv("_NIX_TEST_PRIMOPS") == "1") {
+        addPrimOp({
+            .name = "__testThrowErrorMaybe",
+            .args = { "e" },
+            .arity = 1,
+            .doc = "throw dummy error if _NIX_TEST_DO_THROW_ERROR == 1, return arg otherwise",
+            .fun = prim_testThrowErrorMaybe
+        });
+    }
+
     /* Add a value containing the current Nix expression search path. */
     auto list = buildList(lookupPath.elements.size());
     for (const auto & [n, i] : enumerate(lookupPath.elements)) {
diff --git a/tests/functional/flakes/eval-cache.sh b/tests/functional/flakes/eval-cache.sh
index 75a2c8cac..a007393b4 100755
--- a/tests/functional/flakes/eval-cache.sh
+++ b/tests/functional/flakes/eval-cache.sh
@@ -28,6 +28,8 @@ cat >"$flake1Dir/flake.nix" <<EOF
       in
         assert (f 100); self.drv;
     ifd = assert (import self.drv); self.drv;
+    artificialTwoDifferentThrows = builtins.testThrowErrorMaybe (throw "real throw");
+    artificialThrowOrDrv = builtins.testThrowErrorMaybe self.drv;
   };
 }
 EOF
@@ -48,3 +50,35 @@ nix build --no-link "$flake1Dir#stack-depth"
 expect 1 nix build "$flake1Dir#ifd" --option allow-import-from-derivation false 2>&1 \
   | grepQuiet 'error: cannot build .* during evaluation because the option '\''allow-import-from-derivation'\'' is disabled'
 nix build --no-link "$flake1Dir#ifd"
+
+# make sure this builtin does not reach production
+nix eval --expr 'assert ! (builtins ? testThrowErrorMaybe); null'
+
+# and that it does reach the testing environment
+export _NIX_TEST_PRIMOPS=1
+nix eval --expr 'assert (builtins.testThrowErrorMaybe true); null'
+
+# and that it works
+_NIX_TEST_DO_THROW_ERROR=1 expect 1 nix eval --expr 'builtins.testThrowErrorMaybe true' 2>&1 \
+  | grepQuiet 'this is a dummy error'
+
+
+# make sure error details do not get cached
+_NIX_TEST_DO_THROW_ERROR=1 expect 1 nix eval "$flake1Dir#artificialTwoDifferentThrows" 2>&1 \
+  | grepQuiet "this is a dummy error"
+expect 1 nix eval "$flake1Dir#artificialTwoDifferentThrows" 2>&1 \
+  | grepQuiet "real throw"
+
+# If a cached error cannot be reproduced, do not continue as usual.
+# Instead, throw an error.
+#
+# To be clear, the following test case should never occur in production.
+# But it might, due to an implementation error in Nix or plugins.
+
+# create an artificial exception cache entry
+_NIX_TEST_DO_THROW_ERROR=1 expect 1 nix build "$flake1Dir#artificialThrowOrDrv" 2>&1 \
+  | grepQuiet "this is a dummy error"
+
+# Invoke again but without the artificial throwing of an exception
+expect 1 nix build "$flake1Dir#artificialThrowOrDrv" 2>&1 \
+  | grepQuiet "unexpected evaluation success despite the existence of a cached error. This should not happen. It is a bug in the implementation of Nix or a plugin. A transient exception should not have been cached."