From e3c55dd0d3c32f77b6fdb77c4b9e5564b565523b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 24 May 2024 16:16:38 +0200 Subject: [PATCH 1/7] Add test for the evaluation cache (cherry picked from commit 8b86f415c1a8565085e70475933e459a29d67283) --- tests/functional/flakes/eval-cache.sh | 22 ++++++++++++++++++++++ tests/functional/local.mk | 1 + 2 files changed, 23 insertions(+) create mode 100644 tests/functional/flakes/eval-cache.sh diff --git a/tests/functional/flakes/eval-cache.sh b/tests/functional/flakes/eval-cache.sh new file mode 100644 index 000000000..90c7abd3c --- /dev/null +++ b/tests/functional/flakes/eval-cache.sh @@ -0,0 +1,22 @@ +source ./common.sh + +requireGit + +flake1Dir="$TEST_ROOT/eval-cache-flake" + +createGitRepo "$flake1Dir" "" + +cat >"$flake1Dir/flake.nix" <&1 | grepQuiet 'error: breaks' +expect 1 nix build "$flake1Dir#foo.bar" 2>&1 | grepQuiet 'error: breaks' diff --git a/tests/functional/local.mk b/tests/functional/local.mk index d60f1e08e..b6933c368 100644 --- a/tests/functional/local.mk +++ b/tests/functional/local.mk @@ -14,6 +14,7 @@ nix_tests = \ flakes/absolute-paths.sh \ flakes/build-paths.sh \ flakes/flake-in-submodule.sh \ + flakes/eval-cache.sh \ gc.sh \ nix-collect-garbage-d.sh \ remote-store.sh \ From 8e5ec819f8027ecedc5173f49c705195efda2a1c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 19 Apr 2024 19:34:07 +0200 Subject: [PATCH 2/7] AttrCursor: Remove forceErrors Instead, force evaluation of the original value only if we need to show the exception to the user. (cherry picked from commit 2c88930ef298f6804eb9c064ca918aef01fcd2e3) --- src/libexpr/eval-cache.cc | 40 +++++++++++++++++++++++++++------------ src/libexpr/eval-cache.hh | 25 +++++++++++++++++++----- src/nix/main.cc | 11 ++++++++++- 3 files changed, 58 insertions(+), 18 deletions(-) diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 2c9aa5532..c058cfb3a 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -6,6 +6,25 @@ namespace nix::eval_cache { +CachedEvalError::CachedEvalError(ref cursor, Symbol attr) + : EvalError(cursor->root->state, "cached failure of attribute '%s'", cursor->getAttrPathStr(attr)) + , cursor(cursor), attr(attr) +{ } + +void CachedEvalError::force() +{ + auto & v = cursor->forceValue(); + + if (v.type() == nAttrs) { + auto a = v.attrs()->get(this->attr); + + state.forceValue(*a->value, a->pos); + } + + // Shouldn't happen. + throw EvalError(state, "evaluation of cached failed attribute '%s' unexpectedly succeeded", cursor->getAttrPathStr(attr)); +} + static const char * schema = R"sql( create table if not exists Attributes ( parent integer not null, @@ -469,7 +488,7 @@ Suggestions AttrCursor::getSuggestionsForAttr(Symbol name) return Suggestions::bestMatches(strAttrNames, root->state.symbols[name]); } -std::shared_ptr AttrCursor::maybeGetAttr(Symbol name, bool forceErrors) +std::shared_ptr AttrCursor::maybeGetAttr(Symbol name) { if (root->db) { if (!cachedValue) @@ -486,12 +505,9 @@ std::shared_ptr AttrCursor::maybeGetAttr(Symbol name, bool forceErro if (attr) { if (std::get_if(&attr->second)) return nullptr; - else if (std::get_if(&attr->second)) { - if (forceErrors) - debug("reevaluating failed cached attribute '%s'", getAttrPathStr(name)); - else - throw CachedEvalError("cached failure of attribute '%s'", getAttrPathStr(name)); - } else + else if (std::get_if(&attr->second)) + throw CachedEvalError(ref(shared_from_this()), name); + else return std::make_shared(root, std::make_pair(shared_from_this(), name), nullptr, std::move(attr)); } @@ -536,9 +552,9 @@ std::shared_ptr AttrCursor::maybeGetAttr(std::string_view name) return maybeGetAttr(root->state.symbols.create(name)); } -ref AttrCursor::getAttr(Symbol name, bool forceErrors) +ref AttrCursor::getAttr(Symbol name) { - auto p = maybeGetAttr(name, forceErrors); + auto p = maybeGetAttr(name); if (!p) throw Error("attribute '%s' does not exist", getAttrPathStr(name)); return ref(p); @@ -549,11 +565,11 @@ ref AttrCursor::getAttr(std::string_view name) return getAttr(root->state.symbols.create(name)); } -OrSuggestions> AttrCursor::findAlongAttrPath(const std::vector & attrPath, bool force) +OrSuggestions> AttrCursor::findAlongAttrPath(const std::vector & attrPath) { auto res = shared_from_this(); for (auto & attr : attrPath) { - auto child = res->maybeGetAttr(attr, force); + auto child = res->maybeGetAttr(attr); if (!child) { auto suggestions = res->getSuggestionsForAttr(attr); return OrSuggestions>::failed(suggestions); @@ -750,7 +766,7 @@ bool AttrCursor::isDerivation() StorePath AttrCursor::forceDerivation() { - auto aDrvPath = getAttr(root->state.sDrvPath, true); + auto aDrvPath = getAttr(root->state.sDrvPath); auto drvPath = root->state.store->parseStorePath(aDrvPath->getString()); if (!root->state.store->isValidPath(drvPath) && !settings.readOnlyMode) { /* The eval cache contains 'drvPath', but the actual path has diff --git a/src/libexpr/eval-cache.hh b/src/libexpr/eval-cache.hh index 46c4999c8..cac985829 100644 --- a/src/libexpr/eval-cache.hh +++ b/src/libexpr/eval-cache.hh @@ -10,14 +10,28 @@ namespace nix::eval_cache { -MakeError(CachedEvalError, EvalError); - struct AttrDb; class AttrCursor; +struct CachedEvalError : EvalError +{ + const ref cursor; + const Symbol attr; + + CachedEvalError(ref cursor, Symbol attr); + + /** + * Evaluate this attribute, which should result in a regular + * `EvalError` exception being thrown. + */ + [[noreturn]] + void force(); +}; + class EvalCache : public std::enable_shared_from_this { friend class AttrCursor; + friend class CachedEvalError; std::shared_ptr db; EvalState & state; @@ -73,6 +87,7 @@ typedef std::variant< class AttrCursor : public std::enable_shared_from_this { friend class EvalCache; + friend class CachedEvalError; ref root; typedef std::optional, Symbol>> Parent; @@ -102,11 +117,11 @@ public: Suggestions getSuggestionsForAttr(Symbol name); - std::shared_ptr maybeGetAttr(Symbol name, bool forceErrors = false); + std::shared_ptr maybeGetAttr(Symbol name); std::shared_ptr maybeGetAttr(std::string_view name); - ref getAttr(Symbol name, bool forceErrors = false); + ref getAttr(Symbol name); ref getAttr(std::string_view name); @@ -114,7 +129,7 @@ public: * Get an attribute along a chain of attrsets. Note that this does * not auto-call functors or functions. */ - OrSuggestions> findAlongAttrPath(const std::vector & attrPath, bool force = false); + OrSuggestions> findAlongAttrPath(const std::vector & attrPath); std::string getString(); diff --git a/src/nix/main.cc b/src/nix/main.cc index eb510eecc..079518d4a 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -12,6 +12,7 @@ #include "finally.hh" #include "loggers.hh" #include "markdown.hh" +#include "eval-cache.hh" #include #include @@ -480,7 +481,15 @@ void mainWrapped(int argc, char * * argv) if (args.command->second->forceImpureByDefault() && !evalSettings.pureEval.overridden) { evalSettings.pureEval = false; } - args.command->second->run(); + + try { + args.command->second->run(); + } catch (eval_cache::CachedEvalError & e) { + /* Evaluate the original attribute that resulted in this + cached error so that we can show the original error to the + user. */ + e.force(); + } } } From fc141979355c9b9f50f7719f0f14c5587123de24 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 19 Apr 2024 19:42:05 +0200 Subject: [PATCH 3/7] Typo (cherry picked from commit eeb4c408670a6037bd4085fde654bf4a5945c4c7) --- src/nix/flake.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 87dd4da1b..7dad728f1 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1110,7 +1110,7 @@ struct CmdFlakeShow : FlakeCommand, MixJSON // If we don't recognize it, it's probably content return true; } catch (EvalError & e) { - // Some attrs may contain errors, eg. legacyPackages of + // Some attrs may contain errors, e.g. legacyPackages of // nixpkgs. We still want to recurse into it, instead of // skipping it at all. return true; From ef63ba10a27a2b9a04c321d395939ba327107695 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=A9clairevoyant?= <848000+eclairevoyant@users.noreply.github.com> Date: Sun, 8 Sep 2024 11:55:34 -0400 Subject: [PATCH 4/7] fix --- src/libexpr/eval-cache.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index c058cfb3a..210f43f4d 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -7,7 +7,7 @@ namespace nix::eval_cache { CachedEvalError::CachedEvalError(ref cursor, Symbol attr) - : EvalError(cursor->root->state, "cached failure of attribute '%s'", cursor->getAttrPathStr(attr)) + : EvalError("cached failure of attribute '%s'", cursor->getAttrPathStr(attr)) , cursor(cursor), attr(attr) { } @@ -16,13 +16,13 @@ void CachedEvalError::force() auto & v = cursor->forceValue(); if (v.type() == nAttrs) { - auto a = v.attrs()->get(this->attr); + auto a = v.attrs->get(this->attr); state.forceValue(*a->value, a->pos); } // Shouldn't happen. - throw EvalError(state, "evaluation of cached failed attribute '%s' unexpectedly succeeded", cursor->getAttrPathStr(attr)); + throw EvalError("evaluation of cached failed attribute '%s' unexpectedly succeeded", cursor->getAttrPathStr(attr)); } static const char * schema = R"sql( From 5bcc7069bec00ee86669f98961dbe2a319d6f6d9 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 9 Sep 2024 00:38:28 +0200 Subject: [PATCH 5/7] Store ref in CachedEvalError This makes the previous commits work. Newer versions store it in all EvalErrors. --- src/libexpr/eval-cache.cc | 8 ++++---- src/libexpr/eval-cache.hh | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 210f43f4d..8705a5851 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -6,9 +6,9 @@ namespace nix::eval_cache { -CachedEvalError::CachedEvalError(ref cursor, Symbol attr) +CachedEvalError::CachedEvalError(ref state, ref cursor, Symbol attr) : EvalError("cached failure of attribute '%s'", cursor->getAttrPathStr(attr)) - , cursor(cursor), attr(attr) + , state(state), cursor(cursor), attr(attr) { } void CachedEvalError::force() @@ -18,7 +18,7 @@ void CachedEvalError::force() if (v.type() == nAttrs) { auto a = v.attrs->get(this->attr); - state.forceValue(*a->value, a->pos); + state->forceValue(*a->value, a->pos); } // Shouldn't happen. @@ -506,7 +506,7 @@ std::shared_ptr AttrCursor::maybeGetAttr(Symbol name) if (std::get_if(&attr->second)) return nullptr; else if (std::get_if(&attr->second)) - throw CachedEvalError(ref(shared_from_this()), name); + throw CachedEvalError(ref(root->state.shared_from_this()), ref(shared_from_this()), name); else return std::make_shared(root, std::make_pair(shared_from_this(), name), nullptr, std::move(attr)); diff --git a/src/libexpr/eval-cache.hh b/src/libexpr/eval-cache.hh index cac985829..d025872fa 100644 --- a/src/libexpr/eval-cache.hh +++ b/src/libexpr/eval-cache.hh @@ -15,10 +15,11 @@ class AttrCursor; struct CachedEvalError : EvalError { + const ref state; const ref cursor; const Symbol attr; - CachedEvalError(ref cursor, Symbol attr); + CachedEvalError(ref, ref cursor, Symbol attr); /** * Evaluate this attribute, which should result in a regular From 1b0805d4510c986e815a2f453b8d7da685a2994c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 9 Sep 2024 17:42:20 +0200 Subject: [PATCH 6/7] installerScriptForGHA: aarch64-darwin Backport of https://github.com/NixOS/nix/pull/11009 --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 230bb6031..02a37d748 100644 --- a/flake.nix +++ b/flake.nix @@ -550,7 +550,7 @@ # tarball for the user's system and calls the second half of the # installation script. installerScript = installScriptFor [ "x86_64-linux" "i686-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" "armv6l-linux" "armv7l-linux" ]; - installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" "armv6l-linux" "armv7l-linux"]; + installerScriptForGHA = installScriptFor [ "x86_64-linux" "aarch64-darwin" "armv6l-linux" "armv7l-linux"]; # docker image with Nix inside dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); From c42e3e5b33a6caa55ad8c7d893d699ac686cf87e Mon Sep 17 00:00:00 2001 From: Mel Zuser Date: Mon, 18 Dec 2023 15:02:26 -0800 Subject: [PATCH 7/7] installer: allow overriding of NIX_FIRST_BUILD_ID on darwin because there are often already users in the 300 range and it's painful to work around. revives #6466 (cherry picked from commit fa4bbe53e837a138c382468601cd769736f7d1dc) --- scripts/install-darwin-multi-user.sh | 6 ++++-- scripts/install-multi-user.sh | 19 +++++++++++++++---- scripts/install-systemd-multi-user.sh | 4 ++++ 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh index 0326d3415..766f81bde 100644 --- a/scripts/install-darwin-multi-user.sh +++ b/scripts/install-darwin-multi-user.sh @@ -3,11 +3,13 @@ set -eu set -o pipefail +# System specific settings +export NIX_FIRST_BUILD_UID="${NIX_FIRST_BUILD_UID:-301}" +export NIX_BUILD_USER_NAME_TEMPLATE="_nixbld%d" + readonly NIX_DAEMON_DEST=/Library/LaunchDaemons/org.nixos.nix-daemon.plist # create by default; set 0 to DIY, use a symlink, etc. readonly NIX_VOLUME_CREATE=${NIX_VOLUME_CREATE:-1} # now default -NIX_FIRST_BUILD_UID="301" -NIX_BUILD_USER_NAME_TEMPLATE="_nixbld%d" # caution: may update times on / if not run as normal non-root user read_only_root() { diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 656769d84..d846f5b5c 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -25,9 +25,9 @@ readonly RED='\033[31m' readonly NIX_USER_COUNT=${NIX_USER_COUNT:-32} readonly NIX_BUILD_GROUP_ID="${NIX_BUILD_GROUP_ID:-30000}" readonly NIX_BUILD_GROUP_NAME="nixbld" -# darwin installer needs to override these -NIX_FIRST_BUILD_UID="${NIX_FIRST_BUILD_UID:-30001}" -NIX_BUILD_USER_NAME_TEMPLATE="nixbld%d" +# each system specific installer must set these: +# NIX_FIRST_BUILD_UID +# NIX_BUILD_USER_NAME_TEMPLATE # Please don't change this. We don't support it, because the # default shell profile that comes with Nix doesn't support it. readonly NIX_ROOT="/nix" @@ -699,6 +699,12 @@ EOF fi } +check_required_system_specific_settings() { + if [ -z "${NIX_FIRST_BUILD_UID+x}" ] || [ -z "${NIX_BUILD_USER_NAME_TEMPLATE+x}" ]; then + failure "Internal error: System specific installer for $(uname) ($1) does not export required settings." + fi +} + welcome_to_nix() { local -r NIX_UID_RANGES="${NIX_FIRST_BUILD_UID}..$((NIX_FIRST_BUILD_UID + NIX_USER_COUNT - 1))" local -r RANGE_TEXT=$(echo -ne "${BLUE}(uids [${NIX_UID_RANGES}])${ESC}") @@ -718,7 +724,9 @@ manager. This will happen in a few stages: if you are ready to continue. 3. Create the system users ${RANGE_TEXT} and groups ${GROUP_TEXT} - that the Nix daemon uses to run builds. + that the Nix daemon uses to run builds. To create system users + in a different range, exit and run this tool again with + NIX_FIRST_BUILD_UID set. 4. Perform the basic installation of the Nix files daemon. @@ -960,13 +968,16 @@ main() { if is_os_darwin; then # shellcheck source=./install-darwin-multi-user.sh . "$EXTRACTED_NIX_PATH/install-darwin-multi-user.sh" + check_required_system_specific_settings "install-darwin-multi-user.sh" elif is_os_linux; then # shellcheck source=./install-systemd-multi-user.sh . "$EXTRACTED_NIX_PATH/install-systemd-multi-user.sh" # most of this works on non-systemd distros also + check_required_system_specific_settings "install-systemd-multi-user.sh" else failure "Sorry, I don't know what to do on $(uname)" fi + welcome_to_nix if ! is_root; then diff --git a/scripts/install-systemd-multi-user.sh b/scripts/install-systemd-multi-user.sh index 07b34033a..202a9bb54 100755 --- a/scripts/install-systemd-multi-user.sh +++ b/scripts/install-systemd-multi-user.sh @@ -3,6 +3,10 @@ set -eu set -o pipefail +# System specific settings +export NIX_FIRST_BUILD_UID="${NIX_FIRST_BUILD_UID:-30001}" +export NIX_BUILD_USER_NAME_TEMPLATE="nixbld%d" + readonly SERVICE_SRC=/lib/systemd/system/nix-daemon.service readonly SERVICE_DEST=/etc/systemd/system/nix-daemon.service